fileminer 1.0.2 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bc8657cdddb39cdeb63ba485063c29e8b743f1ab333b440b830eeb0b1ab40ffb
-  data.tar.gz: 429b76ea2aaa66be2b13d25314f3d85f68b0fdb6c8acea79d8dd6468ab357d92
+  metadata.gz: 721d984d3d2e8c905f0bf86d52b2d89fb0adbb70c651ddd7d48cac16720d3584
+  data.tar.gz: 12015a1aba0b44df10b4033ff476a4d7818d3db250e5bbbe9b0a9c4d1afc4c3b
 SHA512:
-  metadata.gz: 941055a2163c105d3cd58aaeb0c6fdf83df66a2a11f7001ae7ef5639a4b0edbc62271335bdcf43283f03e198a8bd86e5a342ff10dfe626420fa254c23c630370
-  data.tar.gz: 3b684114ebd59fcd11684ec1b132a7e535d96df47e498fad0a9cc70ef1126687f51081cd922a19021fba60ab7ae283a95736c2416df9b87cc24f24c1405aaa81
+  metadata.gz: b0c9305e8e2ca63f13812593cdd1d9697de6fd51d9e633a5249accd704dbbc487924e2a36b2d2ff40981d7a271d79325a2a607b4618d1b82ca56693d7f9dad7a
+  data.tar.gz: 017dddf1edc9b4859124504e05bb1d2ed46b6d52c9bec053e56b838e9ded56027d6cbeedd768578ebfa9249c6e716f2b872157f77ad0cf06073f95b471cb2510

data/README.md

@@ -1,2 +1,183 @@
 # fileminer
-A simple file/log transfer tool coding by ruby.
+[![Gem Version](https://d25lcipzij17d.cloudfront.net/badge.svg?id=rb&type=6&v=1.1.0&x2=0)](https://rubygems.org/gems/fileminer)
+[![MIT licensed](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/fmjsjx/fileminer/blob/master/LICENSE)
+
+
+A simple line based file/log transfer tool coding by ruby.
+
+# quick start
+1. installation
+
+Install fileminer from RubyGems:
+```
+$ gem install fileminer
+Fetching fileminer-1.1.0.gem
+Successfully installed fileminer-1.1.0
+Parsing documentation for fileminer-1.1.0
+Installing ri documentation for fileminer-1.1.0
+Done installing documentation for fileminer after 0 seconds
+1 gem installed
+$ _
+```
+
+2. setting up fileminer
+
+* generate configuration file
+
+At first, we should generate the fileminer configuration file.
+
+We provide a command tool 'fileminer-genconf' to generate configurations:
+```
+$ fileminer-genconf -h
+Usage: fileminer-genconf [options]
+
+Options:
+    -t, --type fileminer|supervisor  Type of the config file to be generated
+                                     Default is fileminer
+    -o, --out [path]                 Output content to a file
+                                     For type fileminer, default is /etc/fileminer/fileminer.yml
+                                     For type supervisor, default is ./fileminer.ini
+    -h, --help                       Print help
+$ _
+```
+
+Generation with default options:
+```
+$ fileminer-genconf -o
+generated config file: /etc/fileminer/fileminer.yml
+$ _
+```
+
+* config input on fileminer.yml
+
+Edit fileminer.inputs on fileminer.yml:
+```yaml
+fileminer.inputs:
+  paths:
+    - /path/to/*.log
+```
+
+* configure output on fileminer.yml
+
+In current version, fileminer provides three prefab output plugins: redis, kafka & mysql. One process can only choose one of them.
+
+**Output to redis:**
+
+Redis output plugin now using LPUSH to send messages for each line.
+
+Install redis client, fileminer use hiredis by default:
+```
+$ gem install redis hiredis
+Fetching redis-4.1.0.gem
+Successfully installed redis-4.1.0
+Parsing documentation for redis-4.1.0
+Installing ri documentation for redis-4.1.0
+Done installing documentation for redis after 0 seconds
+Fetching hiredis-0.6.3.gem
+Building native extensions. This could take a while...
+Successfully installed hiredis-0.6.3
+Parsing documentation for hiredis-0.6.3
+Installing ri documentation for hiredis-0.6.3
+Done installing documentation for hiredis after 0 seconds
+2 gems installed
+$ _
+```
+
+Edit fileminer.yml:
+```yaml
+output.redis:
+  # target redis server URI
+  uri: redis://localhost:6379/0
+  # target redis key, type must be LIST
+  key: fileminer
+```
+
+
+**Output to kafka:**
+
+Install kafka ruby client:
+```
+$ gem install ruby-kafka
+Successfully installed ruby-kafka-0.7.5
+Parsing documentation for ruby-kafka-0.7.5
+Done installing documentation for ruby-kafka after 0 seconds
+1 gem installed
+$ _
+```
+
+Edit fileminer.yml
+```yaml
+output.kafka:
+  brokers: ['host1:9092','host2:9092','host3:9092']
+  client_id: fileminer
+  topic: fileminer
+```
+
+
+**Output to MySQL:**
+
+Install mysql2:
+```
+$ gem install mysql2
+Building native extensions. This could take a while...
+Successfully installed mysql2-0.5.2
+Parsing documentation for mysql2-0.5.2
+Done installing documentation for mysql2 after 0 seconds
+1 gem installed
+$ _
+```
+
+Edit fileminer.yml
+```yaml
+output.mysql:
+  host: hostname
+  port: 3306
+  username: someuser
+  password: somepwd
+  database: somedb
+  table: sometable
+```
+
+
+*Since 1.1.0, you can also use customized output plugin.*
+
+**Output using customized output plugin:**
+
+Get your script ready:
+```ruby
+require 'fileminer/plugins'
+
+class YourPlugin < Output::OutputPlugin
+  def initialize options
+    ...
+  end
+  ...
+  public
+  def send_all(lines, &listener)
+    ...
+  end
+  ...
+end
+```
+
+Edit fileminer.yml
+```yaml
+output.script:
+  script: /path/to/your_script.rb
+  plugin_class: YourPlugin
+  init_options:
+    some_field: some value
+```
+
+3. runnning fileminer
+
+Run in command line:
+```
+$ fileminer
+
+```
+or
+```
+$ fileminer /path/to/fileminer.yml
+
+```

data/conf/fileminer_default.yml

@@ -139,3 +139,15 @@ output.redis:
 
   # table name
   #table: sometable
+
+# using customized ruby script
+#output.script:
+  # customized ruby script path
+  #script: /path/to/script.rb
+
+  # class name of the custom plugin extends Output::OutputPlugin
+  #plugin_class: Output::CustomPlugin
+
+  # options to be input when initialize the custom plugin
+  #init_options:
+    # some fields

data/lib/fileminer/output/mysql.rb

@@ -14,7 +14,6 @@ module Output
       ssl_mode: :disabled
     }
 
-    #
     # Create a mysql output plugin instance
     #
     # @param [Hash] options
@@ -37,30 +36,20 @@ module Output
       @encoding = conf[:encoding]
       conf[:ssl_mode] = :disabled if conf[:ssl_mode] != :enabled
       @mysql = Mysql2::Client.new conf
-      @mysql_conf = conf
       create_table_if_not_exists
       @sqls = Hash.new { |hash, key| hash[key] = generate_batch_sql key }
     end
 
     private
     def create_table_if_not_exists
-      mysql_client = get_mysql_client
-      rs = mysql_client.query 'SHOW TABLES'
+      rs = @mysql.query 'SHOW TABLES'
       tables = rs.map { |row| row.values[0] }
       unless tables.include? @table
         sql = create_table_sql
-        mysql_client.query sql
+        @mysql.query sql
       end
     end
 
-    def get_mysql_client
-      mysql_client = @mysql
-      if mysql_client.closed?
-        @mysql = mysql_client = Mysql2::Client.new @mysql_conf
-      end
-      mysql_client
-    end
-
     def create_table_sql
       <<-EOS
         CREATE TABLE `#@table` (
@@ -79,7 +68,14 @@ module Output
       "INSERT IGNORE INTO `#@table`(`host`,`path`,`pos`,`end`,`data`) VALUES " << (['(?,?,?,?,?)'] * size).join(',')
     end
 
-    #
+    def get_batch_sql(size)
+      if @sqls.key? size
+        @sqls[size]
+      else
+        @sqls[size] = generate_batch_sql size
+      end
+    end
+
     # Send all lines to mysql
     #
     # @param [Array] lines
@@ -87,12 +83,18 @@ module Output
     public
     def send_all(lines, &listener)
       values = lines.flat_map { |line| [line[:host], line[:path], line[:pos], line[:end], line[:data]] }
-      sql = @sqls[lines.size]
-      mysql_client = get_mysql_client
-      stmt = @mysql_client.prepare sql
-      stmt.execute *values
-      stmt.close
-      listener.call
+      sql = get_batch_sql lines.size
+      @mysql.query 'BEGIN'
+      begin
+        stat = @mysql.prepare sql
+        stat.execute *values
+        stat.close
+        @mysql.query 'COMMIT'
+        listener.call
+      rescue => err
+        @mysql.query 'ROLLBACK'
+        raise err
+      end
     end
 
   end

data/lib/fileminer/version.rb

@@ -1,5 +1,5 @@
 class FileMiner
 
-  VERSION = '1.0.2'
+  VERSION = '1.1.0'
 
 end

data/lib/fileminer.rb

@@ -13,7 +13,7 @@ class FileMiner
     refresh_files_time_trigger: '30s',
     max_time_of_each_mining: '5s',
     max_lines_of_each_mining: -1,
-    max_lines_of_each_file: -1, 
+    max_lines_of_each_file: -1,
   }
 
   attr_reader :miner, :output, :running
@@ -103,6 +103,9 @@ class FileMiner
     when conf.key?('output.mysql')
       mysql_conf = conf['output.mysql'].keys_to_sym
       init_output_mysql mysql_conf
+    when conf.key?('output.script')
+      script_conf = conf['output.script'].keys_to_sym
+      init_output_script script_conf
     else
       raise 'Missing config for output'
     end
@@ -136,6 +139,15 @@ class FileMiner
     Output::MysqlPlugin.new mysql_conf
   end
 
+  def init_output_script(script_conf)
+    script_path = script_conf[:script]
+    plugin_class_name = script_conf[:plugin_class]
+    init_options = script_conf[:init_options] || {}
+    require script_path
+    plugin_class = Object.const_get plugin_class_name
+    plugin_class.new init_options.keys_to_sym
+  end
+
   def send_lines(record, lines)
     if @output.batch?
       @output.send_all lines do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fileminer
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.1.0
 platform: ruby
 authors:
 - Fang MinJie
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-06-24 00:00:00.000000000 Z
+date: 2019-06-20 00:00:00.000000000 Z
 dependencies: []
 description: A simple file/log transfer tool coding by ruby.
 email:
@@ -54,7 +54,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.4
+rubygems_version: 3.0.1
 signing_key: 
 specification_version: 4
 summary: A simple file/log transfer tool coding by ruby.