fileminer 1.0.0RC1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 7237a574207f6cb4913c5be6f2ef70dda7c735ffbf786fba9bfaf3941366fd88
+  data.tar.gz: 9e2a23b7b2a49fb78af7ae1ba969f562df855196b5d1aa066b84063e11611afe
+SHA512:
+  metadata.gz: a47639237446113e7dcbf9dd5f79b7008ea102cb3993597743555714fe1ff92e6bb268deee20b8e0ef63c15618e4d745191e6a116333105a743f8ab0da5b32d9
+  data.tar.gz: e60806ee50da1f42d619c9fd01febb239790cb1ebc05938fa7d8634ec52e4da4d17ac3e3d316b8e2f184d45676cfc4a7bea8079e49e5cf5f215e33c9dcfd479a

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018 fmjsjx
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,2 @@
+# fileminer
+A simple file/log transfer tool coding by ruby.

data/bin/fileminer

@@ -0,0 +1,31 @@
+#!/usr/bin/env ruby
+
+
+require 'logger'
+require 'yaml'
+require 'fileminer'
+
+
+logger = Logger.new STDERR
+
+begin
+  # Usage:
+  #   ruby -Ilib ./bin/fileminer /path/to/fileminer.yml
+  yml = File.open(ARGV[0]) { |io| io.read }
+  conf = YAML.load yml
+  # create fileminer instance
+  fileminer = FileMiner.new conf
+
+  # trap INT signal to stop mining
+  trap(:INT) { fileminer.stop_mining }
+
+  # start mining
+  fileminer.start_mining
+rescue => e
+  logger.error e
+  usage = <<-EOS
+  Usage:
+    ruby -Ilib ./bin/fileminer /path/to/fileminer.yml
+  EOS
+  puts usage
+end

data/conf/fileminer_default.yml

@@ -0,0 +1,146 @@
+# general settings
+#fileminer.settings:
+
+  # default value is 30s
+  #refresh_files_time_trigger: 30s
+
+  # default value is 5s
+  #max_time_of_each_mining: 5s
+
+  # default value is -1
+  #max_lines_of_each_mining: -1
+
+  # default value is -1
+  #max_lines_of_each_file: -1
+
+  # TODO
+
+# fileminer inputs
+fileminer.inputs:
+
+  # setup the path of the registry file
+  # default value is /var/lib/fileminer/registry
+  #registry_path: /var/lib/fileminer/registry
+
+  # file paths
+  paths:
+    - /path/to/*.log
+
+    # other path
+    #- /other/path/*.log
+
+  # EOF seconds
+  # default value is 86400(1 day)
+  #eof_seconds: 86400
+
+  # Batch lines to read
+  # default value is 200
+  #batch_lines: 200
+
+  # Host
+  # default value is `Socket.gethostname`
+  #host: 127.0.0.1
+
+# -- output --
+
+# output to redis
+# based on redis-rb, please make sure that redis has already been installed
+# install redis: gem install redis
+output.redis:
+  # the URI of the redis server
+  uri: redis://localhost:6379/0
+
+  # host of the redis server
+  # effective only if uri is not be set
+  # default value is localhost
+  #host: localhost
+
+  # port of the redis server
+  # default value is 6379
+  # effective only if uri is not be set
+  #port: 6379
+
+  # db of the redis server
+  # default value is 0
+  # effective only if uri is not be set
+  #db: 0
+
+  # password of the redis server
+  # effective only if uri is not be set
+  #password:
+
+  # key of the redis server
+  # required
+  key: fileminer
+
+# output to kafka
+# based on ruby-kafka, please make sure that ruby-kafka has already been installed
+# install ruby-kafka: gem install ruby-kafka
+#output.kafka:
+  # kafka brokers
+  # default value is ['localhost:9092']
+  #brokers: ['localhost:9092']
+
+  # producer client_id
+  # default value is fileminer
+  #client_id: fileminer
+
+  # kafka topic
+  # default value is fileminer
+  #topic: fileminer
+
+  # producer mode
+  # sync or async
+  # default is sync
+  #mode: sync
+
+  # auto delivery
+  # enabled or disabled
+  # default value is disabled
+  # effective only if mode is async
+  # when effective, at least one of delivery_threshold and delivery_interval should be set
+  #auto_delivery: disabled
+
+  # delivery threshold, trigger a delivery once x messages have been buffered
+  # effective only if mode is async and auto_deliver is enabled
+  #delivery_threshold: 100
+
+  # delivery interval, trigger a delivery every x seconds
+  # effective only if mode is async and auto_deliver is enabled
+  #delivery_interval: 30
+
+# output to mysql
+# based on mysql2, please make sure that mysql2 has already been installed
+# install ruby-kafka: gem install mysql2
+#output.mysql:
+  # host
+  # default value is localhost
+  #host: localhost
+
+  # port
+  # default value is 3306
+  #port: 3306
+
+  # username
+  #username: someuser
+
+  # password
+  #password: somepwd
+
+  # database name
+  #database: somedb
+
+  # encoding
+  # default value is utf8mb4
+  #encoding: utf8mb4
+
+  # SSL mode
+  # enabled or disabled
+  # default value is disabled
+  #ssl_mode: disabled
+
+  # table name
+  #table: sometable
+
+# other outputs
+# TODO

data/lib/fileminer/miner.rb

@@ -0,0 +1,128 @@
+require 'set'
+require 'json'
+
+
+class Dir
+
+  class << self
+
+    # Creates the directory with the path given, including anynecessary but nonexistent parent directories.
+    #
+    # @param [String] path
+    def mkdirs(path)
+      parent = File.dirname path
+      mkdirs parent unless Dir.exist? parent
+      Dir.mkdir path
+    end
+
+  end
+
+end
+
+
+class Miner
+
+  DEFAULTS = {
+    registry_path: '/var/lib/fileminer/registry',
+    eof_seconds: 86400,
+    batch_lines: 200,
+  }
+
+  attr_reader :registry_path, :paths, :eof_seconds, :batch_lines, :files, :active_files
+
+  # Create a new file miner instance
+  #
+  # @param [Hash] options
+  # @option options [String] :registry_path (/var/lib/fileminer/registry)
+  # @option options [Array] :paths
+  # @option options [Integer] :eof_seconds (86400)
+  # @option options [Integer] :batch_lines (50)
+  # @option options [String] :host (Socket.gethostname)
+  def initialize(options = {})
+    # fix options by DEFAULTS
+    DEFAULTS.each { |k, v| options[k] = v unless options.key? k }
+    @registry_path = options[:registry_path]
+    @paths = options[:paths]
+    @eof_seconds = options[:eof_seconds]
+    @batch_lines = options[:batch_lines]
+    @host = options[:host]
+    if @host.nil?
+      require 'socket'
+      @host = Socket.gethostname
+    end
+    @files = []
+    @active_files = []
+    if File.exist? @registry_path
+      File.open(@registry_path) { |io| @files = JSON.parse(io.read, {symbolize_names: true}) }
+      @active_files = @files.select { |record| !record[:eof] }
+    else
+      parent_dir = File.dirname @registry_path
+      Dir.mkdirs parent_dir unless Dir.exist? parent_dir
+    end
+  end
+
+  # Save registry
+  def save_registry
+    File.open(@registry_path, 'w') { |io| io.write @files.to_json }
+  end
+
+  # Refresh
+  def refresh_files
+    now = Time.now
+    file_paths = Set.new
+    file_paths.merge Dir[*@paths].select { |path| File.file? path }
+    @active_file = @files.select do |record|
+      path = record[:path]
+      file_exists = file_paths.delete? path
+      unless record[:eof]
+        if file_exists
+          # check if EOF
+          if record[:pos] == File.size(path) && now - File.mtime(path) > @eof_seconds
+            record[:eof] = true
+          end
+        else
+          # missing file, set :eof to true
+          record[:eof] = true
+        end
+      end
+      !record[:eof]
+    end
+    file_paths.each do |path|
+      record = {path: path, pos: 0, eof: false}
+      @files << record
+      @active_files << record
+    end
+    @files_refresh_time = now
+  end
+
+  # Read lines
+  def read_lines(record)
+    file_path = record[:path]
+    File.open file_path do |io|
+      lines = []
+      io.pos = record[:pos]
+      while lines.size < @batch_lines
+        line = {host: @host, path: file_path, pos: io.pos}
+        begin
+          data = io.readline
+          break if data.nil?
+          if data[-1] != "\n"
+            io.pos = line[:pos]
+            break
+          end
+        rescue EOFError
+          break
+        end
+        line[:end] = io.pos
+        line[:data] = data
+        lines << line
+      end
+      lines
+    end
+  end
+
+  def files_need_refresh?(refresh_files_time_trigger)
+    Time.now - @files_refresh_time >= refresh_files_time_trigger
+  end
+
+end

data/lib/fileminer/output/kafka.rb

@@ -0,0 +1,62 @@
+require 'kafka'
+require 'json'
+require_relative '../output'
+
+
+module Output
+
+  class KafkaPlugin < OutputPlugin
+
+    # Create a kafka output plugin instance
+    #
+    # @param [Hash] options
+    # @option options [Array] :brokers (['localhost:9092'])
+    # @option options [String] :client_id ('fileminer')
+    # @option options [String] :topic ('fileminer')
+    # @option options [Symbol] :mode (:sync) :sync or :async
+    # @option options [Symbol] :auto_delivery (:disabled) :disabled or :enabled
+    # @option options [Hash] :delivery_conf
+    def initialize(options)
+      brokers = options[:brokers] || ['localhost:9092']
+      client_id = options[:client_id] || 'fileminer'
+      @topic = options[:topic] || 'fileminer'
+      @kafka = Kafka.new(brokers, client_id: client_id)
+      case @mode = options[:mode]
+      when :sync
+        @producer = @kafka.producer
+      when :async
+        case @auto_delivery = options[:auto_delivery]
+        when :disabled
+          @producer = @kafka.async_producer
+        when :enabled
+          @producer = @kafka.async_producer options[:delivery_conf]
+        else
+          raise "invalid value #@auto_delivery of auto_delivery"
+        end
+      else
+        raise "unsupported mode #@mode"
+      end
+    end
+
+    # Send all lines to kafka using producer API
+    #
+    # @param [Array] lines
+    # @yield a listener to be called after all lines just be delivered
+    def send_all(lines, &listener)
+      lines.each do |line|
+        message = line.to_json
+        @producer.produce(message, topic: @topic)
+      end
+      @producer.deliver_messages unless @mode == :async and @auto_delivery == :enabled
+      listener.call
+    end
+
+    # close the kafka producer
+    def close
+      @producer.shutdown
+      @kafka.close
+    end
+
+  end
+
+end

data/lib/fileminer/output/mysql.rb

@@ -0,0 +1,101 @@
+require 'mysql2'
+require_relative '../output'
+
+
+module Output
+
+  class MysqlPlugin < OutputPlugin
+
+    DEFAULT_MYSQL = {
+      host: 'localhost',
+      port: 3306,
+      password: '',
+      encoding: 'utf8mb4',
+      ssl_mode: :disabled
+    }
+
+    # Create a mysql output plugin instance
+    #
+    # @param [Hash] options
+    # @option options [String] :host ('localhost')
+    # @option options [Integer] :port (3306)
+    # @option options [String] :username
+    # @option options [String] :password ('')
+    # @option options [String] :database
+    # @option options [String] :encoding ('utf8mb4')
+    # @option options [Symbol] :ssl_mode (:disabled)
+    # @option options [String] :table
+    def initialize(options)
+      raise 'Missing config username on output.mysql' unless options.key? :username
+      raise 'Missing config database on output.mysql' unless options.key? :database
+      raise 'Missing config table on output.mysql' unless options.key? :table
+      conf = DEFAULT_MYSQL.merge options
+      @table = conf.delete :table
+      conf[:port] = conf[:port].to_i
+      conf[:password] = conf[:password].to_s
+      @encoding = conf[:encoding]
+      conf[:ssl_mode] = :disabled if conf[:ssl_mode] != :enabled
+      @mysql = Mysql2::Client.new conf
+      create_table_if_not_exists
+      @sqls = Hash.new { |hash, key| hash[key] = generate_batch_sql key }
+    end
+
+    private
+    def create_table_if_not_exists
+      rs = @mysql.query 'SHOW TABLES'
+      tables = rs.map { |row| row.values[0] }
+      unless tables.include? @table
+        sql = create_table_sql
+        @mysql.query sql
+      end
+    end
+
+    def create_table_sql
+      <<-EOS
+        CREATE TABLE `#@table` (
+          `id` bigint(20) PRIMARY KEY AUTO_INCREMENT,
+          `host` varchar(255) NOT NULL,
+          `path` varchar(255) NOT NULL,
+          `pos` bigint(20) NOT NULL,
+          `end` bigint(20) NOT NULL,
+          `data` text NOT NULL,
+          UNIQUE KEY `UNIQUE_host_path_pos` (`host`,`path`,`pos`)
+        ) ENGINE=InnoDB DEFAULT CHARSET=#@encoding
+      EOS
+    end
+
+    def generate_batch_sql(size)
+      "INSERT IGNORE INTO `#@table`(`host`,`path`,`pos`,`end`,`data`) VALUES " << (['(?,?,?,?,?)'] * size).join(',')
+    end
+
+    def get_batch_sql(size)
+      if @sqls.key? size
+        @sqls[size]
+      else
+        @sqls[size] = generate_batch_sql size
+      end
+    end
+
+    # Send all lines to mysql
+    #
+    # @param [Array] lines
+    # @yield a listener to be called after all lines just be sent
+    public
+    def send_all(lines, &listener)
+      values = lines.flat_map { |line| [line[:host], line[:path], line[:pos], line[:end], line[:data]] }
+      sql = get_batch_sql lines.size
+      @mysql.query 'BEGIN'
+      begin
+        stat = @mysql.prepare sql
+        stat.execute *values
+        @mysql.query 'COMMIT'
+        listener.call
+      rescue => err
+        @mysql.query 'ROLLBACK'
+        raise err
+      end
+    end
+
+  end
+
+end

data/lib/fileminer/output/redis.rb

@@ -0,0 +1,63 @@
+require 'redis'
+require 'json'
+require_relative '../output'
+
+
+module Output
+
+  class RedisPlugin < OutputPlugin
+
+    # Create a redis output plugin instance
+    #
+    # @param [Hash] options
+    # @option options [String] :uri  redis URI string
+    # @option options [String] :host
+    # @option options [Integer] :port
+    # @option options [Integer] :db
+    # @option options [String] :password
+    # @option options [String] :key  redis key
+    def initialize(options)
+      uri = options[:uri]
+      if uri.nil?
+        uri = parse_uri options
+      end
+      @key = options[:key]
+      raise 'Missing key config on output.redis' if @key.nil?
+      driver = require_lib?('hiredis') ? :hiredis : :ruby
+      @redis = Redis.new url: uri, driver: driver
+    end
+
+    private
+    def parse_uri(options)
+      host = options[:host] || 'localhost'
+      port = options[:port] || 6379
+      db = options[:db] || 0
+      password = options[:password]
+      if password.nil?
+        "redis://#{host}:#{port}/#{db}"
+      else
+        "redis://:#{password}@#{host}:#{port}/#{db}"
+      end
+    end
+
+    def require_lib?(name)
+      require name
+    rescue LoadError
+      false
+    end
+
+
+    # Send all lines to redis using LPUSH @key
+    #
+    # @param [Array] lines
+    # @yield a listener to be called after all lines just be sent
+    public
+    def send_all(lines, &listener)
+      messages = lines.map { |line| line.to_json }
+      @redis.lpush @key, messages
+      listener.call
+    end
+
+  end
+
+end

data/lib/fileminer/output.rb

@@ -0,0 +1,17 @@
+# Output module
+module Output
+  # abstract class OutputPlugin
+  class OutputPlugin
+
+    # If plugin is in batch mode
+    def batch?
+      true
+    end
+
+    def close
+      # do nothing default
+    end 
+
+  end
+
+end

data/lib/fileminer/plugins.rb

@@ -0,0 +1 @@
+require_relative 'output'

data/lib/fileminer/version.rb

@@ -0,0 +1,5 @@
+class FileMiner
+
+  VERSION = '1.0.0RC1'
+
+end

data/lib/fileminer.rb

@@ -0,0 +1,243 @@
+#!/usr/bin/env ruby
+
+
+require 'logger'
+require_relative 'fileminer/miner'
+require_relative 'fileminer/plugins'
+
+FILEMINER_SETTINGS = 'fileminer.settings'
+FILEMINER_INPUTS = 'fileminer.inputs'
+
+
+class Hash
+
+  def keys_to_sym
+    map { |k, v| [k.to_sym, v] }.to_h
+  end
+
+  def keys_to_sym!
+    new_hash = keys_to_sym
+    clear
+    merge! new_hash
+  end
+
+end
+
+
+class FileMiner
+
+  DEFAULT_SETTINGS = {
+    refresh_files_time_trigger: '30s',
+    max_time_of_each_mining: '5s',
+    max_lines_of_each_mining: -1,
+    max_lines_of_each_file: -1, 
+  }
+
+  attr_reader :miner, :output, :running
+
+  # Create a new FileMiner instance
+  #
+  # @param [Hash] conf
+  def initialize(conf)
+    init_settings conf['fileminer.settings']
+    @output = init_output conf
+    raise 'Missing config fileminer.inputs' unless conf.key? 'fileminer.inputs'
+    @miner = Miner.new conf['fileminer.inputs'].keys_to_sym
+    @miner.refresh_files
+    @miner.save_registry
+    @running = false
+  end
+
+  private
+  def init_settings(conf)
+    if conf.nil?
+      conf = DEFAULT_SETTINGS.clone
+    else
+      conf = DEFAULT_SETTINGS.merge conf.keys_to_sym
+    end
+    # default logger to stderr
+    # TODO make logger configurable in future
+    @logger = Logger.new STDERR
+    @logger.level = Logger::WARN
+    # mining break trigger
+    max_time_of_each_mining = parse_time conf[:max_time_of_each_mining], 'max_time_of_each_mining on fileminer.settings'
+    max_lines_of_each_mining = conf[:max_lines_of_each_mining]
+    if max_lines_of_each_mining >= 0
+      @mining_break_trigger = lambda { |start_time, lines| Time.now - start_time > max_time_of_each_mining || lines >= max_lines_of_each_mining }
+    else
+      @mining_break_trigger = lambda { |start_time, lines| Time.now - start_time > max_time_of_each_mining }
+    end
+    # file break trigger
+    max_lines_of_each_file = conf[:max_lines_of_each_file]
+    if max_lines_of_each_file >= 0
+      @file_break_trigger = lambda { |lines| lines < @miner.batch_lines || lines >= max_lines_of_each_file }
+    else
+      @file_break_trigger = lambda { |lines| lines < @miner.batch_lines }
+    end
+    # refresh_files_time_trigger
+    @refresh_files_time_trigger = parse_time conf[:refresh_files_time_trigger], 'refresh_files_time_trigger on fileminer.settings'
+  end
+
+  def parse_time(value, conf_name)
+    if /^(\d+)(\w+)$/ =~ value
+      num = $1.to_i
+      unit = $2
+      case unit
+      when 'd'
+        num * 86400
+      when 'h'
+        num * 3600
+      when 'min'
+        num * 60
+      when 's'
+        num
+      when 'ms'
+        num.to_f / 1000
+      else
+        raise "Unsupported time unit '#{unit}' of #{conf_name}"
+      end
+    else
+      raise "Error format '#{value}' of #{conf_name}"
+    end
+  end
+
+  def mining_break?(start_time, lines)
+    @mining_break_trigger.call start_time, lines
+  end
+
+  def file_break?(lines)
+    @file_break_trigger.call lines
+  end
+
+  def init_output(conf)
+    case
+    when conf.key?('output.redis')
+      redis_conf = conf['output.redis'].keys_to_sym
+      init_output_redis redis_conf
+    when conf.key?('output.kafka')
+      kafka_conf = conf['output.kafka'].keys_to_sym
+      init_output_kafka kafka_conf
+    when conf.key?('output.mysql')
+      mysql_conf = conf['output.mysql'].keys_to_sym
+      init_output_mysql mysql_conf
+    else
+      raise 'Missing config for output'
+    end
+  end
+
+  def init_output_redis(redis_conf)
+    require_relative 'fileminer/output/redis'
+    Output::RedisPlugin.new redis_conf
+  end
+
+  def init_output_kafka(kafka_conf)
+    require_relative 'fileminer/output/kafka'
+    kafka_conf[:mode] = kafka_conf[:mode] == 'async' ? :async : :sync
+    if kafka_conf[:mode] == :async
+      kafka_conf[:auto_delivery] = kafka_conf[:auto_delivery] == 'enabled' ? :enabled : :disabled
+      if kafka_conf[:auto_delivery] == :enabled
+        delivery_threshold = kafka_conf.delete :delivery_threshold
+        delivery_interval = kafka_conf.delete :delivery_interval
+        raise 'Missing conf delivery_threshold or delivery_interval' if delivery_threshold.nil? && delivery_interval.nil?
+        kafka_conf[:delivery_conf] = delivery_conf = Hash.new
+        delivery_conf[:delivery_threshold] = delivery_threshold unless delivery_threshold.nil?
+        delivery_conf[:delivery_interval] = delivery_interval unless delivery_interval.nil?
+      end
+    end
+    Output::KafkaPlugin.new kafka_conf
+  end
+
+  def init_output_mysql(mysql_conf)
+    require_relative 'fileminer/output/mysql'
+    mysql_conf[:ssl_mode] = mysql_conf[:ssl_mode] == 'enabled' ? :enabled : :disabled
+    Output::MysqlPlugin.new mysql_conf
+  end
+
+  def send_lines(record, lines)
+    if @output.batch?
+      @output.send_all lines do
+        record[:pos] = lines[-1][:end]
+        @miner.save_registry
+      end
+    else
+      lines.each do |line|
+        @output.send line do
+          record[:pos] = line[:end]
+          @miner.save_registry
+        end
+      end
+    end
+  end
+
+  public
+  def mine_once
+    start_time = Time.now
+    full_lines = 0
+    @miner.active_files.all? do |record|
+      mining_next = true
+      if record[:pos] < File.size(record[:path])
+        file_lines = 0
+        loop do
+          lines = @miner.read_lines record
+          break if lines.empty?
+          send_lines record, lines
+          file_lines += lines.size
+          full_lines += lines.size
+          if mining_break? start_time, full_lines
+            mining_next = false
+            break
+          end
+          break if file_break? file_lines
+        end
+      end
+      mining_next
+    end
+    full_lines
+  end
+
+  def start_mining
+    unless @running
+      @running = true
+      while @running
+        begin
+          files_refreshed = check_files
+          sent_lines = mine_once
+          # sleep 5 seconds if no more data
+          # TODO using settings instead in future
+          if sent_lines == 0
+            @miner.save_registry if files_refreshed
+            sleep 5
+          end
+        rescue => e
+          @logger.error e
+          # sleep for a little while to wait output recover
+          sleep 5 if @running
+        end
+      end
+      @miner.save_registry
+    end
+  end
+
+  def check_files
+    if @miner.files_need_refresh? @refresh_files_time_trigger
+      @miner.refresh_files
+    end
+  end
+
+  def stop_mining
+    @running = false if @running
+  end
+
+end
+
+
+if __FILE__ == $0
+  # Usage:
+  #     ruby fileminer.rb /etc/fileminer/fileminer.yml
+  require 'yaml'
+  yml = File.open(ARGV[0]) { |io| io.read }
+  conf = YAML.load yml
+  fileminer = FileMiner.new conf
+  trap(:INT) { fileminer.stop_mining }
+  fileminer.start_mining
+end

metadata

@@ -0,0 +1,56 @@
+--- !ruby/object:Gem::Specification
+name: fileminer
+version: !ruby/object:Gem::Version
+  version: 1.0.0RC1
+platform: ruby
+authors:
+- Fang MinJie
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-01-17 00:00:00.000000000 Z
+dependencies: []
+description: A simple file/log transfer tool coding by ruby.
+email:
+- fmjsjx@163.com
+executables:
+- fileminer
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- bin/fileminer
+- conf/fileminer_default.yml
+- lib/fileminer.rb
+- lib/fileminer/miner.rb
+- lib/fileminer/output.rb
+- lib/fileminer/output/kafka.rb
+- lib/fileminer/output/mysql.rb
+- lib/fileminer/output/redis.rb
+- lib/fileminer/plugins.rb
+- lib/fileminer/version.rb
+homepage: https://github.com/fmjsjx/fileminer
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">"
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubygems_version: 3.0.1
+signing_key: 
+specification_version: 4
+summary: A simple file/log transfer tool coding by ruby.
+test_files: []