fileminer 1.1.1 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 40b0403c4675a8340fee70e155fd7e39ccc902144ca4ab1af03a08db4b8ef8d9
-  data.tar.gz: f344c468064be5e3a594befbb62a43e5937f4351029fa88a7fef3a0d6310ce87
+  metadata.gz: 93670c0973d05575fdab215da684c633c9367f2288168cecab359b0464c539de
+  data.tar.gz: 55eaa5c573e308585da35ec30423988c049ef25669ea161a069a5c3392d75ee5
 SHA512:
-  metadata.gz: 9f72907fd4e74d2bbbd93130fd1762101c1aa757a44923067b47eb62a515d7e3a18e4452cd2eb4ad73cb66f36d06ca2b3bc6bb4fe0a04c6131ba2b29256d40fc
-  data.tar.gz: '022318a184a41042136a96d2691890ca3d78f4a3d12b2e9ea2ffe2774430d98861460e9e3a0fb0ae2a3e6f820791af24e39e7c7bc1bac2a53406fa5e74d82aed'
+  metadata.gz: 3786c83518c206f3a1bec0ed6b13ce46f3c1604655be8a2d48afa66e8bf202d40fb3bbda5c4c3521e5bbd7265ce7773fa114f03520f162876f6a19ce7ca0c474
+  data.tar.gz: 729cba5d996c05301a8ed4870180f89f74511b0709d96c666df43ba7755d17c4a56d5531f19d98e5180029ff8283268671a4ed258582dc8ef0d437d34d22272a

data/README.md

@@ -1,5 +1,5 @@
 # fileminer
-[![Gem Version](https://d25lcipzij17d.cloudfront.net/badge.svg?id=rb&type=6&v=1.1.0&x2=0)](https://rubygems.org/gems/fileminer)
+[![Gem Version](https://d25lcipzij17d.cloudfront.net/badge.svg?id=rb&type=6&v=1.2.0&x2=0)](https://rubygems.org/gems/fileminer)
 [![MIT licensed](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/fmjsjx/fileminer/blob/master/LICENSE)
 
 
@@ -11,10 +11,10 @@ A simple line based file/log transfer tool coding by ruby.
 Install fileminer from RubyGems:
 ```
 $ gem install fileminer
-Fetching fileminer-1.1.0.gem
-Successfully installed fileminer-1.1.0
-Parsing documentation for fileminer-1.1.0
-Installing ri documentation for fileminer-1.1.0
+Fetching fileminer-1.2.0.gem
+Successfully installed fileminer-1.2.0
+Parsing documentation for fileminer-1.2.0
+Installing ri documentation for fileminer-1.2.0
 Done installing documentation for fileminer after 0 seconds
 1 gem installed
 $ _
@@ -32,7 +32,13 @@ At first, we should generate the fileminer configuration file.
 We provide a command tool 'fileminer-genconf' to generate configurations:
 ```
 $ fileminer-genconf -h
-Usage: fileminer-genconf [options]
+Usage:
+    fileminer-genconf [options]
+
+Samples:
+    fileminer-genconf -t fileminer -o      Generate config on /etc/fileminer/fileminer.yml
+    fileminer-genconf -t supervisor -o -l  Generate ./fileminer.ini with logfile on
+                                           /var/log/fileminer/stderr.log
 
 Options:
     -t, --type fileminer|supervisor  Type of the config file to be generated
@@ -40,7 +46,10 @@ Options:
     -o, --out [path]                 Output content to a file
                                      For type fileminer, default is /etc/fileminer/fileminer.yml
                                      For type supervisor, default is ./fileminer.ini
+    -l, --logfile [path]             Logfile configured on supervisor config file
+                                     Default is /var/log/fileminer/stderr.log
     -h, --help                       Print help
+
 $ _
 ```
 

data/bin/fileminer-genconf

@@ -29,6 +29,20 @@ def gen_fileminer_conf(options)
 end
 
 def gen_supervisor_conf(options)
+  logfile = options[:logfile]
+  if logfile.nil? && options.key?(:logfile)
+    logfile = '/var/log/fileminer/stderr.log'
+  end
+  stdouts = ''
+  unless logfile.nil?
+    stdouts = <<-EOS
+stderr_logfile=#{logfile}
+stderr_logfile_maxbytes=10MB
+stderr_logfile_backups=10
+    EOS
+    log_dir = File.dirname logfile
+    Dir.mkdirs log_dir unless Dir.exists? log_dir
+  end
   wrapper_home = File.join ENV['GEM_HOME'], 'wrappers'
   content = <<-EOS
 [program:fileminer]
@@ -37,6 +51,7 @@ autostart=true
 autorestart=true
 stopsinal=INT
 user=root
+#{stdouts}
   EOS
   if options.key? :out
     out = options[:out]
@@ -44,7 +59,6 @@ user=root
     unless dst.end_with?('.ini') || dst.end_with?('.conf')
       dst = File.join dst, 'fileminer.ini'
     end
-
     dst_dir = File.dirname dst
     Dir.mkdirs dst_dir unless Dir.exists? dst_dir
     File.open(dst, 'w') { |io| io.write content }
@@ -57,7 +71,13 @@ end
 options = Hash.new
 OptionParser.new do |opts|
   # banner
-  opts.banner = "Usage: fileminer-genconf [options]"
+  opts.banner = "Usage:"
+  opts.separator '    fileminer-genconf [options]'
+  opts.separator ''
+  opts.separator 'Samples:'
+  opts.separator '    fileminer-genconf -t fileminer -o      Generate config on /etc/fileminer/fileminer.yml'
+  opts.separator '    fileminer-genconf -t supervisor -o -l  Generate ./fileminer.ini with logfile on'
+  opts.separator '                                           /var/log/fileminer/stderr.log'
   opts.separator ''
   opts.separator 'Options:'
 
@@ -83,6 +103,13 @@ OptionParser.new do |opts|
     options[:out] = value
   end
 
+  # logfile, just for type=supervisor
+  opts.on('-l', '--logfile [path]',
+          'Logfile configured on supervisor config file',
+          'Default is /var/log/fileminer/stderr.log') do |value|
+    options[:logfile] = value
+  end
+
 end.parse!
 
 case options[:type]
@@ -90,4 +117,4 @@ when :fileminer
   gen_fileminer_conf options
 when :supervisor
   gen_supervisor_conf options
-end
+end

data/conf/fileminer_default.yml

@@ -13,13 +13,27 @@
   # default value is -1
   #max_lines_of_each_file: -1
 
+  # default value is 5s
+  #sleep_time_when_no_more_data: 5s
+
 # fileminer inputs
 fileminer.inputs:
 
+  # setup the path of work dir
+  # default value is /var/lib/fileminer
+  #work_dir: /var/lib/fileminer
+
   # setup the path of the registry file
-  # default value is /var/lib/fileminer/registry
+  # default value is ${work_dir}/registry
   #registry_path: /var/lib/fileminer/registry
 
+  # setup the path of the history file
+  # default value is ${work_dir}/history
+  #history_path: /var/lib/fileminer/history
+
+  # default value is 20
+  #max_eof_files: 20
+
   # file paths
   paths:
     - /path/to/*.log

data/lib/fileminer/miner.rb

@@ -6,7 +6,8 @@ require_relative 'tools/io'
 class Miner
 
   DEFAULTS = {
-    registry_path: '/var/lib/fileminer/registry',
+    work_dir: '/var/lib/fileminer',
+    max_eof_files: 20,
     eof_seconds: 86400,
     batch_lines: 200,
   }
@@ -25,6 +26,14 @@ class Miner
     # fix options by DEFAULTS
     DEFAULTS.each { |k, v| options[k] = v unless options.key? k }
     @registry_path = options[:registry_path]
+    if @registry_path.nil?
+      @registry_path = "#{options[:work_dir]}/registry"
+    end
+    @history_path = options[:history_path]
+    if @history_path.nil?
+      @history_path = "#{options[:work_dir]}/history"
+    end
+    @max_eof_files = options[:max_eof_files]
     @paths = options[:paths]
     @eof_seconds = options[:eof_seconds]
     @batch_lines = options[:batch_lines]
@@ -35,8 +44,15 @@ class Miner
     end
     @files = []
     @active_files = []
+    @history = []
+    if File.exist? @history_path
+      @history = File.open(@history_path) { |io| JSON.parse(io.read) }
+    else
+      parent_dir = File.dirname @history_path
+      Dir.mkdirs parent_dir unless Dir.exist? parent_dir
+    end
     if File.exist? @registry_path
-      File.open(@registry_path) { |io| @files = JSON.parse(io.read, {symbolize_names: true}) }
+      @files = File.open(@registry_path) { |io| JSON.parse(io.read, {symbolize_names: true}) }
       @active_files = @files.select { |record| !record[:eof] }
     else
       parent_dir = File.dirname @registry_path
@@ -44,37 +60,40 @@ class Miner
     end
   end
 
+  # Save work status
+  def save_work_status
+    save_history
+    save_registry
+  end
+
   # Save registry
   def save_registry
     File.open(@registry_path, 'w') { |io| io.write @files.to_json }
   end
 
+  # Save history
+  def save_history
+    File.open(@history_path, 'w') { |io| io.write @history.to_json }
+  end
+
   # Refresh
   def refresh_files
     now = Time.now
     file_paths = Set.new
     file_paths.merge Dir[*@paths].select { |path| File.file? path }
-    @active_file = @files.select do |record|
-      path = record[:path]
-      file_exists = file_paths.delete? path
-      unless record[:eof]
-        if file_exists
-          # check if EOF
-          if record[:pos] == File.size(path) && now - File.mtime(path) > @eof_seconds
-            record[:eof] = true
-          end
-        else
-          # missing file, set :eof to true
-          record[:eof] = true
-        end
-      end
-      !record[:eof]
+    active_files, eof_size = select_active_files now, file_paths
+    if eof_size > @max_eof_files
+      move_eof_to_history
     end
-    file_paths.each do |path|
+    history_files = Set.new @history
+    file_paths.select do |path|
+      ! history_files.member? path
+    end.each do |path|
       record = {path: path, pos: 0, eof: false}
       @files << record
-      @active_files << record
+      active_files << record
     end
+    @active_files = active_files
     @files_refresh_time = now
   end
 
@@ -108,4 +127,43 @@ class Miner
     Time.now - @files_refresh_time >= refresh_files_time_trigger
   end
 
+  private
+  def select_active_files(now, file_paths)
+    eof_seconds = @eof_seconds
+    eof_size = 0
+    active_files = @files.select do |record|
+      path = record[:path]
+      file_exists = file_paths.delete? path
+      if record[:eof]
+        eof_size += 1
+      else
+        if file_exists
+          # check if EOF
+          if record[:pos] == File.size(path) && now - File.mtime(path) > eof_seconds
+            record[:eof] = true
+            eof_size += 1
+          end
+        else
+          # missing file, set :eof to true
+          record[:eof] = true
+          eof_size += 1
+        end
+      end
+      !record[:eof]
+    end
+    [active_files, eof_size]
+  end
+
+  private
+  def move_eof_to_history
+    to_removed_paths = @files.select do |record|
+      record[:eof]
+    end.map do |record|
+      record[:path]
+    end
+    path_set = Set.new to_removed_paths
+    @files.delete_if { |record| path_set.member? record[:path] }
+    @history.concat to_removed_paths
+  end
+
 end

data/lib/fileminer/version.rb

@@ -1,5 +1,5 @@
 class FileMiner
 
-  VERSION = '1.1.1'
+  VERSION = '1.2.0'
 
 end

data/lib/fileminer.rb

@@ -14,6 +14,7 @@ class FileMiner
     max_time_of_each_mining: '5s',
     max_lines_of_each_mining: -1,
     max_lines_of_each_file: -1,
+    sleep_time_when_no_more_data: '5s',
   }
 
   attr_reader :miner, :output, :running
@@ -59,6 +60,8 @@ class FileMiner
     end
     # refresh_files_time_trigger
     @refresh_files_time_trigger = parse_time conf[:refresh_files_time_trigger], 'refresh_files_time_trigger on fileminer.settings'
+    # sleep seconds when no more data
+    @sleep_seconds_when_no_more_data = parse_time conf[:sleep_time_when_no_more_data], 'sleep_time_when_no_more_data on fileminer.settings'
   end
 
   def parse_time(value, conf_name)
@@ -168,12 +171,13 @@ class FileMiner
   def mine_once
     start_time = Time.now
     full_lines = 0
-    @miner.active_files.all? do |record|
+    miner = @miner
+    miner.active_files.all? do |record|
       mining_next = true
       if record[:pos] < File.size(record[:path])
         file_lines = 0
         loop do
-          lines = @miner.read_lines record
+          lines = miner.read_lines record
           break if lines.empty?
           send_lines record, lines
           file_lines += lines.size
@@ -193,6 +197,7 @@ class FileMiner
   def start_mining
     unless @running
       @running = true
+      sleep_seconds = @sleep_seconds_when_no_more_data
       while @running
         begin
           files_refreshed = check_files
@@ -200,13 +205,13 @@ class FileMiner
           # sleep 5 seconds if no more data
           # TODO using settings instead in future
           if sent_lines == 0
-            @miner.save_registry if files_refreshed
-            sleep 5
+            @miner.save_work_status if files_refreshed
+            sleep sleep_seconds
           end
         rescue => e
           @logger.error e
           # sleep for a little while to wait output recover
-          sleep 5 if @running
+          sleep sleep_seconds if @running
         end
       end
       @miner.save_registry

metadata

@@ -1,16 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: fileminer
 version: !ruby/object:Gem::Version
-  version: 1.1.1
+  version: 1.2.0
 platform: ruby
 authors:
 - Fang MinJie
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-06-24 00:00:00.000000000 Z
+date: 2019-06-26 00:00:00.000000000 Z
 dependencies: []
-description: A simple file/log transfer tool coding by ruby.
+description: |2
+      FileMiner is a simple file/log transfer tool implemented in Ruby.
+      Up to now, FileMiner has only tested on CRuby.
 email:
 - fmjsjx@163.com
 executables:
@@ -57,5 +59,5 @@ requirements: []
 rubygems_version: 3.0.4
 signing_key: 
 specification_version: 4
-summary: A simple file/log transfer tool coding by ruby.
+summary: A simple file/log transfer tool.
 test_files: []