rforward 0.1.1 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b604a36bc1961d87954e6cd92266934969ace196
-  data.tar.gz: da76ae995aaf1f648da48ec8c03b1db8a26f5274
+  metadata.gz: 9f7548dcc4d9f4012d9098335e6c06c5e105b812
+  data.tar.gz: 467cce2e9839510f5691b44a9d729843551aadd5
 SHA512:
-  metadata.gz: 20a8dd075a2b6fb29e9ed00dd8a648b05debe226c641096a7c1f8221357571d1ea42872aef2866bc947a80f1efe9f693bc8a68b2ef871a3a5cbee5707670051a
-  data.tar.gz: 93b501de72c5e6cc2df5e63fd9dc7999cfa34b5f429ce469da7abfec3fae5d192f3d2616abb6c4ac57c9963899e95832c181f16c1364d2dd05d07882eceb0cdd
+  metadata.gz: 1c78ce72a5d8dc00eead64a2f29c18dea7f74b498a656e8fc1ea72d93586a91bf7151f02a2c449d73749d03fdf665fbdc918cf68f59e45782008924a8f46b057
+  data.tar.gz: da5cea88320a45468c48ef986cf462fdfb7e3c946f72e8804633aaf718a33b670c13b6486ae60d464d0b5a30b7043841d8303cf4132b7d9fcda3655d4387ca5d

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    rforward (0.1.1)
+    rforward (0.1.3)
       dry-container
       fluent-logger
       thor (~> 0.20)

data/lib/rforward/config.rb

@@ -29,7 +29,12 @@ class Config
     hash = {
       'fluentd_host' => 'localhost',
       'fluentd_port' => '24224',
-      'tag' => 'event'
+      'tag' => 'event',
+      'flush_delay' => '20',
+      'flush_threshold' => '100000',
+      'time_key' => 'time',
+      'time_format' => '%Y-%m-%dT%H:%M:%S%z',
+      'index_key' => 'index_key'
     }
   end
 end

data/lib/rforward/ffluentd_line.rb

@@ -10,12 +10,23 @@ class FFluentdLine
   def call line
     Stat.instance.total += 1
     json = JSON.parse line
+    json = mutate json
+    puts json
     @client.post Config.instance[:tag], json
     Stat.instance.success += 1
+    Stat.instance.flush_counter += 1
     true
   rescue Exception => e
     Stat.instance.failed += 1
     RLogger.instance.error "(#{e.message}) (line: #{line})"
     false
   end
+
+  def mutate record
+    return record unless record[Config.instance['time_key']]
+    time = Time.strptime record[Config.instance['time_key']], Config.instance['time_format']
+    record[Config.instance['index_key']] = "fluentd-#{time.strftime('%Y-%m')}"
+    record
+  end
+
 end

data/lib/rforward/file_processor.rb

@@ -13,6 +13,16 @@ class FileProcessor
     Stat.instance.files_current += 1
     RLogger.instance.info "finish working on #{filepath}"
     RLogger.instance.stat
+    delay
+  end
+
+
+  def delay
+    if Config.instance['flush_threshold'].to_i < Stat.instance.flush_counter
+      RLogger.instance.info "Sleep for #{Config.instance['flush_delay']} seconds"
+      sleep(Config.instance['flush_delay'].to_i)
+      Stat.instance.flush_counter = 0
+    end
   end
 
   def self.call filepath

data/lib/rforward/stat.rb

@@ -1,9 +1,9 @@
 class Stat
   include Singleton
-  attr_accessor :success, :failed, :total, :files_total, :files_current
+  attr_accessor :success, :failed, :total, :files_total, :files_current, :flush_counter
 
   def initialize
-    @success, @failed, @total, @files_total, @files_current = 0, 0, 0, 0, 0
+    @success, @failed, @total, @files_total, @files_current, @flush_counter = 0, 0, 0, 0, 0, 0
   end
 
   def to_s

data/lib/rforward/version.rb

@@ -1,3 +1,3 @@
 module Rforward
-  VERSION = "0.1.1"
+  VERSION = "0.1.3"
 end

data/lib/rforward.rb

@@ -3,6 +3,7 @@ require 'singleton'
 require 'yaml'
 require 'json'
 require 'dry-container'
+require 'time'
 require "rforward/exceptions"
 require "rforward/stat"
 require "rforward/rlogger"
@@ -14,17 +15,19 @@ require 'thor'
 
 module Rforward
   class CLI < Thor
-    desc "Process JSON logs into Fluentd Forward", ""
+    desc "process_logs", "Process JSON logs into Fluentd Forward"
     def process_logs path, ext=".log"
       check_config
       dependencies
       DirectoryProcessor.call path, ext
       RLogger.instance.info "#{path} logs extensions #{ext}"
+      puts "Work finidhed press any key"
+      STDIN.gets
     rescue ConfigNotFoundEx => e
       RLogger.instance.error e.message
     end
 
-    desc "Create config file", ""
+    desc "create_config", "Create config file in current working directory"
     def create_config
       Config.instance.create_sample_config config_path
     rescue ConfigNotFoundEx => e

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rforward
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.3
 platform: ruby
 authors:
 - gingray