dummy_log_generator 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 239561651394dd1e8a834b4d6c96d4b40aa2477d
-  data.tar.gz: 5359dbbd8aac2e4a62f6d6eceaac9bd8bbc39013
+  metadata.gz: 4b77ee9d48bae1cb615e7b45a3c3cc2b58388c48
+  data.tar.gz: fbd14739b3aaf235d0e9e095c6d1777a4d6ae14c
 SHA512:
-  metadata.gz: 71dc67b5580ff0ece0e3d908728b3babfdc35755235e6577f94d3ff2b84dd45a93491f85117c4439d596bc0d54f48ef7aad69ba37e57c4d6408bffe40c077917
-  data.tar.gz: 345b86a8f396f711ad09886df24af56eecc4f9c30edb63acb16b7d51b48c7b4557ff257c7646c2caa29ef39dfbd993616b2055f412a6780724dab4a56b4ff297
+  metadata.gz: ae54272e33cec7b6871acf5f6b63402ae4f83bc264fe48abddd63fe6741fd2ad71dae41a380e827f23a8ce91a65189f84ddccb3d5edd0a5b2b512974aad8e9f0
+  data.tar.gz: 26ff83fa64e28e9c16b8163d3a2117f414da3add07f6082adb2f54af8706e497753a3a727b5ffedf704179d09d3b11a1343ba70cd6d3d4ce4be5bccc49aeba5e

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+### 0.0.2
+
+Enhancement:
+
+  * Add `countup: true` suppert
+
 ### 0.0.1
 
 First version

data/README.md

@@ -30,6 +30,7 @@ configure 'sample' do
   rate 500
   delimiter "\t"
   labeled false
+  field :id, type: :integer, countup: true
   field :time, type: :datetime, format: "[%Y-%m-%d %H:%M:%S]", random: true
   field :level, type: :string, any: %w[DEBUG INFO WARN ERROR]
   field :method, type: :string, any: %w[GET POST PUT]
@@ -42,9 +43,9 @@ end
 Running dummy_log_generator outputs like
 
 ```
-[1984-05-25 05:10:03]   DEBUG   GET     /api/v1/people  4.451362369925074       fl8nmh4f
-[2027-11-04 02:58:01]   INFO    GET     /api/v1/people  3.984084722909503       WhsvwEeF
-[1973-02-17 09:09:34]   WARN    GET     /api/v1/people  2.290704255755689       3UyK3jgi
+360     [2031-02-09 15:01:07]   DEBUG   GET     /api/v1/people  1.105236938958904       PcGw8kEi
+361     [1991-08-26 13:03:05]   WARN    GET     /api/v1/people  1.8938518088450287      RGOuoydG
+362     [2037-11-04 15:32:38]   DEBUG   POST    /api/v1/people  2.7431863060457538      rVij0nWP
 ```
 
 ## Configuration Parameters
@@ -97,6 +98,10 @@ You can specify following data types to your `field` parameters:
 
     You can specify a range of integers, then the generator picks one in the range (uniform) randomly
 
+  * :countup
+
+    Generate countup data. Default: false
+
 * :float
 
   * :range
@@ -112,6 +117,7 @@ There is a [fluent-plugin-dummydata-producer](https://github.com/tagomoris/fluen
 1. write tests
 2. make it slim (remove active_support, etc)
 3. outputs to a file (currently, outputs to STDOUT)
+4. speed up (evaluate fields at only starting up)
 
 ## Contributing
 

data/dummy_log_generator.conf

@@ -2,7 +2,8 @@ configure 'sample' do
   rate 500
   delimiter "\t"
   labeled false
-  field :time, type: :datetime, format: "[%Y-%m-%d %H:%M:%S]", random: true
+  field :id, type: :integer, countup: true
+  field :time, type: :datetime, format: "[%Y-%m-%d %H:%M:%S]", random: false
   field :level, type: :string, any: %w[DEBUG INFO WARN ERROR]
   field :method, type: :string, any: %w[GET POST PUT]
   field :uri, type: :string, any: %w[/api/v1/people /api/v1/textdata /api/v1/messages]

data/lib/dummy_log_generator.rb

@@ -2,6 +2,7 @@ require 'logger'
 require "dummy_log_generator/version"
 require "dummy_log_generator/error"
 require "dummy_log_generator/generator"
+require "dummy_log_generator/formatter"
 require "dummy_log_generator/worker"
 require "dummy_log_generator/dsl"
 

data/lib/dummy_log_generator/cli.rb

@@ -13,6 +13,7 @@ module DummyLogGenerator
       @options = @options.dup # avoid frozen
       if options[:config] && File.exists?(options[:config])
         dsl = instance_eval(File.read(options[:config]), options[:config])
+        @options[:generator] = dsl.generator
         @options[:formatter] = dsl.formatter
         @options[:rate]      = dsl.config.rate
       end

data/lib/dummy_log_generator/dsl.rb

@@ -6,32 +6,16 @@ module DummyLogGenerator
       @rate = 500
     end
   end
-
-  class Formatter
-    attr_accessor :labeled, :delimiter, :fields
-
-    def initialize
-      @labeled = true
-      @delimiter = "\t"
-      @fields = {}
-    end
-
-    def output(generated_params)
-      if labeled
-        generated_params.map {|key, val| "#{key}:#{val}" }.join(delimiter)
-      else
-        generated_params.values.join(delimiter)
-      end
-    end
-  end
 end
 
 module DummyLogGenerator
   class Dsl
+    attr_reader :generator
     attr_reader :formatter
     attr_reader :config
 
     def initialize
+      @generator = Generator.new
       @formatter = Formatter.new
       @config = Config.new
     end
@@ -41,7 +25,7 @@ module DummyLogGenerator
     end
 
     def field(name, opts)
-      formatter.fields[name] = opts
+      generator.fields[name] = opts
     end
 
     def delimiter(delimiter)

data/lib/dummy_log_generator/formatter.rb

@@ -0,0 +1,19 @@
+module DummyLogGenerator
+  class Formatter
+    attr_accessor :labeled, :delimiter, :fields
+
+    def initialize
+      @labeled = true
+      @delimiter = "\t"
+      @fields = {}
+    end
+
+    def format(fields)
+      if labeled
+        fields.map {|key, val| "#{key}:#{val}" }.join(delimiter)
+      else
+        fields.values.join(delimiter)
+      end
+    end
+  end
+end

data/lib/dummy_log_generator/generator.rb

@@ -1,32 +1,33 @@
 module DummyLogGenerator
   class Generator
-    attr_reader :formatter
+    attr_accessor :fields
     attr_reader :rand
 
-    def initialize(formatter)
-      @formatter = formatter
+    def initialize
+      @fields = {}
       @rand = ::DummyLogGenerator::Random.new
     end
 
-    def generate
-      fields = {}
-      formatter.fields.each do |key, opts|
+    def generate(prev_data = {})
+      data = {}
+      fields.each do |key, opts|
         opts = opts.dup
         type = opts.delete(:type)
-        fields[key] = case type
-                      when :datetime
-                        rand.datetime(opts)
-                      when :string
-                        rand.string(opts)
-                      when :integer
-                        rand.integer(opts)
-                      when :float
-                        rand.float(opts)
-                      else
-                        raise ConfigError.new(type)
-                      end
+        opts[:prev] = prev_data[key]
+        data[key] = case type
+                    when :datetime
+                      rand.datetime(opts)
+                    when :string
+                      rand.string(opts)
+                    when :integer
+                      rand.integer(opts)
+                    when :float
+                      rand.float(opts)
+                    else
+                      raise ConfigError.new(type)
+                    end
       end
-      formatter.output(fields)
+      data
     end
     alias_method :gen, :generate
   end
@@ -37,24 +38,27 @@ module DummyLogGenerator
       @chars = ('a'..'z').to_a + ('A'..'Z').to_a + ('0'..'9').to_a # no symbols and multi-bytes for now
     end
 
-    def string(length: 8, any: nil)
-      unless any.nil?
+    def string(length: 8, any: nil, prev: nil)
+      if any
         self.any(any)
       else
         Array.new(length){@chars[rand(@chars.size-1)]}.join
       end
     end
 
-    def interger(range: nil)
-      unless range.nil?
+    def integer(range: nil, countup: false, prev: nil)
+      if range
         self.range(range)
+      elsif countup
+        prev ||= -1
+        prev + 1
       else
         rand(0..2,147,483,647)
       end
     end
 
-    def float(range: nil)
-      unless range.nil?
+    def float(range: nil, prev: nil)
+      if range
         self.range(range)
       else
         r = rand(1..358)
@@ -62,7 +66,7 @@ module DummyLogGenerator
       end
     end
 
-    def datetime(format: "%Y-%m-%d %H:%M:%S.%3N", random: false)
+    def datetime(format: "%Y-%m-%d %H:%M:%S.%3N", random: false, prev: nil)
       time = if random
                y = rand(1970..2037);
                m = rand(1..12);

data/lib/dummy_log_generator/version.rb

@@ -1,3 +1,3 @@
 module DummyLogGenerator
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/lib/dummy_log_generator/worker.rb

@@ -7,13 +7,15 @@ module DummyLogGenerator
     end
 
     def reload
-      @rate = config[:rate] || 500 # msgs / sec
+      @rate = config[:rate]
       @formatter = config[:formatter]
-      @generator = Generator.new(@formatter)
+      @generator = config[:generator]
     end
 
+    # thanks! ref. https://github.com/tagomoris/fluent-plugin-dummydata-producer/blob/a550fd4424f71cd9227e138c3c89f600ba40a0d5/lib/fluent/plugin/in_dummydata_producer.rb#L63
     def run
       batch_num = (@rate / 9).to_i + 1
+      prev_data  = {}
       while !@stop
         current_time = Time.now.to_i
         rate_count = 0
@@ -21,7 +23,9 @@ module DummyLogGenerator
         while !@stop && rate_count < @rate && Time.now.to_i == current_time
           batch_num.times do
             # ToDo: what if generation is slower than I/O?
-            STDOUT.puts @generator.generate
+            # We may should generate data and output in parallel
+            prev_data = @generator.generate(prev_data)
+            STDOUT.puts @formatter.format(prev_data)
           end
           rate_count += batch_num
           sleep 0.1

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dummy_log_generator
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - sonots
@@ -158,6 +158,7 @@ files:
 - lib/dummy_log_generator/cli.rb
 - lib/dummy_log_generator/dsl.rb
 - lib/dummy_log_generator/error.rb
+- lib/dummy_log_generator/formatter.rb
 - lib/dummy_log_generator/generator.rb
 - lib/dummy_log_generator/version.rb
 - lib/dummy_log_generator/worker.rb