dummer 0.3.5 → 0.4.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 36f913a41901d9441ff565d14ef17c5c122d8a3f
+  data.tar.gz: 50a3dc5517b2c40095f5e36da51ef1ad8ae83530
+SHA512:
+  metadata.gz: 052bc0589e628b86cd5dbf00a895948a39c9b9cb1dce20c2e9957dec65ceb9d6825c9d01d12573162cf9a8697fc7d0e312ceaa39ed1594bd24006a4082bc3e90
+  data.tar.gz: 0164d36f4d12507bf955248dc63941ce518446d4dd19239494cf206159186c6174836aa2ef4b24ddf1a3eb0099a95b96332d59b044fac110d2fce51c0f6c6072

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+### 0.4.0 (2015/08/08)
+
+Enhancements:
+
+  * Make countup integer field stores integer instead of formatted string as prev_data (thanks to mururu)
+
+Changes:
+
+  * Refactoring: split generator codes
+
 ### 0.3.5 (2014/04/23)
 
 Enhancements:

data/Gemfile

@@ -1,3 +1,4 @@
 source 'https://rubygems.org'
 
 gemspec
+gem 'fluentd' # for example

data/README.md

@@ -75,14 +75,14 @@ id:0424  time:[2013-11-19 02:34:58]  level:WARN  method:POST uri:/api/v1/textdat
 
 (experimental)
 
-Create a configuration file. Assume that a fluentd process is running on localhost:20000.
+Create a configuration file. Assume that a fluentd process is running on localhost:24224.
 A sample configuration is as follows:
 
 ```ruby
 # dummer.conf
 configure 'sample' do
   host "localhost" # define `host` and `port` instead of `output`
-  port 20000
+  port 24224
   rate 500
   tag type: string, any: %w[raw.syslog raw.message raw.nginx] # configure tag
   field :id, type: :integer, countup: true, format: "%04d"
@@ -175,15 +175,15 @@ Following parameters in the configuration file are available:
 
 * field
 
-    Define data fields to generate. `message` and `input` options are ignored. 
+    Random field generator mode. Define data fields to generate. `message` and `input` options are ignored. See also `Field Data Types` section below.
 
-* input
+* message
 
-    Use this if you want to write messages by reading lines of an input file in rotation. `message` is ignored.
+    Specific message generation mode. See [message.conf](./example/message.conf) as an example. This mode works pretty fast because it does not require to generate values randomly. 
 
-* message
+* input
 
-    Use this if you want to write only a specific message. 
+    Messages taken from an input file mode.  Use this if you want to write messages by reading lines of an input file in rotation. `message` option is ignored. See [input.conf](./example/input.conf) as an example. This mode also works fast.
 
 ### Field Data Types
 
@@ -322,7 +322,6 @@ and I wanted a standalone separated tool to do benchmark.
 ## ToDO
 
 1. write tests
-2. Add an option to send data to fluentd directly
 
 ## Contributing
 

data/example/countup.conf

@@ -0,0 +1,4 @@
+configure 'sample' do
+  output "dummer.log"
+  field :time, type: :integer, countup: true, format: "2013-11-25 00:23:52.%09d"
+end

data/example/fluent.conf

@@ -0,0 +1,8 @@
+<source>
+  type forward
+  port 20000
+</source>
+
+<match raw.**>
+  type stdout
+</match>

data/lib/dummer.rb

@@ -1,6 +1,12 @@
 require "dummer/version"
 require "dummer/error"
 require "dummer/setting"
+require 'dummer/random'
 require "dummer/generator"
+require 'dummer/generator/abstract_mode'
+require 'dummer/generator/message_mode'
+require 'dummer/generator/input_mode'
+require 'dummer/generator/field_mode'
+require 'dummer/generator/factory'
 require "dummer/worker"
 require "dummer/dsl"

data/lib/dummer/generator.rb

@@ -1,23 +1,12 @@
 module Dummer
   class Generator
     def initialize(setting)
-      @message_proc =
-        if fields = setting.fields
-          Field.message_proc(fields, setting.labeled, setting.delimiter, setting.label_delimiter)
-        elsif input = setting.input
-          Input.message_proc(input)
-        else
-          Message.message_proc(setting.message)
-        end
-      @record_proc =
-        if fields = setting.fields
-          Field.record_proc(fields)
-        elsif input = setting.input
-          Input.record_proc(input)
-        else
-          Message.record_proc(setting.message)
-        end
-      @tag_proc = Field.tag_proc(setting.tag)
+      factory = Factory.new(setting)
+      # fluent-logger
+      @tag_proc = factory.create_tag_proc
+      @record_proc = factory.create_record_proc
+      # file
+      @message_proc = factory.create_message_proc
     end
 
     # @return [String] message
@@ -34,207 +23,5 @@ module Dummer
     def record
       @record_proc.call
     end
-
-    class Message
-      def self.message_proc(message)
-        message = "#{message.chomp}\n"
-        Proc.new { message }
-      end
-
-      def self.record_proc(message)
-        # ToDo: implement parser
-        message_proc = message_proc(message)
-        Proc.new { { "message" => message_proc.call } }
-      end
-    end
-
-    class Input
-      def self.message_proc(input)
-        messages = nil
-        begin
-          open(input) do |in_file|
-            messages = in_file.readlines
-          end
-        rescue Errno::ENOENT
-          raise ConfigError.new("Input file `#{input}` is not readable")
-        end
-        idx = -1
-        size = messages.size
-        Proc.new {
-          idx = (idx + 1) % size
-          messages[idx]
-        }
-      end
-
-      def self.record_proc(input)
-        # ToDo: implement parser
-        message_proc = message_proc(input)
-        Proc.new { { "message" => message_proc.call } }
-      end
-    end
-
-    class Field
-      def self.message_proc(fields, labeled, delimiter, label_delimiter)
-        format_proc = format_proc(labeled, delimiter, label_delimiter)
-        record_proc = record_proc(fields)
-
-        Proc.new {
-          hash = record_proc.call
-          format_proc.call(hash)
-        }
-      end
-
-      def self.record_proc(fields)
-        field_procs = field_procs(fields)
-
-        prev_data = {}
-        Proc.new {
-          data = {}
-          field_procs.each do |key, proc|
-            prev = prev_data[key] || -1
-            data[key] = proc.call(prev)
-          end
-          prev_data = data
-        }
-      end
-
-      def self.tag_proc(tag_opts)
-        field_procs({"tag" => tag_opts})["tag"]
-      end
-
-      def self.format_proc(labeled, delimiter, label_delimiter)
-        if labeled
-          Proc.new {|fields| "#{fields.map {|key, val| "#{key}#{label_delimiter}#{val}" }.join(delimiter)}\n" }
-        else
-          Proc.new {|fields| "#{fields.values.join(delimiter)}\n" }
-        end
-      end
-
-      def self.field_procs(fields)
-        rand = ::Dummer::Random.new
-        field_procs = {}
-        fields.each do |key, opts|
-          opts = opts.dup
-          type = opts.delete(:type)
-          field_procs[key] =
-            case type
-            when :string
-              rand.string(opts)
-            when :integer
-              rand.integer(opts)
-            when :float
-              rand.float(opts)
-            when :datetime
-              rand.datetime(opts)
-            else
-              raise ConfigError.new("type: `#{type}` is not defined.")
-            end
-        end
-        field_procs
-      end
-    end
-  end
-
-  class Random
-    def initialize
-      @rand = ::Random.new(0)
-      @chars = ('a'..'z').to_a + ('A'..'Z').to_a + ('0'..'9').to_a # no symbols and multi-bytes for now
-    end
-
-    def range(range)
-      rand(range)
-    end
-
-    def any(any)
-      any[rand(any.size)]
-    end
-
-    def rand(arg = nil)
-      @rand.rand(arg)
-    end
-
-    # belows are data types
-
-    def string(opts = {})
-      length, any, value = (opts[:length] || 8), opts[:any], opts[:value]
-      if value
-        string = value.to_s
-        Proc.new { string }
-      elsif any
-        Proc.new { self.any(any) }
-      else
-        Proc.new { Array.new(length){@chars[rand(@chars.size-1)]}.join }
-      end
-    end
-
-    def integer(opts = {})
-      format, range, countup, value = opts[:format], opts[:range], opts[:countup], opts[:value]
-      if format
-        if value
-          integer = sprintf(format, value.to_i)
-          Proc.new { integer }
-        elsif range
-          Proc.new { sprintf(format, self.range(range)) }
-        elsif countup
-          Proc.new {|prev| sprintf(format, prev.to_i + 1) }
-        else
-          Proc.new { sprintf(format, rand(0..2,147,483,647)) }
-        end
-      else
-        if value
-          integer = value.to_i
-          Proc.new { integer }
-        elsif range
-          Proc.new { self.range(range) }
-        elsif countup
-          Proc.new {|prev| prev + 1 }
-        else
-          Proc.new { rand(0..2,147,483,647) }
-        end
-      end
-    end
-
-    def float(opts = {})
-      format, range, value = opts[:format], opts[:range], opts[:value]
-      if format
-        if value
-          float = value.to_f
-          Proc.new { sprintf(format, float) }
-        elsif range
-          Proc.new { sprintf(format, self.range(range)) }
-        else
-          Proc.new { r = rand(1..358); sprintf(format, r * Math.cos(r)) }
-        end
-      else
-        if value
-          float = value.to_f
-          Proc.new { float }
-        elsif range
-          Proc.new { self.range(range) }
-        else
-          Proc.new { r = rand(1..358); r * Math.cos(r) }
-        end
-      end
-    end
-
-    def datetime(opts = {})
-      format, random, value = (opts[:format] || "%Y-%m-%d %H:%M:%S.%3N"), (opts[:random] || false), opts[:value]
-      if value
-        Proc.new { value.strftime(format) }
-      elsif random
-        Proc.new {
-          y = rand(1970..2037);
-          m = rand(1..12);
-          d = rand(1..27);
-          h = rand(0..23);
-          min = rand(0..59);
-          s = rand(0..59);
-          usec = rand(0..999999);
-          Time.local(y, m, d, h, min, s, usec).strftime(format)
-        }
-      else
-        Proc.new { Time.now.strftime(format) }
-      end
-    end
   end
 end

data/lib/dummer/generator/abstract_mode.rb

@@ -0,0 +1,20 @@
+module Dummer
+  class Generator
+    class AbstractMode
+      # file
+      def self.message_proc(message)
+        raise NotImplementedError
+      end
+
+      # fluent-logger
+      def self.record_proc(message)
+        raise NotImplementedError
+      end
+
+      # fluent-logger
+      def self.tag_proc(tag_opts)
+        FieldMode.field_procs({"tag" => tag_opts})["tag"]
+      end
+    end
+  end
+end

data/lib/dummer/generator/factory.rb

@@ -0,0 +1,43 @@
+module Dummer
+  class Generator
+    class Factory
+      attr_reader :setting
+
+      def initialize(setting)
+        @setting = setting
+      end
+
+      # file
+      def create_message_proc
+        if fields = setting.fields
+          FieldMode.message_proc(
+            fields,
+            setting.labeled,
+            setting.delimiter,
+            setting.label_delimiter
+          )
+        elsif input = setting.input
+          InputMode.message_proc(input)
+        else
+          MessageMode.message_proc(setting.message)
+        end
+      end
+
+      # fluent-logger
+      def create_record_proc
+        if fields = setting.fields
+          FieldMode.record_proc(fields)
+        elsif input = setting.input
+          InputMode.record_proc(input)
+        else
+          MessageMode.record_proc(setting.message)
+        end
+      end
+
+      # fluent-logger
+      def create_tag_proc
+        FieldMode.tag_proc(setting.tag)
+      end
+    end
+  end
+end

data/lib/dummer/generator/field_mode.rb

@@ -0,0 +1,74 @@
+module Dummer
+  class Generator
+    class FieldMode < AbstractMode
+      def self.message_proc(fields, labeled, delimiter, label_delimiter)
+        format_proc = format_proc(labeled, delimiter, label_delimiter)
+        record_proc = record_proc(fields)
+
+        Proc.new {
+          hash = record_proc.call
+          format_proc.call(hash)
+        }
+      end
+
+      def self.record_proc(fields)
+        field_procs = field_procs(fields)
+
+        prev_data = {}
+        Proc.new {
+          data = {}
+          field_procs.each do |key, proc|
+            prev = prev_data[key] || -1
+            value, raw = proc.call(prev)
+            data[key] = value
+            prev_data[key] = raw || value
+          end
+          data
+        }
+      end
+
+      def self.tag_proc(tag_opts)
+        proc = field_procs({"tag" => tag_opts})["tag"]
+        prev = -1
+        Proc.new {
+          value, raw = proc.call(prev)
+          prev = raw || value
+          value
+        }
+      end
+
+      # private
+
+      def self.format_proc(labeled, delimiter, label_delimiter)
+        if labeled
+          Proc.new {|fields| "#{fields.map {|key, val| "#{key}#{label_delimiter}#{val}" }.join(delimiter)}\n" }
+        else
+          Proc.new {|fields| "#{fields.values.join(delimiter)}\n" }
+        end
+      end
+
+      def self.field_procs(fields)
+        rand = ::Dummer::Random.new
+        field_procs = {}
+        fields.each do |key, opts|
+          opts = opts.dup
+          type = opts.delete(:type)
+          field_procs[key] =
+            case type
+            when :string
+              rand.string(opts)
+            when :integer
+              rand.integer(opts)
+            when :float
+              rand.float(opts)
+            when :datetime
+              rand.datetime(opts)
+            else
+              raise ConfigError.new("type: `#{type}` is not defined.")
+            end
+        end
+        field_procs
+      end
+    end
+  end
+end

data/lib/dummer/generator/input_mode.rb

@@ -0,0 +1,28 @@
+module Dummer
+  class Generator
+    class InputMode < AbstractMode
+      def self.message_proc(input)
+        messages = nil
+        begin
+          open(input) do |in_file|
+            messages = in_file.readlines
+          end
+        rescue Errno::ENOENT
+          raise ConfigError.new("Input file `#{input}` is not readable")
+        end
+        idx = -1
+        size = messages.size
+        Proc.new {
+          idx = (idx + 1) % size
+          messages[idx]
+        }
+      end
+
+      def self.record_proc(input)
+        # ToDo: implement parser
+        message_proc = message_proc(input)
+        Proc.new { { "message" => message_proc.call } }
+      end
+    end
+  end
+end

data/lib/dummer/generator/message_mode.rb

@@ -0,0 +1,16 @@
+module Dummer
+  class Generator
+    class MessageMode < AbstractMode
+      def self.message_proc(message)
+        message = "#{message.chomp}\n"
+        Proc.new { message }
+      end
+
+      def self.record_proc(message)
+        # ToDo: implement parser
+        message_proc = message_proc(message)
+        Proc.new { { "message" => message_proc.call } }
+      end
+    end
+  end
+end

data/lib/dummer/random.rb

@@ -0,0 +1,107 @@
+module Dummer
+  class Random
+    def initialize
+      @rand = ::Random.new(0)
+      @chars = ('a'..'z').to_a + ('A'..'Z').to_a + ('0'..'9').to_a # no symbols and multi-bytes for now
+    end
+
+    # belows are data types
+    # @return Proc object which returns a random generated value, or [formatted_value, raw_value]
+
+    def string(opts = {})
+      length, any, value = (opts[:length] || 8), opts[:any], opts[:value]
+      if value
+        string = value.to_s
+        Proc.new { string }
+      elsif any
+        Proc.new { self.any(any) }
+      else
+        Proc.new { Array.new(length){@chars[rand(@chars.size-1)]}.join }
+      end
+    end
+
+    def integer(opts = {})
+      format, range, countup, value = opts[:format], opts[:range], opts[:countup], opts[:value]
+      if format
+        if value
+          integer = sprintf(format, value.to_i)
+          Proc.new { integer }
+        elsif range
+          Proc.new { sprintf(format, self.range(range)) }
+        elsif countup
+          Proc.new {|prev| v = prev + 1; [sprintf(format, v), v] }
+        else
+          Proc.new { sprintf(format, rand(0..2,147,483,647)) }
+        end
+      else
+        if value
+          integer = value.to_i
+          Proc.new { integer }
+        elsif range
+          Proc.new { self.range(range) }
+        elsif countup
+          Proc.new {|prev| prev + 1 }
+        else
+          Proc.new { rand(0..2,147,483,647) }
+        end
+      end
+    end
+
+    def float(opts = {})
+      format, range, value = opts[:format], opts[:range], opts[:value]
+      if format
+        if value
+          float = value.to_f
+          Proc.new { sprintf(format, float) }
+        elsif range
+          Proc.new { sprintf(format, self.range(range)) }
+        else
+          Proc.new { r = rand(1..358); sprintf(format, r * Math.cos(r)) }
+        end
+      else
+        if value
+          float = value.to_f
+          Proc.new { float }
+        elsif range
+          Proc.new { self.range(range) }
+        else
+          Proc.new { r = rand(1..358); r * Math.cos(r) }
+        end
+      end
+    end
+
+    def datetime(opts = {})
+      format, random, value = (opts[:format] || "%Y-%m-%d %H:%M:%S.%3N"), (opts[:random] || false), opts[:value]
+      if value
+        Proc.new { value.strftime(format) }
+      elsif random
+        Proc.new {
+          y = rand(1970..2037);
+          m = rand(1..12);
+          d = rand(1..27);
+          h = rand(0..23);
+          min = rand(0..59);
+          s = rand(0..59);
+          usec = rand(0..999999);
+          Time.local(y, m, d, h, min, s, usec).strftime(format)
+        }
+      else
+        Proc.new { Time.now.strftime(format) }
+      end
+    end
+
+    # private
+
+    def range(range)
+      rand(range)
+    end
+
+    def any(any)
+      any[rand(any.size)]
+    end
+
+    def rand(arg = nil)
+      @rand.rand(arg)
+    end
+  end
+end

data/lib/dummer/version.rb

@@ -1,3 +1,3 @@
 module Dummer
-  VERSION = "0.3.5"
+  VERSION = "0.4.0"
 end

metadata

@@ -1,142 +1,125 @@
 --- !ruby/object:Gem::Specification
 name: dummer
 version: !ruby/object:Gem::Version
-  version: 0.3.5
-  prerelease: 
+  version: 0.4.0
 platform: ruby
 authors:
 - sonots
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-23 00:00:00.000000000 Z
+date: 2015-08-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: serverengine
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: fluent-logger
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: pry-nav
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: Generates dummy log data for Fluentd benchmark
@@ -149,8 +132,8 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .travis.yml
+- ".gitignore"
+- ".travis.yml"
 - CHANGELOG.md
 - Gemfile
 - LICENSE.txt
@@ -160,6 +143,8 @@ files:
 - bin/dummer_simple
 - bin/dummer_yes
 - dummer.gemspec
+- example/countup.conf
+- example/fluent.conf
 - example/fluent_logger.conf
 - example/input.conf
 - example/input.txt
@@ -170,6 +155,12 @@ files:
 - lib/dummer/dsl.rb
 - lib/dummer/error.rb
 - lib/dummer/generator.rb
+- lib/dummer/generator/abstract_mode.rb
+- lib/dummer/generator/factory.rb
+- lib/dummer/generator/field_mode.rb
+- lib/dummer/generator/input_mode.rb
+- lib/dummer/generator/message_mode.rb
+- lib/dummer/random.rb
 - lib/dummer/setting.rb
 - lib/dummer/version.rb
 - lib/dummer/worker.rb
@@ -179,32 +170,25 @@ files:
 homepage: https://github.com/sonots/dummer
 licenses:
 - MIT
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: -916554641
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: -916554641
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 2.2.2
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Generates dummy log data for Fluentd benchmark
 test_files: []