dummer 0.3.0

data/bin/dummer_simple

@@ -0,0 +1,81 @@
+#!/usr/bin/env ruby
+require 'thor'
+
+module DummerSimple
+  class CLI < Thor
+    default_command :start
+    desc "start", "Start a dummer"
+    option :sync,                         :type => :boolean, :desc => 'Set `IO#sync=true`'
+    option :second,   :aliases => ["-s"], :type => :numeric, :default => 1, :desc => 'Duration of running in second'
+    option :parallel, :aliases => ["-p"], :type => :numeric, :default => 1, :desc => 'Number of processes to run in parallel'
+    option :output,   :aliases => ["-o"], :type => :string,  :default => 'dummy.log', :desc => 'Output file'
+    option :input,    :aliases => ["-i"], :type => :string,  :desc => 'Input file (Output messages by reading lines of the file in rotation)'
+    option :message,  :aliases => ["-m"], :type => :string,  :desc => 'Output message',
+      :default => "time:2013-11-20 23:39:42 +0900\tlevel:ERROR\tmethod:POST\turi:/api/v1/people\treqtime:3.1983877060667103\n"
+    def start
+      sync     = @options[:sync]
+      parallel = @options[:parallel]
+      output   = @options[:output]
+      message  = "#{@options[:message].chomp}\n"
+      messages = nil
+      if input = @options[:input] and (messages = readlines(input)).nil?
+        STDERR.puts "Input file `#{input}` does not exist or is not readable."
+        exit 1
+      end
+
+      open(output, (File::WRONLY | File::APPEND | File::CREAT)) do |out_file|
+        out_file.sync = true if sync
+        if messages
+          wait_and_kill { write_messages(out_file, messages, parallel) }
+        else
+          wait_and_kill { write_message(out_file, message, parallel) }
+        end
+      end
+    end
+
+    no_commands {
+      def wait_and_kill(&block)
+        second = Time.now + @options[:second]
+        pids = yield # fork
+        while Time.now < second do; sleep 0.01; end
+        pids.each {|pid| Process.kill(:TERM, pid) }
+      end
+
+      def write_message(out_file, message, parallel = 1)
+        parallel.times.map do
+          Process.fork do
+            while true do
+              out_file.write message
+            end
+          end
+        end
+      end
+
+      def write_messages(out_file, messages, parallel = 1)
+        size = messages.size
+        parallel.times.map do
+          Process.fork do
+            idx = -1 # create this in child-process not to be shared
+            while true do
+              idx = (idx + 1) % size
+              out_file.write messages[idx]
+            end
+          end
+        end
+      end
+
+      def readlines(input)
+        messages = nil
+        begin
+          open(input) do |in_file|
+            messages = in_file.readlines
+          end
+        rescue Errno::ENOENT
+        end
+        messages
+      end
+    }
+  end
+end
+
+DummerSimple::CLI.start(ARGV)

data/bin/dummer_yes

@@ -0,0 +1,36 @@
+#!/usr/bin/env ruby
+require 'thor'
+
+module DummerYes
+  class CLI < Thor
+    default_command :start
+    desc "start", "Start a dummer"
+    option :second,   :aliases => ["-s"], :type => :numeric, :default => 1, :desc => 'Duration of running in second'
+    option :parallel, :aliases => ["-p"], :type => :numeric, :default => 1, :desc => 'Number of processes to run in parallel'
+    option :output,   :aliases => ["-o"], :type => :string,  :default => 'dummy.log', :desc => 'Output file'
+    option :message,  :aliases => ["-m"], :type => :string,  :desc => 'Output message',
+      :default => "time:2013-11-20 23:39:42 +0900\tlevel:ERROR\tmethod:POST\turi:/api/v1/people\treqtime:3.1983877060667103\n"
+    def start
+      parallel = @options[:parallel]
+      output   = @options[:output]
+      message  = "#{@options[:message].chomp}\n"
+      wait_and_kill {
+        parallel.times.map do
+          spawn("yes #{message}", :out => [output, (File::WRONLY | File::APPEND | File::CREAT)])
+        end
+      }
+    end
+
+    no_commands {
+      def wait_and_kill(&block)
+        second = Time.now + @options[:second]
+        pids = yield # fork
+        while Time.now < second do; sleep 0.01; end
+        pids.each {|pid| Process.kill(:TERM, pid) }
+      end
+    }
+  end
+end
+
+DummerYes::CLI.start(ARGV)
+

data/dummer.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'dummer/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "dummer"
+  spec.version       = Dummer::VERSION
+  spec.authors       = ["sonots"]
+  spec.email         = ["sonots@gmail.com"]
+  spec.description   = %q{Generates dummy log data for Fluentd benchmark}
+  spec.summary       = %q{Generates dummy log data for Fluentd benchmark}
+  spec.homepage      = "https://github.com/sonots/dummer"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "thor"
+  spec.add_dependency "serverengine"
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "pry"
+  spec.add_development_dependency "pry-nav"
+end

data/example/input.conf

@@ -0,0 +1,6 @@
+configure 'input' do
+  output "dummer.log"
+  rate 500
+  input "example/input.txt"
+end
+

data/example/input.txt

@@ -0,0 +1,10 @@
+id:0	time:2013-11-26 02:45:33 +0900	level:WARN	method:GET	uri:/api/v1/people	reqtime:2.3955637407914354	foobar:3xwwVG3T
+id:1	time:2013-11-26 02:45:33 +0900	level:INFO	method:POST	uri:/api/v1/people	reqtime:4.954494736221596	foobar:kRP5SiBW
+id:2	time:2013-11-26 02:45:33 +0900	level:DEBUG	method:POST	uri:/api/v1/people	reqtime:1.719978260834408	foobar:0EFaal3L
+id:3	time:2013-11-26 02:45:33 +0900	level:DEBUG	method:POST	uri:/api/v1/people	reqtime:3.2873443102110755	foobar:sGMFl2xs
+id:4	time:2013-11-26 02:45:33 +0900	level:INFO	method:POST	uri:/api/v1/textdata	reqtime:1.8776513072710397	foobar:dTlmjln3
+id:5	time:2013-11-26 02:45:33 +0900	level:INFO	method:POST	uri:/api/v1/people	reqtime:2.7644992372264974	foobar:jLL6zTdt
+id:6	time:2013-11-26 02:45:33 +0900	level:WARN	method:POST	uri:/api/v1/people	reqtime:4.609761906663468	foobar:k2hegeOG
+id:7	time:2013-11-26 02:45:33 +0900	level:DEBUG	method:POST	uri:/api/v1/people	reqtime:1.7312535151311619	foobar:JpwUkYKs
+id:8	time:2013-11-26 02:45:33 +0900	level:WARN	method:POST	uri:/api/v1/people	reqtime:2.2088570489185217	foobar:eOSmfjCb
+id:9	time:2013-11-26 02:45:33 +0900	level:WARN	method:POST	uri:/api/v1/people	reqtime:2.6351789530831637	foobar:cFBI2Gsm

data/example/message.conf

@@ -0,0 +1,5 @@
+configure 'message' do
+  output "dummer.log"
+  rate 500
+  message "time:2013-11-25 00:23:52 +0900\tlevel:ERROR\tmethod:POST\turi:/api/v1/people\treqtime:3.1983877060667103\n"
+end

data/example/sample.conf

@@ -0,0 +1,13 @@
+configure 'sample' do
+  output "dummer.log"
+  rate 500
+  delimiter "\t"
+  labeled true
+  field :id, type: :integer, countup: true, format: "%04d"
+  field :time, type: :datetime, format: "[%Y-%m-%d %H:%M:%S]", random: false
+  field :level, type: :string, any: %w[DEBUG INFO WARN ERROR]
+  field :method, type: :string, any: %w[GET POST PUT]
+  field :uri, type: :string, any: %w[/api/v1/people /api/v1/textdata /api/v1/messages]
+  field :reqtime, type: :float, range: 0.1..5.0
+  field :foobar, type: :string, length: 8
+end

data/lib/dummer.rb

@@ -0,0 +1,6 @@
+require "dummer/version"
+require "dummer/error"
+require "dummer/setting"
+require "dummer/generator"
+require "dummer/worker"
+require "dummer/dsl"

data/lib/dummer/cli.rb

@@ -0,0 +1,96 @@
+require 'thor'
+require 'dummer'
+require 'ext/hash/keys'
+require 'ext/hash/except'
+
+module Dummer
+  class CLI < Thor
+    # options for serverengine
+    class_option :pid_path, :aliases => ["-p"], :type => :string, :default => 'dummer.pid'
+    default_command :start
+
+    def initialize(args = [], opts = [], config = {})
+      super(args, opts, config)
+    end
+
+    desc "start", "Start a dummer"
+    option :config,    :aliases => ["-c"], :type => :string, :default => 'dummer.conf', :desc => 'Config file'
+    option :rate,      :aliases => ["-r"], :type => :numeric, :desc => 'Number of generating messages per second'
+    option :output,    :aliases => ["-o"], :type => :string, :desc => 'Output file'
+    option :message,   :aliases => ["-m"], :type => :string, :desc => 'Output message'
+    # options for serverengine
+    option :daemonize, :aliases => ["-d"], :type => :boolean, :desc => 'Daemonize. Stop with `dummer stop`'
+    option :workers,   :aliases => ["-w"], :type => :numeric, :desc => 'Number of parallels'
+    option :worker_type,                   :type => :string, :default => 'process'
+    def start
+      @options = @options.dup # avoid frozen
+      dsl =
+        if options[:config] && File.exists?(options[:config])
+          instance_eval(File.read(options[:config]), options[:config])
+        else
+          Dummer::Dsl.new
+        end
+      @options[:setting] = dsl.setting
+      dsl.setting.rate = options[:rate] if options[:rate]
+      dsl.setting.output = options[:output] if options[:output]
+      dsl.setting.message = options[:message] if options[:message]
+      # options for serverengine
+      @options[:workers] ||= dsl.setting.workers
+
+      opts = @options.symbolize_keys.except(:config)
+      se = ServerEngine.create(nil, Dummer::Worker, opts)
+      se.run
+    end
+
+    desc "stop", "Stops a dummer"
+    def stop
+      pid = File.read(@options["pid_path"]).to_i
+
+      begin
+        Process.kill("QUIT", pid)
+        puts "Stopped #{pid}"
+      rescue Errno::ESRCH
+        puts "Dummer #{pid} not running"
+      end
+    end
+
+    desc "graceful_stop", "Gracefully stops a dummer"
+    def graceful_stop
+      pid = File.read(@options["pid_path"]).to_i
+
+      begin
+        Process.kill("TERM", pid)
+        puts "Gracefully stopped #{pid}"
+      rescue Errno::ESRCH
+        puts "Dummer #{pid} not running"
+      end
+    end
+
+    desc "restart", "Restarts a dummer"
+    def restart
+      pid = File.read(@options["pid_path"]).to_i
+
+      begin
+        Process.kill("HUP", pid)
+        puts "Restarted #{pid}"
+      rescue Errno::ESRCH
+        puts "Dummer #{pid} not running"
+      end
+    end
+
+    desc "graceful_restart", "Graceful restarts a dummer"
+    def graceful_restart
+      pid = File.read(@options["pid_path"]).to_i
+
+      begin
+        Process.kill("USR1", pid)
+        puts "Gracefully restarted #{pid}"
+      rescue Errno::ESRCH
+        puts "Dummer #{pid} not running"
+      end
+    end
+
+  end
+end
+
+Dummer::CLI.start(ARGV)

data/lib/dummer/dsl.rb

@@ -0,0 +1,28 @@
+module Dummer
+  class Dsl
+    attr_reader :setting
+
+    def initialize
+      @setting = Setting.new
+    end
+
+    def method_missing(name, *args)
+      if @setting.respond_to?("#{name}=")
+        @setting.__send__("#{name}=", *args)
+      else
+        raise ConfigError.new("Config parameter `#{name}` does not exist")
+      end
+    end
+
+    def field(name, opts)
+      setting.fields ||= {}
+      setting.fields[name] = opts
+    end
+  end
+end
+
+def configure(title, &block)
+  dsl = Dummer::Dsl.new
+  dsl.instance_eval(&block)
+  dsl
+end

data/lib/dummer/error.rb

@@ -0,0 +1,3 @@
+module Dummer
+  class ConfigError < StandardError; end
+end

data/lib/dummer/generator.rb

@@ -0,0 +1,178 @@
+module Dummer
+  class Generator
+    def initialize(setting)
+      @message_proc =
+        if fields = setting.fields
+          labeled, delimiter = setting.labeled, setting.delimiter
+          prepare_message_proc_for_fields(fields, labeled, delimiter)
+        elsif input = setting.input
+          prepare_message_proc_for_input(input)
+        else
+          message = setting.message
+          prepare_message_proc_for_message(message)
+        end
+    end
+
+    def prepare_message_proc_for_input(input)
+      messages = nil
+      begin
+        open(input) do |in_file|
+          messages = in_file.readlines
+        end
+      rescue Errno::ENOENT
+        raise ConfigError.new("Input file `#{input}` is not readable")
+      end
+      idx = -1
+      size = messages.size
+      Proc.new {
+        idx = (idx + 1) % size
+        messages[idx]
+      }
+    end
+
+    def prepare_message_proc_for_message(message)
+      message = "#{message.chomp}\n"
+      Proc.new { message }
+    end
+
+    def prepare_message_proc_for_fields(fields, labeled, delimiter)
+      format_proc = prepare_format_proc(labeled, delimiter)
+      field_procs = prepare_field_procs(fields)
+
+      prev_data = {}
+      Proc.new {
+        data = {}
+        field_procs.each do |key, proc|
+          prev = prev_data[key] || -1
+          data[key] = proc.call(prev)
+        end
+        prev_data = data
+        format_proc.call(data)
+      }
+    end
+
+    def prepare_format_proc(labeled, delimiter)
+      if labeled
+        Proc.new {|fields| "#{fields.map {|key, val| "#{key}:#{val}" }.join(delimiter)}\n" }
+      else
+        Proc.new {|fields| "#{fields.values.join(delimiter)}\n" }
+      end
+    end
+
+    def prepare_field_procs(fields)
+      rand = ::Dummer::Random.new
+      field_procs = {}
+      fields.each do |key, opts|
+        opts = opts.dup
+        type = opts.delete(:type)
+        if rand.respond_to?(type)
+          field_procs[key] = rand.send(type, opts)
+        else
+          raise ConfigError.new(type)
+        end
+      end
+      field_procs
+    end
+
+    def generate
+      @message_proc.call
+    end
+  end
+
+  class Random
+    def initialize
+      @rand = ::Random.new(0)
+      @chars = ('a'..'z').to_a + ('A'..'Z').to_a + ('0'..'9').to_a # no symbols and multi-bytes for now
+    end
+
+    def string(length: 8, any: nil, value: nil)
+      if value
+        string = value.to_s
+        Proc.new { string }
+      elsif any
+        Proc.new { self.any(any) }
+      else
+        Proc.new { Array.new(length){@chars[rand(@chars.size-1)]}.join }
+      end
+    end
+
+    def integer(format: nil, range: nil, countup: false, value: nil)
+      if format
+        if value
+          integer = sprintf(format, value.to_i)
+          Proc.new { integer }
+        elsif range
+          Proc.new { sprintf(format, self.range(range)) }
+        elsif countup
+          Proc.new {|prev| sprintf(format, prev.to_i + 1) }
+        else
+          Proc.new { sprintf(format, rand(0..2,147,483,647)) }
+        end
+      else
+        if value
+          integer = value.to_i
+          Proc.new { integer }
+        elsif range
+          Proc.new { self.range(range) }
+        elsif countup
+          Proc.new {|prev| prev + 1 }
+        else
+          Proc.new { rand(0..2,147,483,647) }
+        end
+      end
+    end
+
+    def float(format: nil, range: nil, value: nil)
+      if format
+        if value
+          float = value.to_f
+          Proc.new { sprintf(format, float) }
+        elsif range
+          Proc.new { sprintf(format, self.range(range)) }
+        else
+          Proc.new { r = rand(1..358); sprintf(format, r * Math.cos(r)) }
+        end
+      else
+        if value
+          float = value.to_f
+          Proc.new { float }
+        elsif range
+          Proc.new { self.range(range) }
+        else
+          Proc.new { r = rand(1..358); r * Math.cos(r) }
+        end
+      end
+    end
+
+    def datetime(format: "%Y-%m-%d %H:%M:%S.%3N", random: false, value: nil)
+      if value
+        Proc.new { value.strftime(format) }
+      elsif random
+        Proc.new {
+          y = rand(1970..2037);
+          m = rand(1..12);
+          d = rand(1..27);
+          h = rand(0..23);
+          min = rand(0..59);
+          s = rand(0..59);
+          usec = rand(0..999999);
+          Time.local(y, m, d, h, min, s, usec).strftime(format)
+        }
+      else
+        Proc.new { Time.now.strftime(format) }
+      end
+    end
+
+    def range(range)
+      rand(range)
+    end
+
+    def any(any)
+      any[rand(any.size-1)]
+    end
+
+    def rand(arg = nil)
+      @rand.rand(arg)
+    end
+  end
+end