dummy_log_generator 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 239561651394dd1e8a834b4d6c96d4b40aa2477d
+  data.tar.gz: 5359dbbd8aac2e4a62f6d6eceaac9bd8bbc39013
+SHA512:
+  metadata.gz: 71dc67b5580ff0ece0e3d908728b3babfdc35755235e6577f94d3ff2b84dd45a93491f85117c4439d596bc0d54f48ef7aad69ba37e57c4d6408bffe40c077917
+  data.tar.gz: 345b86a8f396f711ad09886df24af56eecc4f9c30edb63acb16b7d51b48c7b4557ff257c7646c2caa29ef39dfbd993616b2055f412a6780724dab4a56b4ff297

data/.gitignore

@@ -0,0 +1,26 @@
+*.gem
+*.rbc
+~*
+#*
+*~
+.bundle
+Gemfile.lock
+.rbenv-version
+.ruby-version
+vendor
+doc/*
+tmp/*
+.yardoc
+pkg
+.config
+.yardoc
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.travis.yml

@@ -0,0 +1,4 @@
+rvm:
+  - 2.0.0
+gemfile:
+  - Gemfile

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+### 0.0.1
+
+First version

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in dummy_log_generator.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 sonots
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,127 @@
+# DummyLogGenerator
+
+Generates dummy log data for Fluentd benchmark 
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'dummy_log_generator'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install dummy_log_generator
+
+Run as
+
+    $ dummy_log_generator -c dummy_data_generator.conf
+
+## Usage
+
+Sample configuration is as follows:
+
+```ruby
+# dummy_log_generator.conf
+configure 'sample' do
+  rate 500
+  delimiter "\t"
+  labeled false
+  field :time, type: :datetime, format: "[%Y-%m-%d %H:%M:%S]", random: true
+  field :level, type: :string, any: %w[DEBUG INFO WARN ERROR]
+  field :method, type: :string, any: %w[GET POST PUT]
+  field :uri, type: :string, any: %w[/api/v1/people /api/v1/textdata /api/v1/messages]
+  field :reqtime, type: :float, range: 0.1..5.0
+  field :foobar, type: :string, length: 8
+end 
+```
+
+Running dummy_log_generator outputs like
+
+```
+[1984-05-25 05:10:03]   DEBUG   GET     /api/v1/people  4.451362369925074       fl8nmh4f
+[2027-11-04 02:58:01]   INFO    GET     /api/v1/people  3.984084722909503       WhsvwEeF
+[1973-02-17 09:09:34]   WARN    GET     /api/v1/people  2.290704255755689       3UyK3jgi
+```
+
+## Configuration Parameters
+
+Following parameters for configuration is available
+
+* rate
+
+    Specify how many messages to generate per second. Default: 500 msgs / sec
+
+* delimiter
+
+    Specify the delimiter between each columns. Default: tab
+
+* labeled
+
+    Add label or not. Default: true
+
+* field
+
+    Define data fields to generate
+
+## Data Types
+
+You can specify following data types to your `field` parameters:
+
+* :datetime
+
+  * :format
+
+    You can specify format of datetime as `%Y-%m-%d %H:%M:%S`. See [Time#strftime](http://www.ruby-doc.org/core-2.0.0/Time.html#method-i-strftime) for details. 
+
+  * :random
+
+    Generate datetime randomly or not (Time.now). Default: false
+
+* :string
+
+  * :any
+
+    You can specify an array of strings, then the generator picks one from them randomly
+
+  * :length
+
+    You can specify the length of string to generate randomly
+
+* :integer
+
+  * :range
+
+    You can specify a range of integers, then the generator picks one in the range (uniform) randomly
+
+* :float
+
+  * :range
+
+    You can specify a range of integers, then the generator picks one in the range (uniform) randomly
+
+## Relatives
+
+There is a [fluent-plugin-dummydata-producer](https://github.com/tagomoris/fluent-plugin-dummydata-producer), but I wanted to output dummy data to a log file, and I wanted a standalone tool.
+
+## ToDO
+
+1. write tests
+2. make it slim (remove active_support, etc)
+3. outputs to a file (currently, outputs to STDOUT)
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+## Licenses
+
+See [LICENSE.txt](LICENSE.txt)
+

data/Rakefile

@@ -0,0 +1,15 @@
+# encoding: utf-8
+require "bundler/gem_tasks"
+
+require 'rspec/core'
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec) do |spec|
+    spec.pattern = FileList['spec/**/*_spec.rb']
+end
+task :default => :spec
+
+desc 'Open an irb session preloaded with the gem library'
+task :console do
+    sh 'irb -rubygems -I lib -r dummy_log_generator.rb'
+end
+task :c => :console

data/bin/dummy_log_generator

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+#
+# This file was generated by Bundler.
+#
+# The application 'yohoushi' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require 'pathname'
+ENV['BUNDLE_GEMFILE'] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+require 'rubygems'
+require 'bundler/setup'
+
+load File.expand_path("../../lib/dummy_log_generator/cli.rb", __FILE__)

data/dummy_log_generator.conf

@@ -0,0 +1,11 @@
+configure 'sample' do
+  rate 500
+  delimiter "\t"
+  labeled false
+  field :time, type: :datetime, format: "[%Y-%m-%d %H:%M:%S]", random: true
+  field :level, type: :string, any: %w[DEBUG INFO WARN ERROR]
+  field :method, type: :string, any: %w[GET POST PUT]
+  field :uri, type: :string, any: %w[/api/v1/people /api/v1/textdata /api/v1/messages]
+  field :reqtime, type: :float, range: 0.1..5.0
+  field :foobar, type: :string, length: 8
+end

data/dummy_log_generator.gemspec

@@ -0,0 +1,31 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'dummy_log_generator/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "dummy_log_generator"
+  spec.version       = DummyLogGenerator::VERSION
+  spec.authors       = ["sonots"]
+  spec.email         = ["sonots@gmail.com"]
+  spec.description   = %q{Generates dummy log data for Fluentd benchmark}
+  spec.summary       = %q{Generates dummy log data for Fluentd benchmark}
+  spec.homepage      = "https://github.com/sonots/dummy_log_generator"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "thor"
+  spec.add_dependency "serverengine"
+  spec.add_dependency "active_support"
+  spec.add_dependency "i18n"
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "pry"
+  spec.add_development_dependency "pry-nav"
+end

data/lib/dummy_log_generator/cli.rb

@@ -0,0 +1,89 @@
+require 'thor'
+require 'dummy_log_generator'
+require 'active_support/core_ext'
+
+module DummyLogGenerator
+  class CLI < Thor
+    class_option :config, :aliases => ["-c"], :type => :string, :default => 'dummy_log_generator.conf'
+    default_command :start
+
+    def initialize(args = [], opts = [], config = {})
+      super(args, opts, config)
+
+      @options = @options.dup # avoid frozen
+      if options[:config] && File.exists?(options[:config])
+        dsl = instance_eval(File.read(options[:config]), options[:config])
+        @options[:formatter] = dsl.formatter
+        @options[:rate]      = dsl.config.rate
+      end
+    end
+
+    desc "start", "Start a dummy_log_generator"
+    option :require,     :aliases => ["-r"], :type => :string
+    option :daemonize,   :aliases => ["-d"], :type => :boolean
+    option :module_name, :aliases => ["-m"], :type => :string, :default => 'DummyLogGenerator::Worker'
+    # options for dummy_log_generator
+    option :rate,        :aliases => ["-i"], :type => :numeric
+    def start
+      opts = @options.symbolize_keys.except(:require, :config, :module_name)
+
+      se = ServerEngine.create(nil, @options["module_name"].constantize, opts)
+      se.run
+    end
+
+    desc "stop", "Stops a dummy_log_generator"
+    option :pid_path,    :aliases => ["-p"], :type => :string
+    def stop
+      pid = File.read(@options["pid_path"]).to_i
+
+      begin
+        Process.kill("QUIT", pid)
+        puts "Stopped #{pid}"
+      rescue Errno::ESRCH
+        puts "DummyLogGenerator #{pid} not running"
+      end
+    end
+
+    desc "graceful_stop", "Gracefully stops a dummy_log_generator"
+    option :pid_path,    :aliases => ["-p"], :type => :string
+    def graceful_stop
+      pid = File.read(@options["pid_path"]).to_i
+
+      begin
+        Process.kill("TERM", pid)
+        puts "Gracefully stopped #{pid}"
+      rescue Errno::ESRCH
+        puts "DummyLogGenerator #{pid} not running"
+      end
+    end
+
+    desc "restart", "Restarts a dummy_log_generator"
+    option :pid_path,    :aliases => ["-p"], :type => :string
+    def restart
+      pid = File.read(@options["pid_path"]).to_i
+
+      begin
+        Process.kill("HUP", pid)
+        puts "Restarted #{pid}"
+      rescue Errno::ESRCH
+        puts "DummyLogGenerator #{pid} not running"
+      end
+    end
+
+    desc "graceful_restart", "Graceful restarts a dummy_log_generator"
+    option :pid_path,    :aliases => ["-p"], :type => :string
+    def graceful_restart
+      pid = File.read(@options["pid_path"]).to_i
+
+      begin
+        Process.kill("USR1", pid)
+        puts "Gracefully restarted #{pid}"
+      rescue Errno::ESRCH
+        puts "DummyLogGenerator #{pid} not running"
+      end
+    end
+
+  end
+end
+
+DummyLogGenerator::CLI.start(ARGV)

data/lib/dummy_log_generator/dsl.rb

@@ -0,0 +1,61 @@
+module DummyLogGenerator
+  class Config
+    attr_accessor :rate
+
+    def initiaize
+      @rate = 500
+    end
+  end
+
+  class Formatter
+    attr_accessor :labeled, :delimiter, :fields
+
+    def initialize
+      @labeled = true
+      @delimiter = "\t"
+      @fields = {}
+    end
+
+    def output(generated_params)
+      if labeled
+        generated_params.map {|key, val| "#{key}:#{val}" }.join(delimiter)
+      else
+        generated_params.values.join(delimiter)
+      end
+    end
+  end
+end
+
+module DummyLogGenerator
+  class Dsl
+    attr_reader :formatter
+    attr_reader :config
+
+    def initialize
+      @formatter = Formatter.new
+      @config = Config.new
+    end
+
+    def rate(rate)
+      config.rate = rate
+    end
+
+    def field(name, opts)
+      formatter.fields[name] = opts
+    end
+
+    def delimiter(delimiter)
+      formatter.delimiter = delimiter
+    end
+
+    def labeled(labeled)
+      formatter.labeled = labeled
+    end
+  end
+end
+
+def configure(title, &block)
+  dsl = DummyLogGenerator::Dsl.new
+  dsl.instance_eval(&block)
+  dsl
+end

data/lib/dummy_log_generator/error.rb

@@ -0,0 +1,3 @@
+module DummyLogGenerator
+  class ConfigError < StandardError; end
+end

data/lib/dummy_log_generator/generator.rb

@@ -0,0 +1,93 @@
+module DummyLogGenerator
+  class Generator
+    attr_reader :formatter
+    attr_reader :rand
+
+    def initialize(formatter)
+      @formatter = formatter
+      @rand = ::DummyLogGenerator::Random.new
+    end
+
+    def generate
+      fields = {}
+      formatter.fields.each do |key, opts|
+        opts = opts.dup
+        type = opts.delete(:type)
+        fields[key] = case type
+                      when :datetime
+                        rand.datetime(opts)
+                      when :string
+                        rand.string(opts)
+                      when :integer
+                        rand.integer(opts)
+                      when :float
+                        rand.float(opts)
+                      else
+                        raise ConfigError.new(type)
+                      end
+      end
+      formatter.output(fields)
+    end
+    alias_method :gen, :generate
+  end
+
+  class Random
+    def initialize
+      @rand = ::Random.new(0)
+      @chars = ('a'..'z').to_a + ('A'..'Z').to_a + ('0'..'9').to_a # no symbols and multi-bytes for now
+    end
+
+    def string(length: 8, any: nil)
+      unless any.nil?
+        self.any(any)
+      else
+        Array.new(length){@chars[rand(@chars.size-1)]}.join
+      end
+    end
+
+    def interger(range: nil)
+      unless range.nil?
+        self.range(range)
+      else
+        rand(0..2,147,483,647)
+      end
+    end
+
+    def float(range: nil)
+      unless range.nil?
+        self.range(range)
+      else
+        r = rand(1..358)
+        r * Math.cos(r) # cheat
+      end
+    end
+
+    def datetime(format: "%Y-%m-%d %H:%M:%S.%3N", random: false)
+      time = if random
+               y = rand(1970..2037);
+               m = rand(1..12);
+               d = rand(1..27);
+               h = rand(0..23);
+               min = rand(0..59);
+               s = rand(0..59);
+               usec = rand(0..999999);
+               Time.local(y, m, d, h, min, s, usec)
+             else
+               Time.now
+             end
+      time.strftime(format)
+    end
+
+    def range(range)
+      rand(range)
+    end
+
+    def any(any)
+      any[rand(any.size-1)]
+    end
+
+    def rand(arg = nil)
+      @rand.rand(arg)
+    end
+  end
+end

data/lib/dummy_log_generator/version.rb

@@ -0,0 +1,3 @@
+module DummyLogGenerator
+  VERSION = "0.0.1"
+end

data/lib/dummy_log_generator/worker.rb

@@ -0,0 +1,40 @@
+require 'serverengine'
+
+module DummyLogGenerator
+  module Worker
+    def initialize
+      reload
+    end
+
+    def reload
+      @rate = config[:rate] || 500 # msgs / sec
+      @formatter = config[:formatter]
+      @generator = Generator.new(@formatter)
+    end
+
+    def run
+      batch_num = (@rate / 9).to_i + 1
+      while !@stop
+        current_time = Time.now.to_i
+        rate_count = 0
+
+        while !@stop && rate_count < @rate && Time.now.to_i == current_time
+          batch_num.times do
+            # ToDo: what if generation is slower than I/O?
+            STDOUT.puts @generator.generate
+          end
+          rate_count += batch_num
+          sleep 0.1
+        end
+        # wait for next second
+        while !@stop && Time.now.to_i == current_time
+          sleep 0.04
+        end
+      end
+    end
+
+    def stop
+      @stop = true
+    end
+  end
+end

data/lib/dummy_log_generator.rb

@@ -0,0 +1,7 @@
+require 'logger'
+require "dummy_log_generator/version"
+require "dummy_log_generator/error"
+require "dummy_log_generator/generator"
+require "dummy_log_generator/worker"
+require "dummy_log_generator/dsl"
+

metadata

@@ -0,0 +1,188 @@
+--- !ruby/object:Gem::Specification
+name: dummy_log_generator
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- sonots
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-11-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: serverengine
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: active_support
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: i18n
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-nav
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Generates dummy log data for Fluentd benchmark
+email:
+- sonots@gmail.com
+executables:
+- dummy_log_generator
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .travis.yml
+- CHANGELOG.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/dummy_log_generator
+- dummy_log_generator.conf
+- dummy_log_generator.gemspec
+- lib/dummy_log_generator.rb
+- lib/dummy_log_generator/cli.rb
+- lib/dummy_log_generator/dsl.rb
+- lib/dummy_log_generator/error.rb
+- lib/dummy_log_generator/generator.rb
+- lib/dummy_log_generator/version.rb
+- lib/dummy_log_generator/worker.rb
+homepage: https://github.com/sonots/dummy_log_generator
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Generates dummy log data for Fluentd benchmark
+test_files: []