embulk-input-random 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 344eddc7c080632f70da8dacc8d9d96f38954e07
+  data.tar.gz: c03c9dd6a7e71f2eace49339f0fa7f240d4f931c
+SHA512:
+  metadata.gz: 281fb6c47bf7763fa9e0609c34b3c99aa3db634419962b38d025bfcbaf33acb8638a4b75689f974de7cad7bac20fcb087a41ec3a562cee1704ebd60c22c8aa72
+  data.tar.gz: 9d82e8a8656c33271560aa57b0dfd2262f52e4de30488bf76999b1af81412d94b4b7ce0201cec5699a07bbe31502a869710c1bfba47377c7c4a54172b0eedee2

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 KUMAZAKI Hiroki
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,109 @@
+# Embulk::Input::Random
+
+Random data generator for [Embulk](https://github.com/embulk/embulk).
+Expected to used to test or benchmark.
+
+## Installation
+
+Run this command with your embulk binary.
+
+```ruby
+$ embulk gem install embulk-input-random
+```
+
+## Usage
+
+Specify in your config.yml file
+
+```yaml
+in:
+  type: random
+  rows: 100
+  threads: 2
+  schema:
+    myid: primary_key
+    name: string
+    score: integer
+```
+
+- type: specify this plugin as `random`
+- rows: number of inserting rows (required)
+- threads: number of thread (optional)
+- schema: specify the attribute of table and data type (required)
+
+
+### Try
+
+You can try this plugin with saving below as random.yaml
+
+```
+exec: {}
+in:
+  type: random
+  rows: 100
+  threads: 1
+  schema:
+    id: primary_key
+    name: string
+    score: integer
+out:
+  type: stdout
+```
+
+and just run
+
+```
+$ java -jar embulk.jar preview random.yml
+```
+
+will generate result like
+
+```
+Random generation started.
+Random generator input thread 0...
++---------+---------------------------------------------+------------+
+| id:long |                                 name:string | score:long |
++---------+---------------------------------------------+------------+
+|       0 | UPPYQ0S1oiKDddasQxOlXPhZ9ys-FtVwH6-DIywnHG8 |        875 |
+|       1 | IT8KHcI48wM_0ygtm8OVSZQSR1xA4g5lntZ9xAQwY5Y |      2,652 |
+|       2 | 6HOLiPz9-srgwV8bgBX0Whd7Dq6HRUPKusZdONRxesw |      8,560 |
+|       3 | r8X3G5iVZsJJEAp5Wqy8LdUte-2wmnz2Zb9gMiiTp-Q |      2,288 |
+|       4 | g7DDPm6J0y6G9FYGcDgsMk-V6Rewz03sLIu3VUfmp5M |      2,065 |
+|       5 | fwNJWwztnraaa9MH01sq1Uhx2iz66djdkeUSw18DFnQ |      9,214 |
+|       6 | EE2WZ3Z7UIFN4U93fgjWYmGqzWEruVBVBaWJXGjfCsQ |      9,972 |
+|       7 | 70WHrnDYAPx5qNtRxcG2HF-Y4yMO1SXigMep0NFtOo8 |      2,988 |
+|       8 | wbbi1qQlC3x0WY8uksUc_b0PvJjN6e6QhTrykMF7BJE |        831 |
+|       9 | zNBjwP_l1Fu7t8b4xIiYz7dfEO0v0BHS5vZd-xqdmCk |      8,596 |
+|      10 | Rj_NNmf4MG0UASImGbmEHPKf_MUOZe97Jyrs5RQA3q4 |      1,129 |
+...
+```
+
+You can insert arbitrary storage via Embulk!
+
+### Data Type
+
+Now supported types are belows
+- string: 32bytes of ascii code string
+- integer: random integer 0 to 10000
+- primary_key: increasing number for each rows
+- float: random floating point 0 to 10000
+- date: random date from 1970 to now
+
+More and more types will be appended...
+
+## Todo
+
+- Add more data type to generate
+  - fake user names
+  - flexible length of strings
+  - flexible range of numerics
+  - random generator with gaussian
+  - binary
+
+## Contributing
+
+1. Fork it ( https://github.com/kumagi]/embulk-input-random/fork )
+2. Create your feature branch (`git checkout -b my-cool-feature`)
+3. Commit your changes (`git commit -am 'Add cool feature'`)
+4. Push to the branch (`git push origin my-cool-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/embulk-input-random.gemspec

@@ -0,0 +1,22 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+Gem::Specification.new do |spec|
+  spec.name          = "embulk-input-random"
+  spec.version       = "0.0.2"
+  spec.authors       = ["KUMAZAKI Hiroki"]
+  spec.email         = ["hiroki.kumazaki@gmail.com"]
+  spec.summary       = %q{Random Table Generator for Embulk}
+  spec.description   = %q{Create dummy table}
+  spec.homepage      = "https://github.com/kumagi/embulk-input-random"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

data/lib/embulk/input/random.rb

@@ -0,0 +1,77 @@
+# -*- coding:utf-8 -*-
+
+module Embulk
+  require 'securerandom'
+  class InputRandom < InputPlugin
+    require 'json'
+
+    Plugin.register_input('random', self)
+
+    def self.transaction(config, &control)
+      schema = config.param('schema', :hash)
+      rows = config.param('rows', :integer, default: 5000)
+      threads = config.param('threads', :integer, default: 1)
+
+      columns = schema.each_with_index.map{|column, index|
+        attr, type = column
+        # TODO: type should more flexible
+        case type.downcase
+        when "boolean"
+          Column.new(index, attr, :boolean)
+        when "string"
+          Column.new(index, attr, :string)
+        when "integer", "int", "long", "primary_key"
+          Column.new(index, attr, :long)
+        when "double", "float"
+          Column.new(index, attr, :double)
+        when "date"
+          Column.new(index, attr, :timestamp)
+        end
+      }
+
+      task = {'schema' => schema, 'rows' => rows}
+
+      puts "Random generation started."
+      commit_reports = yield(task, columns, threads)
+      puts "Random input finished. Commit reports = #{commit_reports.to_json}"
+
+      return {}
+    end
+
+    def initialize(task, schema, index, page_builder)
+      super
+    end
+
+    def run
+      puts "Random generator input thread #{@index}..."
+      rows = @task['rows']
+      schema = @task['schema']
+
+      rows.times{|n|
+        @page_builder.add(schema.map{|attr, type|
+                            case type
+                            when "string"
+                              SecureRandom.urlsafe_base64(32)
+                            when "integer", "int", "long"
+                              (Random.rand * 10000).to_i
+                            when "primary_key"
+                              n
+                            when 'float', 'double'
+                              Random.rand * 10000
+                            when 'date'
+                              Time.at(rand * Time.now.to_f)
+                            else
+                              raise "unknown type: #{type}"
+                            end
+                          })
+      }
+      @page_builder.finish
+
+      {  # commit report
+        "rows" => rows,
+        "columns" => schema.size
+      }
+    end
+  end
+
+end

metadata

@@ -0,0 +1,80 @@
+--- !ruby/object:Gem::Specification
+name: embulk-input-random
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- KUMAZAKI Hiroki
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-02-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: Create dummy table
+email:
+- hiroki.kumazaki@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- embulk-input-random.gemspec
+- lib/embulk/input/random.rb
+homepage: https://github.com/kumagi/embulk-input-random
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Random Table Generator for Embulk
+test_files: []
+has_rdoc: