embulk-plugin-input-random 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d8364449a38b8532afc93e4e0335068a5d95bfc0
+  data.tar.gz: bc62e978be480abf942719fd05395c638c58346d
+SHA512:
+  metadata.gz: 4ba0469a8a7ab6c71deba01e9675e73a5e2ddc48c6711e457a670ac48e998e26c87ab2557686591814a3e3f114c0014a1bfdea9f2d793de51b324fb554f31ab6
+  data.tar.gz: e3d3f693849b4a54176ad13391fba844ed7af0b0d3c9a2c203aa305782946c9f954e3a7ab92e9d49534933a9729f9c0e01f644dfecf13986b0b75eee958d6776

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 KUMAZAKI Hiroki
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,105 @@
+# Embulk::Plugin::Input::Random
+
+Random data generator for [Embulk](https://github.com/embulk/embulk).
+Expected to used to test or benchmark.
+
+## Installation
+
+Run this command with your embulk binary.
+
+```ruby
+java -jar embulk.jar gem install embulk-plugin-input-random
+```
+
+## Usage
+
+Specify in your config.yml file
+
+```yaml
+in:
+  type: random
+  rows: 100
+  threads: 2
+  schema:
+    myid: primary_key
+    name: string
+    score: integer
+```
+
+- type: specify this plugin as `random`
+- rows: number of inserting rows (required)
+- threads: number of thread (optional)
+- schema: specify the attribute of table and data type (required)
+
+
+### Try
+
+You can try this plugin with saving below as random.yaml
+
+```
+exec: {}
+in:
+  type: random
+  rows: 100
+  threads: 1
+  schema:
+    id: primary_key
+    name: string
+    score: integer
+out:
+  type: stdout
+```
+
+and just run
+
+```
+$ java -jar embulk.jar run random.yml
+```
+
+will generate result like
+
+```
+Random generation started.
+2015-01-29 02:13:15,497 [INFO]: main:org.embulk.exec.LocalExecutor: Running 4 tasks using 8 local threads
+2015-01-29 02:13:15,498 [INFO]: main:org.embulk.exec.LocalExecutor: {done:  0 / 4, running: 0}
+Random generator input thread 2...
+Random generator input thread 0...
+Random generator input thread 1...
+Random generator input thread 3...
+0,FamwnrENyzxADpecnTQP2lClzJPO1acRC1506d3VUf4,3791
+1,aAA6K0T8L6plLrux9dqd2oFLZWBkYyNnZKGstN3sRRA,9942
+2,tgGcwBPxZ-tQgJmTdIJXgiv-MDBvbUKnMMn1KjNixlo,5907
+3,SyxFHOepjo5WbNgukUEZY7CVT5Y6XGeVb8Gdsdq_EI8,6378
+4,5vfOwlw_xLhaYkd5dXCUaC3TsAyzDbD12GHohy2SXsM,6132
+5,-C_6oNt1Z07RKlGKQ8_OVbCCuHbC-LOLFl5HrIE8y9c,7520
+```
+
+You can insert arbitrary storage via Embulk!
+
+### Data Type
+
+Now supported types are belows
+- string: 32bytes of ascii code string
+- integer: random integer 0 to 10000
+- primary_key: increasing number for each rows
+- float: random floating point 0 to 10000
+- date: random date from 1970 to now
+
+More and more types will be appended...
+
+## Todo
+
+- Add more data type to generate
+  - fake user names
+  - flexible length of strings
+  - flexible range of numerics
+  - random generator with gaussian
+  - binary
+
+## Contributing
+
+1. Fork it ( https://github.com/kumagi]/embulk-plugin-input-random/fork )
+2. Create your feature branch (`git checkout -b my-cool-feature`)
+3. Commit your changes (`git commit -am 'Add cool feature'`)
+4. Push to the branch (`git push origin my-cool-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/embulk-plugin-input-random.gemspec

@@ -0,0 +1,23 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'embulk/plugin/input/random/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "embulk-plugin-input-random"
+  spec.version       = "0.0.1"
+  spec.authors       = ["KUMAZAKI Hiroki"]
+  spec.email         = ["hiroki.kumazaki@gmail.com"]
+  spec.summary       = %q{Random Table Generator for Embulk}
+  spec.description   = %q{Create dummy table for test or benchmark}
+  spec.homepage      = "https://github.com/kumagi/embulk-plugin-input-random"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+end

data/lib/embulk/input_random.rb

@@ -0,0 +1,75 @@
+module Embulk
+  require 'securerandom'
+  class InputRandom < InputPlugin
+    require 'json'
+
+    Plugin.register_input('random', self)
+
+    def self.transaction(config, &control)
+      schema = config.param('schema', :hash)
+      rows = config.param('rows', :integer, default: 5000)
+      threads = config.param('threads', :integer, default: 1)
+
+      columns = schema.each_with_index.map{|column, index|
+        attr, type = column
+        # TODO: type should more flexible
+        case type.downcase
+        when "boolean"
+          Column.new(index, attr, :boolean)
+        when "string"
+          Column.new(index, attr, :string)
+        when "integer", "int", "long", "primary_key"
+          Column.new(index, attr, :long)
+        when "double", "float"
+          Column.new(index, attr, :double)
+        when "date"
+          Column.new(index, attr, :timestamp)
+        end
+      }
+
+      task = {'schema' => schema, 'rows' => rows}
+
+      puts "Random generation started."
+      commit_reports = yield(task, columns, threads)
+      puts "Random input finished. Commit reports = #{commit_reports.to_json}"
+
+      return {}
+    end
+
+    def initialize(task, schema, index, page_builder)
+      super
+    end
+
+    def run
+      puts "Random generator input thread #{@index}..."
+      rows = @task['rows']
+      schema = @task['schema']
+
+      rows.times{|n|
+        @page_builder.add(schema.map{|attr, type|
+                            case type
+                            when "string"
+                              SecureRandom.urlsafe_base64(32)
+                            when "integer", "int", "long"
+                              (Random.rand * 10000).to_i
+                            when "primary_key"
+                              n
+                            when 'float', 'double'
+                              Random.rand * 10000
+                            when 'date'
+                              Time.at(rand * Time.now.to_f)
+                            else
+                              raise "unknown type: #{type}"
+                            end
+                          })
+      }
+      @page_builder.finish
+
+      {  # commit report
+        "rows" => rows,
+        "columns" => schema.size
+      }
+    end
+  end
+
+end

data/lib/embulk/plugin/input/random/version.rb

@@ -0,0 +1,3 @@
+module EmbulkInputRandom
+  VERSION = "0.0.1"
+end

metadata

@@ -0,0 +1,81 @@
+--- !ruby/object:Gem::Specification
+name: embulk-plugin-input-random
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- KUMAZAKI Hiroki
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-01-28 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: Create dummy table for test or benchmark
+email:
+- hiroki.kumazaki@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- embulk-plugin-input-random.gemspec
+- lib/embulk/input_random.rb
+- lib/embulk/plugin/input/random/version.rb
+homepage: https://github.com/kumagi/embulk-plugin-input-random
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Random Table Generator for Embulk
+test_files: []
+has_rdoc: