embulk 0.7.0-java

data/lib/embulk/command/embulk_selfupdate.rb

@@ -0,0 +1,84 @@
+module Embulk
+  def self.selfupdate(options={})
+    require 'uri'
+
+    jar, resource = __FILE__.split("!", 2)
+    jar_path = URI.parse(jar).path rescue jar
+    unless resource && File.file?(jar_path)
+      STDERR.puts ""
+      STDERR.puts "Embulk is not installed by a single jar. Selfupdate is not supported."
+      STDERR.puts "If you used gem to install embulk, please run: "
+      STDERR.puts ""
+      STDERR.puts "  $ gem install embulk"
+      STDERR.puts ""
+      raise SystemExit.new(1)
+    end
+
+    puts "Checking the latest version..."
+    latest_version = check_latest_version
+
+    if Gem::Version.new(latest_version) <= Gem::Version.new(Embulk::VERSION)
+      puts "Already up-to-date. #{latest_version} is the latest version."
+      return
+    end
+
+    puts "Found new version #{latest_version}."
+
+    unless File.writable?(jar_path)
+      STDERR.puts ""
+      STDERR.puts "Installation path #{jar_path} is not writable."
+      STDERR.puts "Dou you need to run with sudo?"
+      STDERR.puts ""
+      raise SystemExit.new(1)
+    end
+
+    url = "https://dl.bintray.com/embulk/maven/embulk-#{latest_version}.jar"
+    puts "Downloading #{url} ..."
+
+    require 'open-uri'
+    require 'tempfile'
+    Tempfile.open(["embulk-selfupdate", ".jar"]) do |tmp|
+      tmp.chmod(File.stat(jar_path).mode)
+      OpenURI.open_uri(url) do |f|
+        IO.copy_stream(f, tmp)
+      end
+      tmp.close(false)
+
+      # check corruption
+      unless options[:force]
+        begin
+          data = File.read("jar:#{java.io.File.new(tmp.path).toURI.toURL}!/embulk/version.rb")
+          m = Module.new
+          m.module_eval(data)
+          unless m::Embulk::VERSION == latest_version
+            raise "Embulk::VERSION does not match with #{latest_version}"
+          end
+        rescue => e
+          STDERR.puts "Corruption checking failed (#{e})."
+          STDERR.puts "This version might include incompatible changes."
+          STDERR.puts "Please add '-f' argument to selfupdate command to skip checking."
+          raise SystemExit.new(1, e.to_s)
+        end
+      end
+
+      File.rename(tmp.path, jar_path)
+    end
+
+    puts "Updated to #{latest_version}."
+  end
+
+  def self.check_latest_version
+    require 'net/https'
+    bintray = Net::HTTP.new('bintray.com', 443)
+    bintray.use_ssl = true
+    bintray.verify_mode = OpenSSL::SSL::VERIFY_NONE
+    bintray.start do
+      response = bintray.get('/embulk/maven/embulk/_latestVersion')
+      raise "Expected response code 302 Found but got #{response.code}" if response.code != "302"
+      location = response["Location"].to_s
+      m = /(\d+\.\d+[^\/]+)/.match(location)
+      raise "Cound not find version number in Location header '#{location}'" unless m
+      return m[1]
+    end
+  end
+end

data/lib/embulk/data/bundle/.bundle/config

@@ -0,0 +1,3 @@
+---
+BUNDLE_PATH: .
+BUNDLE_DISABLE_SHARED_GEMS: '1'

data/lib/embulk/data/bundle/.ruby-version

@@ -0,0 +1 @@
+jruby-9.0.0.0

data/lib/embulk/data/bundle/Gemfile

@@ -0,0 +1,26 @@
+source 'https://rubygems.org/'
+gem 'embulk', '~> 0.7.0'
+
+#
+# Note: After updating this file, run this command to update Gemfile.lock:
+#
+#       cd this_directory
+#       embulk bundle
+#
+
+#
+# RubyGems Plugins
+#
+#gem 'embulk-output-mysql'
+#gem 'embulk-output-postgresql', '>= 0.1.0'
+#gem 'embulk-input-baz', '= 0.2.0'
+#gem 'embulk-input-xyz', '~> 0.3.0'
+#
+
+#
+# Private plugins
+#
+#gem 'embulk-inupt-awesome', path: '/path/to/your/embulk-input-awesome'
+#gem 'embulk-output-awesome', git: 'https://github.com/you/embulk-output-awesome.git', branch: 'master'
+#
+

data/lib/embulk/data/bundle/embulk/filter/example.rb

@@ -0,0 +1,42 @@
+module Embulk
+  module Filter
+
+    class ExampleFilterPlugin < FilterPlugin
+      # filter plugin file name must be: embulk/filter/<name>.rb
+      Plugin.register_filter('example', self)
+
+      def self.transaction(config, in_schema, &control)
+        task = {
+          'key' => config.param('key', :string, default: "filter_key"),
+          'value' => config.param('value', :string, default: "filter_value")
+        }
+
+        idx = in_schema.size
+        out_columns = in_schema + [Column.new(idx, task['key'], :string)]
+
+        puts "Example filter started."
+        yield(task, out_columns)
+        puts "Example filter finished."
+      end
+
+      def initialize(task, in_schema, out_schema, page_builder)
+        super
+        @value = task['value']
+      end
+
+      def close
+      end
+
+      def add(page)
+        page.each do |record|
+          @page_builder.add(record + [@value])
+        end
+      end
+
+      def finish
+        @page_builder.finish
+      end
+    end
+
+  end
+end

data/lib/embulk/data/bundle/embulk/input/example.rb

@@ -0,0 +1,54 @@
+module Embulk
+  module Input
+
+    class ExampleInputPlugin < InputPlugin
+      # input plugin file name must be: embulk/input/<name>.rb
+      Plugin.register_input('example', self)
+
+      def self.transaction(config, &control)
+        files = ['file1', 'file2']
+        task = {
+          'files' => files,
+          'hostname' => config.param('hostname', :string, default: nil)
+        }
+
+        columns = [
+          Column.new(0, 'file', :string),
+          Column.new(1, 'hostname', :string),
+          Column.new(2, 'col0', :long),
+          Column.new(3, 'col1', :double),
+        ]
+
+        resume(task, columns, files.length, &control)
+      end
+
+      def self.resume(task, columns, count, &control)
+        puts "Example input started."
+        task_reports = yield(task, columns, count)
+        puts "Example input finished. Commit reports = #{task_reports.to_json}"
+
+        next_config_diff = {}
+        return next_config_diff
+      end
+
+      def initialize(task, schema, index, page_builder)
+        super
+        @file = task['files'][index]
+        @hostname = task['hostname']
+      end
+
+      def run
+        puts "Example input thread #{@index}..."
+
+        10.times do |i|
+          @page_builder.add([@file, @hostname, i, 10.0])
+        end
+        @page_builder.finish  # don't forget to call finish :-)
+
+        task_report = {}
+        return task_report
+      end
+    end
+
+  end
+end

data/lib/embulk/data/bundle/embulk/output/example.rb

@@ -0,0 +1,58 @@
+module Embulk
+  module Output
+
+    class ExampleOutputPlugin < OutputPlugin
+      # output plugin file name must be: embulk/output/<name>.rb
+      Plugin.register_output('example', self)
+
+      def self.transaction(config, schema, count, &control)
+        task = {
+          'message' => config.param('message', :string, default: "record")
+        }
+
+        resume(task, schema, count, &control)
+      end
+
+      def self.resume(task, schema, count, &control)
+        puts "Example output started."
+        task_reports = yield(task)
+        puts "Example output finished. Commit reports = #{task_reports.to_json}"
+
+        next_config_diff = {}
+        return next_config_diff
+      end
+
+      def initialize(task, schema, index)
+        puts "Example output thread #{index}..."
+        super
+        @message = task.param('message', :string)
+        @records = 0
+      end
+
+      def close
+      end
+
+      def add(page)
+        page.each do |record|
+          hash = Hash[schema.names.zip(record)]
+          STDOUT.write "#{@message}: #{hash.to_json}\n"
+          @records += 1
+        end
+      end
+
+      def finish
+      end
+
+      def abort
+      end
+
+      def commit
+        task_report = {
+          "records" => @records
+        }
+        return task_report
+      end
+    end
+
+  end
+end

data/lib/embulk/data/new/LICENSE.txt

@@ -0,0 +1,21 @@
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/lib/embulk/data/new/README.md.erb

@@ -0,0 +1,111 @@
+# <%= display_name %> <%= display_category %> plugin for Embulk
+
+%case language
+%when :ruby
+TODO: Write short description here and <%= full_project_name %>.gemspec file.
+%when :java
+TODO: Write short description here and build.gradle file.
+%else
+TODO: Write short description here.
+%end
+
+## Overview
+
+%case category
+%when :output, :file_output
+* **Plugin type**: <%= display_category %>
+* **Load all or nothing**: no
+* **Resume supported**: no
+* **Cleanup supported**: yes
+%when :file_input
+* **Plugin type**: <%= display_category %>
+* **Resume supported**: yes
+* **Cleanup supported**: yes
+%when :input
+* **Plugin type**: <%= display_category %>
+* **Resume supported**: yes
+* **Cleanup supported**: yes
+* **Guess supported**: no
+%when :parser, :decoder
+* **Plugin type**: <%= display_category %>
+* **Guess supported**: no
+%else
+* **Plugin type**: <%= display_category %>
+%end
+
+## Configuration
+
+- **option1**: description (integer, required)
+- **option2**: description (string, default: `"myvalue"`)
+- **option3**: description (string, default: `null`)
+
+## Example
+
+```yaml
+%case category
+%when :input, :file_input
+in:
+  type: <%= name %>
+  option1: example1
+  option2: example2
+%when :output, :file_output
+out:
+  type: <%= name %>
+  option1: example1
+  option2: example2
+%when :filter
+filters:
+  - type: <%= name %>
+    option1: example1
+    option2: example2
+%when :parser
+in:
+  type: any file input plugin type
+  parser:
+    type: <%= name %>
+    option1: example1
+    option2: example2
+%when :formatter
+out:
+  type: any output input plugin type
+  formatter:
+    type: <%= name %>
+    option1: example1
+    option2: example2
+%when :decoder
+in:
+  type: any output input plugin type
+  decoders:
+    - type: <%= name %>
+      option1: example1
+      option2: example2
+%when :encoder
+out:
+  type: any output input plugin type
+  encoders:
+    - type: <%= name %>
+      option1: example1
+      option2: example2
+%end
+```
+
+%case category
+%when :parser, :decoder
+(If guess supported) you don't have to write `<%= category %>:` section in the configuration file. After writing `in:` section, you can let embulk guess `<%= category %>:` section using this command:
+
+```
+$ embulk gem install <%= full_project_name %>
+$ embulk guess -g <%= name %> config.yml -o guessed.yml
+```
+%end
+
+## Build
+
+```
+%case language
+%when :ruby
+$ rake
+%when :java
+$ ./gradlew gem  # -t to watch change of files and rebuild continuously
+% end
+```

data/lib/embulk/data/new/gitignore.erb

@@ -0,0 +1,13 @@
+*~
+/pkg/
+/tmp/
+%if language == :java
+*.gemspec
+.gradle/
+/classpath/
+build/
+.idea
+%elsif language == :ruby
+/.bundle/
+/Gemfile.lock
+%end

data/lib/embulk/data/new/java/build.gradle.erb

@@ -0,0 +1,73 @@
+plugins {
+    id "com.jfrog.bintray" version "1.1"
+    id "com.github.jruby-gradle.base" version "0.1.5"
+    id "java"
+}
+import com.github.jrubygradle.JRubyExec
+repositories {
+    mavenCentral()
+    jcenter()
+}
+configurations {
+    provided
+}
+
+version = "0.1.0"
+
+dependencies {
+    compile  "org.embulk:embulk-core:<%= Embulk::VERSION %>"
+    provided "org.embulk:embulk-core:<%= Embulk::VERSION %>"
+    // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
+    testCompile "junit:junit:4.+"
+}
+
+task classpath(type: Copy, dependsOn: ["jar"]) {
+    doFirst { file("classpath").deleteDir() }
+    from (configurations.runtime - configurations.provided + files(jar.archivePath))
+    into "classpath"
+}
+clean { delete "classpath" }
+
+task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) {
+    jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
+    script "${project.name}.gemspec"
+    doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
+}
+
+task gemPush(type: JRubyExec, dependsOn: ["gem"]) {
+    jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "push"
+    script "pkg/${project.name}-${project.version}.gem"
+}
+
+task "package"(dependsOn: ["gemspec", "classpath"]) << {
+    println "> Build succeeded."
+    println "> You can run embulk with '-L ${file(".").absolutePath}' argument."
+}
+
+task gemspec {
+    ext.gemspecFile = file("${project.name}.gemspec")
+    inputs.file "build.gradle"
+    outputs.file gemspecFile
+    doLast { gemspecFile.write($/
+Gem::Specification.new do |spec|
+  spec.name          = "${project.name}"
+  spec.version       = "${project.version}"
+  spec.authors       = [<%= author.dump %>]
+  spec.summary       = %[<%= display_name %> <%= display_category %> plugin for Embulk]
+  spec.description   = %[<%= description %>]
+  spec.email         = [<%= email.dump %>]
+  spec.licenses      = ["MIT"]
+  # TODO set this: spec.homepage      = <%= "https://github.com/#{email[/([^@]*)/]}/#{full_project_name}".dump %>
+
+  spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
+  spec.test_files    = spec.files.grep(%r"^(test|spec)/")
+  spec.require_paths = ["lib"]
+
+  #spec.add_dependency 'YOUR_GEM_DEPENDENCY', ['~> YOUR_GEM_DEPENDENCY_VERSION']
+  spec.add_development_dependency 'bundler', ['~> 1.0']
+  spec.add_development_dependency 'rake', ['>= 10.0']
+end
+/$)
+    }
+}
+clean { delete "${project.name}.gemspec" }