embulk 0.7.0-java

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d8c4435b7efc6bed5410df4855329fa2b4a10d2d
+  data.tar.gz: e6ca52300e459ab476be4d4c63d3e5288a5a4f5d
+SHA512:
+  metadata.gz: 95e963ff9bb277488d64f17d8d79eed664d3200534508b63d0eb72c9f1cf0ea01643ac2c29a27fb316d9f811368de7ff4f0c235212462c9eba0d3c131423eb5e
+  data.tar.gz: 7a84fa41e977cc1d9be79d554ba3e8d91b3ce83bc204a9ff51a043ea779ed49eac893e6b442a26fba64fc01c35d48dd582b8072cc086947165f90c99f96dc7b3

data/.gitignore

@@ -0,0 +1,15 @@
+target/
+build/
+pkg/
+*.iml
+*~
+._*
+.idea
+tmp/
+vendor/
+/classpath/
+/.bundle
+.yardoc
+/embulk-*.jar
+/.gradle
+/coverage

data/.ruby-version

@@ -0,0 +1 @@
+2.2.2

data/.travis.yml

@@ -0,0 +1,18 @@
+language: java
+jdk:
+  - oraclejdk8
+cache:
+  directories:  # run "travis cache --delete" to delete caches
+    - $HOME/.gradle
+sudo: false
+install:
+  - pip install --user sphinx
+script:
+  - ./gradlew --info check rubyTest
+after_success:
+  - PATH="$HOME/.local/bin:$PATH" ./embulk-docs/push-gh-pages.sh
+env:
+  global:
+    - GIT_USER_NAME=travis
+    - GIT_USER_EMAIL=travis@embulk.org
+    - secure: K5qT2PcCP/40dNW+1H4NZ6y1+GAZbyP/lMQ1tSMsAICGkMQ/A+Mp5wtnIGIsAf6JGcJ1PvpCoLE1V6wKFL5fEwxi4SRcTnZTh9PkeAk8dgezOMoX4EqeZiQAYv4MM2zKL+Gr6QivjmRA7I5jrZCo8JyaA5XfQ7ygjICKNJy8NaE=

data/COPYING

@@ -0,0 +1,14 @@
+Copyright (C) 2015 Sadayuki Furuhashi
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org/'
+gemspec

data/Gemfile.lock

@@ -0,0 +1,31 @@
+PATH
+  remote: .
+  specs:
+    embulk (0.7.0)
+      jruby-jars (= 9.0.0.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    jruby-jars (9.0.0.0)
+    kramdown (1.5.0)
+    power_assert (0.2.2)
+    rake (10.4.2)
+    test-unit (3.0.9)
+      power_assert
+    yard (0.8.7.6)
+
+PLATFORMS
+  java
+  ruby
+
+DEPENDENCIES
+  bundler (>= 1.0)
+  embulk!
+  kramdown (~> 1.5.0)
+  rake (>= 0.10.0)
+  test-unit (~> 3.0.9)
+  yard (~> 0.8.7)
+
+BUNDLED WITH
+   1.10.6

data/README.md

@@ -0,0 +1,206 @@
+# What's Embulk?
+
+Embulk is a parallel bulk data loader that **helps data transfer between various storages, databases, NoSQL and cloud services**.
+
+**Embulk supports plugins** to add functions. You can [share the plugins](http://www.embulk.org/plugins/) to keep your custom scripts readable, maintainable, and reusable.
+
+[![Embulk](https://gist.githubusercontent.com/frsyuki/f322a77ee2766a508ba9/raw/e8539b6b4fda1b3357e8c79d3966aa8148dbdbd3/embulk-overview.png)](http://www.slideshare.net/frsyuki/embuk-making-data-integration-works-relaxed/12)
+[Embulk, an open-source plugin-based parallel bulk data loader](http://www.slideshare.net/frsyuki/embuk-making-data-integration-works-relaxed) at Slideshare
+
+# Document
+
+Embulk documents: http://www.embulk.org/docs/
+
+## Quick Start
+
+### Linux & Mac & BSD
+
+Embulk is a Java application. Please make sure that [Java](http://www.oracle.com/technetwork/java/javase/downloads/index.html) is installed.
+
+Following 4 commands install embulk to your home directory:
+
+```sh
+curl --create-dirs -o ~/.embulk/bin/embulk -L "http://dl.embulk.org/embulk-latest.jar"
+chmod +x ~/.embulk/bin/embulk
+echo 'export PATH="$HOME/.embulk/bin:$PATH"' >> ~/.bashrc
+source ~/.bashrc
+```
+
+Next step: [Running example in 4 commands](#running-example)
+
+### Windows
+
+Embulk is a Java application. Please make sure that [Java](http://www.oracle.com/technetwork/java/javase/downloads/index.html) is installed.
+
+You can download `embulk.bat` using this command on cmd.exe or PowerShell.exe:
+
+```
+PowerShell -Command "& {Invoke-WebRequest http://dl.embulk.org/embulk-latest.jar -OutFile embulk.bat}"
+```
+
+Next step: [Running example in 4 commands](#running-example)
+
+### Running example
+
+`embulk example` command generates a sample CSV file so that you can try embulk quickly:
+
+```
+embulk example ./try1
+embulk guess   ./try1/example.yml -o config.yml
+embulk preview config.yml
+embulk run     config.yml
+```
+
+Next step: [Using plugins](#using-plugins)
+
+### Using plugins
+
+You can use plugins to load data from/to various systems and file formats. Here is the list of publicly released plugins: [list of plugins by category](http://www.embulk.org/plugins/).
+
+An example is [embulk-output-command](https://github.com/embulk/embulk-output-command) plugin. It executes an external command to output the records.
+
+To install plugins, you can use `embulk gem install <name>` command:
+
+```
+embulk gem install embulk-output-command
+embulk gem list
+```
+
+Embulk bundles some built-in plugins such as `embulk-encoder-gzip` or `embulk-formatter-csv`. You can use those plugins with following configuration file:
+
+```yaml
+in:
+  type: file
+  path_prefix: "./try1/csv/sample_"
+out:
+  type: command
+  command: "cat - > task.$INDEX.$SEQID.csv.gz"
+  encoders:
+    - {type: gzip}
+  formatter:
+    type: csv
+```
+
+### Resuming a failed transaction
+
+Embulk supports resuming failed transactions.
+To enable resuming, you need to start transaction with `-r PATH` option:
+
+```
+embulk run config.yml -r resume-state.yml
+```
+
+If the transaction fails, embulk stores state some states to the yaml file. You can retry the transaction using exactly same command:
+
+```
+embulk run config.yml -r resume-state.yml
+```
+
+If you give up on resuming the transaction, you can use `embulk cleanup` subcommand to delete intermediate data:
+
+```
+embulk cleanup config.yml -r resume-state.yml
+```
+
+### Using plugin bundle
+
+`embulk bundle new` subcommand creates (or updates if already exists) a private (isolated) bundle of a plugins.
+You can use the bundle using `-b <bundle_dir>` option. `embulk bundle` also generates some example plugins to \<bundle_dir>/embulk/\*.rb directory.
+
+See the generated \<bundle_dir>/Gemfile file how to plugin bundles work.
+
+```
+embulk bundle new ./embulk_bundle
+embulk guess  -b ./embulk_bundle ...
+embulk run    -b ./embulk_bundle ...
+```
+
+## Use cases
+
+* [Scheduled bulk data loading to Elasticsearch + Kibana 4 from CSV files](http://www.embulk.org/docs/recipe/scheduled-csv-load-to-elasticsearch-kibana4.html)
+
+For further details, visit [Embulk documentation](http://www.embulk.org/docs/).
+
+## Upgrading to the latest version
+
+Following command updates embulk itself to the latest released version.
+
+```sh
+embulk selfupdate
+```
+
+## Embulk Development
+
+### Build
+
+```
+./gradlew cli  # creates pkg/embulk-VERSION.jar
+./gradlew gem  # creates pkg/embulk-VERSION.gem
+```
+
+You can see JaCoCo's test coverage report at `${project}/build/reports/tests/index.html`
+You can see Findbug's report at `${project}/build/reports/findbug/main.html`  # FIXME coverage information is not included somehow
+
+You can use `classpath` task to use `bundle exec ./bin/embulk` for development:
+
+```
+./gradlew -t classpath  # -x test: skip test
+./bin/embulk
+```
+
+To deploy artifacts to your local maven repository at ~/.m2/repository/:
+
+```
+./gradlew install
+```
+
+To compile the source code of embulk-core project only:
+
+```
+./gradlew :embulk-core:compileJava
+```
+
+Task `dependencies` shows dependency tree of embulk-core project:
+
+```
+./gradlew :embulk-core:dependencies
+```
+
+### Documents
+
+Embulk uses Sphinx, YARD (Ruby API) and JavaDoc (Java API) for document generation.
+
+```
+brew install python
+pip install sphinx
+gem install yard
+./gradlew site
+# documents are: embulk-docs/build/html
+```
+
+### Release
+
+You need to add your bintray account information to ~/.gradle/gradle.properties
+
+```
+bintray_user=(bintray user name)
+bintray_api_key=(bintray api key)
+```
+
+Run following commands and follow its instruction:
+
+```
+./gradlew set_version -Pto=$VERSION
+```
+
+```
+./gradlew releaseCheck
+./gradlew release
+git commit -am v$VERSION
+git tag v$VERSION
+```
+
+See also:
+* [Bintray](https://bintray.com)
+* [How to acquire bintray API Keys](https://bintray.com/docs/usermanual/interacting/interacting_editingyouruserprofile.html#anchorAPIKEY)
+

data/Rakefile

@@ -0,0 +1,26 @@
+#!/usr/bin/env rake
+
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+require 'rake/clean'
+require 'bundler'
+require 'yard'
+
+desc 'Generate YARD document'
+YARD::Rake::YardocTask.new(:doc) do |t|
+  t.files   = ['lib/**/*.rb']
+  t.options = %w[-M kramdown]
+  t.options << '--debug' << '--verbose' if $trace
+end
+
+desc "Clean java code and copy files"
+task :clean do
+  sh "./gradlew clean"
+end
+
+desc "Build and copy jar files to ./classpath"
+task :classpath do
+  sh "./gradlew classpath"
+end
+
+task :default => :classpath

data/appveyor.yml

@@ -0,0 +1,20 @@
+version: "{branch} {build}"
+
+build:
+  verbosity: detailed
+
+build_script:
+  - gradlew.bat --info --no-daemon check rubyTest
+
+cache:
+  - C:\Users\appveyor\.gradle
+
+environment:
+  matrix:
+  - JAVA_HOME: C:\Program Files\Java\jdk1.8.0
+  # - JAVA_HOME: C:\Program Files (x86)\Java\jdk1.8.0
+
+test: off
+
+matrix:
+  fast_finish: true

data/bin/embulk

@@ -0,0 +1,106 @@
+#!/usr/bin/env ruby
+
+if RUBY_PLATFORM =~ /java/i
+  # Enable embulk_bundle if run by CRuby.
+  # Disable embulk_bundle if run by JRuby.
+  if ENV['EMBULK_BIN_ENABLE_BUNDLE'] == File.expand_path(__FILE__)
+    ENV.delete('EMBULK_BIN_ENABLE_BUNDLE')
+    require_relative '../lib/embulk/command/embulk_bundle'
+  else
+    $LOAD_PATH << File.expand_path('../lib', File.dirname(__FILE__))
+    require 'embulk/command/embulk_main'
+  end
+  exit 0
+end
+
+java_args = []
+jruby_args = []
+default_optimize = false
+overwrite_optimize = nil
+
+until ARGV.empty?
+  v = ARGV[0]
+  case v
+  when "-J+O"
+    overwrite_optimize = true
+    ARGV.shift
+  when "-J-O"
+    overwrite_optimize = false
+    ARGV.shift
+  when "-J"
+    prop_path = ARGV[1]
+    unless prop_path
+      STDERR.puts "-J option requires an option"
+      exit 1
+    end
+    props = File.read(prop_path)
+    java_props = props.split("\n").reject {|prop| prop.strip.empty? }
+    java_args = java_props + java_args
+    ARGV.shift
+    ARGV.shift
+  when /-J(.*)/
+    java_args << v[2..-1]
+    ARGV.shift
+  when /-R(.*)/
+    jruby_args << v[2..-1]
+    ARGV.shift
+  when "run"
+    default_optimize = true
+    break
+  else
+    break
+  end
+end
+
+# TODO cygwin check
+cygwin = false
+
+java_cmd = ENV['JAVACMD']
+unless java_cmd
+  if java_home = ENV['JAVA_HOME']
+    if cygwin
+      java_cmd = "#{`cygpath -u "#{java_home}"`.strip}/bin/java"
+    else
+      java_cmd = "#{java_home}/bin/java"
+    end
+  else
+    java_cmd = "java"
+  end
+end
+
+if overwrite_optimize == true || (default_optimize == true && overwrite_optimize != false)
+  java_args = %w[-XX:+AggressiveOpts -XX:+UseConcMarkSweepGC] + java_args
+else
+  java_args = %w[-XX:+AggressiveOpts -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -Xverify:none] + java_args
+end
+
+# Disable bundler options of CRuby before starting JRuby
+ENV.delete('GEM_HOME')
+ENV.delete('GEM_PATH')
+ENV.delete('BUNDLE_GEMFILE')
+ENV.delete('BUNDLE_BIN_PATH')
+ENV.delete('RUBYLIB')
+ENV.delete('RUBYOPT')
+
+env = {
+  'EMBULK_BIN_ENABLE_BUNDLE' => File.expand_path(__FILE__)
+}
+
+begin
+  require 'jruby-jars'
+rescue LoadError => e
+  STDERR.puts  "Could not load jruby-jar.gem. Did you run with \"bundle exec\"?"
+  raise e
+end
+jruby_cp = "#{File.dirname(JRubyJars.core_jar_path)}/*"
+
+# java ... -jar ruby-complete.jar bin/embulk "$@"
+cmdline = [java_cmd]
+cmdline.concat java_args
+cmdline << '-cp' << jruby_cp << 'org.jruby.Main'
+cmdline.concat jruby_args
+cmdline << __FILE__
+cmdline.concat ARGV
+exec env, *cmdline
+exit 127
+