embulk 0.1.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    MjM4MTAxYzU5YTZlYWZkMWM0NjYzYTZlMGRkMjgwN2I0Yjc0MDU5YQ==
+  data.tar.gz: !binary |-
+    MGJkZmQ2NDM0NWQ3NWIwNmY2MWUyYjY0OWM0MzA2MmIwM2RmMDYwOQ==
+SHA512:
+  metadata.gz: !binary |-
+    MGY5Y2E2YjBiMjBhZTkxMzljNGU4MGNhZGE0MWM2N2YzZmM2YWM1MWRkZjUx
+    NTExZmEyN2Q5NDIyNThiNGNiYTE3ZDM0MzA3ZDMwNTE2NDI0N2E2ODM5ODE2
+    OTQxN2FjNWY1MzY3ODc5NGU5YTk3NmEwYjA4NDVjMTkwZTAxNjE=
+  data.tar.gz: !binary |-
+    NDZlMzkxOGVhN2VjY2U1ZGY4OTNlZWZlMzYxMWEyNDYyMDM3ZGMxNDlkOGY4
+    MWFhYzdhNzgyNTUxY2VkNGE4NWIzZDJkNmU1Nzc1ZTYyMjc0NTlkYzc3MGUx
+    ZWIwNTYzMzc3ZDA1NWQ0MzMwNmViZjZkNTdhZWQ0YjE1MDg0ZTA=

data/.gitignore

@@ -0,0 +1,13 @@
+target/
+build/
+*.iml
+*~
+._*
+.idea
+tmp/
+vendor/
+/classpath/
+/.bundle
+.yardoc
+/embulk-*.jar
+/.gradle

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org/'
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,33 @@
+PATH
+  remote: .
+  specs:
+    embulk (0.1.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    diff-lcs (1.2.5)
+    json (1.8.1)
+    kramdown (1.5.0)
+    rake (10.4.2)
+    rspec (2.99.0)
+      rspec-core (~> 2.99.0)
+      rspec-expectations (~> 2.99.0)
+      rspec-mocks (~> 2.99.0)
+    rspec-core (2.99.2)
+    rspec-expectations (2.99.2)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.99.2)
+    yard (0.8.7.6)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (>= 1.0)
+  embulk!
+  json (~> 1.7)
+  kramdown (~> 1.5.0)
+  rake (>= 0.10.0)
+  rspec (~> 2.11)
+  yard (~> 0.8.7)

data/README.md

@@ -0,0 +1,117 @@
+# Embulk: plugin-based parallel bulk data loader
+
+## What's Embulk?
+
+TODO
+
+## Quick Start
+
+The single-file package is the simplest way to try Embulk. You can download the latest embulk.jar from [releases]() and run it with java:
+
+```
+wget https://github.com/embulk/embulk/releases .... /latest
+java -jar embulk.jar --help
+```
+
+Let's load a CSV file, for example. `embulk example` subcommand generates a csv file and config file for you.
+
+```
+java -jar embulk.jar example ./try1
+java -jar embulk.jar guess   ./try1/example.yml -o config.yml
+java -jar embulk.jar preview config.yml
+java -jar embulk.jar run     config.yml
+```
+
+### Using plugins
+
+You can use plugins to load data from/to various systems and file formats.
+An example is [embulk-output-postgres-json]() plugin. It outputs data into PostgreSQL server using "json" column type.
+
+```
+java -jar embulk.jar gem install embulk-output-postgres-json
+java -jar embulk.jar gem list
+```
+
+You can search plugins on RubyGems: [search for "embulk-"](https://rubygems.org/search?utf8=%E2%9C%93&query=embulk-).
+
+### Using plugin bundle
+
+`embulk bundle` subcommand creates (or updates if already exists) a *plugin bundle* directory.
+You can use the bundle using `-b <bundle_dir>` option. `embulk bundle` also generates some example plugins to \<bundle_dir>/embulk/\*.rb directory.
+
+See generated \<bundle_dir>/Gemfile file how to plugin bundles work.
+
+```
+java -jar embulk.jar bundle ./embulk_bundle
+java -jar embulk.jar guess  -b ./embulk_bundle ...
+java -jar embulk.jar run    -b ./embulk_bundle ...
+```
+
+### Releasing plugins to RubyGems
+
+TODO: documents
+
+```
+embulk-plugin-xyz
+```
+
+## Embulk Development
+
+### Build
+
+```
+rake  # creates embulk-VERSION.jar
+```
+
+You can see JaCoCo's test coverage report at ${project}/target/site/jacoco/index.html
+
+To build by Gradle, run:
+```
+./gradlew build
+```
+If you want to deploy artifacts on local maven repository like ~/.m2/repository/, run:
+```
+./gradlew install
+```
+If you want to compile the source code of embulk-core project only, run:
+```
+./gradlew :embulk-core:compileJava
+```
+The following command allows use to see the dependency tree of embulk-core project
+```
+./gradlew :embulk-core:dependencies
+```
+
+### Release
+
+You need to add your bintray account information to ~/.gradle/gradle.properties
+
+```
+bintray_user=(bintray user name)
+bintray_api_key=(bintray api key)
+```
+
+Increment version number written at following 3 files (TODO improve this):
+
+* build.gradle
+* pom.xml
+* lib/embulk/version.rb
+
+Then, build and upload using gradle:
+
+```
+./gradlew bintrayUpload
+```
+
+Finally, you need to manually upload the single-file jar package to bintray.
+Run `rake` and upload embulk-VERSION.jar from "Upload Files" link at https://bintray.com/embulk/maven/embulk/VERSION/upload.
+
+```
+rake
+# embulk-VERSION.jar is built
+```
+
+See also:
+* [Bintray](https://bintray.com)
+* [How to acquire bintray API Keys](https://bintray.com/docs/usermanual/interacting/interacting_apikeys.html)
+

data/Rakefile

@@ -0,0 +1,58 @@
+#!/usr/bin/env rake
+
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+require 'rake/clean'
+require 'bundler'
+require 'yard'
+
+desc 'Generate YARD document'
+YARD::Rake::YardocTask.new(:doc) do |t|
+  t.files   = ['lib/**/*.rb']
+  t.options = %w[-M kramdown]
+  t.options << '--debug' << '--verbose' if $trace
+end
+
+desc "Clean java code and copy files"
+task :clean do
+  sh "mvn clean"
+  rm_rf "classpath"
+end
+
+desc "Compile java code and copy jar files to classpath/"
+task :compile do
+  sh "mvn package dependency:copy-dependencies"
+  rm_rf "classpath"
+  mkdir_p "classpath"
+  cp Dir["embulk-cli/target/dependency/*.jar"], "classpath"
+  targets = Dir["embulk-cli/target/embulk-cli-*.jar"]
+  targets.reject! {|target| target =~ /-sources.jar$/ || target =~ /-executable.jar$/ }
+  cp targets, "classpath"
+end
+
+desc "Create embulk-{version}.jar"
+task :jar do
+  require_relative 'lib/embulk/version'
+  executable = Dir["embulk-cli/target/embulk-cli-*-executable.jar"].sort.last
+  executable_data = File.read(executable).force_encoding('ASCII-8BIT')
+  header = <<EOF
+#!/bin/sh
+exec java -jar "$0" "$@"
+exit 127
+EOF
+  data = header.force_encoding('ASCII-8BIT') + executable_data
+  path = "embulk-#{Embulk::VERSION}.jar"
+  rm_f path
+  File.open(path, 'wb', 0755) {|f| f.write data }
+  puts "Created #{path}"
+end
+
+desc "Run clean, compile, build and jar"
+task :all do
+  Rake::Task["clean"].invoke
+  Rake::Task["compile"].invoke
+  Rake::Task["build"].invoke
+  Rake::Task["jar"].invoke
+end
+
+task :default => :all

data/bin/embulk

@@ -0,0 +1,63 @@
+#!/usr/bin/env ruby
+
+# TODO cygwin check
+cygwin = false
+
+java_args = []
+while i = ARGV.find_index {|arg| arg =~ /^\-D/ }
+  if ARGV[i] == "-D"
+    prop_path = ARGV[i + 1]
+    unless prop_path
+      STDERR.puts "-D option requires an option"
+      exit 1
+    end
+    ARGV.slice!(i, 2)
+  else
+    java_args << ARGV[i]
+    ARGV.slice!(i)
+  end
+end
+
+if prop_path
+  props = File.read(prop_path)
+  java_props = props.split("\n").reject {|prop| prop.strip.empty? }
+  java_args = java_props + java_args
+end
+
+java_cmd = ENV['JAVACMD']
+unless java_cmd
+  if java_home = ENV['JAVA_HOME']
+    if cygwin
+      java_cmd = "#{`cygpath -u "#{java_home}"`.strip}/bin/java"
+    else
+      java_cmd = "#{java_home}/bin/java"
+    end
+  else
+    java_cmd = "java"
+  end
+end
+
+embulk_home = ENV['EMBULK_HOME']
+unless embulk_home
+  embulk_home = File.dirname(File.dirname(__FILE__))
+end
+ENV['EMBULK_HOME'] = File.expand_path(embulk_home)
+
+classpath_dir = File.join(embulk_home, 'classpath')
+lib_dir = File.join(embulk_home, 'lib')
+
+jruby_complete = Dir.entries(classpath_dir).find {|jar| jar =~ /jruby-complete-[\d\.]+\.jar/ }
+unless jruby_complete
+  STDERR.puts "Could not find jruby-complete at $EMBULK_HOME/classpath directory."
+  STDERR.puts "Please confirm EMBULK_HOME is correctly set"
+  STDERR.puts "Current EMBULK_HOME = #{ENV['EMBULK_HOME'].to_s.dump}"
+  raise SystemExit.new(1)
+end
+
+# java ... -jar ruby-complete.jar $EMBULK_HOME/lib/embulk/command/embulk.rb "$@"
+cmdline = [java_cmd]
+cmdline.concat java_args
+cmdline << '-jar' << File.join(classpath_dir, jruby_complete)
+cmdline << File.join(lib_dir, 'embulk/command/embulk.rb')
+cmdline.concat ARGV
+exec *cmdline

data/build.gradle

@@ -0,0 +1,149 @@
+// TODO this build.gradle file just works but not well-written yet...
+
+//plugins {
+//    id "com.jfrog.bintray" version "1.0"
+//}
+buildscript {
+    repositories {
+        jcenter()
+    }
+    dependencies {
+        classpath 'com.jfrog.bintray.gradle:gradle-bintray-plugin:1.0'
+    }
+}
+
+allprojects {
+    repositories {
+        mavenCentral()
+        jcenter()
+    }
+
+    apply plugin: 'idea'
+    apply plugin: 'maven-publish'
+    apply plugin: 'com.jfrog.bintray'
+
+    group = 'org.embulk'
+    version = '0.1.0'
+
+    // to upload artifacts to Bintray by gradle-bintray-plugin
+    // $ gradle bintrayUpload
+    bintray {
+        // write at your bintray user name and api key to ~/.gradle/gradle.properties file:
+        // bintray_user=frsyuki
+        // bintray_api_key=xxxxxxxxxxx
+        user = project.hasProperty('bintray_user') ? bintray_user : ''
+        key = project.hasProperty('bintray_api_key') ? bintray_api_key : ''
+
+        publications = ['mavenJava']
+
+        dryRun = false
+        publish = false  // TODO publish = project.publish
+
+        pkg {
+            userOrg = 'embulk'
+            repo = 'maven'
+            name = 'embulk'
+            desc = 'Embulk, a plugin-based parallel bulk data loader'
+            websiteUrl = 'https://github.com/embulk/embulk'
+            issueTrackerUrl = 'https://github.com/embulk/embulk/issues'
+            vcsUrl = 'https://github.com/embulk/embulk.git'
+            licenses = ['Apache-2.0']
+            labels = ['embulk', 'ruby', 'java']
+            publicDownloadNumbers = true
+
+            version {
+                name = project.version
+                // TODO
+                //mavenCentralSync {
+                //    sync = true
+                //    user = 'userToken'
+                //    password = 'paasword'
+                //    close = '1'
+                //}
+            }
+        }
+    }
+}
+
+subprojects {
+    apply plugin: 'java'
+
+    // to install jar files to the local repo
+    // $ gradle install
+    apply plugin: 'maven'
+
+    compileJava.options.encoding = 'UTF-8' // source encoding
+    sourceCompatibility = 1.7
+    targetCompatibility = 1.7
+
+    dependencies {
+        compile 'commons-logging:commons-logging:1.2',
+                'com.google.guava:guava:17.0',
+                'com.google.inject:guice:3.0',
+                'com.google.inject.extensions:guice-multibindings:3.0',
+                'javax.inject:javax.inject:1',
+                'com.fasterxml.jackson.core:jackson-annotations:2.4.3',
+                'com.fasterxml.jackson.core:jackson-core:2.4.3',
+                'com.fasterxml.jackson.core:jackson-databind:2.4.3',
+                'com.fasterxml.jackson.datatype:jackson-datatype-guava:2.4.3',
+                'com.fasterxml.jackson.datatype:jackson-datatype-joda:2.4.3',
+                'com.fasterxml.jackson.module:jackson-module-guice:2.4.3',
+                'log4j:log4j:1.2.17',
+                'org.slf4j:slf4j-api:1.7.9',
+                'org.slf4j:slf4j-log4j12:1.7.9',
+                'org.jruby:jruby-complete:1.7.16.1',
+                'org.yaml:snakeyaml:1.14',
+                'javax.validation:validation-api:1.1.0.Final',
+                'org.apache.bval:bval-jsr303:0.5',
+                'io.airlift:slice:0.7',
+                'joda-time:joda-time:2.3',
+                'io.netty:netty-buffer:4.0.24.Final',
+                'com.ibm.icu:icu4j:53.1'
+
+        testCompile 'junit:junit:4.10',
+                    'org.mockito:mockito-core:1.9.5'
+    }
+
+    // custom tasks for creating source/javadoc jars
+    task sourcesJar(type: Jar, dependsOn: classes) {
+        classifier = 'sources'
+        from sourceSets.main.allSource
+    }
+
+    task javadocJar(type: Jar, dependsOn: javadoc) {
+        classifier = 'javadoc'
+        from javadoc.destinationDir
+    }
+
+    // add javadoc/source jar tasks as artifacts
+    artifacts {
+        archives sourcesJar, javadocJar
+    }
+
+    // $ gradle copyDependencies
+    task copyDependencies(type: Copy) {
+        println ":copy depend to $buildDir/libs/dependencies"
+        new File("$buildDir/libs/dependencies").mkdirs()
+
+        into "$buildDir/libs/dependencies"
+        from configurations.runtime.filter {
+            //  TODO avoid bytecode compatibility error
+            it.name != 'asm-3.1.jar' && it.name != 'cglib-2.2.1-v20090111.jar'
+        }
+    }
+
+    publishing {
+        publications {
+            mavenJava(MavenPublication) {
+                groupId project.group
+                artifactId project.name
+                version project.version
+
+                from components.java
+
+                artifact sourcesJar
+                artifact javadocJar
+            }
+        }
+    }
+}