embulk 0.3.2 → 0.4.0

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    NzUyMzU0ODY2OWE5N2U4NTM3NjIwYjUzNTlmNjUwMGRkNWE4N2UxZg==
-  data.tar.gz: !binary |-
-    NzkyM2I3NzI2YmY4M2EyNGI3MzA1ODYyYmIwMzgxZjdkMzYxODg3Yg==
+SHA1:
+  metadata.gz: bf78740441995394000d1ada6d8ca939cda77754
+  data.tar.gz: 9da2b0b76a8b702dffa2063fcc626e1ea25698af
 SHA512:
-  metadata.gz: !binary |-
-    MWE0YjBiYzY4ZDc2MDc0ODFmY2QxZDY2ODljZTBhOTBmYjMwZjhhMWY5M2Q5
-    N2JjODJhOTE1M2EwYmQxMjBiNWEzNmNjNjJkZDRiOTUyMjMzMTBlMGNiMTYx
-    YzIxNGRjMTk2OTI5YzQwYjQ5NTcyODQ0ZGQzOTJhYzA2YTU3NmM=
-  data.tar.gz: !binary |-
-    ZGUxMzk0ZjcyNWRmMTNhY2IwYzFlODg3NGMzOGMyMzlhNzM5YjllZjkyMWMy
-    ZTc5Yjk2YzY0MWE2MTBmODZhNmVjMWMzZjdjMDg0YWRkNDBmZGIwZWVjN2Rl
-    NzFiN2EwNmU2Y2M3MzdiMWVhMDUxZDRmZDE1YzBlYWVjYzk0YTU=
+  metadata.gz: 12cec9ae89bab146559eaa68e2cc57a6ac71a48e18b81b3c393077842dc7f30b7fb8ea41502538c23d1ae0ca2e64f428cacd534e04f06d59d39d6e096b9b2d79
+  data.tar.gz: 513f463830ac564a873bf11a0a58905f57c4de91f4415718b428721cc836ca54d25c3293869b1974652189c9c29f4969c120d8ce1ada72e5d55e7025e7108234

data/.travis.yml

@@ -0,0 +1,16 @@
+language: java
+jdk:
+  - oraclejdk8
+sudo: false
+addons:
+  apt_packages:
+    - sphinx
+    - yardoc
+script:
+  - ./gradlew check
+  - ./embulk-docs/push-gh-pages.sh
+env:
+  global:
+    GIT_USER_NAME: travis
+    GIT_USER_NAME: travis@embulk.org
+    secure: ZiDtC073/oAXtPkXZYP9MoVBidLrrjeAV8PeXICtTzVMxR4Td5QDifiTy2NNStVLT+1g6NgzjSK9XFpDyVFb2cNy+9EOP+FoSC6Q0gRD/9nIrvB/LAtiDDfLqRtUrvPjN6U8BF3Zm2IvwTP77ml8bZZy3SH9ob4NQhxxEXVtynI=

data/Gemfile

@@ -1,3 +1,2 @@
 source 'https://rubygems.org/'
-
 gemspec

data/README.md

@@ -97,28 +97,50 @@ java -jar embulk.jar cleanup config.yml -r resume-state.yml
 ### Build
 
 ```
-rake  # creates embulk-VERSION.jar
+./gradlew cli  # creates pkg/embulk-VERSION.jar
+./gradlew gem  # creates pkg/embulk-VERSION.gem
 ```
 
-You can see JaCoCo's test coverage report at ${project}/target/site/jacoco/index.html
+You can see JaCoCo's test coverage report at `${project}/build/reports/tests/index.html`
+You can see Findbug's report at `${project}/build/reports/findbug/main.html`  # FIXME coverage information is not included somehow
+
+You can use `classpath` task to use `./bin/embulk` for development:
 
-To build by Gradle, run:
 ```
-./gradlew build
+./gradlew classpath  # -x test: skip test
+./bin/embulk
 ```
-If you want to deploy artifacts on local maven repository like ~/.m2/repository/, run:
+
+To deploy artifacts to your local maven repository at ~/.m2/repository/:
+
 ```
 ./gradlew install
 ```
-If you want to compile the source code of embulk-core project only, run:
+
+To compile the source code of embulk-core project only:
+
 ```
 ./gradlew :embulk-core:compileJava
 ```
-The following command allows use to see the dependency tree of embulk-core project
+
+Task `dependencies` shows dependency tree of embulk-core project:
+
 ```
 ./gradlew :embulk-core:dependencies
 ```
 
+### Documents
+
+Embulk uses Sphinx, YARD (Ruby API) and JavaDoc (Java API) for document generation.
+
+```
+brew install python
+pip install sphinx
+gem install yard
+./gradlew site
+# documents are: embulk-docs/build/html
+```
+
 ### Release
 
 You need to add your bintray account information to ~/.gradle/gradle.properties
@@ -128,24 +150,20 @@ bintray_user=(bintray user name)
 bintray_api_key=(bintray api key)
 ```
 
-Increment version number written at following 3 files (TODO improve this):
+Update following files:
 
-* build.gradle
-* pom.xml
-* lib/embulk/version.rb
+* embulk-docs/src/release/release-VERSION.rst (release note)
+* build.gradle (version number)
+* lib/embulk/version.rb (version number)
 
 Then, build and upload using gradle:
 
 ```
+./gradlew releaseCheck
+./gradlew cli gem
 ./gradlew bintrayUpload
-```
-
-Finally, you need to manually upload the single-file jar package to bintray.
-Run `rake` and upload embulk-VERSION.jar from "Upload Files" link at https://bintray.com/embulk/maven/embulk/VERSION/upload.
-
-```
-rake
-# embulk-VERSION.jar is built
+gem push pkg/embulk-....gem
+open "https://bintray.com/embulk/maven/embulk"  # and upload pkg/embulk-....jar
 ```
 
 See also:

data/Rakefile

@@ -15,44 +15,12 @@ end
 
 desc "Clean java code and copy files"
 task :clean do
-  sh "mvn clean"
-  rm_rf "classpath"
+  sh "./gradlew clean"
 end
 
-desc "Compile java code and copy jar files to classpath/"
-task :compile do
-  sh "mvn package dependency:copy-dependencies"
-  rm_rf "classpath"
-  mkdir_p "classpath"
-  cp Dir["embulk-cli/target/dependency/*.jar"], "classpath"
-  targets = Dir["embulk-cli/target/embulk-cli-*.jar"]
-  targets.reject! {|target| target =~ /-sources.jar$/ || target =~ /-executable.jar$/ }
-  cp targets, "classpath"
+desc "Build and copy jar files to ./classpath"
+task :classpath do
+  sh "./gradlew classpath"
 end
 
-desc "Create embulk-{version}.jar"
-task :jar do
-  require_relative 'lib/embulk/version'
-  executable = Dir["embulk-cli/target/embulk-cli-*-executable.jar"].sort.last
-  executable_data = File.read(executable).force_encoding('ASCII-8BIT')
-  header = <<EOF
-#!/bin/sh
-exec java -jar "$0" "$@"
-exit 127
-EOF
-  data = header.force_encoding('ASCII-8BIT') + executable_data
-  path = "embulk-#{Embulk::VERSION}.jar"
-  rm_f path
-  File.open(path, 'wb', 0755) {|f| f.write data }
-  puts "Created #{path}"
-end
-
-desc "Run clean, compile, build and jar"
-task :all do
-  Rake::Task["clean"].invoke
-  Rake::Task["compile"].invoke
-  Rake::Task["build"].invoke
-  Rake::Task["jar"].invoke
-end
-
-task :default => :all
+task :default => :classpath

data/bin/embulk

@@ -45,7 +45,7 @@ jruby_complete = Dir.entries(classpath_dir).find {|jar| jar =~ /jruby-complete-[
 unless jruby_complete
   STDERR.puts "Could not find jruby-complete at #{embulk_home}/classpath directory."
   if embulk_home == '.'
-    STDERR.puts "Did you run \`rake\`? You need to build java code and create ./classpath directory first."
+    STDERR.puts "Did you run ./gradlew classpath? You need to build java code and create ./classpath directory first."
   end
   raise SystemExit.new(1)
 end

data/build.gradle

@@ -1,37 +1,104 @@
-// TODO this build.gradle file just works but not well-written yet...
 
-//plugins {
-//    id "com.jfrog.bintray" version "1.0"
-//}
-buildscript {
-    repositories {
-        jcenter()
-    }
-    dependencies {
-        classpath 'com.jfrog.bintray.gradle:gradle-bintray-plugin:1.0'
-        classpath 'com.github.ben-manes:gradle-versions-plugin:0.7'
-    }
+plugins {
+    id 'com.jfrog.bintray' version '1.1'
+    id 'com.github.ben-manes.versions' version '0.7'
+    id 'com.github.jruby-gradle.base' version '0.1.5'
+    id 'com.github.johnrengelman.shadow' version '1.2.0'
+    id 'java'
 }
 
 allprojects {
+    group = 'org.embulk'
+    version = '0.4.0'
+}
+
+subprojects {
+    apply plugin: 'maven'  // install jar files to the local repo: $ gradle install
+    apply plugin: 'java'
+    apply plugin: 'findbugs'
+    apply plugin: 'jacoco'
+
     repositories {
         mavenCentral()
         jcenter()
     }
 
-    apply plugin: 'idea'
-    apply plugin: 'maven-publish'
-    apply plugin: 'com.jfrog.bintray'
+    compileJava.options.encoding = 'UTF-8' // source encoding
+    sourceCompatibility = 1.7
+    targetCompatibility = 1.7
 
-    // determine which dependencies have updates
-    // $ gradle dependencyUpdates
-    apply plugin: 'com.github.ben-manes.versions'
+    configurations {
+        // guice depends on asm and cglib but version of the libraries conflict
+        // with ones bundled in jruby-complete and cause bytecode compatibility error
+        compile.exclude group: 'asm', module: 'asm'
+        compile.exclude group: 'org.sonatype.sisu.inject', module: 'cglib'
+    }
 
-    group = 'org.embulk'
-    version = '0.3.2'
+    // determine which dependencies have updates: $ gradle dependencyUpdates
+    dependencies {
+        compile 'com.google.guava:guava:18.0'
+        compile 'com.google.inject:guice:3.0'
+        compile 'com.google.inject.extensions:guice-multibindings:3.0'
+        compile 'javax.inject:javax.inject:1'
+        compile 'com.fasterxml.jackson.core:jackson-annotations:2.5.0'
+        compile 'com.fasterxml.jackson.core:jackson-core:2.5.0'
+        compile 'com.fasterxml.jackson.core:jackson-databind:2.5.0'
+        compile 'com.fasterxml.jackson.datatype:jackson-datatype-guava:2.5.0'
+        compile 'com.fasterxml.jackson.datatype:jackson-datatype-joda:2.5.0'
+        compile 'com.fasterxml.jackson.module:jackson-module-guice:2.5.0'
+        compile 'log4j:log4j:1.2.17'
+        compile 'org.slf4j:slf4j-api:1.7.10'
+        compile 'org.slf4j:slf4j-log4j12:1.7.10'
+        compile 'org.jruby:jruby-complete:1.7.19'
+        compile 'com.google.code.findbugs:annotations:3.0.0'
+        compile 'org.yaml:snakeyaml:1.14'
+        compile 'javax.validation:validation-api:1.1.0.Final'
+        compile 'org.apache.bval:bval-jsr303:0.5'
+        compile 'io.airlift:slice:0.9'
+        compile 'joda-time:joda-time:2.7'
+        compile 'io.netty:netty-buffer:5.0.0.Alpha1'
+        compile 'com.ibm.icu:icu4j:54.1.1'
+
+        testCompile 'junit:junit:4.12'
+    }
+
+    gradle.projectsEvaluated {
+        tasks.withType(JavaCompile) {
+            options.compilerArgs << "-Xlint:unchecked" //<< "-Xlint:deprecation"
+        }
+        tasks.withType(FindBugs) {
+            reports {
+                xml.enabled = false
+                html.enabled = true
+            }
+        }
+    }
+
+    findbugs {
+        ignoreFailures = true
+    }
+
+    // add javadoc/source jar tasks as artifacts to be released
+    task sourcesJar(type: Jar, dependsOn: classes) {
+        classifier = 'sources'
+        from sourceSets.main.allSource
+    }
+    task javadocJar(type: Jar, dependsOn: javadoc) {
+        classifier = 'javadoc'
+        from javadoc.destinationDir
+    }
+    artifacts {
+        archives sourcesJar, javadocJar
+    }
 
-    // to upload artifacts to Bintray by gradle-bintray-plugin
-    // $ gradle bintrayUpload
+    javadoc {
+        options {
+            locale = 'en_US'
+            encoding = 'UTF-8'
+        }
+    }
+
+    // upload artifacts to Bintray: $ gradle bintrayUpload
     bintray {
         // write at your bintray user name and api key to ~/.gradle/gradle.properties file:
         // bintray_user=frsyuki
@@ -42,7 +109,7 @@ allprojects {
         publications = ['mavenJava']
 
         dryRun = false
-        publish = false  // TODO publish = project.publish
+        publish = false  // TODO automate uploading embulk.jar and make this true
 
         pkg {
             userOrg = 'embulk'
@@ -70,86 +137,102 @@ allprojects {
     }
 }
 
-subprojects {
-    apply plugin: 'java'
-
-    // to install jar files to the local repo
-    // $ gradle install
-    apply plugin: 'maven'
-
-    compileJava.options.encoding = 'UTF-8' // source encoding
-    sourceCompatibility = 1.7
-    targetCompatibility = 1.7
-
-    dependencies {
-        compile 'commons-logging:commons-logging:1.2',
-                'com.google.guava:guava:17.0',
-                'com.google.inject:guice:3.0',
-                'com.google.inject.extensions:guice-multibindings:3.0',
-                'javax.inject:javax.inject:1',
-                'com.fasterxml.jackson.core:jackson-annotations:2.4.3',
-                'com.fasterxml.jackson.core:jackson-core:2.4.3',
-                'com.fasterxml.jackson.core:jackson-databind:2.4.3',
-                'com.fasterxml.jackson.datatype:jackson-datatype-guava:2.4.3',
-                'com.fasterxml.jackson.datatype:jackson-datatype-joda:2.4.3',
-                'com.fasterxml.jackson.module:jackson-module-guice:2.4.3',
-                'log4j:log4j:1.2.17',
-                'org.slf4j:slf4j-api:1.7.9',
-                'org.slf4j:slf4j-log4j12:1.7.9',
-                'org.jruby:jruby-complete:1.7.16.1',
-                'com.google.code.findbugs:annotations:3.0.0',
-                'org.yaml:snakeyaml:1.14',
-                'javax.validation:validation-api:1.1.0.Final',
-                'org.apache.bval:bval-jsr303:0.5',
-                'io.airlift:slice:0.7',
-                'joda-time:joda-time:2.3',
-                'io.netty:netty-buffer:4.0.24.Final',
-                'com.ibm.icu:icu4j:53.1'
-
-        testCompile 'junit:junit:4.10',
-                    'org.mockito:mockito-core:1.9.5'
-    }
-
-    // custom tasks for creating source/javadoc jars
-    task sourcesJar(type: Jar, dependsOn: classes) {
-        classifier = 'sources'
-        from sourceSets.main.allSource
+project(':embulk-cli') {
+    apply plugin: 'com.github.johnrengelman.shadow'
+
+    shadowJar {
+        manifest {
+            attributes 'Implementation-Title': project.name,
+                       'Implementation-Version': project.version,
+                       'Implementation-Vendor-Id': project.group,
+                       'Specification-Title': project.name,
+                       'Specification-Version': project.version,
+                       'Main-Class': 'org.embulk.cli.Main'
+        }
+        append("${parent.projectDir}/COPYING")
     }
 
-    task javadocJar(type: Jar, dependsOn: javadoc) {
-        classifier = 'javadoc'
-        from javadoc.destinationDir
+    task classpath(type: Copy) {
+        doFirst { file("${parent.projectDir}/classpath").mkdirs() }
+        from configurations.runtime
+        into "${parent.projectDir}/classpath"
     }
+}
 
-    // add javadoc/source jar tasks as artifacts
-    artifacts {
-        archives sourcesJar, javadocJar
-    }
+task classpath(dependsOn: ['build', ':embulk-cli:classpath']) << { }
+clean { delete 'classpath' }
+
+task cli(dependsOn: ':embulk-cli:shadowJar') << {
+    file('pkg').mkdirs()
+    File f = file("pkg/embulk-${project.version}.jar")
+    f.write('''\
+#!/bin/sh
+exec java -jar "$0" "$@"
+exit 127
+''')
+    f.append(file("embulk-cli/build/libs/embulk-cli-${project.version}-all.jar").readBytes())
+    f.setExecutable(true)
+}
 
-    // $ gradle copyDependencies
-    task copyDependencies(type: Copy) {
-        println ":copy depend to $buildDir/libs/dependencies"
-        new File("$buildDir/libs/dependencies").mkdirs()
+import com.github.jrubygradle.JRubyExec
+task gem(type: JRubyExec) {
+    jrubyArgs '-rrubygems/gem_runner', '-eGem::GemRunner.new.run(ARGV)', 'build'
+    script 'build/gemspec'
+    doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
+}
+gem.dependsOn('gemspec')
+gem.dependsOn('classpath')
 
-        into "$buildDir/libs/dependencies"
-        from configurations.runtime.filter {
-            //  TODO avoid bytecode compatibility error
-            it.name != 'asm-3.1.jar' && it.name != 'cglib-2.2.1-v20090111.jar'
-        }
+task releaseCheck << {
+    if (!file("lib/embulk/version.rb").getText().contains("${project.version}")) {
+        throw new GradleException("lib/embulk/version.rb doesn't include ${project.version}")
+    }
+    String date = new Date().format("yyyy-MM-dd")
+    if (!file("embulk-docs/src/release/release-${project.version}.rst").getText().contains(date)) {
+        throw new GradleException("ChangeLog doesn't include entry for ${project.version}")
     }
+    println "Ready. Run ./gradlew release"
+}
 
-    publishing {
-        publications {
-            mavenJava(MavenPublication) {
-                groupId project.group
-                artifactId project.name
-                version project.version
+task release(dependsOn: ["cli", "gem"]) << {
+    println """
+manual operations:
 
-                from components.java
+  git commit -a -m v${project.version}
+  git tag v${project.version}
+  gem push pkg/embulk-${project.version}.gem"
+  ./gradlew bintrayUpload
+  open "https://bintray.com/embulk/maven/embulk/${project.version}/view"  # and upload pkg/embulk-${project.version}.jar
 
-                artifact sourcesJar
-                artifact javadocJar
-            }
-        }
-    }
+"""
+}
+
+task gemspec << {
+    file('build').mkdirs()
+    file('build/gemspec').write($/
+Gem::Specification.new do |gem|
+  gem.name          = "embulk"
+  gem.version       = "${project.version}"
+
+  gem.summary       = "Embulk, a plugin-based parallel bulk data loader"
+  gem.description   = "Embulk is an open-source, plugin-based bulk data loader to scale and simplify data management across heterogeneous data stores. It can collect and ship any kinds of data in high throughput with transaction control."
+  gem.authors       = ["Sadayuki Furuhashi"]
+  gem.email         = ["frsyuki@gmail.com"]
+  gem.license       = "Apache 2.0"
+  gem.homepage      = "https://github.com/embulk/embulk"
+
+  gem.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
+  gem.test_files    = gem.files.grep(%r"^(test|spec)/")
+  gem.executables   = gem.files.grep(%r"^bin/").map{ |f| File.basename(f) }
+  gem.require_paths = ["lib"]
+  gem.has_rdoc      = false
+
+  gem.add_development_dependency "bundler", [">= 1.0"]
+  gem.add_development_dependency "rake", [">= 0.10.0"]
+  gem.add_development_dependency "rspec", ["~> 2.11"]
+  gem.add_development_dependency "json", ["~> 1.7"]
+  gem.add_development_dependency "yard", ["~> 0.8.7"]
+  gem.add_development_dependency "kramdown", ["~> 1.5.0"]
+end
+/$)
 }