embulk 0.8.15 → 0.8.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c89d18ccf5e8011ff1071e37d9fa0016bd97f179
-  data.tar.gz: d1137ad2c115b1bc006c5669e4c9a69aeb82a21d
+  metadata.gz: 8343679bf498ccc8d1519185737a9c54ced0d669
+  data.tar.gz: 3b737c7e3886c6915f503bd2960efb993ac3c4e6
 SHA512:
-  metadata.gz: 5da8fb3f31164ea935cb28a437f6dde726b44443ce73aa1274bb39a72d98f465126c58ebc5c49499a1aa18858a166532e1625a38a8699eba0ccd48f7ff13f4f8
-  data.tar.gz: e23084c691a1493e662149233c59acd1a6226413de86651c4c97348cb6921ed81d31808cdef4a664111dc86d41a851441562159c8fee0c966cc805dc5a80dbe1
+  metadata.gz: 5e11e26e3e9025d669a9cb9761d883e9470bd67e52a4019aa6d2b08b6ae2c44822fdea0be435862ebce21c7267ba2e08293e3e03441297e063b220a9fe9a29a6
+  data.tar.gz: 31b5b9f4f008951efbec4b84e75a823c4832d2ec5bd495bb4024c97687160008566aed4dbccfe25638da18995e3b46da4e885cd942e2e29e02709c672b56b694

data/README.md

@@ -72,6 +72,7 @@ Embulk bundles some built-in plugins such as `embulk-encoder-gzip` or `embulk-fo
 in:
   type: file
   path_prefix: "./try1/csv/sample_"
+  ...
 out:
   type: command
   command: "cat - > task.$INDEX.$SEQID.csv.gz"
@@ -199,7 +200,7 @@ bintray_api_key=(bintray api key)
 Run following commands and follow its instruction:
 
 ```
-./gradlew set_version -Pto=$VERSION
+./gradlew setVersion -Pto=$VERSION
 ```
 
 ```

data/appveyor.yml

@@ -3,7 +3,15 @@ version: "{branch} {build}"
 build:
   verbosity: detailed
 
+install:
+ - ps: $zipPath = "$($env:USERPROFILE)\Handle.zip"
+ - ps: (New-Object Net.WebClient).DownloadFile('https://download.sysinternals.com/files/Handle.zip', $zipPath)
+ - ps: 7z x $zipPath -y -o"$env:APPVEYOR_BUILD_FOLDER" | Out-Null
+
+# To avoid call-selfrun.bat FileNotFound exception, it installs and uses handle.exe as workaround provided by AppVeyor support.
+# see http://help.appveyor.com/discussions/problems/5975-the-process-cannot-access-the-file-because-it-is-being-used-by-another-process
 build_script:
+  - handle.exe -a -u C:\projects\embulk\embulk-cli\build\classes\test\org\embulk\cli\call-selfrun.bat -nobanner
   - gradlew.bat --info --no-daemon check rubyTest
 
 cache:

data/build.gradle

@@ -4,7 +4,7 @@ buildscript {
         classpath 'com.jfrog.bintray.gradle:gradle-bintray-plugin:1.3.1'
         classpath "com.github.jruby-gradle:jruby-gradle-jar-plugin:1.0.1"
         classpath 'com.github.ben-manes:gradle-versions-plugin:0.11.3'
-        classpath 'com.github.jengelman.gradle.plugins:shadow:1.2.2'
+        classpath 'com.github.jengelman.gradle.plugins:shadow:1.2.3'
     }
 }
 apply plugin: "com.github.jruby-gradle.jar"
@@ -16,7 +16,7 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards"), p
 
 allprojects {
     group = 'org.embulk'
-    version = '0.8.15'
+    version = '0.8.16'
 
     ext {
         jrubyVersion = '9.1.5.0'
@@ -151,7 +151,7 @@ subprojects {
 }
 
 task wrapper(type: Wrapper) {
-    gradleVersion = '2.6'
+    gradleVersion = '3.2.1'
 }
 
 import com.github.jrubygradle.JRubyExec
@@ -201,19 +201,22 @@ task updateResources(dependsOn: 'classpath') {
 //
 // classpath task
 //
-task classpath(dependsOn: ['build', ':embulk-cli:classpath']) << { }
+task classpath(dependsOn: ['build', ':embulk-cli:classpath']) { doLast {} }
+
 clean { delete 'classpath' }
 
 //
 // cli task
 //
-task cli(dependsOn: ':embulk-cli:shadowJar') << {
-    file('pkg').mkdirs()
-    File f = file("pkg/embulk-${project.version}.jar")
-    f.write("")
-    f.append(file("embulk-cli/src/main/sh/selfrun.sh").readBytes())
-    f.append(file("embulk-cli/build/libs/embulk-cli-${project.version}-all.jar").readBytes())
-    f.setExecutable(true)
+task cli(dependsOn: ':embulk-cli:shadowJar') {
+    doLast {
+        file('pkg').mkdirs()
+        File f = file("pkg/embulk-${project.version}.jar")
+        f.write("")
+        f.append(file("embulk-cli/src/main/sh/selfrun.sh").readBytes())
+        f.append(file("embulk-cli/build/libs/embulk-cli-${project.version}-all.jar").readBytes())
+        f.setExecutable(true)
+    }
 }
 bintrayUpload.dependsOn(['cli'])
 
@@ -282,58 +285,96 @@ task rubyGemsUpload(type: Exec, dependsOn: ["gem", "rubyGemsUploadJRuby"]) {
 //
 // releaseCheck and release tasks
 //
-task releaseCheck << {
-    if (!file("lib/embulk/version.rb").getText().contains("${project.version}")) {
-        throw new GradleException("lib/embulk/version.rb doesn't include ${project.version}")
-    }
-    if (!file("embulk-docs/src/release/release-${project.version}.rst").getText().contains("${project.version}")) {
-        throw new GradleException("Release note for ${project.version} doesn't exist")
-    }
-    if (!file("embulk-docs/src/release.rst").getText().contains("release-${project.version}")) {
-        throw new GradleException("embulk-docs/src/release.rst doesn't include release-${project.version}")
-    }
-    String date = new Date().format("yyyy-MM-dd")
-    if (!file("embulk-docs/src/release/release-${project.version}.rst").getText().contains(date)) {
-        throw new GradleException("embulk-docs/src/release/release-${project.version}.rst doesn't include today's release date")
+task releaseCheck {
+    doLast {
+        if (!file("lib/embulk/version.rb").getText().contains("${project.version}")) {
+            throw new GradleException("lib/embulk/version.rb doesn't include ${project.version}")
+        }
+        if (!file("embulk-docs/src/release/release-${project.version}.rst").getText().contains("${project.version}")) {
+            throw new GradleException("Release note for ${project.version} doesn't exist")
+        }
+        if (!file("embulk-docs/src/release.rst").getText().contains("release-${project.version}")) {
+            throw new GradleException("embulk-docs/src/release.rst doesn't include release-${project.version}")
+        }
+        String date = new Date().format("yyyy-MM-dd")
+        if (!file("embulk-docs/src/release/release-${project.version}.rst").getText().contains(date)) {
+            throw new GradleException("embulk-docs/src/release/release-${project.version}.rst doesn't include today's release date")
+        }
+        // TODO check git-ls-files includes release-<version>.rst file
+        println "Ready. Run 'release' task."
     }
-    // TODO check git-ls-files includes release-<version>.rst file
-    println "Ready. Run 'release' task."
 }
 
-task release(dependsOn: ["cli", "releaseCheck", "bintrayUpload", "rubyGemsUpload"]) << {
-    println """
+task release(dependsOn: ["cli", "releaseCheck", "bintrayUpload", "rubyGemsUpload"]) {
+    doLast {
+        println """
 Manual operations:
 
   git commit -am v${project.version}
   git tag v${project.version}
 
 """
+    }
 }
 bintrayUpload.mustRunAfter('releaseCheck')
 rubyGemsUpload.mustRunAfter('releaseCheck')
 rubyGemsUploadJRuby.mustRunAfter('releaseCheck')
 
-task setVersion << {
-    if (!project.hasProperty("to")) {
-        throw new GradleException("Usage: ./gradlew setVersion -Pto=VERSION")
-    }
+task setVersion {
+    doLast {
+        if (!project.hasProperty("to")) {
+            throw new GradleException("Usage: ./gradlew setVersion -Pto=VERSION")
+        }
+
+        File gradle_ver = file('build.gradle')
+        gradle_ver.write(gradle_ver.getText().replaceFirst("version = '(\\d+)(\\.\\d+){2}'", "version = '${to}'"))
 
-    File gradle_ver = file('build.gradle')
-    gradle_ver.write(gradle_ver.getText().replaceFirst("version = '(\\d+)(\\.\\d+){2}'", "version = '${to}'"))
+        File ruby_ver = file('lib/embulk/version.rb')
+        ruby_ver.write(ruby_ver.getText().replaceFirst("VERSION = '(\\d+)(\\.\\d+){2}'", "VERSION = '${to}'"))
+
+        List<String> docs = [
+            'README.md',
+        ]
+        docs.each() { path ->
+            File doc = file(path)
+            doc.write(doc.getText().replaceAll('embulk-(\\d+)(\\.\\d+){2}', "embulk-${to}"))
+        }
 
-    File ruby_ver = file('lib/embulk/version.rb')
-    ruby_ver.write(ruby_ver.getText().replaceFirst("VERSION = '(\\d+)(\\.\\d+){2}'", "VERSION = '${to}'"))
+        file("embulk-docs/src/release/release-${to}.rst").append("")
+        "git add embulk-docs/src/release/release-${to}.rst".execute().waitFor()
 
-    List<String> docs = [
-        'README.md',
-    ]
-    docs.each() { path ->
-        File doc = file(path)
-        doc.write(doc.getText().replaceAll('embulk-(\\d+)(\\.\\d+){2}', "embulk-${to}"))
+        println "add 'release/release-${to}' line to embulk-docs/src/release.rst"
     }
+}
+
+task updateJRuby {
+    doLast {
+        if (!project.hasProperty("to")) {
+            throw new GradleException("Usage: ./gradlew updateJRuby -Pto=VERSION")
+        }
 
-    file("embulk-docs/src/release/release-${to}.rst").append("")
-    "git add embulk-docs/src/release/release-${to}.rst".execute().waitFor()
+        File gradle_ver = file('build.gradle')
+        gradle_ver.write(gradle_ver.getText().replaceFirst("jrubyVersion = '(\\d+)(\\.\\d+){3}'", "jrubyVersion = '${to}'"))
 
-    println "add 'release/release-${to}' line to embulk-docs/src/release.rst"
+        File gemspec_ver  = file('embulk.gemspec')
+        gemspec_ver.write(gemspec_ver.getText().replaceFirst("gem.add_dependency \"jruby-jars\", '= (\\d+)(\\.\\d+){3}'", "gem.add_dependency \"jruby-jars\", '= ${to}'"))
+
+        File migrate_plugin_ver  = file('lib/embulk/command/embulk_migrate_plugin.rb')
+        migrate_plugin_ver.write(migrate_plugin_ver.getText().replaceFirst("\".ruby-version\", \"jruby-(\\d+)(\\.\\d+){3}\"", "\".ruby-version\", \"jruby-${to}\""))
+
+        File gemfile_lock_ver  = file('Gemfile.lock')
+        gemfile_lock_ver.write(gemfile_lock_ver.getText().replaceFirst("jruby-jars \\(= (\\d+)(\\.\\d+){3}\\)", "jruby-jars (= ${to})"))
+
+        gemfile_lock_ver.write(gemfile_lock_ver.getText().replaceFirst("jruby-jars \\((\\d+)(\\.\\d+){3}\\)", "jruby-jars (${to})"))
+
+        List<String> dot_ruby_vers = [
+            'lib/embulk/data/bundle/.ruby-version',
+            'lib/embulk/data/new/ruby/.ruby-version'
+        ]
+        dot_ruby_vers.each() { path ->
+            File dot_ruby_ver = file(path)
+            dot_ruby_ver.write(dot_ruby_ver.getText().replaceAll('jruby-(\\d+)(\\.\\d+){3}', "jruby-${to}"))
+        }
+
+    }
 }

data/embulk-core/src/main/java/org/embulk/config/TaskValidationException.java

@@ -4,7 +4,7 @@ import java.util.Set;
 import javax.validation.ConstraintViolation;
 
 public class TaskValidationException
-        extends RuntimeException
+        extends ConfigException
 {
     @SuppressWarnings("unchecked")
     private final Set violations;

data/embulk-core/src/main/java/org/embulk/exec/SamplingParserPlugin.java

@@ -37,6 +37,8 @@ public class SamplingParserPlugin
                     if (taskCount == 0) {
                         throw new NoSampleException("No input files to read sample data");
                     }
+                    int maxSize = -1;
+                    int maxSizeTaskIndex = -1;
                     for (int taskIndex=0; taskIndex < taskCount; taskIndex++) {
                         try {
                             runner.run(taskSource, schema, taskIndex, new PageOutput() {
@@ -51,10 +53,33 @@ public class SamplingParserPlugin
                                 public void close() { }
                             });
                         } catch (NotEnoughSampleError ex) {
+                            if (maxSize < ex.getSize()) {
+                                maxSize = ex.getSize();
+                                maxSizeTaskIndex = taskIndex;
+                            }
                             continue;
                         }
                     }
-                    throw new NoSampleException("All input files are smaller than minimum sampling size");  // TODO include minSampleSize in message
+                    if (maxSize <= 0) {
+                        throw new NoSampleException("All input files are empty");
+                    }
+                    taskSource.getNested("ParserTaskSource").set("force", true);
+                    try {
+                        runner.run(taskSource, schema, maxSizeTaskIndex, new PageOutput() {
+                            @Override
+                            public void add(Page page)
+                            {
+                                throw new RuntimeException("Input plugin must be a FileInputPlugin to guess parser configuration");  // TODO exception class
+                            }
+
+                            public void finish() { }
+
+                            public void close() { }
+                        });
+                    } catch (NotEnoughSampleError ex) {
+                        throw new NoSampleException("All input files are smaller than minimum sampling size");
+                    }
+                    throw new NoSampleException("All input files are smaller than minimum sampling size");
                 }
             });
             throw new AssertionError("SamplingParserPlugin must throw SampledNoticeError");
@@ -81,7 +106,19 @@ public class SamplingParserPlugin
 
     public static class NotEnoughSampleError
             extends Error
-    { }
+    {
+        private final int size;
+
+        public NotEnoughSampleError(int size)
+        {
+            this.size = size;
+        }
+
+        public int getSize()
+        {
+            return size;
+        }
+    }
 
     private final int minSampleSize;
     private final int sampleSize;
@@ -105,8 +142,10 @@ public class SamplingParserPlugin
             FileInput input, PageOutput output)
     {
         Buffer buffer = readSample(input, sampleSize);
-        if (buffer.limit() < minSampleSize) {
-            throw new NotEnoughSampleError();
+        if (!taskSource.get(boolean.class, "force", false)) {
+            if (buffer.limit() < minSampleSize) {
+                throw new NotEnoughSampleError(buffer.limit());
+            }
         }
         throw new SampledNoticeError(buffer);
     }

data/embulk-core/src/main/java/org/embulk/spi/PageBuilder.java

@@ -123,6 +123,11 @@ public class PageBuilder
 
     public void setString(int columnIndex, String value)
     {
+        if (value == null) {
+            setNull(columnIndex);
+            return;
+        }
+
         Integer reuseIndex = stringReferences.get(value);
         if (reuseIndex != null) {
             bufferSlice.setInt(getOffset(columnIndex), reuseIndex);
@@ -143,6 +148,11 @@ public class PageBuilder
 
     public void setJson(int columnIndex, Value value)
     {
+        if (value == null) {
+            setNull(columnIndex);
+            return;
+        }
+
         int index = valueReferences.size();
         valueReferences.add(value.immutableValue());
         bufferSlice.setInt(getOffset(columnIndex), index);
@@ -158,6 +168,11 @@ public class PageBuilder
 
     public void setTimestamp(int columnIndex, Timestamp value)
     {
+        if (value == null) {
+            setNull(columnIndex);
+            return;
+        }
+
         int offset = getOffset(columnIndex);
         bufferSlice.setLong(offset, value.getEpochSecond());
         bufferSlice.setInt(offset + 8, value.getNano());

data/embulk-core/src/main/java/org/embulk/spi/util/ResumableInputStream.java

@@ -15,6 +15,8 @@ public class ResumableInputStream
     protected InputStream in;
     private long offset;
     private long markedOffset;
+    private Exception lastClosedCause;
+    private boolean closed;
 
     public ResumableInputStream(InputStream initialInputStream, Reopener reopener)
     {
@@ -22,6 +24,7 @@ public class ResumableInputStream
         this.in = initialInputStream;
         this.offset = 0L;
         this.markedOffset = 0L;
+        this.lastClosedCause = null;
     }
 
     public ResumableInputStream(Reopener reopener) throws IOException
@@ -32,6 +35,7 @@ public class ResumableInputStream
     private void reopen(Exception closedCause) throws IOException
     {
         if (in != null) {
+            lastClosedCause = closedCause;
             try {
                 in.close();
             } catch (IOException ignored) {
@@ -39,11 +43,13 @@ public class ResumableInputStream
             in = null;
         }
         in = reopener.reopen(offset, closedCause);
+        lastClosedCause = null;
     }
 
     @Override
     public int read() throws IOException
     {
+        ensureOpened();
         while (true) {
             try {
                 int v = in.read();
@@ -58,6 +64,7 @@ public class ResumableInputStream
     @Override
     public int read(byte[] b) throws IOException
     {
+        ensureOpened();
         while (true) {
             try {
                 int r = in.read(b);
@@ -72,6 +79,7 @@ public class ResumableInputStream
     @Override
     public int read(byte[] b, int off, int len) throws IOException
     {
+        ensureOpened();
         while (true) {
             try {
                 int r = in.read(b, off, len);
@@ -86,6 +94,7 @@ public class ResumableInputStream
     @Override
     public long skip(long n) throws IOException
     {
+        ensureOpened();
         while (true) {
             try {
                 long r = in.skip(n);
@@ -100,18 +109,29 @@ public class ResumableInputStream
     @Override
     public int available() throws IOException
     {
+        ensureOpened();
         return in.available();
     }
 
     @Override
     public void close() throws IOException
     {
-        in.close();
+        if (in != null) {
+            in.close();
+            closed = true;
+            in = null;
+        }
     }
 
     @Override
     public void mark(int readlimit)
     {
+        try {
+            ensureOpened();
+        }
+        catch (IOException ex) {
+            throw new RuntimeException(ex);
+        }
         in.mark(readlimit);
         markedOffset = offset;
     }
@@ -119,6 +139,7 @@ public class ResumableInputStream
     @Override
     public void reset() throws IOException
     {
+        ensureOpened();
         in.reset();
         offset = markedOffset;
     }
@@ -126,6 +147,22 @@ public class ResumableInputStream
     @Override
     public boolean markSupported()
     {
+        try {
+            ensureOpened();
+        }
+        catch (IOException ex) {
+            throw new RuntimeException(ex);
+        }
         return in.markSupported();
     }
+
+    private void ensureOpened() throws IOException
+    {
+        if (in == null) {
+            if (closed) {
+                throw new IOException("stream closed");
+            }
+            reopen(lastClosedCause);
+        }
+    }
 }