embulk 0.6.1 → 0.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6de3d95e030e043dd82676af00373a34a2e50e47
-  data.tar.gz: 554e7b89b160fe65d540eeffd1dd03403b08fdaf
+  metadata.gz: 148c75f1b85e1d85859d0be3293d85e5fc3f1bab
+  data.tar.gz: c07d2801fc7cc4b0caa60d2ff692553b29827cfe
 SHA512:
-  metadata.gz: ad8d669c190e0d468c44803fbf5b4614302e19e582a1f66cf99540a4a00ae1baf9261ad568cd6116c8c22fbefe3a7d1d13585ab25e8cbc38ec79837ce823fd40
-  data.tar.gz: 562d3cff28a2ba94de942a36e768e20f3205cf44a8b95551223702fce8fe1cfe974eb9ce6c6a8cb4b5416189f9b5dadf0b4c5e2657282c90b0932bf4d9cf955c
+  metadata.gz: 6f499389219a19663e1e3789b68bf1eea7f650df7921347ec5d0641395a1a1b11a5087f571a9e0ac3536cc1156592cb15b1f5b4283e069f68d456449c90d4c69
+  data.tar.gz: 78eeb275c30689f63947961fd1d2f0b01d96abfef0c16b1e68370fdca0570c809f1e4cb38924e67c10f11e86759c88e706dcb32c45f55eaf10876055bdedf8ea

data/README.md

@@ -30,7 +30,7 @@ Embulk is a Java application. Please make sure that you installed [Java](http://
 Following 4 commands install embulk to your home directory:
 
 ```
-curl --create-dirs -o ~/.embulk/bin/embulk -L https://bintray.com/artifact/download/embulk/maven/embulk-0.6.1.jar
+curl --create-dirs -o ~/.embulk/bin/embulk -L "http://dl.embulk.org/embulk-latest.jar"
 chmod +x ~/.embulk/bin/embulk
 echo 'export PATH="$HOME/.embulk/bin:$PATH"' >> ~/.bashrc
 source ~/.bashrc
@@ -45,7 +45,7 @@ Embulk is a Java application. Please make sure that you installed [Java](http://
 You can assume the jar file is a .bat file.
 
 ```
-PowerShell -Command "& {Invoke-WebRequest https://bintray.com/artifact/download/embulk/maven/embulk-0.6.1.jar -OutFile embulk.bat}"
+PowerShell -Command "& {Invoke-WebRequest http://dl.embulk.org/embulk-latest.jar -OutFile embulk.bat}"
 ```
 
 Next step: [Trying the example](#trying-the-example)

data/build.gradle

@@ -12,7 +12,7 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards")]
 
 allprojects {
     group = 'org.embulk'
-    version = '0.6.1'
+    version = '0.6.2'
 
     apply plugin: 'java'
     apply plugin: 'maven-publish'
@@ -252,7 +252,6 @@ task set_version << {
 
     List<String> docs = [
         'README.md',
-        'embulk-docs/src/recipe/scheduled-csv-load-to-elasticsearch-kibana4.rst'
     ]
     docs.each() { path ->
         File doc = file(path)

data/embulk-core/src/main/java/org/embulk/exec/BulkLoader.java

@@ -327,28 +327,6 @@ public class BulkLoader
         }
     }
 
-    protected ExecutorPlugin newExecutorPlugin(BulkLoaderTask task)
-    {
-        return Exec.newPlugin(ExecutorPlugin.class,
-                task.getExecConfig().get(PluginType.class, "type", new PluginType("local")));
-    }
-
-    protected InputPlugin newInputPlugin(BulkLoaderTask task)
-    {
-        return Exec.newPlugin(InputPlugin.class, task.getInputConfig().get(PluginType.class, "type"));
-    }
-
-    protected List<FilterPlugin> newFilterPlugins(BulkLoaderTask task)
-    {
-        return Filters.newFilterPlugins(Exec.session(),
-                Filters.getPluginTypes(task.getFilterConfigs()));
-    }
-
-    protected OutputPlugin newOutputPlugin(BulkLoaderTask task)
-    {
-        return Exec.newPlugin(OutputPlugin.class, task.getOutputConfig().get(PluginType.class, "type"));
-    }
-
     public ExecutionResult run(ExecSession exec, final ConfigSource config)
     {
         try {
@@ -400,32 +378,6 @@ public class BulkLoader
         }
     }
 
-    public void doCleanup(ConfigSource config, ResumeState resume)
-    {
-        BulkLoaderTask task = config.loadConfig(BulkLoaderTask.class);
-        InputPlugin inputPlugin = newInputPlugin(task);
-        OutputPlugin outputPlugin = newOutputPlugin(task);
-
-        ImmutableList.Builder<CommitReport> successfulInputCommitReports = ImmutableList.builder();
-        ImmutableList.Builder<CommitReport> successfulOutputCommitReports = ImmutableList.builder();
-        for (Optional<CommitReport> inputCommitReport : resume.getInputCommitReports()) {
-            if (inputCommitReport.isPresent()) {
-                successfulInputCommitReports.add(inputCommitReport.get());
-            }
-        }
-        for (Optional<CommitReport> outputCommitReport : resume.getOutputCommitReports()) {
-            if (outputCommitReport.isPresent()) {
-                successfulOutputCommitReports.add(outputCommitReport.get());
-            }
-        }
-
-        inputPlugin.cleanup(resume.getInputTaskSource(), resume.getInputSchema(),
-                resume.getInputCommitReports().size(), successfulInputCommitReports.build());
-
-        outputPlugin.cleanup(resume.getOutputTaskSource(), resume.getOutputSchema(),
-                resume.getOutputCommitReports().size(), successfulOutputCommitReports.build());
-    }
-
     private static class ProcessPluginSet
     {
         private final PluginType inputPluginType;
@@ -477,6 +429,37 @@ public class BulkLoader
         }
     }
 
+    public void doCleanup(ConfigSource config, ResumeState resume)
+    {
+        BulkLoaderTask task = config.loadConfig(BulkLoaderTask.class);
+        ProcessPluginSet plugins = new ProcessPluginSet(task);  // TODO don't create filter plugins
+
+        ImmutableList.Builder<CommitReport> successfulInputCommitReports = ImmutableList.builder();
+        ImmutableList.Builder<CommitReport> successfulOutputCommitReports = ImmutableList.builder();
+        for (Optional<CommitReport> inputCommitReport : resume.getInputCommitReports()) {
+            if (inputCommitReport.isPresent()) {
+                successfulInputCommitReports.add(inputCommitReport.get());
+            }
+        }
+        for (Optional<CommitReport> outputCommitReport : resume.getOutputCommitReports()) {
+            if (outputCommitReport.isPresent()) {
+                successfulOutputCommitReports.add(outputCommitReport.get());
+            }
+        }
+
+        plugins.getInputPlugin().cleanup(resume.getInputTaskSource(), resume.getInputSchema(),
+                resume.getInputCommitReports().size(), successfulInputCommitReports.build());
+
+        plugins.getOutputPlugin().cleanup(resume.getOutputTaskSource(), resume.getOutputSchema(),
+                resume.getOutputCommitReports().size(), successfulOutputCommitReports.build());
+    }
+
+    private ExecutorPlugin newExecutorPlugin(BulkLoaderTask task)
+    {
+        return Exec.newPlugin(ExecutorPlugin.class,
+                task.getExecConfig().get(PluginType.class, "type", new PluginType("local")));
+    }
+
     private ExecutionResult doRun(ConfigSource config)
     {
         final BulkLoaderTask task = config.loadConfig(BulkLoaderTask.class);

data/embulk-docs/push-gh-pages.sh

@@ -16,7 +16,7 @@ function r() {
 [ "$TRAVIS_BRANCH" != "master" -a "$TRAVIS_BRANCH" != "$(git describe --tags --always HEAD)" ] && exit 0
 
 revision="$(git rev-parse HEAD)"
-remote="$(git config remote.origin.url | sed "s+^git:+https:+")"
+remote="https://github.com/embulk/embulk.github.io.git"
 re ./gradlew site
 
 r git fetch --unshallow || echo "using complete repository."
@@ -28,7 +28,7 @@ re cd gh_pages
 re git remote add travis_push "$remote"
 re git fetch travis_push
 
-re git checkout -b gh-pages travis_push/gh-pages
+re git checkout -b gh-pages travis_push/master
 re rm -rf docs
 re cp -a ../embulk-docs/build/html docs
 re git add --all docs
@@ -46,4 +46,4 @@ fi
 re git config credential.helper "store --file=$HOME/.git_credentials"
 echo "https://$GITHUB_TOKEN:@github.com" > "$HOME/.git_credentials"
 trap "rm -rf $HOME/.git_credentials" EXIT
-re git push travis_push gh-pages
+re git push travis_push gh-pages:master

data/embulk-docs/src/recipe/scheduled-csv-load-to-elasticsearch-kibana4.rst

@@ -56,7 +56,7 @@ You can find the latest embulk binary from the `releases <https://bintray.com/em
 
 .. code-block:: console
 
-    $ sudo wget https://bintray.com/artifact/download/embulk/maven/embulk-0.6.1.jar -O /usr/local/bin/embulk
+    $ sudo wget http://dl.embulk.org/embulk-latest.jar -O /usr/local/bin/embulk
     $ sudo chmod +x /usr/local/bin/embulk
 
 Step 2. Install Elasticsearch plugin

data/embulk-docs/src/release.rst

@@ -4,6 +4,7 @@ Release Notes
 .. toctree::
     :maxdepth: 1
 
+    release/release-0.6.2
     release/release-0.6.1
     release/release-0.6.0
     release/release-0.5.5

data/embulk-docs/src/release/release-0.6.2.rst

@@ -0,0 +1,17 @@
+Release 0.6.2
+==================================
+
+Built-in plugins
+------------------
+
+* ``guess-csv`` guesses charset as ``"MS932"`` instead of ``"Shift_JIS"`` because practically almost all of documents encoded by ``Shift_JIS`` are created by Windows and ``Shift_JIS`` implemented by Microsoft means ``MS932`` in Java. (@kosaki55tea++, @nalsh++)
+* ``parser-csv`` recovers errors by invalid number formats and skips the row rather than making entire transaction failed (@hito4t++)
+
+General Changes
+------------------
+
+* Changed download URL. We can use the consitent URL to download the latest jar.
+
+Release Date
+------------------
+2015-04-13

data/embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java

@@ -233,7 +233,7 @@ public class CsvParserPlugin
                         });
                         pageBuilder.addRecord();
 
-                    } catch (CsvTokenizer.InvalidFormatException e) {
+                    } catch (CsvTokenizer.InvalidFormatException | CsvRecordValidateException e) {
                         long lineNumber = tokenizer.getCurrentLineNumber();
                         String skippedLine = tokenizer.skipCurrentLine();
                         log.warn(String.format("Skipped (line %d): %s", lineNumber, skippedLine), e);

data/lib/embulk/guess/charset.rb

@@ -4,6 +4,16 @@ module Embulk
     class CharsetGuessPlugin < GuessPlugin
       Plugin.register_guess('charset', self)
 
+      STATIC_MAPPING = {
+        # ISO-8859-1 means ASCII which is a subset of UTF-8 in most of cases
+        # due to lack of sample data set.
+        "ISO-8859-1" => "UTF-8",
+
+        # Shift_JIS is used almost only by Windows that uses "CP932" in fact.
+        # And "CP932" called by Microsoft actually means "MS932" in Java.
+        "Shift_JIS" => "MS932",
+      }
+
       def guess(config, sample_buffer)
         # ICU4J
         detector = com.ibm.icu.text.CharsetDetector.new
@@ -13,11 +23,8 @@ module Embulk
           name = "UTF-8"
         else
           name = best_match.getName
-          if name == "ISO-8859-1"
-            # ISO-8859-1 means ASCII which is a subset
-            # of UTF-8 in most of cases due to lack of
-            # sample data set
-            name = "UTF-8"
+          if mapped_name = STATIC_MAPPING[name]
+            name = mapped_name
           end
         end
         return {"parser" => {"charset" => name}}

data/lib/embulk/version.rb

@@ -1,3 +1,3 @@
 module Embulk
-  VERSION = '0.6.1'
+  VERSION = '0.6.2'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 0.6.2
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-04-09 00:00:00.000000000 Z
+date: 2015-04-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -290,6 +290,7 @@ files:
 - embulk-docs/src/release/release-0.5.5.rst
 - embulk-docs/src/release/release-0.6.0.rst
 - embulk-docs/src/release/release-0.6.1.rst
+- embulk-docs/src/release/release-0.6.2.rst
 - embulk-standards/build.gradle
 - embulk-standards/src/main/java/org/embulk/standards/CsvFormatterPlugin.java
 - embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java
@@ -395,8 +396,8 @@ files:
 - classpath/bval-jsr303-0.5.jar
 - classpath/commons-beanutils-core-1.8.3.jar
 - classpath/commons-lang3-3.1.jar
-- classpath/embulk-core-0.6.1.jar
-- classpath/embulk-standards-0.6.1.jar
+- classpath/embulk-core-0.6.2.jar
+- classpath/embulk-standards-0.6.2.jar
 - classpath/guava-18.0.jar
 - classpath/guice-3.0.jar
 - classpath/guice-multibindings-3.0.jar