embulk 0.6.27 → 0.7.0

data/embulk-core/src/main/java/org/embulk/spi/util/Filters.java

@@ -4,7 +4,6 @@ import java.util.List;
 import com.google.common.collect.ImmutableList;
 import org.embulk.config.TaskSource;
 import org.embulk.config.ConfigSource;
-import org.embulk.config.CommitReport;
 import org.embulk.config.ConfigDiff;
 import org.embulk.plugin.PluginType;
 import org.embulk.spi.ExecSession;

data/embulk-core/src/test/java/org/embulk/spi/TestFileInputRunner.java

@@ -13,7 +13,7 @@ import org.junit.Test;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import org.embulk.EmbulkTestRuntime;
-import org.embulk.config.CommitReport;
+import org.embulk.config.TaskReport;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.ConfigDiff;
 import org.embulk.config.TaskSource;
@@ -62,7 +62,7 @@ public class TestFileInputRunner
         @Override
         public void cleanup(TaskSource taskSource,
                 int taskCount,
-                List<CommitReport> successCommitReports)
+                List<TaskReport> successTaskReports)
         {
         }
 
@@ -95,7 +95,7 @@ public class TestFileInputRunner
                 }
 
                 @Override
-                public CommitReport commit()
+                public TaskReport commit()
                 {
                     transactionCompleted = true;
                     return null;
@@ -126,10 +126,10 @@ public class TestFileInputRunner
         final MockPageOutput output = new MockPageOutput();
         runner.transaction(config, new InputPlugin.Control()
         {
-            public List<CommitReport> run(TaskSource inputTaskSource,
+            public List<TaskReport> run(TaskSource inputTaskSource,
                     Schema schema, int taskCount)
             {
-                List<CommitReport> reports = new ArrayList<>();
+                List<TaskReport> reports = new ArrayList<>();
                 reports.add(runner.run(inputTaskSource, schema, 0, output));
                 return reports;
             }
@@ -180,10 +180,10 @@ public class TestFileInputRunner
         try {
             runner.transaction(config, new InputPlugin.Control()
             {
-                public List<CommitReport> run(TaskSource inputTaskSource,
+                public List<TaskReport> run(TaskSource inputTaskSource,
                         Schema schema, int taskCount)
                 {
-                    List<CommitReport> reports = new ArrayList<>();
+                    List<TaskReport> reports = new ArrayList<>();
                     reports.add(runner.run(inputTaskSource, schema, 0, output));
                     return reports;
                 }

data/embulk-core/src/test/java/org/embulk/spi/TestFileOutputRunner.java

@@ -6,7 +6,7 @@ import java.util.ArrayList;
 import java.util.List;
 
 import org.embulk.EmbulkTestRuntime;
-import org.embulk.config.CommitReport;
+import org.embulk.config.TaskReport;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.ConfigDiff;
 import org.embulk.config.Task;
@@ -52,7 +52,7 @@ public class TestFileOutputRunner
         @Override
         public void cleanup(TaskSource taskSource,
                 int taskCount,
-                List<CommitReport> successCommitReports)
+                List<TaskReport> successTaskReports)
         {
         }
 
@@ -90,10 +90,10 @@ public class TestFileOutputRunner
                 }
 
                 @Override
-                public CommitReport commit()
+                public TaskReport commit()
                 {
                     transactionCompleted = true;
-                    return Exec.newCommitReport();
+                    return Exec.newTaskReport();
                 }
             };
         }
@@ -122,7 +122,7 @@ public class TestFileOutputRunner
 
         runner.transaction(config, schema, 1, new OutputPlugin.Control()
         {
-            public List<CommitReport> run(final TaskSource outputTask)
+            public List<TaskReport> run(final TaskSource outputTask)
             {
                 TransactionalPageOutput tran = runner.open(outputTask, schema,
                         1);
@@ -142,7 +142,7 @@ public class TestFileOutputRunner
                     }
                     tran.close();
                 }
-                return new ArrayList<CommitReport>();
+                return new ArrayList<TaskReport>();
             }
         });
 
@@ -181,7 +181,7 @@ public class TestFileOutputRunner
         try {
             runner.transaction(config, schema, 1, new OutputPlugin.Control()
             {
-                public List<CommitReport> run(final TaskSource outputTask)
+                public List<TaskReport> run(final TaskSource outputTask)
                 {
                     TransactionalPageOutput tran = runner.open(outputTask,
                             schema, 1);
@@ -196,7 +196,7 @@ public class TestFileOutputRunner
                         }
                         tran.close();
                     }
-                    return new ArrayList<CommitReport>();
+                    return new ArrayList<TaskReport>();
                 }
             });
         } catch (NullPointerException npe) {

data/embulk-docs/src/built-in.rst

@@ -59,34 +59,33 @@ A configuration file consists of following sections:
 In many cases, what you need to write is **in:**, **out**: and **formatter** sections only because ``guess`` command guesses **parser** and **decoder** options for you. See also the `Quick Start <https://github.com/embulk/embulk#quick-start>`_.
 
 
-..
-  Using variables
-  ~~~~~~~~~~~~~~~~~~
-  
-  You can embed environment variables in configuration file using `Liquid template engine <http://liquidmarkup.org/>`_ (This is experimental feature. Behavior might change or be removed in future releases).
-  
-  To use template engine, configuration file name must end with ``.yml.liquid``.
-  
-  Environment variables are set to ``env`` variable.
-  
-  .. code-block:: yaml
-  
-      in:
-        type: file
-        path_prefix: {{ env.path_prefix }}
-        decoders:
-        - {type: gzip}
-        parser:
-          ...
-      out:
-        type: postgresql
-        host: {{ env.pg_host }}
-        port: {{ env.pg_port }}
-        user: {{ env.pg_user }}
-        password: "{{ env.pg_password }}"
-        database: embulk_load
-        mode: insert
-        table: {{ env.pg_table }}
+Using variables
+~~~~~~~~~~~~~~~~~~
+
+You can embed environment variables in configuration file using `Liquid template engine <http://liquidmarkup.org/>`_ (This is experimental feature. Behavior might change or be removed in future releases).
+
+To use template engine, configuration file name must end with ``.yml.liquid``.
+
+Environment variables are set to ``env`` variable.
+
+.. code-block:: yaml
+
+    in:
+      type: file
+      path_prefix: {{ env.path_prefix }}
+      decoders:
+      - {type: gzip}
+      parser:
+        ...
+    out:
+      type: postgresql
+      host: {{ env.pg_host }}
+      port: {{ env.pg_port }}
+      user: {{ env.pg_user }}
+      password: "{{ env.pg_password }}"
+      database: embulk_load
+      mode: insert
+      table: {{ env.pg_table }}
 
 
 Local file input plugin
@@ -181,8 +180,6 @@ Options
 +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
 | max\_quoted\_size\_limit   | integer  | Maximum number of bytes of a quoted value. If a value exceeds the limit, the row will be skipped               | ``131072`` by default  |
 +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
-| stop\_on\_invalid\_record  | boolean  | Stop bulk load transaction if a file includes invalid record (such as invalid timestamp)                       | ``false`` by default   |
-+----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
 | default\_timezone          | string   | Time zone of timestamp columns if the value itself doesn't include time zone description (eg. Asia/Tokyo)      | ``UTC`` by default     |
 +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
 | newline                    | enum     | Newline character (CRLF, LF or CR)                                                                             | ``CRLF`` by default    |

data/embulk-docs/src/conf.py

@@ -53,9 +53,9 @@ copyright = u'2015, Embulk Project'
 # built documents.
 #
 # The short X.Y version.
-version = '0.6'
+version = '0.7'
 # The full version, including alpha/beta/rc tags.
-release = '0.6'
+release = '0.7'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

data/embulk-docs/src/release.rst

@@ -4,8 +4,7 @@ Release Notes
 .. toctree::
     :maxdepth: 1
 
-    release/release-0.6.27
-    release/release-0.6.26
+    release/release-0.7.0
     release/release-0.6.25
     release/release-0.6.24
     release/release-0.6.23

data/embulk-docs/src/release/release-0.7.0.rst

@@ -0,0 +1,96 @@
+Release 0.7.0
+==================================
+
+General Changes
+------------------
+
+* Upraded JRuby version to 9.0.0.0. Ruby scripting is compatible with Ruby 2.2 syntax.
+
+* Added ``embulk migrate <plugin directory>`` subcommand. This command modifies plugin code to use the latest Embulk API.
+
+* Enambed Liquid template engine. If configuration file name ends with ``.yml.liquid``, embulk embeds environment variables using Liquid template engine.
+
+* Embulk gem package for JRuby doesn't include JRuby itself any more. Size of a gem package is reduced from 38MB to 7.6MB.
+
+* Embulk gem is also released for CRuby. This enables us to install ``embulk`` command using ``gem install embulk``.
+
+* **IMPORTANT**: ``embulk bundle`` command runs bundler. To create a new plugin bundle, use ``embulk bundle new <directory>`` command. To update gems, use ``embulk bundle`` command at the directory. Instructions are written at generated Gemfile file.
+
+
+Ruby Plugin API
+------------------
+
+* Added experimental ``Embulk.setup`` and ``Embulk::Runner``.
+
+  This enables ruby scripts to run embulk easily. This is also good for test code. For example, you can use this code:
+
+.. code-block:: ruby
+
+    require 'embulk'
+    Embulk.setup
+    Embulk::Runner.run(YAML.load_file("config.yml"))
+
+* ``embulk new`` generates .ruby-version file with jruby-9.0.0.0 for ruby-based plugins.
+
+  This makes plugin development easy as following:
+
+.. code-block:: console
+
+    # 1. Create plugin template
+    $ embulk new ruby-parser awesome
+    $ cd embulk-parser-awesome
+    # or upgrade existent plugin: embulk migrate embulk-parser-awesome
+   
+    # 2. Install dependency gems including embulk itself at vendor/bundle directory
+    $ bundle install --path vendor/bundle
+   
+    # 3. Create an example configuration file
+    $ vi config.yml
+   
+    # 4. You can run embulk without building & installing gem
+    $ bundle exec embulk run config.yml
+
+* Constants defined at ``Embulk::Java`` are deprecated. They're still kept for backward compatibility but will be removed at a future release.
+
+* Added ``Embulk::Java::Config`` and ``Embulk::SPI`` namespaces to access Java classes.
+
+
+Java Plugin API
+------------------
+
+* **IMPORTANT**: Renamed CommitReport class to TaskReport. Binary backward compatibility is kept so that old plugins built with embulk 0.6.x can run with embulk 0.7.0. But this backward compatibility code will be removed at future release.
+
+  To upgrade your plugin code, you can use ``embulk migrate <plugin directory>`` command.
+
+* Upgraded gradle version to 2.6. This version supports ``./gradlew -t <task>`` command that watches changes of files and rebuild continuously.
+
+  This makes plugin development easy as following:
+
+.. code-block:: console
+
+    # 1. Create plugin template
+    $ embulk new java-input awesome
+    $ cd embulk-input-awesome
+    # or upgrade existent plugin: embulk migrate embulk-input-awesome
+   
+    # 2. Build code continously
+    $ ./gradlew -t package
+   
+    # 3. Create an example configuration file
+    $ vi config.yml
+   
+    # 4. Run embulk with -L option
+    $ embulk -L . run config.yml
+
+* Added ``EmbulkEmbed.Bootstrap`` class to build ``EmbulkEmbed`` instance.
+
+* Added ``ConfigLoader.fromJsonString(String)`` and ``ConfigLoader.fromYamlString(String)`` methods.
+
+* Added guess, preview, and run methods at ``EmbulkEmbed`` don't need ExecSession instance any more.
+
+* EmbulkService is now deprecated. Replacement is EmbulkEmbed.
+
+
+Release Date
+------------------
+2015-08-18

data/embulk-standards/src/main/java/org/embulk/standards/CsvFormatterPlugin.java

@@ -1,11 +1,8 @@
 package org.embulk.standards;
 
 import com.google.common.base.Optional;
-import com.google.common.collect.ImmutableBiMap;
-import com.google.common.collect.ImmutableMap;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
-import org.embulk.spi.type.TimestampType;
 import org.embulk.spi.time.Timestamp;
 import org.embulk.spi.time.TimestampFormatter;
 import org.embulk.config.Task;
@@ -18,7 +15,6 @@ import org.embulk.spi.FormatterPlugin;
 import org.embulk.spi.Page;
 import org.embulk.spi.PageOutput;
 import org.embulk.spi.PageReader;
-import org.embulk.spi.Exec;
 import org.embulk.spi.FileOutput;
 import org.embulk.spi.util.LineEncoder;
 import org.embulk.spi.util.Timestamps;

data/embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java

@@ -11,11 +11,9 @@ import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.ConfigException;
 import org.embulk.config.TaskSource;
-import org.embulk.spi.type.TimestampType;
 import org.embulk.spi.time.TimestampParser;
 import org.embulk.spi.time.TimestampParseException;
 import org.embulk.spi.Column;
-import org.embulk.spi.ColumnConfig;
 import org.embulk.spi.Schema;
 import org.embulk.spi.SchemaConfig;
 import org.embulk.spi.ColumnVisitor;
@@ -91,10 +89,6 @@ public class CsvParserPlugin
         @Config("allow_extra_columns")
         @ConfigDefault("false")
         boolean getAllowExtraColumns();
-
-        @Config("stop_on_invalid_record")
-        @ConfigDefault("false")
-        boolean getStopOnInvalidRecord();
     }
 
     public static class QuoteCharacter
@@ -235,7 +229,6 @@ public class CsvParserPlugin
         final String nullStringOrNull = task.getNullString().orNull();
         final boolean allowOptionalColumns = task.getAllowOptionalColumns();
         final boolean allowExtraColumns = task.getAllowExtraColumns();
-        final boolean stopOnInvalidRecord = task.getStopOnInvalidRecord();
         int skipHeaderLines = task.getSkipHeaderLines();
 
         try (final PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), schema, output)) {
@@ -359,9 +352,6 @@ public class CsvParserPlugin
                     } catch (CsvTokenizer.InvalidFormatException | CsvTokenizer.InvalidValueException | CsvRecordValidateException e) {
                         long lineNumber = tokenizer.getCurrentLineNumber();
                         String skippedLine = tokenizer.skipCurrentLine();
-                        if (stopOnInvalidRecord) {
-                            throw new ConfigException(String.format("Invalid record at line %d: %s", lineNumber, skippedLine), e);
-                        }
                         log.warn(String.format("Skipped line %d (%s): %s", lineNumber, e.getMessage(), skippedLine));
                         //exec.notice().skippedLine(skippedLine);
 

data/embulk-standards/src/main/java/org/embulk/standards/CsvTokenizer.java

@@ -5,9 +5,7 @@ import java.util.List;
 import java.util.ArrayList;
 import java.util.Deque;
 import java.util.ArrayDeque;
-import org.embulk.config.ConfigException;
 import org.embulk.spi.util.LineDecoder;
-import org.embulk.spi.Exec;
 
 public class CsvTokenizer
 {
@@ -71,9 +69,7 @@ public class CsvTokenizer
             // recover lines of quoted value
             skippedLine = quotedValueLines.remove(0);  // TODO optimize performance
             unreadLines.addAll(quotedValueLines);
-            if (line != null) {
-                unreadLines.add(line);
-            }
+            unreadLines.add(line);
             lineNumber -= quotedValueLines.size();
             quotedValueLines.clear();
         }

data/embulk-standards/src/main/java/org/embulk/standards/LocalFileInputPlugin.java

@@ -13,7 +13,6 @@ import java.nio.file.Files;
 import java.nio.file.SimpleFileVisitor;
 import java.nio.file.FileVisitResult;
 import java.nio.file.attribute.BasicFileAttributes;
-import javax.validation.constraints.NotNull;
 import com.google.common.collect.ImmutableList;
 import com.google.common.base.Optional;
 import org.embulk.config.Config;
@@ -23,7 +22,7 @@ import org.embulk.config.Task;
 import org.embulk.config.TaskSource;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.ConfigDiff;
-import org.embulk.config.CommitReport;
+import org.embulk.config.TaskReport;
 import org.embulk.spi.BufferAllocator;
 import org.embulk.spi.Exec;
 import org.embulk.spi.FileInputPlugin;
@@ -100,7 +99,7 @@ public class LocalFileInputPlugin
     @Override
     public void cleanup(TaskSource taskSource,
             int taskCount,
-            List<CommitReport> successCommitReports)
+            List<TaskReport> successTaskReports)
     { }
 
     public List<String> listFiles(PluginTask task)
@@ -195,9 +194,9 @@ public class LocalFileInputPlugin
             { }
 
             @Override
-            public CommitReport commit()
+            public TaskReport commit()
             {
-                return Exec.newCommitReport();
+                return Exec.newTaskReport();
             }
         };
     }

data/embulk-standards/src/main/java/org/embulk/standards/LocalFileOutputPlugin.java

@@ -4,7 +4,6 @@ import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
 import java.io.IOException;
-import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Locale;
@@ -14,7 +13,7 @@ import org.embulk.config.ConfigException;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.ConfigDiff;
-import org.embulk.config.CommitReport;
+import org.embulk.config.TaskReport;
 import org.embulk.config.Task;
 import org.embulk.config.TaskSource;
 import org.embulk.spi.Buffer;
@@ -70,7 +69,7 @@ public class LocalFileOutputPlugin
     @Override
     public void cleanup(TaskSource taskSource,
             int taskCount,
-            List<CommitReport> successCommitReports)
+            List<TaskReport> successTaskReports)
     { }
 
     @Override
@@ -136,9 +135,9 @@ public class LocalFileOutputPlugin
             public void abort()
             { }
 
-            public CommitReport commit()
+            public TaskReport commit()
             {
-                CommitReport report = Exec.newCommitReport();
+                TaskReport report = Exec.newTaskReport();
                 // TODO better setting for Report
                 // report.set("file_names", fileNames);
                 // report.set("file_sizes", fileSizes);