embulk 0.4.10 → 0.5.0

data/embulk-docs/plugins/index.html.erb

@@ -29,7 +29,7 @@
                   <th>Name</th>
                   <th>Author</th>
                   <th>About</th>
-                  <th>Version</th>
+                  <th>Download</th>
               </tr>
             </thead>
             <tbody>
@@ -54,7 +54,7 @@
                 </td>
                 <% end %>
                 <td><%=h gem[:info] %></td>
-                <td style="width:3em"><%=h gem[:version] %></td>
+                <td style="width:3em"><%=h gem[:downloads] %></td>
               </tr>
               <% end %>
             </tbody>

data/embulk-docs/src/recipe/scheduled-csv-load-to-elasticsearch-kibana4.rst

@@ -56,7 +56,7 @@ You can find the latest embulk binary from the `releases <https://bintray.com/em
 
 .. code-block:: console
 
-    $ sudo wget https://bintray.com/artifact/download/embulk/maven/embulk-0.4.10.jar -O /usr/local/bin/embulk
+    $ sudo wget https://bintray.com/artifact/download/embulk/maven/embulk-0.5.0.jar -O /usr/local/bin/embulk
     $ sudo chmod +x /usr/local/bin/embulk
 
 Step 2. Install Elasticsearch plugin
@@ -84,7 +84,7 @@ Create this configuration file and save as ``config.yml``:
       path_prefix: ./mydata/csv/
     out:
       type: elasticsearch
-      index_name: embulk
+      index: embulk
       index_type: embulk
       nodes:
         - host: localhost
@@ -121,7 +121,7 @@ The generated config-complete.yml file should include complete information as fo
         - {name: comment, type: string}
     out:
       type: elasticsearch
-      index_name: embulk
+      index: embulk
       index_type: embulk
       nodes:
       - {host: localhost}

data/embulk-docs/src/release.rst

@@ -21,4 +21,5 @@ Release Notes
     release/release-0.4.8
     release/release-0.4.9
     release/release-0.4.10
+    release/release-0.5.0
 

data/embulk-docs/src/release/release-0.5.0.rst

@@ -0,0 +1,81 @@
+Release 0.5.0
+==================================
+
+New Guess Plugin Architecture
+------------------
+
+Embulk v0.5.0 supports dynamically loadable guess plugins.
+
+* For parser and decoder plugins:
+
+  * CLI subcommand ``guess`` accepts new ``-g, --guess NAMES`` argument to load parser and decoder guess plugins.
+
+  * Plugin template generator creates stub code of guess plugin for parser and decoder plugins.
+
+* For input plugins:
+
+  * Added ``Embulk::InputPlugin.guess(config)`` and ``spi.InputPlugin#guess(config)`` methods for Ruby and Java plugins.
+
+  * ``guess`` subcommand executes the new guess method and takes the return value of the method.
+
+For example, if you write a parser plugin named ``myparser``, you can use this configuration file first:
+
+.. code-block:: yaml
+
+    in:
+      type: file
+      path_prefix: path/to/myfiles
+    out:
+      type: stdout
+
+The ``embulk guess`` command with ``-g myparser`` argument calls the guess plugin bundled in the plugin:
+
+.. code-block:: console
+
+    $ embulk gem install embulk-parser-myparser
+    $ embulk guess config.yml -o guessed.yml -g myparser
+
+On the other hand, if the plugin type is input, you don't need additional command-line arguments. For example, if the input plugin name is ``myinput``, you can use this configuration file:
+
+.. code-block:: yaml
+
+    in:
+      type: myinput
+    out:
+      type: stdout
+
+The ``embulk guess`` command finds the ``InputPlugin.guess`` of the input plugin and calls it:
+
+.. code-block:: console
+
+    $ embulk gem install embulk-input-myinput
+    $ embulk guess config.yml -o guessed.yml
+
+Plugin SPI
+------------------
+
+* Added ``Embulk::InputPlugin.guess(config)`` method for Ruby input plugins.
+
+  * Backward compatibility: existent plugins don't have to implement the method. The default behavior is raising ``NotImplementedError``.
+
+* Added ``spi.InputPlugin#guess(ConfigSource config)`` method for Java input plugins.
+
+  * Backward compatibility: existent plugins don't have to implement the method. Mehtod linkage errors are handled at embulk-core. The default behavior is raising ``UnsupportedOperationException``.
+
+Built-in plugins
+------------------
+
+* ``csv`` parser plugin implements ``max_quoted_size_limit`` option. Default value is 131072 (128KB).
+
+  * This option is useful when a column value includes a quote character accidentally. If the plugin detects those values, it skips the line and continues parsing from the next line.
+
+General Changes
+------------------
+
+* ``spi.util.FileInputInputStream#skip`` method never returns -1 to follow the Java API standard (@hata++).
+* Plugin template generator creates appropriate "Overview" section in README.md file depending on the plugin type.
+
+
+Release Date
+------------------
+2015-03-02

data/embulk-standards/src/main/java/org/embulk/standards/CsvTokenizer.java

@@ -27,7 +27,7 @@ public class CsvTokenizer
     private final char escape;
     private final String newline;
     private final boolean trimIfNotQuoted;
-    private final long maxQuotedSizeLimit;  // TODO not used yet
+    private final long maxQuotedSizeLimit;
     private final LineDecoder input;
 
     private RecordState recordState = RecordState.END;  // initial state is end of a record. nextRecord() must be called first
@@ -274,6 +274,9 @@ public class CsvTokenizer
                         }
 
                     } else {
+                        if ((linePos - valueStartPos) + quotedValue.length() > maxQuotedSizeLimit) {
+                            throw new QuotedSizeLimitExceededException("The size of the quoted value exceeds the limit size ("+maxQuotedSizeLimit+")");
+                        }
                         // keep QUOTED_VALUE state
                     }
                     break;
@@ -351,4 +354,13 @@ public class CsvTokenizer
     {
         return c == escape;
     }
+
+    static class QuotedSizeLimitExceededException
+            extends RuntimeException
+    {
+        QuotedSizeLimitExceededException(String message)
+        {
+            super(message);
+        }
+    }
 }

data/embulk-standards/src/main/java/org/embulk/standards/StandardPluginModule.java

@@ -4,13 +4,17 @@ import com.google.common.base.Preconditions;
 import com.google.inject.Binder;
 import com.google.inject.Module;
 import com.google.inject.name.Names;
+import com.google.inject.multibindings.Multibinder;
 import org.embulk.spi.FormatterPlugin;
 import org.embulk.spi.InputPlugin;
 import org.embulk.spi.OutputPlugin;
 import org.embulk.spi.ParserPlugin;
 import org.embulk.spi.DecoderPlugin;
 import org.embulk.spi.EncoderPlugin;
+import org.embulk.exec.GuessExecutor;
+import org.embulk.plugin.PluginType;
 import static org.embulk.plugin.InjectedPluginSource.registerPluginTo;
+import static org.embulk.exec.GuessExecutor.registerDefaultGuessPluginTo;
 
 public class StandardPluginModule
         implements Module
@@ -39,5 +43,10 @@ public class StandardPluginModule
 
         // file encoder plugins
         registerPluginTo(binder, EncoderPlugin.class, "gzip", GzipFileEncoderPlugin.class);
+
+        // default guess plugins
+        registerDefaultGuessPluginTo(binder, new PluginType("gzip"));
+        registerDefaultGuessPluginTo(binder, new PluginType("csv"));
+        // charset and newline guess plugins are loaded and invoked by CsvGuessPlugin
     }
 }

data/embulk-standards/src/test/java/org/embulk/standards/TestCsvTokenizer.java

@@ -6,6 +6,8 @@ import java.nio.charset.UnsupportedCharsetException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Random;
+
 import com.fasterxml.jackson.databind.node.JsonNodeFactory;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
@@ -13,6 +15,9 @@ import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
 import org.embulk.EmbulkTestRuntime;
 import org.embulk.config.ConfigSource;
 import org.embulk.spi.Buffer;
@@ -255,21 +260,73 @@ public class TestCsvTokenizer
                     "\"trailing\n3\"  ,\"trailing\n4\"  "));
     }
 
-    /*
-    @Test(expected = CsvTokenizer.CsvValueValidateException.class)
-    public void parseTooLargeSizedValues() throws Exception
+    @Test
+    public void throwQuotedSizeLimitExceededException() throws Exception
     {
-        config.set("max_quoted_column_size", 8L);
+        config.set("max_quoted_size_limit", 8);
         reloadPluginTask();
-        List<List<String>> parsed = doParse(task, bufferList("utf-8",
-                "aaa,bbb", "\n", "\"cccccccc\",ddd", "\n"));
 
-        assertEquals(Arrays.asList(
-                        Arrays.asList("aaa", "bbb"),
-                        Arrays.asList("ccc", "ddd")),
-                parsed);
+        try {
+            parse(task,
+                    "v1,v2",
+                    "v3,\"0123456789\"");
+            fail();
+        } catch (Exception e) {
+            assertTrue(e instanceof CsvTokenizer.QuotedSizeLimitExceededException);
+        }
+
+        // multi-line
+        try {
+            parse(task,
+                    "v1,v2",
+                    "\"012345\n6789\",v3");
+            fail();
+        } catch (Exception e) {
+            assertTrue(e instanceof CsvTokenizer.QuotedSizeLimitExceededException);
+        }
+    }
+
+    @Test
+    public void recoverFromQuotedSizeLimitExceededException() throws Exception
+    {
+        config.set("max_quoted_size_limit", 12);
+        reloadPluginTask();
+
+        String[] lines = new String[] {
+            "v1,v2",
+            "v3,\"0123",  // this is a broken line and should be skipped
+            "v4,v5",      // this line should be not be skiped
+            "v6,v7",      // this line should be not be skiped
+        };
+        FileInput input = newFileInputFromLines(task, lines);
+        LineDecoder decoder = new LineDecoder(input, task);
+        CsvTokenizer tokenizer = new CsvTokenizer(decoder, task);
+        Schema schema = task.getSchemaConfig().toSchema();
+
+        tokenizer.nextFile();
+
+        assertTrue(tokenizer.nextRecord());
+        assertEquals("v1", tokenizer.nextColumn());
+        assertEquals("v2", tokenizer.nextColumn());
+
+        assertTrue(tokenizer.nextRecord());
+        assertEquals("v3", tokenizer.nextColumn());
+        try {
+            tokenizer.nextColumn();
+            fail();
+        } catch (Exception e) {
+            assertTrue(e instanceof CsvTokenizer.QuotedSizeLimitExceededException);
+        }
+        assertEquals("v3,\"0123", tokenizer.skipCurrentLine());
+
+        assertTrue(tokenizer.nextRecord());
+        assertEquals("v4", tokenizer.nextColumn());
+        assertEquals("v5", tokenizer.nextColumn());
+
+        assertTrue(tokenizer.nextRecord());
+        assertEquals("v6", tokenizer.nextColumn());
+        assertEquals("v7", tokenizer.nextColumn());
     }
-    */
 
     /*
     @Test

data/lib/embulk/column.rb

@@ -1,20 +1,43 @@
 module Embulk
 
-  class Column < Struct.new(:index, :name, :type)
-    def initialize(index, name, type)
-      super(index, name, type)
+  class Column < Struct.new(:index, :name, :type, :format)
+    def initialize(*args)
+      if args.length == 1 && args[0].is_a?(Hash)
+        # initialize(hash)
+        hash = args.first
+        super(hash[:index], hash[:name], hash[:type], hash[:format])
+      else
+        # initialize(index, name, type, format)
+        super(*args)
+      end
+    end
+
+    def to_json(*args)
+      if type == :timestamp && format
+        {"index"=>index, "name"=>name, "type"=>type, "format"=>format}.to_json(*args)
+      else
+        {"index"=>index, "name"=>name, "type"=>type}.to_json(*args)
+      end
     end
 
     if Embulk.java?
       def self.from_java(java_column)
-        Column.new(
-          java_column.getIndex,
-          java_column.getName,
-          Type.from_java(java_column.getType))
+        type = Type.from_java(java_column.getType)
+        if type == :timestamp
+          format = java_column.getType.getFormat
+        else
+          format = nil
+        end
+
+        Column.new(java_column.getIndex, java_column.getName, type, format)
       end
 
       def to_java
-        Java::Column.new(index, name, Type.new_java_type(type))
+        if type == :timestamp && format
+          Java::Column.new(index, name, Type.new_java_type(type).withFormat(format))
+        else
+          Java::Column.new(index, name, Type.new_java_type(type))
+        end
       end
     end
   end

data/lib/embulk/command/embulk_new_plugin.rb

@@ -9,7 +9,8 @@ module Embulk
     embulk_category = :output if category == :file_output
 
     project_name = "embulk-#{embulk_category}-#{name}"
-    plugin_path = "lib/embulk/#{embulk_category}/#{name}.rb"
+    plugin_dir = "lib/embulk"
+    plugin_path = "#{plugin_dir}/#{embulk_category}/#{name}.rb"
 
     if File.exist?(project_name)
       raise "./#{project_name} already exists. Please delete it first."
@@ -31,27 +32,30 @@ module Embulk
       display_name = name.split('-').map {|a| a.capitalize }.join(' ')
       display_category = category.to_s.gsub('_', ' ')
 
-      description =
-        case category
-        when :input
-          %[Loads records from #{display_name}.]
-        when :file_input
-          %[Reads files stored on #{display_name}.]
-        when :parser
-          %[Parses #{display_name} files read by other file input plugins.]
-        when :decoder
-          %[Decodes #{display_name}-encoded files read by other file input plugins.]
-        when :output
-          %[Dumps records to #{display_name}.]
-        when :file_output
-          %[Stores files on #{display_name}.]
-        when :formatter
-          %[Formats #{display_name} files for other file output plugins.]
-        when :encoder
-          %[Encodes files using #{display_name} for other file output plugins.]
-        when :filter
-          %[#{display_name}]
-        end
+      extra_guess_erb = {}
+
+      case category
+      when :input
+        description = %[Loads records from #{display_name}.]
+      when :file_input
+        description = %[Reads files stored on #{display_name}.]
+      when :parser
+        description = %[Parses #{display_name} files read by other file input plugins.]
+        extra_guess_erb["ruby/parser_guess.rb.erb"] = "#{plugin_dir}/guess/#{name}.rb"
+      when :decoder
+        description = %[Decodes #{display_name}-encoded files read by other file input plugins.]
+        extra_guess_erb["ruby/decoder_guess.rb.erb"] = "#{plugin_dir}/guess/#{name}.rb"
+      when :output
+        description = %[Dumps records to #{display_name}.]
+      when :file_output
+        description = %[Stores files on #{display_name}.]
+      when :formatter
+        description = %[Formats #{display_name} files for other file output plugins.]
+      when :encoder
+        description = %[Encodes files using #{display_name} for other file output plugins.]
+      when :filter
+        description = %[#{display_name}]
+      end
 
       pkg = Embulk::PackageData.new("new", project_name, binding())
 
@@ -78,6 +82,10 @@ module Embulk
         pkg.cp_erb("java/test.java.erb", "src/test/java/org/embulk/#{embulk_category}/Test#{java_class_name}.java")
       end
 
+      extra_guess_erb.each_pair do |erb,dest|
+        pkg.cp_erb(erb, dest)
+      end
+
       success = true
       puts ""
     ensure

data/lib/embulk/command/embulk_run.rb

@@ -34,7 +34,7 @@ module Embulk
     op = OptionParser.new
     op.version = Embulk::VERSION
 
-    puts "#{Time.now.strftime("%Y-%m-%d %H:%M:%S,%3N %z")}: Embulk v#{Embulk::VERSION}"
+    puts "#{Time.now.strftime("%Y-%m-%d %H:%M:%S.%3N %z")}: Embulk v#{Embulk::VERSION}"
 
     load_paths = []
     classpaths = []
@@ -121,6 +121,9 @@ module Embulk
       op.on('-C', '--classpath PATH', "Add java classpath separated by #{classpath_separator} (CLASSPATH)") do |classpath|
         classpaths.concat classpath.split(classpath_separator)
       end
+      op.on('-g', '--guess NAMES', "Comma-separated list of guess plugin names") do |names|
+        (options[:guessPlugins] ||= []).concat names.split(",")
+      end
       args = 1..1
 
     when :new
@@ -161,7 +164,7 @@ examples:
       args = 0..1
 
     when :exec
-      exec *argv
+      exec(*argv)
       exit 127
 
     else
@@ -292,7 +295,17 @@ examples:
       setup_load_paths(load_paths)
       setup_classpaths(classpaths)
 
-      org.embulk.command.Runner.new(options.to_json).main(subcmd, argv.to_java(:string))
+      begin
+        org.embulk.command.Runner.new(options.to_json).main(subcmd, argv.to_java(:string))
+      rescue => ex
+        puts ex.to_s
+        ex.backtrace.each do |bt|
+          puts "    #{bt}"
+        end
+        puts ""
+        puts "Error: #{ex}"
+        raise SystemExit.new(1, ex.to_s)
+      end
     end
   end