embulk 0.6.16 → 0.6.17

data/embulk-docs/src/release.rst

@@ -4,6 +4,7 @@ Release Notes
 .. toctree::
     :maxdepth: 1
 
+    release/release-0.6.17
     release/release-0.6.16
     release/release-0.6.15
     release/release-0.6.14

data/embulk-docs/src/release/release-0.6.17.rst

@@ -0,0 +1,39 @@
+Release 0.6.17
+==================================
+
+General Changes
+------------------
+
+* '-' in generated plugin name is replaed to '_'.
+
+* Package name of generated java plugins changed from org.embulk.<category> to org.embulk.<category>.<plugin name>.
+
+* Generated plugin templates use recommended utility classes by default.
+
+  * java-decode template generator uses InputStreamFileInput.
+
+  * java-encoder template generator uses FileOutputOutputStream.
+
+  * java-file-input template generator uses InputStreamTransactionalFileInput.
+
+  * guess method of generated java-input plugin returns empty config diff rather throwing an exception.
+
+
+Java Plugin API
+------------------
+
+* Added ``spi.Exec.getTransactionTime()``.
+* Added ``spi.util.Timestamps`` utility class. This method is useful to create ``TimestampParser`` and ``TimestampFormatter`` which are configurable by users.
+* Added ``spi.util.InputStreamFileInput.Opener`` interface to open single file.
+* Added ``spi.util.InputStreamFileInput()`` with ``InputStream`` to use a pre-opend stream.
+* Added ``spi.util.InputStreamTransactionalFileInput`` for convenience of ``FileInputPlugin``.
+
+Ruby Plugin API
+------------------
+
+* Added ``FileInput#to_java`` and ``FileOutput#to_java``.
+
+
+Release Date
+------------------
+2015-07-17

data/embulk-standards/src/main/java/org/embulk/standards/CsvFormatterPlugin.java

@@ -21,6 +21,7 @@ import org.embulk.spi.PageReader;
 import org.embulk.spi.Exec;
 import org.embulk.spi.FileOutput;
 import org.embulk.spi.util.LineEncoder;
+import org.embulk.spi.util.Timestamps;
 
 import org.embulk.spi.util.Newline;
 import java.util.Map;
@@ -101,29 +102,13 @@ public class CsvFormatterPlugin
         control.run(task.dump());
     }
 
-    private TimestampFormatter[] newTimestampFormatters(
-            TimestampFormatter.Task formatterTask, Schema schema,
-            Map<String, TimestampColumnOption> columnOptions)
-    {
-        TimestampFormatter[] formatters = new TimestampFormatter[schema.getColumnCount()];
-        int i = 0;
-        for (Column column : schema.getColumns()) {
-            if (column.getType() instanceof TimestampType) {
-                Optional<TimestampColumnOption> option = Optional.fromNullable(columnOptions.get(column.getName()));
-                formatters[i] = new TimestampFormatter(formatterTask, option);
-            }
-            i++;
-        }
-        return formatters;
-    }
-
     @Override
     public PageOutput open(TaskSource taskSource, final Schema schema,
             FileOutput output)
     {
         final PluginTask task = taskSource.loadTask(PluginTask.class);
         final LineEncoder encoder = new LineEncoder(output, task);
-        final TimestampFormatter[] timestampFormatters = newTimestampFormatters(task, schema, task.getColumnOptions());
+        final TimestampFormatter[] timestampFormatters = Timestamps.newTimestampColumnFormatters(task, schema, task.getColumnOptions());
         final char delimiter = task.getDelimiterChar();
         final QuotePolicy quotePolicy = task.getQuotePolicy();
         final char quote = task.getQuoteChar() != '\0' ? task.getQuoteChar() : '"';

data/embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java

@@ -9,7 +9,6 @@ import org.embulk.config.ConfigSource;
 import org.embulk.config.ConfigException;
 import org.embulk.config.TaskSource;
 import org.embulk.spi.type.TimestampType;
-import org.embulk.spi.time.TimestampFormat;
 import org.embulk.spi.time.TimestampParser;
 import org.embulk.spi.time.TimestampParseException;
 import org.embulk.spi.Column;
@@ -23,6 +22,7 @@ import org.embulk.spi.Exec;
 import org.embulk.spi.FileInput;
 import org.embulk.spi.PageOutput;
 import org.embulk.spi.util.LineDecoder;
+import org.embulk.spi.util.Timestamps;
 import org.slf4j.Logger;
 
 public class CsvParserPlugin
@@ -90,10 +90,6 @@ public class CsvParserPlugin
         public boolean getAllowExtraColumns();
     }
 
-    public interface TimestampColumnOption
-            extends Task, TimestampParser.TimestampColumnOption
-    { }
-
     private final Logger log;
 
     public CsvParserPlugin()
@@ -121,27 +117,12 @@ public class CsvParserPlugin
         control.run(task.dump(), task.getSchemaConfig().toSchema());
     }
 
-    private TimestampParser[] newTimestampParsers(
-            TimestampParser.Task parserTask, SchemaConfig schema)
-    {
-        TimestampParser[] parsers = new TimestampParser[schema.getColumnCount()];
-        int i = 0;
-        for (ColumnConfig column : schema.getColumns()) {
-            if (column.getType() instanceof TimestampType) {
-                TimestampColumnOption option = column.getOption().loadConfig(TimestampColumnOption.class);
-                parsers[i] = new TimestampParser(parserTask, option);
-            }
-            i++;
-        }
-        return parsers;
-    }
-
     @Override
     public void run(TaskSource taskSource, final Schema schema,
             FileInput input, PageOutput output)
     {
         PluginTask task = taskSource.loadTask(PluginTask.class);
-        final TimestampParser[] timestampFormatters = newTimestampParsers(task, task.getSchemaConfig());
+        final TimestampParser[] timestampParsers = Timestamps.newTimestampColumnParsers(task, task.getSchemaConfig());
         LineDecoder lineDecoder = new LineDecoder(input, task);
         final CsvTokenizer tokenizer = new CsvTokenizer(lineDecoder, task);
         final String nullStringOrNull = task.getNullString().orNull();
@@ -225,7 +206,7 @@ public class CsvParserPlugin
                                     pageBuilder.setNull(column);
                                 } else {
                                     try {
-                                        pageBuilder.setTimestamp(column, timestampFormatters[column.getIndex()].parse(v));
+                                        pageBuilder.setTimestamp(column, timestampParsers[column.getIndex()].parse(v));
                                     } catch (TimestampParseException e) {
                                         // TODO support default value
                                         throw new CsvRecordValidateException(e);

data/embulk-standards/src/main/java/org/embulk/standards/GzipFileDecoderPlugin.java

@@ -31,10 +31,10 @@ public class GzipFileDecoderPlugin
     }
 
     @Override
-    public FileInput open(TaskSource taskSource, FileInput input)
+    public FileInput open(TaskSource taskSource, FileInput fileInput)
     {
         PluginTask task = taskSource.loadTask(PluginTask.class);
-        final FileInputInputStream files = new FileInputInputStream(input);
+        final FileInputInputStream files = new FileInputInputStream(fileInput);
         return new InputStreamFileInput(
                 task.getBufferAllocator(),
                 new InputStreamFileInput.Provider() {

data/embulk-standards/src/main/java/org/embulk/standards/GzipFileEncoderPlugin.java

@@ -55,7 +55,6 @@ public class GzipFileEncoderPlugin
                         this.def.setLevel(task.getLevel());
                     }
                 };
-
             }
 
             public void finish() throws IOException

data/embulk-standards/src/main/java/org/embulk/standards/LocalFileInputPlugin.java

@@ -28,7 +28,7 @@ import org.embulk.spi.BufferAllocator;
 import org.embulk.spi.Exec;
 import org.embulk.spi.FileInputPlugin;
 import org.embulk.spi.TransactionalFileInput;
-import org.embulk.spi.util.InputStreamFileInput;
+import org.embulk.spi.util.InputStreamTransactionalFileInput;
 import org.slf4j.Logger;
 
 public class LocalFileInputPlugin
@@ -177,52 +177,28 @@ public class LocalFileInputPlugin
     @Override
     public TransactionalFileInput open(TaskSource taskSource, int taskIndex)
     {
-        PluginTask task = taskSource.loadTask(PluginTask.class);
-        return new LocalFileInput(task, taskIndex);
-    }
+        final PluginTask task = taskSource.loadTask(PluginTask.class);
 
-    public static class LocalFileInput
-            extends InputStreamFileInput
-            implements TransactionalFileInput
-    {
-        // TODO create single-file InputStreamFileInput utility
-        private static class SingleFileProvider
-                implements InputStreamFileInput.Provider
-        {
-            private final File file;
-            private boolean opened = false;
+        final File file = new File(task.getFiles().get(taskIndex));
 
-            public SingleFileProvider(File file)
-            {
-                this.file = file;
-            }
+        return new InputStreamTransactionalFileInput(
+                task.getBufferAllocator(),
+                new InputStreamTransactionalFileInput.Opener() {
+                    public InputStream open() throws IOException
+                    {
+                        return new FileInputStream(file);
+                    }
+                })
+        {
+            @Override
+            public void abort()
+            { }
 
             @Override
-            public InputStream openNext() throws IOException
+            public CommitReport commit()
             {
-                if (opened) {
-                    return null;
-                }
-                opened = true;
-                return new FileInputStream(file);
+                return Exec.newCommitReport();
             }
-
-            @Override
-            public void close() { }
-        }
-
-        public LocalFileInput(PluginTask task, int taskIndex)
-        {
-            super(task.getBufferAllocator(), new SingleFileProvider(new File(task.getFiles().get(taskIndex))));
-        }
-
-        @Override
-        public void abort() { }
-
-        @Override
-        public CommitReport commit()
-        {
-            return Exec.newCommitReport();
-        }
+        };
     }
 }

data/embulk-standards/src/main/java/org/embulk/standards/LocalFileOutputPlugin.java

@@ -82,9 +82,8 @@ public class LocalFileOutputPlugin
         final String pathSuffix = task.getFileNameExtension();
         final String sequenceFormat = task.getSequenceFormat();
 
-        final List<String> fileNames = new ArrayList<>();
-
         return new TransactionalFileOutput() {
+            private final List<String> fileNames = new ArrayList<>();
             private int fileIndex = 0;
             private FileOutputStream output = null;
 
@@ -134,7 +133,8 @@ public class LocalFileOutputPlugin
                 closeFile();
             }
 
-            public void abort() { }
+            public void abort()
+            { }
 
             public CommitReport commit()
             {

data/lib/embulk/command/embulk_new_plugin.rb

@@ -8,28 +8,35 @@ module Embulk
     embulk_category = :input if category == :file_input
     embulk_category = :output if category == :file_output
 
-    project_name = "embulk-#{embulk_category}-#{name}"
+    name = name.gsub(/[^a-zA-Z0-9_]+/, '_')  # replace '-' to '_'
+
+    full_project_name = "embulk-#{embulk_category}-#{name}"
     plugin_dir = "lib/embulk"
     plugin_path = "#{plugin_dir}/#{embulk_category}/#{name}.rb"
 
-    if File.exist?(project_name)
-      raise "./#{project_name} already exists. Please delete it first."
+    if File.exist?(full_project_name)
+      raise "./#{full_project_name} already exists. Please delete it first."
     end
-    FileUtils.mkdir_p(project_name)
+    FileUtils.mkdir_p(full_project_name)
 
-    puts "Creating #{project_name}/"
+    puts "Creating #{full_project_name}/"
 
     success = false
     begin
+      #
+      # Generate gemspec
+      #
       author = `git config user.name`.strip
       author = "YOUR_NAME" if author.empty?
       email = `git config user.email`.strip
       email = "YOUR_NAME" if email.empty?
 
-      ruby_class_name = name.split('-').map {|a| a.capitalize }.join
-      java_iface = category.to_s.split('_').map {|a| a.capitalize }.join
-      java_class_name = name.split('-').map {|a| a.capitalize }.join + java_iface + "Plugin"
-      display_name = name.split('-').map {|a| a.capitalize }.join(' ')
+      # variables used in erb templates
+      ruby_class_name = name.split('_').map {|a| a.capitalize }.join
+      java_iface_name = category.to_s.split('_').map {|a| a.capitalize }.join
+      java_class_name = name.split('_').map {|a| a.capitalize }.join + java_iface_name + "Plugin"
+      java_package_name = "org.embulk.#{embulk_category}.#{name}"
+      display_name = name.split('_').map {|a| a.capitalize }.join(' ')
       display_category = category.to_s.gsub('_', ' ')
 
       extra_guess_erb = {}
@@ -57,7 +64,10 @@ module Embulk
         description = %[#{display_name}]
       end
 
-      pkg = Embulk::PackageData.new("new", project_name, binding())
+      #
+      # Generate project repository
+      #
+      pkg = Embulk::PackageData.new("new", full_project_name, binding())
 
       pkg.cp_erb("README.md.erb", "README.md")
       pkg.cp("LICENSE.txt", "LICENSE.txt")
@@ -67,7 +77,7 @@ module Embulk
       when :ruby
         pkg.cp("ruby/Rakefile", "Rakefile")
         pkg.cp("ruby/Gemfile", "Gemfile")
-        pkg.cp_erb("ruby/gemspec.erb", "#{project_name}.gemspec")
+        pkg.cp_erb("ruby/gemspec.erb", "#{full_project_name}.gemspec")
         pkg.cp_erb("ruby/#{category}.rb.erb", plugin_path)
 
       when :java
@@ -78,18 +88,34 @@ module Embulk
         pkg.set_executable("gradlew")
         pkg.cp_erb("java/build.gradle.erb", "build.gradle")
         pkg.cp_erb("java/plugin_loader.rb.erb", plugin_path)
-        pkg.cp_erb("java/#{category}.java.erb", "src/main/java/org/embulk/#{embulk_category}/#{java_class_name}.java")
-        pkg.cp_erb("java/test.java.erb", "src/test/java/org/embulk/#{embulk_category}/Test#{java_class_name}.java")
+        pkg.cp_erb("java/#{category}.java.erb", "src/main/java/#{java_package_name.gsub(/\./, '/')}/#{java_class_name}.java")
+        pkg.cp_erb("java/test.java.erb", "src/test/java/#{java_package_name.gsub(/\./, '/')}/Test#{java_class_name}.java")
       end
 
       extra_guess_erb.each_pair do |erb,dest|
         pkg.cp_erb(erb, dest)
       end
 
+      puts ""
+      puts "Plugin template is successfully generated."
+
+      case language
+      when :ruby
+        puts "Next steps:"
+        puts ""
+        puts "  $ cd #{full_project_name}"
+        puts "  $ rake"
+      when :java
+        puts "Next steps:"
+        puts ""
+        puts "  $ cd #{full_project_name}"
+        puts "  $ ./gradlew package"
+      end
+
       success = true
       puts ""
     ensure
-      FileUtils.rm_rf project_name unless success
+      FileUtils.rm_rf full_project_name unless success
     end
   end
 end

data/lib/embulk/command/embulk_run.rb

@@ -290,9 +290,9 @@ examples:
       puts ""
       puts "Run following subcommands to try embulk:"
       puts ""
-      puts "   1. guess #{File.join(path, 'example.yml')} -o config.yml"
-      puts "   2. preview config.yml"
-      puts "   3. run config.yml"
+      puts "   1. embulk guess #{File.join(path, 'example.yml')} -o config.yml"
+      puts "   2. embulk preview config.yml"
+      puts "   3. embulk run config.yml"
       puts ""
 
     when :new

data/lib/embulk/data/new/README.md.erb

@@ -35,8 +35,9 @@ TODO: Write short description here.
 
 ## Configuration
 
-- **property1**: description (string, required)
-- **property2**: description (integer, default: default-value)
+- **option1**: description (integer, required)
+- **option2**: description (string, default: `"myvalue"`)
+- **option3**: description (string, default: `null`)
 
 ## Example
 
@@ -45,46 +46,46 @@ TODO: Write short description here.
 %when :input, :file_input
 in:
   type: <%= name %>
-  property1: example1
-  property2: example2
+  option1: example1
+  option2: example2
 %when :output, :file_output
 out:
   type: <%= name %>
-  property1: example1
-  property2: example2
+  option1: example1
+  option2: example2
 %when :filter
 filters:
   - type: <%= name %>
-    property1: example1
-    property2: example2
+    option1: example1
+    option2: example2
 %when :parser
 in:
   type: any file input plugin type
   parser:
     type: <%= name %>
-    property1: example1
-    property2: example2
+    option1: example1
+    option2: example2
 %when :formatter
 out:
   type: any output input plugin type
   formatter:
     type: <%= name %>
-    property1: example1
-    property2: example2
+    option1: example1
+    option2: example2
 %when :decoder
 in:
   type: any output input plugin type
   decoders:
     - type: <%= name %>
-      property1: example1
-      property2: example2
+      option1: example1
+      option2: example2
 %when :encoder
 out:
   type: any output input plugin type
   encoders:
     - type: <%= name %>
-      property1: example1
-      property2: example2
+      option1: example1
+      option2: example2
 %end
 ```
 
@@ -93,7 +94,7 @@ out:
 (If guess supported) you don't have to write `<%= category %>:` section in the configuration file. After writing `in:` section, you can let embulk guess `<%= category %>:` section using this command:
 
 ```
-$ embulk gem install <%= project_name %>
+$ embulk gem install <%= full_project_name %>
 $ embulk guess -g <%= name %> config.yml -o guessed.yml
 ```
 %end