embulk 0.4.10 → 0.5.0

data/lib/embulk/data/new/README.md.erb

@@ -1,12 +1,37 @@
 # <%= display_name %> <%= display_category %> plugin for Embulk
 
-TODO: Write short description here
+%case language
+%when :ruby
+TODO: Write short description here and <%= project_name %>.gemspec file.
+%when :java
+TODO: Write short description here and build.gradle file.
+%else
+TODO: Write short description here.
+%end
 
 ## Overview
 
+%case category
+%when :output, :file_output
 * **Plugin type**: <%= display_category %>
-* **Load all or nothing**: yes
+* **Load all or nothing**: no
 * **Resume supported**: no
+* **Cleanup supported**: yes
+%when :file_input
+* **Plugin type**: <%= display_category %>
+* **Resume supported**: yes
+* **Cleanup supported**: yes
+%when :input
+* **Plugin type**: <%= display_category %>
+* **Resume supported**: yes
+* **Cleanup supported**: yes
+* **Guess supported**: no
+%when :parser, :decoder
+* **Plugin type**: <%= display_category %>
+* **Guess supported**: no
+%else
+* **Plugin type**: <%= display_category %>
+%end
 
 ## Configuration
 
@@ -63,6 +88,16 @@ out:
 %end
 ```
 
+%case category
+%when :parser, :decoder
+(If guess supported) you don't have to write `<%= category %>:` section in the configuration file. After writing `in:` section, you can let embulk guess `<%= category %>:` section using this command:
+
+```
+$ embulk install <%= project_name %>
+$ embulk guess -g <%= name %> config.yml -o guessed.yml
+```
+%end
+
 ## Build
 
 ```

data/lib/embulk/data/new/java/input.java.erb

@@ -32,6 +32,7 @@ public class <%= java_class_name %>
         public SchemaConfig getColumns();
     }
 
+    @Override
     public ConfigDiff transaction(ConfigSource config,
             InputPlugin.Control control)
     {
@@ -43,6 +44,7 @@ public class <%= java_class_name %>
         return resume(task.dump(), schema, taskCount, control);
     }
 
+    @Override
     public ConfigDiff resume(TaskSource taskSource,
             Schema schema, int taskCount,
             InputPlugin.Control control)
@@ -51,12 +53,14 @@ public class <%= java_class_name %>
         return Exec.newConfigDiff();
     }
 
+    @Override
     public void cleanup(TaskSource taskSource,
             Schema schema, int taskCount,
             List<CommitReport> successCommitReports)
     {
     }
 
+    @Override
     public CommitReport run(TaskSource taskSource,
             Schema schema, int taskIndex,
             PageOutput output)
@@ -66,4 +70,14 @@ public class <%= java_class_name %>
         // TODO
         throw new UnsupportedOperationException("The 'run' method needs to be implemented");
     }
+
+    @Override
+    public ConfigDiff guess(ConfigSource config)
+    {
+        // TODO
+        throw new UnsupportedOperationException("'<%= name %>' input plugin does not support guessing.");
+        //ConfigDiff diff = Exec.newConfigDiff();
+        //diff.set("property1", "value");
+        //return diff;
+    }
 }

data/lib/embulk/data/new/java/output.java.erb

@@ -28,6 +28,7 @@ public class <%= java_class_name %>
         public int getProperty2();
     }
 
+    @Override
     public ConfigDiff transaction(ConfigSource config,
             Schema schema, int taskCount,
             OutputPlugin.Control control)
@@ -42,6 +43,7 @@ public class <%= java_class_name %>
         return Exec.newConfigDiff();
     }
 
+    @Override
     public ConfigDiff resume(TaskSource taskSource,
             Schema schema, int taskCount,
             OutputPlugin.Control control)
@@ -49,12 +51,14 @@ public class <%= java_class_name %>
         throw new UnsupportedOperationException("<%= name %> output plugin does not support resuming");
     }
 
+    @Override
     public void cleanup(TaskSource taskSource,
             Schema schema, int taskCount,
             List<CommitReport> successCommitReports)
     {
     }
 
+    @Override
     public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int taskIndex)
     {
         PluginTask task = taskSource.loadTask(PluginTask.class);

data/lib/embulk/data/new/ruby/decoder_guess.rb.erb

@@ -0,0 +1,25 @@
+module Embulk
+  module Guess
+
+    # TODO implement guess plugin to make this command work:
+    #      $ embulk guess -g <%= name.dump %> partial-config.yml
+%if language == :ruby
+
+    #require <%= "embulk/#{embulk_category}/#{name}.rb".dump %>
+%end
+
+    #class <%= ruby_class_name.sub('Plugin', 'GuessPlugin') %> < GuessPlugin
+    #  Plugin.register_guess(<%= name.dump %>, self)
+    #
+    #  FOO_BAR_HEADER = "\x1f\x8b".force_encoding('ASCII-8BIT').freeze
+    #
+    #  def guess(config, sample_buffer)
+    #    if sample_buffer[0,2] == FOO_BAR_HEADER
+    #      return {"decoders" => [{"type" => <%= name.dump %>}]}
+    #    end
+    #    return {}
+    #  end
+    #end
+
+  end
+end

data/lib/embulk/data/new/ruby/input.rb.erb

@@ -14,7 +14,7 @@ module Embulk
         columns = [
           Column.new(0, "example", :string),
           Column.new(1, "column", :long),
-          Column.new(2, "name", :double),
+          Column.new(2, "value", :double),
         ]
 
         resume(task, columns, 1, &control)
@@ -27,6 +27,16 @@ module Embulk
         return next_config_diff
       end
 
+      # TODO
+      #def self.guess(config)
+      #  sample_records = [
+      #    {"example"=>"a", "column"=>1, "value"=>0.1},
+      #    {"example"=>"a", "column"=>2, "value"=>0.2},
+      #  ]
+      #  columns = Guess::SchemaGuess.from_hash_records(sample_records)
+      #  return {"columns" => columns}
+      #end
+
       def init
         # initialization code:
         @property1 = task["property1"]

data/lib/embulk/data/new/ruby/parser_guess.rb.erb

@@ -0,0 +1,65 @@
+module Embulk
+  module Guess
+
+    # TODO implement guess plugin to make this command work:
+    #      $ embulk guess -g <%= name.dump %> partial-config.yml
+    #
+    #      Depending on the file format the plugin uses, you can use choose
+    #      one of binary guess (GuessPlugin), text guess (TextGuessPlugin),
+    #      or line guess (LineGuessPlugin).
+%if language == :ruby
+
+    #require <%= "embulk/#{embulk_category}/#{name}.rb".dump %>
+%end
+
+    #class <%= ruby_class_name.sub('Plugin', 'GuessPlugin') %> < GuessPlugin
+    #  Plugin.register_guess(<%= name.dump %>, self)
+    #
+    #  def guess(config, sample_buffer)
+    #    if sample_buffer[0,2] == GZIP_HEADER
+    #      guessed = {}
+    #      guessed["type"] = <%= name.dump %>
+    #      guessed["property1"] = "guessed-value"
+    #      return {"parser" => guessed}
+    #    else
+    #      return {}
+    #    end
+    #  end
+    #end
+
+    #class <%= ruby_class_name.sub('Plugin', 'GuessPlugin') %> < TextGuessPlugin
+    #  Plugin.register_guess(<%= name.dump %>, self)
+    #
+    #  def guess_text(config, sample_text)
+    #    js = JSON.parse(sample_text) rescue nil
+    #    if js && js["mykeyword"] == "keyword"
+    #      guessed = {}
+    #      guessed["type"] = <%= name.dump %>
+    #      guessed["property1"] = "guessed-value"
+    #      return {"parser" => guessed}
+    #    else
+    #      return {}
+    #    end
+    #  end
+    #end
+
+    #class <%= ruby_class_name.sub('Plugin', 'GuessPlugin') %> < LineGuessPlugin
+    #  Plugin.register_guess(<%= name.dump %>, self)
+    #
+    #  def guess_lines(config, sample_lines)
+    #    all_line_matched = sample_lines.all? do |line|
+    #      line =~ /mypattern/
+    #    end
+    #    if all_line_matched
+    #      guessed = {}
+    #      guessed["type"] = <%= name.dump %>
+    #      guessed["property1"] = "guessed-value"
+    #      return {"parser" => guessed}
+    #    else
+    #      return {}
+    #    end
+    #  end
+    #end
+
+  end
+end

data/lib/embulk/guess/csv.rb

@@ -1,6 +1,6 @@
 module Embulk
   module Guess
-    require_relative 'time_format_guess'
+    require_relative 'schema_guess'
 
     class CsvGuessPlugin < LineGuessPlugin
       Plugin.register_guess('csv', self)
@@ -24,15 +24,6 @@ module Embulk
         "\\N",  # MySQL LOAD, Hive STORED AS TEXTFILE
       ]
 
-      # CsvParserPlugin.TRUE_STRINGS
-      TRUE_STRINGS = Hash[*%w[
-        true True TRUE
-        yes Yes YES
-        y Y
-        on On ON
-        1
-      ].map {|k| [k, true] }]
-
       def guess_lines(config, sample_lines)
         delim = guess_delimiter(sample_lines)
         unless delim
@@ -54,8 +45,8 @@ module Embulk
         # don't even set null_string to avoid confusion of null and 'null' in YAML format
 
         sample_records = sample_lines.map {|line| line.split(delim) }  # TODO use CsvTokenizer
-        first_types = guess_field_types(sample_records[0, 1])
-        other_types = guess_field_types(sample_records[1..-1])
+        first_types = SchemaGuess.types_from_array_records(sample_records[0, 1])
+        other_types = SchemaGuess.types_from_array_records(sample_records[1..-1])
 
         if first_types.size <= 1 || other_types.size <= 1
           # guess failed
@@ -63,7 +54,7 @@ module Embulk
         end
 
         unless parser_config.has_key?("header_line")
-          parser_guessed["header_line"] = (first_types != other_types && !first_types.any? {|t| t != ["string"] })
+          parser_guessed["header_line"] = (first_types != other_types && !first_types.any? {|t| t != "string" })
         end
 
         unless parser_config.has_key?("columns")
@@ -73,10 +64,10 @@ module Embulk
             column_names = (0..other_types.size).to_a.map {|i| "c#{i}" }
           end
           schema = []
-          column_names.zip(other_types).each do |name,(type,format)|
+          column_names.zip(other_types).each do |name,type|
             if name && type
-              if format
-                schema << {"name" => name, "type" => type, "format" => format}
+              if type.is_a?(SchemaGuess::TimestampTypeMatch)
+                schema << {"name" => name, "type" => type, "format" => type.format}
               else
                 schema << {"name" => name, "type" => type}
               end
@@ -163,71 +154,6 @@ module Embulk
         return found ? found[0] : nil
       end
 
-      def guess_field_types(field_lines)
-        column_lines = []
-        field_lines.each do |fields|
-          fields.each_with_index {|field,i| (column_lines[i] ||= []) << guess_type(field) }
-        end
-        columns = column_lines.map do |types|
-          t = types.inject(nil) {|r,t| merge_type(r,t) } || "string"
-          if t.is_a?(TimestampMatch)
-            format = TimeFormatGuess.guess(types.map {|type| type.text })
-            ["timestamp", format]
-          else
-            [t]
-          end
-        end
-        return columns
-      end
-
-      TYPE_COALESCE = Hash[{
-        long: :double,
-        boolean: :long,
-      }.map {|k,v|
-        [[k.to_s, v.to_s].sort, v.to_s]
-      }]
-
-      def merge_type(type1, type2)
-        if type1 == type2
-          type1
-        elsif type1.nil? || type2.nil?
-          type1 || type2
-        else
-          TYPE_COALESCE[[type1, type2].sort] || "string"
-        end
-      end
-
-      class TimestampMatch < String
-        def initialize(text)
-          super("timestamp")
-          @text = text
-        end
-        attr_reader :text
-      end
-
-      def guess_type(str)
-        if TRUE_STRINGS[str]
-          return "boolean"
-        end
-
-        if TimeFormatGuess.guess(str)
-          return TimestampMatch.new(str)
-        end
-
-        if str.to_i.to_s == str
-          return "long"
-        end
-
-        if str.include?('.')
-          a, b = str.split(".", 2)
-          if a.to_i.to_s == a && b.to_i.to_s == b
-            return "double"
-          end
-        end
-
-        return "string"
-      end
-
       def array_sum(array)
         array.inject(0) {|r,i| r += i }
       end

data/lib/embulk/guess/schema_guess.rb

@@ -0,0 +1,107 @@
+module Embulk::Guess
+  require 'embulk/column'
+  require 'embulk/guess/time_format_guess'
+
+  module SchemaGuess
+    class TimestampTypeMatch < String
+      def initialize(format)
+        super("timestamp")
+        @format = format
+      end
+
+      attr_reader :format
+    end
+
+    class << self
+      def from_hash_records(array_of_hash)
+        array_of_hash = Array(array_of_hash)
+        if array_of_hash.empty?
+          raise "SchemaGuess Can't guess schema from no records"
+        end
+        column_names = array_of_hash.first.keys
+        samples = array_of_hash.to_a.map {|hash| column_names.map {|name| hash[name] } }
+        from_array_records(column_names, samples)
+      end
+
+      def from_array_records(column_names, samples)
+        column_types = types_from_array_records(samples)
+        columns = column_types.zip(column_names).map do |(type,name)|
+          hash = {name: name, type: type.to_sym}
+          hash[:format] = type.format if type.is_a?(TimestampTypeMatch)
+          Embulk::Column.new(hash)
+        end
+        return Embulk::Schema.new(columns)
+      end
+
+      # TODO this method will be private once guess/csv is refactored
+      def types_from_array_records(samples)
+        columnar_types = []
+        samples.each do |record|
+          record.each_with_index {|value,i| (columnar_types[i] ||= []) << guess_type(value.to_s) }
+        end
+        columnar_types.map {|types| merge_types(types) }
+      end
+
+      private
+
+      def guess_type(str)
+        if TRUE_STRINGS[str]
+          return "boolean"
+        end
+
+        if TimeFormatGuess.guess(str)
+          return TimestampTypeMatch.new(str)
+        end
+
+        if str.to_i.to_s == str
+          return "long"
+        end
+
+        if str.include?('.')
+          a, b = str.split(".", 2)
+          if a.to_i.to_s == a && b.to_i.to_s == b
+            return "double"
+          end
+        end
+
+        return "string"
+      end
+
+      def merge_types(types)
+        t = types.inject(nil) {|r,t| merge_type(r,t) } || "string"
+        if t.is_a?(TimestampTypeMatch)
+          format = TimeFormatGuess.guess(types.map {|type| type.format })
+          return TimestampTypeMatch.new(format)
+        else
+          return t
+        end
+      end
+
+      # taken from CsvParserPlugin.TRUE_STRINGS
+      TRUE_STRINGS = Hash[*%w[
+        true True TRUE
+        yes Yes YES
+        y Y
+        on On ON
+        1
+      ].map {|k| [k, true] }]
+
+      TYPE_COALESCE = Hash[{
+        long: :double,
+        boolean: :long,
+      }.map {|k,v|
+        [[k.to_s, v.to_s].sort, v.to_s]
+      }]
+
+      def merge_type(type1, type2)
+        if type1 == type2
+          type1
+        elsif type1.nil? || type2.nil?
+          type1 || type2
+        else
+          TYPE_COALESCE[[type1, type2].sort] || "string"
+        end
+      end
+    end
+  end
+end