embulk 0.4.1 → 0.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2aebc5b785b1d7e303dd69fe4b7922abdce2954e
-  data.tar.gz: 99189534152199abb9b280b7f7ce5511691098e7
+  metadata.gz: d63d08592793b691be7f691d842f73d18a202d45
+  data.tar.gz: 5a14e328213b2a97fe6e0abaa044428bb5caa0fd
 SHA512:
-  metadata.gz: fafcf2fcaadaa263619c3f56e90f7a3abf5fa6b257c4ab8338ab0e84130e47ecec5fb3bc09c45b4ccbb7ec207123b1f4517324605000de2525a7632f32ae187f
-  data.tar.gz: 7b72ad6d02f613583624265608ebda2201159b28aaefeed8d5fd9e1a553c2ff361078b4e68d5a2a7f242dc4329536c58925ee4b20739274f9dafb2ba81761d93
+  metadata.gz: 8e447c2ae251e6ef3309c0862344d10074797389ea2c55bce2e2302d82eea2425345316efdf7dae9adcd9f08de5b0132c04e6cad43edaef29ac3cf635ad640f8
+  data.tar.gz: 0b33b0a4f2dc7911d1432828e303c464c8b25bda00dc9eb3df8c9fb905c85011c30132292c581163fb557d2e2eaeb3a32a8bbd61bd486509361b4464aa2477ed

data/README.md

@@ -24,7 +24,7 @@ You can release plugins to share your efforts of data cleaning, error handling,
 The single-file package is the simplest way to try Embulk. You can download the latest embulk-VERSION.jar from [the releases page](https://bintray.com/embulk/maven/embulk/view#files) and run it with java:
 
 ```
-wget https://bintray.com/artifact/download/embulk/maven/embulk-0.4.1.jar -O embulk.jar
+wget https://bintray.com/artifact/download/embulk/maven/embulk-0.4.2.jar -O embulk.jar
 java -jar embulk.jar --help
 ```
 

data/build.gradle

@@ -9,7 +9,7 @@ plugins {
 
 allprojects {
     group = 'org.embulk'
-    version = '0.4.1'
+    version = '0.4.2'
 
     apply plugin: 'maven'  // install jar files to the local repo: $ gradle install
     apply plugin: 'maven-publish'
@@ -205,9 +205,15 @@ task releaseCheck << {
     if (!file("lib/embulk/version.rb").getText().contains("${project.version}")) {
         throw new GradleException("lib/embulk/version.rb doesn't include ${project.version}")
     }
+    if (!file("embulk-docs/src/release/release-${project.version}.rst").getText().contains("${project.version}")) {
+        throw new GradleException("Release note for ${project.version} doesn't exist")
+    }
+    if (!file("embulk-docs/src/release.rst").getText().contains("release-${project.version}")) {
+        throw new GradleException("embulk-docs/src/release.rst doesn't include release-${project.version}")
+    }
     String date = new Date().format("yyyy-MM-dd")
     if (!file("embulk-docs/src/release/release-${project.version}.rst").getText().contains(date)) {
-        throw new GradleException("ChangeLog doesn't include entry for ${project.version}")
+        throw new GradleException("embulk-docs/src/release/release-${project.version}.rst doesn't include today's release date")
     }
     println "Ready. Run ./gradlew release"
 }

data/embulk-core/src/main/java/org/embulk/command/Runner.java

@@ -169,6 +169,7 @@ public class Runner
 
         String yml = writeNextConfig(options.getNextConfigOutputPath(), config, configDiff);
         System.err.println(yml);
+        System.out.println("Created "+options.getNextConfigOutputPath());
     }
 
     private void checkFileWritable(String path)
@@ -191,14 +192,10 @@ public class Runner
     {
         String yml = dumpYaml(obj);
         if (path != null) {
-            if (path.equals("-")) {
-                System.out.print(yml);
-            } else {
-                try (Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(path), "UTF-8"))) {
-                    writer.write(yml);
-                } catch (IOException ex) {
-                    throw new RuntimeException(ex);
-                }
+            try (Writer writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(path), "UTF-8"))) {
+                writer.write(yml);
+            } catch (IOException ex) {
+                throw new RuntimeException(ex);
             }
         }
         return yml;

data/embulk-docs/push-gh-pages.sh

@@ -13,7 +13,7 @@ function r() {
 }
 
 [ "$TRAVIS_PULL_REQUEST" != "false" ] && exit 0
-[ "$TRAVIS_BRANCH" != "master" ] && exit 0
+[ "$TRAVIS_BRANCH" != "master" -a "$TRAVIS_BRANCH" != "$(git describe --tags --always HEAD)" ] && exit 0
 
 revision="$(git rev-parse HEAD)"
 remote="$(git config remote.origin.url | sed "s+^git:+https:+")"

data/embulk-docs/src/release.rst

@@ -11,4 +11,6 @@ Release Notes
     release/release-0.3.1
     release/release-0.3.2
     release/release-0.4.0
+    release/release-0.4.1
+    release/release-0.4.2
 

data/embulk-docs/src/release/release-0.4.1.rst

@@ -1,4 +1,4 @@
-Release 0.4.0
+Release 0.4.1
 ==================================
 
 CLI
@@ -15,4 +15,4 @@ General Changes
 
 Release Date
 ------------------
-2015-02-15
+2015-02-16

data/embulk-docs/src/release/release-0.4.2.rst

@@ -0,0 +1,18 @@
+Release 0.4.2
+==================================
+
+CLI
+------------------
+
+* Removed support for setting ``-`` to ``-o, --output PATH`` option
+
+General Changes
+------------------
+
+* Fixed guess plugin paths (@takei-yuya++)
+* Fixed syntax error in generated gemspec file
+* Fixed syntax error in generated generated ruby filter and input plugins (@takei-yuya++, also reported by hiroyuki-sato)
+
+Release Date
+------------------
+2015-02-16

data/lib/embulk/command/embulk_new_plugin.rb

@@ -34,23 +34,23 @@ module Embulk
       description =
         case category
         when :input
-          %[that loads records from #{display_name} so that any output plugins can receive the records. Search the output plugins by "embulk-output" keyword.]
+          %[that loads records from #{display_name} so that any output plugins can receive the records. Search the output plugins by 'embulk-output' keyword.]
         when :file_input
-          %[that reads files from #{display_name} and parses the file using any parser plugins. Search the parser plugins by "embulk-parser" keyword.]
+          %[that reads files from #{display_name} and parses the file using any parser plugins. Search the parser plugins by 'embulk-parser' keyword.]
         when :parser
-          %[that parses #{display_name} file format read by any file input plugins. Search the file input plugins by "embulk-input file" keywords.]
+          %[that parses #{display_name} file format read by any file input plugins. Search the file input plugins by 'embulk-input file' keywords.]
         when :decoder
-          %[that decodes files encoded by #{display_name} read by any file input plugins. Search the file input plugins by "embulk-input file" keywords.]
+          %[that decodes files encoded by #{display_name} read by any file input plugins. Search the file input plugins by 'embulk-input file' keywords.]
         when :output
-          %[that loads records to #{display_name} read by any input plugins. Search the input plugins by "embulk-input" keyword.]
+          %[that loads records to #{display_name} read by any input plugins. Search the input plugins by 'embulk-input' keyword.]
         when :file_output
-          %[that stores files to #{display_name} formatted by any formatter plugins. Search the formatter plugins by "embulk-formatter" keyword.]
+          %[that stores files to #{display_name} formatted by any formatter plugins. Search the formatter plugins by 'embulk-formatter' keyword.]
         when :formtter
-          %[that formats records using #{display_name} file format and so that any file output plugins can store the files. Search the file output plugins by "embulk-output file" keywords.]
+          %[that formats records using #{display_name} file format and so that any file output plugins can store the files. Search the file output plugins by 'embulk-output file' keywords.]
         when :encoder
-          %[that encodes files using #{display_name} so that any file output plugins can store the files. Search the file output plugins by "embulk-output file" keywords.]
+          %[that encodes files using #{display_name} so that any file output plugins can store the files. Search the file output plugins by 'embulk-output file' keywords.]
         when :filter
-          %[that converts records read by an input plugin before passing it to an output plugins. Search the input and plugins by "embulk-input" and "embulk-output" plugins.]
+          %[that converts records read by an input plugin before passing it to an output plugins. Search the input and plugins by 'embulk-input' and 'embulk-output' plugins.]
         end
 
       pkg = Embulk::PackageData.new("new", project_name, binding())

data/lib/embulk/data/new/java/plugin_loader.rb.erb

@@ -1,3 +1,3 @@
 Embulk::JavaPlugin.register_<%= embulk_category %>(
-  <%= name.to_sym.inspect %>, <%= "org.embulk.#{embulk_category}.#{java_class_name}".dump %>,
+  <%= name.dump %>, <%= "org.embulk.#{embulk_category}.#{java_class_name}".dump %>,
   File.expand_path('../../../../classpath', __FILE__))

data/lib/embulk/data/new/ruby/filter.rb.erb

@@ -2,7 +2,7 @@ module Embulk
   module Filter
 
     class <%= ruby_class_name %> < FilterPlugin
-      Plugin.register_filter(<%= name %>, self)
+      Plugin.register_filter(<%= name.dump %>, self)
 
       def self.transaction(config, in_schema, &control)
         # configuration code:

data/lib/embulk/data/new/ruby/gemspec.erb

@@ -3,8 +3,8 @@ Gem::Specification.new do |spec|
   spec.name          = "<%= project_name %>"
   spec.version       = "0.1.0"
   spec.authors       = [<%= author.dump %>]
-  spec.summary       = "<%= display_name %> <%= display_category %> plugin for Embulk"
-  spec.description   = "<%= display_name %> <%= display_category %> plugin is an Embulk plugin <%= description %>"
+  spec.summary       = <%= "#{display_name} #{display_category} plugin for Embulk".dump %>
+  spec.description   = <%= "#{display_name} #{display_category} plugin is an Embulk plugin #{description}".dump %>
   spec.email         = [<%= email.dump %>]
   spec.licenses      = ["MIT"]
   # TODO: spec.homepage      = "https://github.com/<%= email[/([^@]*)/] %>/<%= project_name %>"

data/lib/embulk/data/new/ruby/input.rb.erb

@@ -2,7 +2,7 @@ module Embulk
   module Input
 
     class <%= ruby_class_name %> < InputPlugin
-      Plugin.register_input(<%= name %>, self)
+      Plugin.register_input(<%= name.dump %>, self)
 
       def self.transaction(config, &control)
         # configuration code:

data/lib/embulk/guess/charset.rb

@@ -0,0 +1,28 @@
+module Embulk
+  module Guess
+
+    class CharsetGuessPlugin < GuessPlugin
+      Plugin.register_guess('charset', self)
+
+      def guess(config, sample_buffer)
+        # ICU4J
+        detector = com.ibm.icu.text.CharsetDetector.new
+        detector.setText(sample_buffer.to_java_bytes)
+        best_match = detector.detect
+        if best_match.getConfidence < 50
+          name = "UTF-8"
+        else
+          name = best_match.getName
+          if name == "ISO-8859-1"
+            # ISO-8859-1 means ASCII which is a subset
+            # of UTF-8 in most of cases due to lack of
+            # sample data set
+            name = "UTF-8"
+          end
+        end
+        return {"parser" => {"charset" => name}}
+      end
+    end
+
+  end
+end

data/lib/embulk/guess/csv.rb

@@ -0,0 +1,206 @@
+module Embulk
+  module Guess
+    require_relative 'time_format_guess'
+
+    class CsvGuessPlugin < LineGuessPlugin
+      Plugin.register_guess('csv', self)
+
+      DELIMITER_CANDIDATES = [
+        ",", "\t", "|"
+      ]
+
+      QUOTE_CANDIDATES = [
+        "\"", "'"
+      ]
+
+      # CsvParserPlugin.TRUE_STRINGS
+      TRUE_STRINGS = Hash[*%w[
+        true True TRUE
+        yes Yes YES
+        y Y
+        on On ON
+        1
+      ].map {|k| [k, true] }]
+
+      def guess_lines(config, sample_lines)
+        delim = guess_delimiter(sample_lines)
+        unless delim
+          # not CSV file
+          return {}
+        end
+
+        parser_config = config["parser"] || {}
+        parser_guessed = {"type" => "csv", "delimiter" => delim}
+
+        quote = guess_quote(sample_lines, delim)
+        parser_guessed["quote"] = quote ? quote : ''
+
+        sample_records = sample_lines.map {|line| line.split(delim) }  # TODO use CsvTokenizer
+        first_types = guess_field_types(sample_records[0, 1])
+        other_types = guess_field_types(sample_records[1..-1])
+
+        if first_types.size <= 1 || other_types.size <= 1
+          # guess failed
+          return {}
+        end
+
+        unless parser_config.has_key?("header_line")
+          parser_guessed["header_line"] = (first_types != other_types && !first_types.any? {|t| t != ["string"] })
+        end
+
+        unless parser_config.has_key?("columns")
+          if parser_guessed["header_line"] || parser_config["header_line"]
+            column_names = sample_records.first
+          else
+            column_names = (0..other_types.size).to_a.map {|i| "c#{i}" }
+          end
+          schema = []
+          column_names.zip(other_types).each do |name,(type,format)|
+            if name && type
+              if format
+                schema << {"name" => name, "type" => type, "format" => format}
+              else
+                schema << {"name" => name, "type" => type}
+              end
+            end
+          end
+          parser_guessed["columns"] = schema
+        end
+
+        return {"parser" => parser_guessed}
+      end
+
+      private
+
+      def guess_delimiter(sample_lines)
+        delim_weights = DELIMITER_CANDIDATES.map do |d|
+          counts = sample_lines.map {|line| line.count(d) }
+          total = array_sum(counts)
+          if total > 0
+            stddev = array_standard_deviation(counts)
+            stddev = 0.000000001 if stddev == 0.0
+            weight = total / stddev
+            [d, weight]
+          else
+            [nil, 0]
+          end
+        end
+
+        delim, weight = *delim_weights.sort_by {|d,weight| weight }.last
+        if delim != nil && weight > 1
+          return delim
+        else
+          return nil
+        end
+      end
+
+      def guess_quote(sample_lines, delim)
+        delim_regexp = Regexp.escape(delim)
+        quote_weights = QUOTE_CANDIDATES.map do |q|
+          weights = sample_lines.map do |line|
+            q_regexp = Regexp.escape(q)
+            count = line.count(q)
+            if count > 0
+              weight = count
+              weight += line.scan(/(?:\A|#{delim_regexp})\s*#{q_regexp}(?:(?!#{q_regexp}).)*\s*#{q_regexp}(?:$|#{delim_regexp})/).size * 20
+              weight += line.scan(/(?:\A|#{delim_regexp})\s*#{q_regexp}(?:(?!#{delim_regexp}).)*\s*#{q_regexp}(?:$|#{delim_regexp})/).size * 40
+              weight
+            else
+              nil
+            end
+          end.compact
+          weights.empty? ? 0 : array_avg(weights)
+        end
+        quote, weight = QUOTE_CANDIDATES.zip(quote_weights).sort_by {|q,w| w }.last
+        if weight >= 10.0
+          return quote
+        else
+          return nil
+        end
+      end
+
+      def guess_field_types(field_lines)
+        column_lines = []
+        field_lines.each do |fields|
+          fields.each_with_index {|field,i| (column_lines[i] ||= []) << guess_type(field) }
+        end
+        columns = column_lines.map do |types|
+          t = types.inject(nil) {|r,t| merge_type(r,t) } || "string"
+          if t.is_a?(TimestampMatch)
+            format = TimeFormatGuess.guess(types.map {|type| type.text })
+            ["timestamp", format]
+          else
+            [t]
+          end
+        end
+        return columns
+      end
+
+      TYPE_COALESCE = Hash[{
+        long: :double,
+        boolean: :long,
+      }.map {|k,v|
+        [[k.to_s, v.to_s].sort, v.to_s]
+      }]
+
+      def merge_type(type1, type2)
+        if type1 == type2
+          type1
+        elsif type1.nil? || type2.nil?
+          type1 || type2
+        else
+          TYPE_COALESCE[[type1, type2].sort] || "string"
+        end
+      end
+
+      class TimestampMatch < String
+        def initialize(text)
+          super("timestamp")
+          @text = text
+        end
+        attr_reader :text
+      end
+
+      def guess_type(str)
+        if TRUE_STRINGS[str]
+          return "boolean"
+        end
+
+        if TimeFormatGuess.guess(str)
+          return TimestampMatch.new(str)
+        end
+
+        if str.to_i.to_s == str
+          return "long"
+        end
+
+        if str.include?('.')
+          a, b = str.split(".", 2)
+          if a.to_i.to_s == a && b.to_i.to_s == b
+            return "double"
+          end
+        end
+
+        return "string"
+      end
+
+      def array_sum(array)
+        array.inject(0) {|r,i| r += i }
+      end
+
+      def array_avg(array)
+        array.inject(0.0) {|r,i| r += i } / array.size
+      end
+
+      def array_variance(array)
+        avg = array_avg(array)
+        array.inject(0.0) {|r,i| r += (i - avg) ** 2 } / array.size
+      end
+
+      def array_standard_deviation(array)
+        Math.sqrt(array_variance(array))
+      end
+    end
+
+  end
+end

data/lib/embulk/guess/gzip.rb

@@ -0,0 +1,18 @@
+module Embulk
+  module Guess
+
+    class GzipGuessPlugin < GuessPlugin
+      Plugin.register_guess('gzip', self)
+
+      GZIP_HEADER = "\x1f\x8b".force_encoding('ASCII-8BIT').freeze
+
+      def guess(config, sample_buffer)
+        if sample_buffer[0,2] == GZIP_HEADER
+          return {"decoders" => [{"type" => "gzip"}]}
+        end
+        return {}
+      end
+    end
+
+  end
+end