embulk 0.4.5 → 0.4.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 57cffd859609745fafc0e644e0e8466c0511c622
-  data.tar.gz: 5df593df4e75dce977ac7ab295f9fe089ababa1a
+  metadata.gz: b02e8762879a557c331e6fba758a1be4bd2d0575
+  data.tar.gz: 1d75577656dbf789d30674e5e689abbb2048f28e
 SHA512:
-  metadata.gz: 2c613261f18c551e6eb1ea9f748148422054d0818a76d8367de8eb68a071445686e871ce304d1413b10eca8570d220b4d53dbd6f3b7f060b008ce3c500903fe9
-  data.tar.gz: 0e6c8086818d6f7d81ac8009ee5fbf76d0aa0a34fc962af60f3647d189ca488cd3c4aa09c8b49f62b3551fcd9b4ad42e2582b69f6a56f2c78f9f6e06f6986439
+  metadata.gz: b3e674c5d0e1a9fa3a09b516339cba7a7f587ff70521287e11c64caa0ec8af1d4a229b88bb168e3d10135c1e574e34c40c264587ab4c3c169e194323ef2040be
+  data.tar.gz: d51979854aa840ff03cbbf0ee118a93aae2066f06485503f3451d31486f222e56203ac11e819e2fa42238a9914b3023a276d0e6a2dbe5010367da837fc504938

data/Gemfile.lock

@@ -1,33 +1,26 @@
 PATH
   remote: .
   specs:
-    embulk (0.1.0)
+    embulk (0.4.5)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    diff-lcs (1.2.5)
-    json (1.8.1)
     kramdown (1.5.0)
+    power_assert (0.2.2)
     rake (10.4.2)
-    rspec (2.99.0)
-      rspec-core (~> 2.99.0)
-      rspec-expectations (~> 2.99.0)
-      rspec-mocks (~> 2.99.0)
-    rspec-core (2.99.2)
-    rspec-expectations (2.99.2)
-      diff-lcs (>= 1.1.3, < 2.0)
-    rspec-mocks (2.99.2)
+    test-unit (3.0.9)
+      power_assert
     yard (0.8.7.6)
 
 PLATFORMS
+  java
   ruby
 
 DEPENDENCIES
   bundler (>= 1.0)
   embulk!
-  json (~> 1.7)
   kramdown (~> 1.5.0)
   rake (>= 0.10.0)
-  rspec (~> 2.11)
+  test-unit (~> 3.0.9)
   yard (~> 0.8.7)

data/README.md

@@ -21,20 +21,36 @@ You can release plugins to share your efforts of data cleaning, error handling,
 
 ## Quick Start
 
-The single-file package is the simplest way to try Embulk. You can download the latest embulk-VERSION.jar from [the releases page](https://bintray.com/embulk/maven/embulk/view#files) and run it with java:
+The single-file package is the simplest way to try Embulk. You can download the latest embulk-VERSION.jar from [the releases page](https://bintray.com/embulk/maven/embulk/view#files) and run it with java.
+
+### Linux & Mac & BSD
+
+Following 4 commans install embulk to your home directory:
+
+```
+curl --create-dirs -o ~/.embulk/bin/embulk -L https://bintray.com/artifact/download/embulk/maven/embulk-0.4.6.jar
+chmod +x ~/.embulk/bin/embulk
+echo 'export PATH="$HOME/.embulk/bin:$PATH"' >> ~/.bashrc
+source ~/.bashrc
+```
+
+### Windows
+
+You can assume the jar file is a .bat file.
 
 ```
-wget https://bintray.com/artifact/download/embulk/maven/embulk-0.4.5.jar -O embulk.jar
-java -jar embulk.jar --help
+curl -o embulk.bat -L https://bintray.com/artifact/download/embulk/maven/embulk-0.4.5.jar
 ```
 
+### Trying examples
+
 Let's load a CSV file, for example. `embulk example` subcommand generates a csv file and config file for you.
 
 ```
-java -jar embulk.jar example ./try1
-java -jar embulk.jar guess   ./try1/example.yml -o config.yml
-java -jar embulk.jar preview config.yml
-java -jar embulk.jar run     config.yml
+embulk example ./try1
+embulk guess   ./try1/example.yml -o config.yml
+embulk preview config.yml
+embulk run     config.yml
 ```
 
 ### Using plugins
@@ -43,8 +59,8 @@ You can use plugins to load data from/to various systems and file formats.
 An example is [embulk-output-postgres-json](https://github.com/frsyuki/embulk-output-postgres-json) plugin. It outputs data into PostgreSQL server using "json" column type.
 
 ```
-java -jar embulk.jar gem install embulk-output-postgres-json
-java -jar embulk.jar gem list
+embulk gem install embulk-output-postgres-json
+embulk gem list
 ```
 
 You can search plugins on RubyGems: [search for "embulk"](https://rubygems.org/search?utf8=%E2%9C%93&query=embulk).
@@ -57,9 +73,9 @@ You can use the bundle using `-b <bundle_dir>` option. `embulk bundle` also gene
 See generated \<bundle_dir>/Gemfile file how to plugin bundles work.
 
 ```
-java -jar embulk.jar bundle ./embulk_bundle
-java -jar embulk.jar guess  -b ./embulk_bundle ...
-java -jar embulk.jar run    -b ./embulk_bundle ...
+embulk bundle ./embulk_bundle
+embulk guess  -b ./embulk_bundle ...
+embulk run    -b ./embulk_bundle ...
 ```
 
 ### Releasing plugins to RubyGems
@@ -76,19 +92,19 @@ Embulk supports resuming failed transactions.
 To enable resuming, you need to start transaction with `-r PATH` option:
 
 ```
-java -jar embulk.jar run config.yml -r resume-state.yml
+embulk run config.yml -r resume-state.yml
 ```
 
 If the transaction fails, embulk stores state some states to the yaml file. You can retry the transaction using exactly same command:
 
 ```
-java -jar embulk.jar run config.yml -r resume-state.yml
+embulk run config.yml -r resume-state.yml
 ```
 
 If you giveup to resume the transaction, you can use `embulk cleanup` subcommand to delete intermediate data:
 
 ```
-java -jar embulk.jar cleanup config.yml -r resume-state.yml
+embulk cleanup config.yml -r resume-state.yml
 ```
 
 

data/build.gradle

@@ -5,13 +5,14 @@ plugins {
     id 'com.github.jruby-gradle.base' version '0.1.5'
     id 'com.github.johnrengelman.shadow' version '1.2.0'
 }
+import com.github.jrubygradle.JRubyExec
 
 def java_projects = [project(":embulk-core"), project(":embulk-standards"), project(":embulk-cli")]
 def release_projects = [project(":embulk-core"), project(":embulk-standards")]
 
 allprojects {
     group = 'org.embulk'
-    version = '0.4.5'
+    version = '0.4.6'
 
     apply plugin: 'java'
     apply plugin: 'maven-publish'
@@ -131,6 +132,10 @@ subprojects {
     }
 }
 
+dependencies {
+    jrubyExec 'rubygems:test-unit:3.0.+'
+}
+
 //
 // classpath task
 //
@@ -143,11 +148,8 @@ clean { delete 'classpath' }
 task cli(dependsOn: ':embulk-cli:shadowJar') << {
     file('pkg').mkdirs()
     File f = file("pkg/embulk-${project.version}.jar")
-    f.write('''\
-#!/bin/sh
-exec java -jar "$0" "$@"
-exit 127
-''')
+    f.write("")
+    f.append(file("embulk-cli/src/main/sh/selfrun.sh").readBytes())
     f.append(file("embulk-cli/build/libs/embulk-cli-${project.version}-all.jar").readBytes())
     f.setExecutable(true)
 }
@@ -175,16 +177,19 @@ project(':embulk-cli') {
     }
 }
 
+task rubyTest(type: JRubyExec) {
+    jrubyArgs '-Ilib', '-Itest', '-rtest/unit', '-eTest::Unit::AutoRunner.run(true, *ARGV)'
+    script 'test'
+}
+
 //
 // gem task
 //
-import com.github.jrubygradle.JRubyExec
 task gem(type: JRubyExec) {
     jrubyArgs '-rrubygems/gem_runner', '-eGem::GemRunner.new.run(ARGV)', 'build'
-    script 'build/gemspec'
+    script 'embulk.gemspec'
     doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
 }
-gem.dependsOn('gemspec')
 gem.dependsOn('classpath')
 
 //
@@ -194,7 +199,6 @@ task rubyGemsUpload(type: JRubyExec, dependsOn: ["gem"]) {
     jrubyArgs '-rrubygems/gem_runner', '-eGem::GemRunner.new.run(ARGV)', 'push'
     script "pkg/embulk-${project.version}.gem"
 }
-gem.dependsOn('gemspec')
 
 //
 // releaseCheck and release tasks
@@ -248,33 +252,3 @@ task set_version << {
 
     println "add 'release/release-${to}' line to embulk-docs/src/release.rst"
 }
-
-task gemspec << {
-    file('build').mkdirs()
-    file('build/gemspec').write($/
-Gem::Specification.new do |gem|
-  gem.name          = "embulk"
-  gem.version       = "${project.version}"
-
-  gem.summary       = "Embulk, a plugin-based parallel bulk data loader"
-  gem.description   = "Embulk is an open-source, plugin-based bulk data loader to scale and simplify data management across heterogeneous data stores. It can collect and ship any kinds of data in high throughput with transaction control."
-  gem.authors       = ["Sadayuki Furuhashi"]
-  gem.email         = ["frsyuki@gmail.com"]
-  gem.license       = "Apache 2.0"
-  gem.homepage      = "https://github.com/embulk/embulk"
-
-  gem.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
-  gem.test_files    = gem.files.grep(%r"^(test|spec)/")
-  gem.executables   = gem.files.grep(%r"^bin/").map{ |f| File.basename(f) }
-  gem.require_paths = ["lib"]
-  gem.has_rdoc      = false
-
-  gem.add_development_dependency "bundler", [">= 1.0"]
-  gem.add_development_dependency "rake", [">= 0.10.0"]
-  gem.add_development_dependency "rspec", ["~> 2.11"]
-  gem.add_development_dependency "json", ["~> 1.7"]
-  gem.add_development_dependency "yard", ["~> 0.8.7"]
-  gem.add_development_dependency "kramdown", ["~> 1.5.0"]
-end
-/$)
-}

data/embulk-cli/src/main/sh/selfrun.sh

@@ -0,0 +1,64 @@
+
+: <<BAT
+@echo off
+
+java -jar %0 %*
+
+exit /B
+BAT
+
+java_args=""
+jruby_args=""
+default_optimize=""
+overwrite_optimize=""
+
+while true; do
+    case "$1" in
+        "-J+O")
+            overwrite_optimize="true"
+            shift
+            break;
+            ;;
+        "-J-O")
+            overwrite_optimize="false"
+            shift
+            break;
+            ;;
+        -J*)
+            v="${1#-J}"
+            if test "$v"; then
+                java_args="$java_args $v"
+            else
+                shift
+                file_args=`cat "$1"`
+                if test $? -ne 0; then
+                    echo "Failed to load java argument file."
+                    exit 1
+                fi
+                java_args="$java_args $file_args"
+            fi
+            shift
+            ;;
+        -R*)
+            v="${1#-R}"
+            jruby_args="$jruby_args $v"
+            shift
+            ;;
+        run)
+            default_optimize="true"
+            break
+            ;;
+        *)
+            break
+            ;;
+    esac
+done
+
+if test "$overwrite_optimize" = "true" -o "$default_optimize" -a "$overwrite_optimize" != "false"; then
+    java_args="-XX:+AggressiveOpts -XX:+UseConcMarkSweepGC $java_args"
+else
+    java_args="-XX:+AggressiveOpts -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -Xverify:none $java_args"
+fi
+
+exec java $java_args -jar "$0" $jruby_args "$@"
+exit 127

data/embulk-core/src/main/java/org/embulk/spi/time/TimestampFormat.java

@@ -43,6 +43,9 @@ public class TimestampFormat
         if(s.startsWith("+") || s.startsWith("-")) {
             return DateTimeZone.forID(s);
 
+        } else if (s.equals("Z")) {
+            return DateTimeZone.UTC;
+
         } else {
             try {
                 int rawOffset = (int) DateTimeFormat.forPattern("z").parseMillis(s);

data/embulk-docs/src/release.rst

@@ -16,4 +16,5 @@ Release Notes
     release/release-0.4.3
     release/release-0.4.4
     release/release-0.4.5
+    release/release-0.4.6
 

data/embulk-docs/src/release/release-0.4.6.rst

@@ -0,0 +1,30 @@
+Release 0.4.6
+==================================
+
+CLI
+------------------
+
+* Updated installation script to install ``embulk`` binary to ~/.embulk/bin directory.
+* Updated boot script to set ``-XX:+TieredCompilation`` and ``-XX:TieredStopAtLevel=1`` JVM options excepting ``run`` mode.
+  * Those change improves startup time of embulk command 1.5 ~ 2x faster.
+* Updated boot script to set ``-XX:+UseConcMarkSweepGC`` JVM option to ``run`` mode.
+* Usage message shows embulk version.
+
+Built-in plugins
+------------------
+
+* ``input/file`` plugin sets ``last_path`` option to the next configuration.
+  * This enables scheduled execution where next execution loads files created after the last execution.
+* ``guess/csv`` fixed guess result when the ordr of dates is not year, month day.
+* ``guess/csv`` guesses dd/mm/yyyy time format if the input data includes at least one day larger than 12.
+* ``guess/csv`` guesses RFC 2822 date and time formats
+* ``guess/csv`` supports "." as the delimiter of year, month or day.
+
+General Changes
+------------------
+
+* embulk command raises a special message if HOME environment variable is not set.
+
+Release Date
+------------------
+2015-02-20

data/embulk-standards/src/main/java/org/embulk/standards/LocalFileInputPlugin.java

@@ -1,6 +1,8 @@
 package org.embulk.standards;
 
 import java.util.List;
+import java.util.ArrayList;
+import java.util.Collections;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.InputStream;
@@ -73,8 +75,14 @@ public class LocalFileInputPlugin
             int taskCount,
             FileInputPlugin.Control control)
     {
+        PluginTask task = taskSource.loadTask(PluginTask.class);
+
         control.run(taskSource, taskCount);
-        return Exec.newConfigDiff();
+
+        List<String> files = new ArrayList<String>(task.getFiles());
+        Collections.sort(files);
+        return Exec.newConfigDiff().
+            set("last_path", files.get(files.size() - 1));
     }
 
     @Override

data/embulk.gemspec

@@ -5,23 +5,22 @@ Gem::Specification.new do |gem|
   gem.name          = "embulk"
   gem.version       = Embulk::VERSION
 
-  gem.summary       = %q{Embulk, a plugin-based parallel bulk data loader}
-  gem.description   = %q{Embulk is an open-source, plugin-based bulk data loader to scale and simplify data management across heterogeneous data stores. It can collect and ship any kinds of data in high throughput with transaction control.}
+  gem.summary       = "Embulk, a plugin-based parallel bulk data loader"
+  gem.description   = "Embulk is an open-source, plugin-based bulk data loader to scale and simplify data management across heterogeneous data stores. It can collect and ship any kinds of data in high throughput with transaction control."
   gem.authors       = ["Sadayuki Furuhashi"]
   gem.email         = ["frsyuki@gmail.com"]
   gem.license       = "Apache 2.0"
   gem.homepage      = "https://github.com/embulk/embulk"
 
   gem.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
-  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
-  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r"^(test|spec)/")
+  gem.executables   = gem.files.grep(%r"^bin/").map{ |f| File.basename(f) }
   gem.require_paths = ["lib"]
   gem.has_rdoc      = false
 
-  gem.add_development_dependency 'bundler', ['>= 1.0']
-  gem.add_development_dependency 'rake', ['>= 0.10.0']
-  gem.add_development_dependency 'rspec', ['~> 2.11']
-  gem.add_development_dependency 'json', ['~> 1.7']
-  gem.add_development_dependency 'yard', ['~> 0.8.7']
-  gem.add_development_dependency 'kramdown', ['~> 1.5.0']
+  gem.add_development_dependency "bundler", [">= 1.0"]
+  gem.add_development_dependency "rake", [">= 0.10.0"]
+  gem.add_development_dependency "test-unit", ["~> 3.0.9"]
+  gem.add_development_dependency "yard", ["~> 0.8.7"]
+  gem.add_development_dependency "kramdown", ["~> 1.5.0"]
 end

data/lib/embulk/command/embulk_run.rb

@@ -10,7 +10,7 @@ module Embulk
     # GEM_HOME is already set by embulk/command/embulk.rb
     gem_home = ENV['GEM_HOME'].to_s
     if gem_home.empty?
-      ENV['GEM_HOME'] = File.expand_path File.join(ENV['HOME'], '.embulk', Gem.ruby_engine, RbConfig::CONFIG['ruby_version'])
+      ENV['GEM_HOME'] = default_gem_home
       Gem.clear_paths  # force rubygems to reload GEM_HOME
     end
 
@@ -298,6 +298,17 @@ examples:
     File.join(home, dir)
   end
 
+  def self.default_gem_home
+    if RUBY_PLATFORM =~ /java/i
+      user_home = java.lang.System.properties["user.home"]
+    end
+    user_home ||= ENV['HOME']
+    unless user_home
+      raise "HOME environment variable is not set."
+    end
+    File.expand_path File.join(user_home, '.embulk', Gem.ruby_engine, RbConfig::CONFIG['ruby_version'])
+  end
+
   private
 
   def self.setup_gem_paths(path)
@@ -322,6 +333,7 @@ examples:
   end
 
   def self.usage(message)
+    STDERR.puts "Embulk v#{Embulk::VERSION}"
     STDERR.puts "usage: <command> [--options]"
     STDERR.puts "commands:"
     STDERR.puts "   bundle    [directory]                              # create or update plugin environment."

data/lib/embulk/data/new/ruby/filter.rb.erb

@@ -26,12 +26,12 @@ module Embulk
       def add(page)
         # filtering code:
         page.each do |record|
-          @page_builder.add(record)
+          page_builder.add(record)
         end
       end
 
       def finish
-        @page_builder.finish
+        page_builder.finish
       end
     end
 

data/lib/embulk/data/new/ruby/formatter.rb.erb

@@ -31,7 +31,7 @@ module Embulk
         # output code:
         page.each do |record|
           if @current_file == nil || @current_file_size > 32*1024
-            @current_file = @file_output.next_file
+            @current_file = file_output.next_file
             @current_file_size = 0
           end
           @current_file.write "|mydata|"
@@ -39,7 +39,7 @@ module Embulk
       end
 
       def finish
-        @file_output.finish
+        file_output.finish
       end
     end
 

data/lib/embulk/data/new/ruby/input.rb.erb

@@ -34,9 +34,9 @@ module Embulk
       end
 
       def run
-        @page_builder.add(["example-value", 1, 0.1])
-        @page_builder.add(["example-value", 2, 0.2])
-        @page_builder.finish
+        page_builder.add(["example-value", 1, 0.1])
+        page_builder.add(["example-value", 2, 0.2])
+        page_builder.finish
 
         commit_report = {}
         return commit_report

data/lib/embulk/data/new/ruby/parser.rb.erb

@@ -31,10 +31,10 @@ module Embulk
           file.each do |buffer|
             # parsering code
             record = ["col1", 2, 3.0]
-            @page_builder.add(record)
+            page_builder.add(record)
           end
         end
-        @page_builder.finish
+        page_builder.finish
       end
     end
 

data/lib/embulk/guess/time_format_guess.rb

@@ -16,6 +16,9 @@ module Embulk::Guess
 
       WEEKDAY_NAME_SHORT = /Sun|Mon|Tue|Wed|Thu|Fri|Sat/
       WEEKDAY_NAME_FULL = /Sunday|Monday|Tuesday|Wednesday|Thursday|Friday|Saturday/
+
+      ZONE_OFF = /(?:Z|[\-\+]\d\d(?::?\d\d)?)/
+      ZONE_ABB = /[A-Z]{1,3}/
     end
 
     class GuessMatch
@@ -119,9 +122,17 @@ module Embulk::Guess
       end
 
       def mergeable_group
-        [@delimiters, @parts]
+        # MDY is mergible with DMY
+        if i = array_sequence_find(@parts, [:day, :month, :year])
+          ps = @parts.dup
+          ps[i, 3] = [:month, :day, :year]
+          [@delimiters, ps]
+        else
+          [@delimiters, @parts]
+        end
       end
 
+      attr_reader :parts
       attr_reader :part_options
 
       def merge!(another_in_group)
@@ -136,27 +147,44 @@ module Embulk::Guess
             [@part_options[i], part_options[i]].sort.last
           end
         end
+
+        # if DMY matches, MDY is likely false match of DMY.
+        dmy = array_sequence_find(another_in_group.parts, [:day, :month, :year])
+        mdy = array_sequence_find(@parts, [:month, :day, :year])
+        if mdy && dmy
+          @parts[mdy, 3] = [:day, :month, :year]
+        end
+      end
+
+      def array_sequence_find(array, seq)
+        (array.size - seq.size + 1).times {|i|
+          return i if array[i, seq.size] == seq
+        }
+        return nil
       end
     end
 
     class GuessPattern
       include Parts
 
-      date_delims = /[\/\-]/
+      date_delims = /[\/\-\.]/
       # yyyy-MM-dd
       YMD         = /(?<year>#{YEAR})(?<date_delim>#{date_delims})(?<month>#{MONTH})\k<date_delim>(?<day>#{DAY})/
       YMD_NODELIM = /(?<year>#{YEAR})(?<month>#{MONTH_NODELIM})(?<day>#{DAY_NODELIM})/
-      # dd/MM/yyyy
-      DMY         = /(?<year>#{YEAR})(?<date_delim>#{date_delims})(?<month>#{MONTH})\k<date_delim>(?<day>#{DAY})/
-      DMY_NODELIM = /(?<year>#{YEAR})(?<month>#{MONTH_NODELIM})(?<day>#{DAY_NODELIM})/
-
-      frac = /[0-9]{1,24}/
+      # MM/dd/yyyy
+      MDY         = /(?<month>#{MONTH})(?<date_delim>#{date_delims})(?<day>#{DAY})\k<date_delim>(?<year>#{YEAR})/
+      MDY_NODELIM = /(?<month>#{MONTH_NODELIM})(?<day>#{DAY_NODELIM})(?<year>#{YEAR})/
+      # dd.MM.yyyy
+      DMY         = /(?<day>#{DAY})(?<date_delim>#{date_delims})(?<month>#{MONTH})\k<date_delim>(?<year>#{YEAR})/
+      DMY_NODELIM = /(?<day>#{DAY_NODELIM})(?<month>#{MONTH_NODELIM})(?<year>#{YEAR})/
+
+      frac = /[0-9]{1,9}/
       time_delims = /[\:\-]/
       frac_delims = /[\.\,]/
-      TIME         = /(?<hour>#{HOUR})(?<time_delim>#{time_delims})(?<minute>#{MINUTE})(?:\k<time_delim>(?<second>#{SECOND})(?:(?<frac_delim>#{frac_delims})(?<frac>#{frac}))?)?/
-      TIME_NODELIM = /(?<hour>#{HOUR_NODELIM})(?<minute>#{MINUTE_NODELIM})((?<second>#{SECOND_NODELIM})(?:(?<frac_delim>#{frac_delims})(?<frac>#{frac}))?)?/
+      TIME         = /(?<hour>#{HOUR})(?:(?<time_delim>#{time_delims})(?<minute>#{MINUTE})(?:\k<time_delim>(?<second>#{SECOND})(?:(?<frac_delim>#{frac_delims})(?<frac>#{frac}))?)?)?/
+      TIME_NODELIM = /(?<hour>#{HOUR_NODELIM})(?:(?<minute>#{MINUTE_NODELIM})((?<second>#{SECOND_NODELIM})(?:(?<frac_delim>#{frac_delims})(?<frac>#{frac}))?)?)?/
 
-      TZ = /(?<zone_space> )?(?<zone>(?<zone_off>[\-\+]\d\d(?::?\d\d)?)|(?<zone_abb>[A-Z]{3}))|(?<z>Z)/
+      ZONE = /(?<zone_space> )?(?<zone>(?<zone_off>#{ZONE_OFF})|(?<zone_abb>#{ZONE_ABB}))/
 
       def match(text)
         delimiters = []
@@ -177,6 +205,21 @@ module Embulk::Guess
           parts << :day
           part_options << part_heading_option(dm["day"])
 
+        elsif dm = (/^#{MDY}(?<rest>.*?)$/.match(text) or /^#{MDY_NODELIM}(?<rest>.*?)$/.match(text))
+          date_delim = dm["date_delim"] rescue ""
+
+          parts << :month
+          part_options << part_heading_option(dm["month"])
+          delimiters << date_delim
+
+          parts << :day
+          part_options << part_heading_option(dm["day"])
+          delimiters << date_delim
+
+          parts << :year
+          part_options << nil
+          delimiters << date_delim
+
         elsif dm = (/^#{DMY}(?<rest>.*?)$/.match(text) or /^#{DMY_NODELIM}(?<rest>.*?)$/.match(text))
           date_delim = dm["date_delim"] rescue ""
 
@@ -198,7 +241,7 @@ module Embulk::Guess
         end
         rest = dm["rest"]
 
-        date_time_delims = /[ _T]/
+        date_time_delims = /(:? |_|T|\. ?)/
         if tm = (
               /^(?<date_time_delim>#{date_time_delims})#{TIME}(?<rest>.*?)?$/.match(rest) or
               /^(?<date_time_delim>#{date_time_delims})#{TIME_NODELIM}(?<rest>.*?)?$/.match(rest) or
@@ -211,31 +254,30 @@ module Embulk::Guess
           parts << :hour
           part_options << part_heading_option(tm["hour"])
 
-          delimiters << time_delim
-          parts << :minute
-          part_options << part_heading_option(tm["minute"])
-
-          if tm["second"]
+          if tm["minute"]
             delimiters << time_delim
-            parts << :second
-            part_options << part_heading_option(tm["second"])
-          end
-
-          if tm["frac"]
-            delimiters << tm["frac_delim"]
-            parts << :frac
-            part_options << tm["frac"].size
+            parts << :minute
+            part_options << part_heading_option(tm["minute"])
+
+            if tm["second"]
+              delimiters << time_delim
+              parts << :second
+              part_options << part_heading_option(tm["second"])
+
+              if tm["frac"]
+                delimiters << tm["frac_delim"]
+                parts << :frac
+                part_options << tm["frac"].size
+              end
+            end
           end
 
           rest = tm["rest"]
         end
 
-        if zm = /^#{TZ}$/.match(rest)
+        if zm = /^#{ZONE}$/.match(rest)
           delimiters << (zm["zone_space"] || '')
-          if zm["z"]
-            # TODO ISO 8601
-            parts << :zone_off
-          elsif zm["zone_off"]
+          if zm["zone_off"]
             parts << :zone_off
           else
             parts << :zone_abb
@@ -265,9 +307,9 @@ module Embulk::Guess
       end
     end
 
-    class RegexpMatch
+    class SimpleMatch
       def initialize(format)
-        @format
+        @format = format
       end
 
       attr_reader :format
@@ -280,10 +322,33 @@ module Embulk::Guess
       end
     end
 
+    class Rfc2822Pattern
+      include Parts
+
+      def initialize
+        @regexp = /^(?<weekday>#{WEEKDAY_NAME_SHORT}, )?\d\d #{MONTH_NAME_SHORT} \d\d\d\d(?<time> \d\d:\d\d(?<second>:\d\d)? (?:(?<zone_off>#{ZONE_OFF})|(?<zone_abb>#{ZONE_ABB})))?$/
+      end
+
+      def match(text)
+        if m = @regexp.match(text)
+          format = ''
+          format << "%a, " if m['weekday']
+          format << "%d %b %Y"
+          format << " %H:%M" if m['time']
+          format << ":%S" if m['second']
+          format << " %z" if m['zone_off']
+          format << " %Z" if m['zone_abb']
+          SimpleMatch.new(format)
+        else
+          nil
+        end
+      end
+    end
+
     class RegexpPattern
       def initialize(regexp, format)
         @regexp = regexp
-        @match = RegexpMatch.new(format)
+        @match = SimpleMatch.new(format)
       end
 
       def match(text)
@@ -298,18 +363,15 @@ module Embulk::Guess
     module StandardPatterns
       include Parts
 
-      RFC_822_1123 = /^#{WEEKDAY_NAME_SHORT}, \d\d #{MONTH_NAME_SHORT} \d\d\d\d \d\d:\d\d:\d\d [a-zA-Z]{3}$/
-      RFC_850_1035 = /^#{WEEKDAY_NAME_FULL}, \d\d-#{MONTH_NAME_SHORT}-\d\d \d\d:\d\d:\d\d [a-zA-Z]{3}$/
-      APACHE_CLF = /^\d\d\/#{MONTH_NAME_SHORT}\/\d\d\d\d \d\d:\d\d:\d\d [\-\+]\d\d(?::?\d\d)?$/
+      APACHE_CLF = /^\d\d\/#{MONTH_NAME_SHORT}\/\d\d\d\d:\d\d:\d\d:\d\d #{ZONE_OFF}?$/
       ANSI_C_ASCTIME = /^#{WEEKDAY_NAME_SHORT} #{MONTH_NAME_SHORT} \d\d? \d\d:\d\d:\d\d \d\d\d\d$/
     end
 
     PATTERNS = [
       GuessPattern.new,
-      RegexpPattern.new(StandardPatterns::RFC_822_1123, "%a, %d %b %Y %H:%M:%S %z"),
-      RegexpPattern.new(StandardPatterns::RFC_850_1035, "%A, %d-%b-%y %H:%M:%S %z"),
-      RegexpPattern.new(StandardPatterns::APACHE_CLF, "%d/%b/%Y %H:%M:%S %Z"),
-      RegexpPattern.new(StandardPatterns::ANSI_C_ASCTIME, "$a %b %e %H:%M:%S %Y"),
+      Rfc2822Pattern.new,
+      RegexpPattern.new(StandardPatterns::APACHE_CLF, "%d/%b/%Y:%H:%M:%S %z"),
+      RegexpPattern.new(StandardPatterns::ANSI_C_ASCTIME, "%a %b %e %H:%M:%S %Y"),
     ]
 
     def self.guess(texts)
@@ -322,8 +384,8 @@ module Embulk::Guess
       elsif matches.size == 1
         return matches[0].format
       else
-        match_groups = matches.group_by {|match| match.mergeable_group }
-        best_match_group = match_groups.sort_by {|group| group.size }.last[1]
+        match_groups = matches.group_by {|match| match.mergeable_group }.values
+        best_match_group = match_groups.sort_by {|group| group.size }.last
         best_match = best_match_group.shift
         best_match_group.each {|m| best_match.merge!(m) }
         return best_match.format

data/lib/embulk/input_plugin.rb

@@ -25,6 +25,8 @@ module Embulk
       init
     end
 
+    attr_reader :task, :schema, :index, :page_builder
+
     def init
     end
 

data/lib/embulk/parser_plugin.rb

@@ -17,6 +17,8 @@ module Embulk
       init
     end
 
+    attr_reader :task, :schema, :page_builder
+
     def init
     end
 

data/lib/embulk/version.rb

@@ -1,3 +1,3 @@
 module Embulk
-  VERSION = '0.4.5'
+  VERSION = '0.4.6'
 end

data/test/guess/test_time_format_guess.rb

@@ -0,0 +1,122 @@
+require 'helper'
+require 'time'
+require 'embulk/guess/time_format_guess'
+
+class TimeFormatGuessTest < ::Test::Unit::TestCase
+  def test_format_delims
+    # date-delim "-"  date-time-delim " "  time-delim ":"  frac delim "."
+    assert_guess "%Y-%m-%d %H:%M:%S.%N",    "2014-01-01 01:01:01.000000001"
+    assert_guess "%Y-%m-%d %H:%M:%S.%N",    "2014-01-01 01:01:01.000001"
+    assert_guess "%Y-%m-%d %H:%M:%S.%L",    "2014-01-01 01:01:01.001"
+    assert_guess "%Y-%m-%d %H:%M:%S",       "2014-01-01 01:01:01"
+    assert_guess "%Y-%m-%d %H:%M",          "2014-01-01 01:01"
+    assert_guess "%Y-%m-%d",                "2014-01-01"
+
+    # date-delim "/"  date-time-delim " "  time-delim "-"  frac delim ","
+    assert_guess "%Y/%m/%d %H-%M-%S,%N",    "2014/01/01 01-01-01,000000001"
+    assert_guess "%Y/%m/%d %H-%M-%S,%N",    "2014/01/01 01-01-01,000001"
+    assert_guess "%Y/%m/%d %H-%M-%S,%L",    "2014/01/01 01-01-01,001"
+    assert_guess "%Y/%m/%d %H-%M-%S",       "2014/01/01 01-01-01"
+    assert_guess "%Y/%m/%d %H-%M",          "2014/01/01 01-01"
+    assert_guess "%Y/%m/%d",                "2014/01/01"
+
+    # date-delim "."  date-time-delim "."  time-delim ":"  frac delim "."
+    assert_guess "%Y.%m.%d.%H:%M:%S.%N",    "2014.01.01.01:01:01.000000001"
+    assert_guess "%Y.%m.%d.%H:%M:%S.%N",    "2014.01.01.01:01:01.000001"
+    assert_guess "%Y.%m.%d.%H:%M:%S.%L",    "2014.01.01.01:01:01.001"
+    assert_guess "%Y.%m.%d.%H:%M:%S",       "2014.01.01.01:01:01"
+    assert_guess "%Y.%m.%d.%H:%M",          "2014.01.01.01:01"
+    assert_guess "%Y.%m.%d",                "2014.01.01"
+
+    # date-delim "."  date-time-delim ". "  time-delim ":"  frac delim ","
+    assert_guess "%Y.%m.%d. %H:%M:%S,%N",    "2014.01.01. 01:01:01,000000001"
+    assert_guess "%Y.%m.%d. %H:%M:%S,%N",    "2014.01.01. 01:01:01,000001"
+    assert_guess "%Y.%m.%d. %H:%M:%S,%L",    "2014.01.01. 01:01:01,001"
+    assert_guess "%Y.%m.%d. %H:%M:%S",       "2014.01.01. 01:01:01"
+    assert_guess "%Y.%m.%d. %H:%M",          "2014.01.01. 01:01"
+    assert_guess "%Y.%m.%d",                 "2014.01.01"
+  end
+
+  def test_format_ymd_orders
+    assert_guess "%Y-%m-%d", "2014-01-01"
+    assert_guess "%Y/%m/%d", "2014/01/01"
+    assert_guess "%Y.%m.%d", "2014.01.01"
+    assert_guess "%m/%d/%Y", "01/01/2014"
+    assert_guess "%m.%d.%Y", "01.01.2014"
+    assert_guess "%d/%m/%Y", "13/01/2014"
+    assert_guess "%d/%m/%Y", "21/01/2014"
+  end
+
+  def test_format_iso8601
+    assert_guess "%Y-%m-%d", "1981-04-05"
+    assert_guess "%Y-%m-%dT%H", "2007-04-06T13"
+    assert_guess "%Y-%m-%dT%H:%M", "2007-04-06T00:00"
+    assert_guess "%Y-%m-%dT%H:%M", "2007-04-05T24:00"
+    assert_guess "%Y-%m-%dT%H:%M:%S", "2007-04-06T13:47:30"
+    assert_guess "%Y-%m-%dT%H:%M:%S%z", "2007-04-06T13:47:30Z"
+    assert_guess "%Y-%m-%dT%H:%M:%S%z", "2007-04-06T13:47:30+00"
+    assert_guess "%Y-%m-%dT%H:%M:%S%z", "2007-04-06T13:47:30+00:00"
+    assert_guess "%Y-%m-%dT%H:%M:%S%z", "2007-04-06T13:47:30+0000"
+    assert_guess "%Y-%m-%dT%H:%M:%S%z", "2007-04-06T13:47:30-01"
+    assert_guess "%Y-%m-%dT%H:%M:%S%z", "2007-04-06T13:47:30-01:30"
+    assert_guess "%Y-%m-%dT%H:%M:%S%z", "2007-04-06T13:47:30-0130"
+  end
+
+  def test_format_rfc_822_2822
+    assert_guess '%a, %d %b %Y %H:%M:%S %Z', "Fri, 20 Feb 2015 14:02:34 PST"
+    assert_guess '%a, %d %b %Y %H:%M:%S %Z', "Fri, 20 Feb 2015 22:02:34 UT"
+    assert_guess '%a, %d %b %Y %H:%M:%S %Z', "Fri, 20 Feb 2015 22:02:34 GMT"
+    assert_guess     '%d %b %Y %H:%M:%S %Z',      "20 Feb 2015 22:02:34 GMT"
+    assert_guess     '%d %b %Y %H:%M %Z',         "20 Feb 2015 22:02 GMT"
+    assert_guess '%a, %d %b %Y %H:%M %Z',    "Fri, 20 Feb 2015 22:02 GMT"
+    assert_guess     '%d %b %Y',                  "20 Feb 2015"
+    assert_guess '%a, %d %b %Y',             "Fri, 20 Feb 2015"
+    assert_guess '%a, %d %b %Y %H:%M %z',    "Fri, 20 Feb 2015 22:02 +0000"
+    assert_guess '%a, %d %b %Y %H:%M %z',    "Fri, 20 Feb 2015 22:02 +00:00"
+    assert_guess '%a, %d %b %Y %H:%M %z',    "Fri, 20 Feb 2015 22:02 +00"
+  end
+
+  def test_format_apache_clf
+    assert_guess '%d/%b/%Y:%H:%M:%S %z', "07/Mar/2004:16:05:50 -0800"
+  end
+
+  def test_format_ansi_c_asctime
+    assert_guess '%a %b %e %H:%M:%S %Y', "Fri May 11 21:44:53 2001"
+  end
+
+  def test_format_merge_frequency
+    assert_guess_partial 2, "%Y-%m-%d %H:%M:%S", ["2014-01-01", "2014-01-01 00:00:00", "2014-01-01 00:00:00"]
+  end
+
+  def test_format_merge_dmy
+    # DMY has higher priority than MDY
+    assert_guess "%m/%d/%Y", ["01/01/2014"]
+    assert_guess "%d/%m/%Y", ["01/01/2014", "01/01/2014", "13/01/2014"]
+    assert_guess "%d.%m.%Y", ["01.01.2014", "01.01.2014", "13.01.2014"]
+    # but frequency is more important if delimiter is different
+    assert_guess_partial 2, "%m/%d/%Y", ["01/01/2014", "01/01/2014", "13.01.2014"]
+  end
+
+  def assert_guess(format, texts)
+    assert_equal format, guess(texts)
+    Array(texts).each do |text|
+      time = Time.strptime(text, format)
+      assert_equal time.to_i, Time.strptime(time.strftime(format), format).to_i
+    end
+  end
+
+  def assert_guess_partial(count, format, texts)
+    assert_equal format, guess(texts)
+    times = Array(texts).map do |text|
+      Time.strptime(text, format) rescue nil
+    end.compact
+    assert_equal count, times.size
+    times.each do |time|
+      assert_equal time.to_i, Time.strptime(time.strftime(format), format).to_i
+    end
+  end
+
+  def guess(texts)
+    Embulk::Guess::TimeFormatGuess.guess(Array(texts))
+  end
+end

data/test/helper.rb

@@ -0,0 +1,6 @@
+require 'test/unit'
+
+module Embulk
+end
+
+# TODO simplecov

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk
 version: !ruby/object:Gem::Version
-  version: 0.4.5
+  version: 0.4.6
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-02-20 00:00:00.000000000 Z
+date: 2015-02-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -39,31 +39,17 @@ dependencies:
   prerelease: false
   type: :development
 - !ruby/object:Gem::Dependency
-  name: rspec
+  name: test-unit
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '2.11'
+        version: 3.0.9
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '2.11'
-  prerelease: false
-  type: :development
-- !ruby/object:Gem::Dependency
-  name: json
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '1.7'
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '1.7'
+        version: 3.0.9
   prerelease: false
   type: :development
 - !ruby/object:Gem::Dependency
@@ -113,6 +99,7 @@ files:
 - build.gradle
 - embulk-cli/build.gradle
 - embulk-cli/src/main/java/org/embulk/cli/Main.java
+- embulk-cli/src/main/sh/selfrun.sh
 - embulk-core/build.gradle
 - embulk-core/src/main/java/org/embulk/EmbulkService.java
 - embulk-core/src/main/java/org/embulk/command/Runner.java
@@ -272,6 +259,7 @@ files:
 - embulk-docs/src/release/release-0.4.3.rst
 - embulk-docs/src/release/release-0.4.4.rst
 - embulk-docs/src/release/release-0.4.5.rst
+- embulk-docs/src/release/release-0.4.6.rst
 - embulk-standards/build.gradle
 - embulk-standards/src/main/java/org/embulk/standards/CsvFormatterPlugin.java
 - embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java
@@ -364,6 +352,8 @@ files:
 - lib/embulk/schema.rb
 - lib/embulk/version.rb
 - settings.gradle
+- test/guess/test_time_format_guess.rb
+- test/helper.rb
 - classpath/annotations-3.0.0.jar
 - classpath/aopalliance-1.0.jar
 - classpath/bval-core-0.5.jar
@@ -371,7 +361,9 @@ files:
 - classpath/commons-beanutils-core-1.8.3.jar
 - classpath/commons-lang3-3.1.jar
 - classpath/embulk-core-0.4.5.jar
+- classpath/embulk-core-0.4.6.jar
 - classpath/embulk-standards-0.4.5.jar
+- classpath/embulk-standards-0.4.6.jar
 - classpath/guava-18.0.jar
 - classpath/guice-3.0.jar
 - classpath/guice-multibindings-3.0.jar
@@ -417,4 +409,6 @@ rubygems_version: 2.1.9
 signing_key:
 specification_version: 4
 summary: Embulk, a plugin-based parallel bulk data loader
-test_files: []
+test_files:
+- test/guess/test_time_format_guess.rb
+- test/helper.rb