embulk 0.6.11 → 0.6.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d45b70ffc7ebef537906a3f471d74f83d40f18ab
-  data.tar.gz: 792c390284e3b949d84b4b9860941620ba6d3f2a
+  metadata.gz: c73174238cf469f8b78f7e7225d35e2ae9c5d0c0
+  data.tar.gz: c3cb6c769f05c1ed0d636b84232018e97a0a955b
 SHA512:
-  metadata.gz: c297663fe08c2c1a83d68942facc2c00801fceab5acdf90a12dcef36b37eac272e10964ddf4f2dd6d869adab6cdeca9bc8925d763f51a6b27feac5c43b78ea65
-  data.tar.gz: ea1a1b4a7c3df3a6f61cb52f2487050d1f2b2a8abf1bf2616b9ee97b2f1d493e2a3b6135f1e32e3d5a0edf40d2a135615be309294e2bc7bdfd6a15c1808a8238
+  metadata.gz: 1c17333b98eb2d249a909d4b7603718090a8d0cd4fc8bb087b7c4c3ad712cf05c7e2db4c51bd0947004fc40d0f0052e3eeac33bb569116e16031d5a0aa256809
+  data.tar.gz: c439b2981678bcb6698173e535f63baa72411f78e1099b89888a25bd96c758f8d9947622b78cc2506b9369d1086da9598a59a462fa5d15ce6c5569fd06602dca

data/.travis.yml

@@ -8,7 +8,7 @@ sudo: false
 install:
   - pip install --user sphinx
 script:
-  - ./gradlew check rubyTest
+  - ./gradlew --info check rubyTest
 after_success:
   - PATH="$HOME/.local/bin:$PATH" ./embulk-docs/push-gh-pages.sh
 env:

data/README.md

@@ -126,7 +126,7 @@ If the transaction fails, embulk stores state some states to the yaml file. You
 embulk run config.yml -r resume-state.yml
 ```
 
-If you giveup to resume the transaction, you can use `embulk cleanup` subcommand to delete intermediate data:
+If you give up on resuming the transaction, you can use `embulk cleanup` subcommand to delete intermediate data:
 
 ```
 embulk cleanup config.yml -r resume-state.yml

data/build.gradle

@@ -11,7 +11,7 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards")]
 
 allprojects {
     group = 'org.embulk'
-    version = '0.6.11'
+    version = '0.6.12'
 
     ext {
         jrubyVersion = '1.7.19'

data/embulk-core/build.gradle

@@ -21,9 +21,8 @@ dependencies {
     compile 'com.fasterxml.jackson.datatype:jackson-datatype-guava:2.5.3'
     compile 'com.fasterxml.jackson.datatype:jackson-datatype-joda:2.5.3'
     compile 'com.fasterxml.jackson.module:jackson-module-guice:2.5.3'
-    compile 'log4j:log4j:1.2.17'
+    compile 'ch.qos.logback:logback-classic:1.1.3'
     compile 'org.slf4j:slf4j-api:1.7.10'
-    compile 'org.slf4j:slf4j-log4j12:1.7.10'
     compile 'org.jruby:jruby-complete:' + project.jrubyVersion
     compile 'com.google.code.findbugs:annotations:3.0.0'
     compile 'org.yaml:snakeyaml:1.14'
@@ -32,6 +31,7 @@ dependencies {
     compile 'io.airlift:slice:0.9'
     compile 'joda-time:joda-time:2.7'
     compile 'io.netty:netty-buffer:5.0.0.Alpha1'
+    compile 'org.fusesource.jansi:jansi:1.11'
 
     // for embulk/guess/charset.rb
     compile 'com.ibm.icu:icu4j:54.1.1'

data/embulk-core/src/main/java/org/embulk/exec/GuessExecutor.java

@@ -139,7 +139,7 @@ public class GuessExecutor
             ConfigSource originalConfig = config.deepCopy().merge(lastGuessed);
             ConfigSource guessInputConfig = originalConfig.deepCopy();
             guessInputConfig.getNestedOrSetEmpty("parser")
-                .set("type", "system_guess")  // override in.parser.type so that FileInputPlugin creates GuessParserPlugin
+                .set("type", "system_guess")  // override in.parser.type so that FileInputRunner.run uses GuessParserPlugin
                 .set("guess_plugins", guessPlugins)
                 .set("orig_config", originalConfig);
 
@@ -153,7 +153,6 @@ public class GuessExecutor
                         if (taskCount == 0) {
                             throw new NoSampleException("No input files to guess");
                         }
-                        // TODO repeat runwith taskIndex++ if NoSampleException happens
                         input.run(inputTaskSource, null, 0, new PageOutput() {
                             @Override
                             public void add(Page page)
@@ -170,6 +169,7 @@ public class GuessExecutor
                         throw new AssertionError("Guess executor must throw GuessedNoticeError");
                     }
                 });
+
                 throw new AssertionError("Guess executor must throw GuessedNoticeError");
 
             } catch (GuessedNoticeError error) {
@@ -319,6 +319,7 @@ public class GuessExecutor
 
         private static Buffer getFirstBuffer(FileInput input)
         {
+            // The first buffer is created by SamplingParserPlugin. See FileInputRunner.guess.
             RuntimeException decodeException = null;
             try {
                 while (input.nextFile()) {

data/embulk-core/src/main/java/org/embulk/exec/LoggerProvider.java

@@ -3,7 +3,11 @@ package org.embulk.exec;
 import java.util.Properties;
 import org.slf4j.ILoggerFactory;
 import org.slf4j.LoggerFactory;
-import org.apache.log4j.PropertyConfigurator;
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.Logger;
+import ch.qos.logback.classic.LoggerContext;
+import ch.qos.logback.classic.joran.JoranConfigurator;
+import ch.qos.logback.core.joran.spi.JoranException;
 import com.google.inject.Inject;
 import com.google.inject.Provider;
 import org.embulk.config.ConfigSource;
@@ -14,13 +18,9 @@ public class LoggerProvider
     @Inject
     public LoggerProvider(@ForSystemConfig ConfigSource systemConfig)
     {
-        // TODO system config
-        Properties prop = new Properties();
-
         final String level;
         String logLevel = systemConfig.get(String.class, "log_level", "info");  // here can't use loadConfig because ModelManager uses LoggerProvider
         switch (logLevel) {
-        case "fatal": level = "FATAL"; break;
         case "error": level = "ERROR"; break;
         case "warn":  level = "WARN";  break;
         case "info":  level = "INFO";  break;
@@ -28,16 +28,30 @@ public class LoggerProvider
         case "trace": level = "TRACE"; break;
         default:
             throw new IllegalArgumentException(String.format(
-                        "System property embulk.logLevel=%s is invalid. Available levels are fatal, error, warn, info, debug and trace.", logLevel));
+                        "System property embulk.logLevel=%s is invalid. Available levels are error, warn, info, debug and trace.", logLevel));
         }
 
-        prop.setProperty("log4j.rootLogger", level+",root");
-        prop.setProperty("log4j.appender.root", "org.apache.log4j.ConsoleAppender");
-        prop.setProperty("log4j.appender.root.layout", "org.apache.log4j.PatternLayout");
-        prop.setProperty("log4j.appender.root.layout.ConversionPattern", "%d{yyyy-MM-dd HH:mm:ss.SSS Z} [%p] (%t): %m%n");
+        LoggerContext context = (LoggerContext) LoggerFactory.getILoggerFactory();
+        JoranConfigurator configurator = new JoranConfigurator();
+        configurator.setContext(context);
+        context.reset();
 
-        // TODO
-        PropertyConfigurator.configure(prop);
+        String name;
+        if (System.console() != null) {
+            name = "/embulk/logback-color.xml";
+        } else {
+            name = "/embulk/logback-console.xml";
+        }
+        try {
+            configurator.doConfigure(getClass().getResource(name));
+        } catch (JoranException ex) {
+            throw new RuntimeException(ex);
+        }
+
+        org.slf4j.Logger logger = LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME);
+        if (logger instanceof Logger) {
+            ((Logger) logger).setLevel(Level.toLevel(level.toUpperCase(), Level.DEBUG));
+        }
     }
 
     public ILoggerFactory get()

data/embulk-core/src/main/java/org/embulk/exec/SamplingParserPlugin.java

@@ -2,6 +2,7 @@ package org.embulk.exec;
 
 import java.util.List;
 import com.google.inject.Inject;
+import com.google.common.base.Preconditions;
 import org.embulk.config.TaskSource;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.CommitReport;
@@ -24,31 +25,9 @@ import static org.embulk.spi.util.Inputs.each;
 public class SamplingParserPlugin
         implements ParserPlugin
 {
-    private final int maxSampleSize;
-
-    @Inject
-    public SamplingParserPlugin(@ForSystemConfig ConfigSource systemConfig)
-    {
-        this.maxSampleSize = 32*1024;  // TODO get sample syze from system config
-    }
-
-    @Override
-    public void transaction(ConfigSource config, ParserPlugin.Control control)
-    {
-        control.run(Exec.newTaskSource(), null);
-    }
-
-    @Override
-    public void run(TaskSource taskSource, Schema schema,
-            FileInput input, PageOutput output)
-    {
-        Buffer buffer = getSample(input, maxSampleSize);
-        throw new SampledNoticeError(buffer);
-    }
-
     public static Buffer runFileInputSampling(final FileInputRunner runner, ConfigSource inputConfig)
     {
-        // override in.parser.type so that FileInputRunner creates GuessParserPlugin
+        // override in.parser.type so that FileInputRunner creates SamplingParserPlugin
         ConfigSource samplingInputConfig = inputConfig.deepCopy();
         samplingInputConfig.getNestedOrSetEmpty("parser").set("type", "system_sampling");
         samplingInputConfig.set("decoders", null);
@@ -60,19 +39,24 @@ public class SamplingParserPlugin
                     if (taskCount == 0) {
                         throw new NoSampleException("No input files to read sample data");
                     }
-                    // TODO repeat runwith taskIndex++ if NoSampleException happens
-                    runner.run(taskSource, schema, 0, new PageOutput() {
-                        @Override
-                        public void add(Page page)
-                        {
-                            throw new RuntimeException("Input plugin must be a FileInputPlugin to guess parser configuration");  // TODO exception class
-                        }
+                    for (int taskIndex=0; taskIndex < taskCount; taskIndex++) {
+                        try {
+                            runner.run(taskSource, schema, taskIndex, new PageOutput() {
+                                @Override
+                                public void add(Page page)
+                                {
+                                    throw new RuntimeException("Input plugin must be a FileInputPlugin to guess parser configuration");  // TODO exception class
+                                }
 
-                        public void finish() { }
+                                public void finish() { }
 
-                        public void close() { }
-                    });
-                    throw new NoSampleException("No input files to guess parser configuration");
+                                public void close() { }
+                            });
+                        } catch (NotEnoughSampleError ex) {
+                            continue;
+                        }
+                    }
+                    throw new NoSampleException("All input files are smaller than minimum sampling size");  // TODO include minSampleSize in message
                 }
             });
             throw new AssertionError("SamplingParserPlugin must throw SampledNoticeError");
@@ -81,29 +65,6 @@ public class SamplingParserPlugin
         }
     }
 
-    private static Buffer getSample(FileInput fileInput, int maxSampleSize)
-    {
-        if (!fileInput.nextFile()) {
-            // no input files
-            return Buffer.EMPTY;
-        }
-
-        Buffer sample = Buffer.allocate(maxSampleSize);
-        int sampleSize = 0;
-
-        for (Buffer buffer : each(fileInput)) {
-            int size = Math.min(buffer.limit(), sample.capacity() - sampleSize);
-            sample.setBytes(sampleSize, buffer, 0, size);
-            sampleSize += size;
-            buffer.release();
-            if (sampleSize >= maxSampleSize) {
-                break;
-            }
-        }
-        sample.limit(sampleSize);
-        return sample;
-    }
-
     public static class SampledNoticeError
             extends Error
     {
@@ -119,4 +80,59 @@ public class SamplingParserPlugin
             return sample;
         }
     }
+
+    public static class NotEnoughSampleError
+            extends Error
+    { }
+
+    private final int minSampleSize;
+    private final int sampleSize;
+
+    @Inject
+    public SamplingParserPlugin(@ForSystemConfig ConfigSource systemConfig)
+    {
+        this.minSampleSize = 40;  // empty gzip file is 33 bytes. // TODO get sample size from system config
+        this.sampleSize = 32*1024;  // TODO get sample size from system config
+        Preconditions.checkArgument(minSampleSize < sampleSize, "minSampleSize must be smaller than sampleSize");
+    }
+
+    @Override
+    public void transaction(ConfigSource config, ParserPlugin.Control control)
+    {
+        control.run(Exec.newTaskSource(), null);
+    }
+
+    @Override
+    public void run(TaskSource taskSource, Schema schema,
+            FileInput input, PageOutput output)
+    {
+        Buffer buffer = readSample(input, sampleSize);
+        if (buffer.limit() < minSampleSize) {
+            throw new NotEnoughSampleError();
+        }
+        throw new SampledNoticeError(buffer);
+    }
+
+    private static Buffer readSample(FileInput fileInput, int sampleSize)
+    {
+        if (!fileInput.nextFile()) {
+            // no input files
+            return Buffer.EMPTY;
+        }
+
+        Buffer sample = Buffer.allocate(sampleSize);
+        int offset = 0;
+
+        for (Buffer buffer : each(fileInput)) {
+            int size = Math.min(buffer.limit(), sample.capacity() - offset);
+            sample.setBytes(offset, buffer, 0, size);
+            offset += size;
+            buffer.release();
+            if (offset >= sampleSize) {
+                break;
+            }
+        }
+        sample.limit(offset);
+        return sample;
+    }
 }

data/embulk-core/src/main/java/org/embulk/jruby/JRubyScriptingModule.java

@@ -4,6 +4,7 @@ import java.util.List;
 import java.util.ArrayList;
 import java.util.Set;
 import java.io.File;
+import org.slf4j.ILoggerFactory;
 import com.google.common.collect.ImmutableSet;
 import com.google.inject.Module;
 import com.google.inject.Binder;
@@ -98,6 +99,14 @@ public class JRubyScriptingModule
             // load embulk.rb
             jruby.runScriptlet("require 'embulk'");
 
+            // initialize logger
+            jruby.callMethod(
+                    jruby.runScriptlet("Embulk"),
+                    "logger=",
+                        jruby.callMethod(
+                            jruby.runScriptlet("Embulk::Logger"),
+                            "new", injector.getInstance(ILoggerFactory.class).getLogger("ruby")));
+
             return jruby;
         }
 

data/embulk-core/src/main/java/org/embulk/spi/Exec.java

@@ -89,7 +89,7 @@ public class Exec
         return session().newTaskSource();
     }
 
-    public boolean isPreview()
+    public static boolean isPreview()
     {
         return session().isPreview();
     }

data/embulk-core/src/main/java/org/embulk/spi/time/TimestampParseException.java

@@ -3,4 +3,8 @@ package org.embulk.spi.time;
 public class TimestampParseException
         extends Exception
 {
+    public TimestampParseException(String message)
+    {
+        super(message);
+    }
 }

data/embulk-core/src/main/java/org/embulk/spi/time/TimestampParser.java

@@ -54,7 +54,7 @@ public class TimestampParser
             // TODO cache parsed zone?
             timeZone = parseDateTimeZone(zone);
             if (timeZone == null) {
-                throw new TimestampParseException();
+                throw new TimestampParseException("Invalid time zone name '" + text + "'");
             }
         }
 

data/embulk-core/src/main/resources/embulk/logback-color.xml

@@ -0,0 +1,72 @@
+<configuration>
+  <appender name="console-error" class="ch.qos.logback.core.ConsoleAppender">
+    <filter class="ch.qos.logback.classic.filter.LevelFilter">
+      <level>ERROR</level>
+      <onMatch>ACCEPT</onMatch>
+      <onMismatch>DENY</onMismatch>
+    </filter>
+    <withJansi>true</withJansi>
+    <encoder>
+      <pattern>%magenta(%d{yyyy-MM-dd HH:mm:ss.SSS Z} [%level] (%thread\): %m%n)</pattern>
+    </encoder>
+  </appender>
+
+  <appender name="console-warn" class="ch.qos.logback.core.ConsoleAppender">
+    <filter class="ch.qos.logback.classic.filter.LevelFilter">
+      <level>WARN</level>
+      <onMatch>ACCEPT</onMatch>
+      <onMismatch>DENY</onMismatch>
+    </filter>
+    <withJansi>true</withJansi>
+    <encoder>
+      <pattern>%yellow(%d{yyyy-MM-dd HH:mm:ss.SSS Z} [%level] (%thread\): %m%n)</pattern>
+    </encoder>
+  </appender>
+
+  <appender name="console-info" class="ch.qos.logback.core.ConsoleAppender">
+    <filter class="ch.qos.logback.classic.filter.LevelFilter">
+      <level>INFO</level>
+      <onMatch>ACCEPT</onMatch>
+      <onMismatch>DENY</onMismatch>
+    </filter>
+    <withJansi>true</withJansi>
+    <encoder>
+      <pattern>%green(%d{yyyy-MM-dd HH:mm:ss.SSS Z} [%level] (%thread\): %m%n)</pattern>
+    </encoder>
+  </appender>
+
+  <appender name="console-debug" class="ch.qos.logback.core.ConsoleAppender">
+    <filter class="ch.qos.logback.classic.filter.LevelFilter">
+      <level>DEBUG</level>
+      <onMatch>ACCEPT</onMatch>
+      <onMismatch>DENY</onMismatch>
+    </filter>
+    <withJansi>true</withJansi>
+    <encoder>
+      <pattern>%white(%d{yyyy-MM-dd HH:mm:ss.SSS Z} [%level] (%thread\): %m%n)</pattern>
+    </encoder>
+  </appender>
+
+  <appender name="console-trace" class="ch.qos.logback.core.ConsoleAppender">
+    <filter class="ch.qos.logback.classic.filter.LevelFilter">
+      <level>TRACE</level>
+      <onMatch>ACCEPT</onMatch>
+      <onMismatch>DENY</onMismatch>
+    </filter>
+    <withJansi>true</withJansi>
+    <encoder>
+      <pattern>%blue(%d{yyyy-MM-dd HH:mm:ss.SSS Z} [%level] (%thread\): %m%n)</pattern>
+    </encoder>
+  </appender>
+
+  <logger name="io.netty.util" level="INFO"/>
+  <logger name="io.netty.buffer" level="INFO"/>
+
+  <root>
+    <appender-ref ref="console-error"/>
+    <appender-ref ref="console-warn"/>
+    <appender-ref ref="console-info"/>
+    <appender-ref ref="console-debug"/>
+    <appender-ref ref="console-trace"/>
+  </root>
+</configuration>

data/embulk-core/src/main/resources/embulk/logback-console.xml

@@ -0,0 +1,14 @@
+<configuration>
+  <appender name="console" class="ch.qos.logback.core.ConsoleAppender">
+    <encoder>
+      <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS Z} [%level] (%thread\): %m%n</pattern>
+    </encoder>
+  </appender>
+
+  <logger name="io.netty.util" level="INFO"/>
+  <logger name="io.netty.buffer" level="INFO"/>
+
+  <root>
+    <appender-ref ref="console"/>
+  </root>
+</configuration>

data/embulk-core/src/main/resources/embulk/logback-file.xml

@@ -0,0 +1,25 @@
+<configuration>
+  <property name="embulk.logFile" value="embulk.log" />
+
+  <appender name="file" class="ch.qos.logback.core.rolling.RollingFileAppender">
+    <file>${embulk.logFile}</file>
+    <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
+      <maxFileSize>5kB</maxFileSize>
+    </triggeringPolicy>
+    <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
+      <fileNamePattern>${embulk.logFile}.%i</fileNamePattern>
+      <minIndex>1</minIndex>
+      <maxIndex>5</maxIndex>
+    </rollingPolicy>
+    <encoder>
+      <pattern>%d{yyyy-MM-dd HH:mm:ss.SSS Z} [%level] (%thread\): %m%n</pattern>
+    </encoder>
+  </appender>
+
+  <logger name="io.netty.util" level="INFO"/>
+  <logger name="io.netty.buffer" level="INFO"/>
+
+  <root>
+    <appender-ref ref="file"/>
+  </root>
+</configuration>

data/embulk-docs/src/release.rst

@@ -4,6 +4,7 @@ Release Notes
 .. toctree::
     :maxdepth: 1
 
+    release/release-0.6.12
     release/release-0.6.11
     release/release-0.6.10
     release/release-0.6.9

data/embulk-docs/src/release/release-0.6.12.rst

@@ -0,0 +1,31 @@
+Release 0.6.12
+==================================
+
+Plugin API
+------------------
+
+* Plugins can use both ``config[:key]`` (``task[:key]``) and ``config['key']`` (``task['key']``).
+* Added ``Embulk.logger`` for Ruby plugins
+
+
+Built-in plugins
+------------------
+
+* ``guess-csv`` plugin does not raise exceptions when a file has only 1 line
+* ``parser-csv`` shows number of lines using 1-origin indexing rather than 0-origin
+
+
+General Changes
+------------------
+
+* Guessing skips files smaller than 40 bytes so that guessing works when there is an empty file at the beginning.
+* Uses ANSI color for log messages
+* Uses logback instead of log4j for logging backend
+* Added ``spi.Exec.isPreview()`` as an alias of ``spi.Exec.session().isPreview()``
+* Plugin template for Ruby uses Xxx instead of XxxInputPlugin for the class name so that class name matches with file name.
+* When parsing timestamp fails, exception message includes the original text (@yyamano++)
+
+
+Release Date
+------------------
+2015-06-22

data/embulk-standards/src/main/java/org/embulk/standards/CsvTokenizer.java

@@ -53,7 +53,8 @@ public class CsvTokenizer
 
     public long getCurrentLineNumber()
     {
-        return lineNumber;
+        // returns actual line number. Internally, lineNumber starts at 0.
+        return lineNumber + 1;
     }
 
     // returns skipped line

data/lib/embulk.rb

@@ -9,8 +9,9 @@ module Embulk
     end
   end
 
+  require 'embulk/logger'
   require 'embulk/error'
-  require 'embulk/plugin'
   require 'embulk/buffer'
   require 'embulk/data_source'
+  require 'embulk/plugin'
 end

data/lib/embulk/command/embulk_new_plugin.rb

@@ -26,7 +26,7 @@ module Embulk
       email = `git config user.email`.strip
       email = "YOUR_NAME" if email.empty?
 
-      ruby_class_name = name.split('-').map {|a| a.capitalize }.join + category.to_s.split('_').map {|a| a.capitalize }.join + "Plugin"
+      ruby_class_name = name.split('-').map {|a| a.capitalize }.join
       java_iface = category.to_s.split('_').map {|a| a.capitalize }.join
       java_class_name = name.split('-').map {|a| a.capitalize }.join + java_iface + "Plugin"
       display_name = name.split('-').map {|a| a.capitalize }.join(' ')

data/lib/embulk/command/embulk_run.rb

@@ -68,7 +68,7 @@ module Embulk
 
     when :run
       op.banner = "Usage: run <config.yml>"
-      op.on('-l', '--log-level LEVEL', 'Log level (fatal, error, warn, info, debug or trace)') do |level|
+      op.on('-l', '--log-level LEVEL', 'Log level (error, warn, info, debug or trace)') do |level|
         options[:logLevel] = level
       end
       op.on('-L', '--load PATH', 'Add a local plugin path') do |plugin_path|
@@ -90,7 +90,7 @@ module Embulk
 
     when :cleanup
       op.banner = "Usage: cleanup <config.yml>"
-      op.on('-l', '--log-level LEVEL', 'Log level (fatal, error, warn, info, debug or trace)') do |level|
+      op.on('-l', '--log-level LEVEL', 'Log level (error, warn, info, debug or trace)') do |level|
         options[:logLevel] = level
       end
       op.on('-L', '--load PATH', 'Add a local plugin path') do |plugin_path|
@@ -109,7 +109,7 @@ module Embulk
 
     when :preview
       op.banner = "Usage: preview <config.yml>"
-      op.on('-l', '--log-level LEVEL', 'Log level (fatal, error, warn, info, debug or trace)') do |level|
+      op.on('-l', '--log-level LEVEL', 'Log level (error, warn, info, debug or trace)') do |level|
         options[:logLevel] = level
       end
       op.on('-L', '--load PATH', 'Add a local plugin path') do |plugin_path|
@@ -128,7 +128,7 @@ module Embulk
 
     when :guess
       op.banner = "Usage: guess <partial-config.yml>"
-      op.on('-l', '--log-level LEVEL', 'Log level (fatal, error, warn, info, debug or trace)') do |level|
+      op.on('-l', '--log-level LEVEL', 'Log level (error, warn, info, debug or trace)') do |level|
         options[:logLevel] = level
       end
       op.on('-o', '--output PATH', 'Path to a file to write the guessed configuration') do |path|

data/lib/embulk/data_source.rb

@@ -1,7 +1,141 @@
 module Embulk
   require 'json'
 
+  module Impl
+    # copied from https://github.com/intridea/hashie/blob/da232547c29673a0d7a79c7bf2670f1ea76813ed/lib/hashie/extensions/indifferent_access.rb
+    module IndifferentAccess
+      def self.included(base)
+        #Hashie::Extensions::Dash::IndifferentAccess::ClassMethods.tap do |extension|
+        #  base.extend(extension) if base <= Hashie::Dash && !base.singleton_class.included_modules.include?(extension)
+        #end
+
+        base.class_eval do
+          alias_method :regular_writer, :[]= unless method_defined?(:regular_writer)
+          alias_method :[]=, :indifferent_writer
+          alias_method :store, :indifferent_writer
+          %w(default update replace fetch delete key? values_at).each do |m|
+            alias_method "regular_#{m}", m unless method_defined?("regular_#{m}")
+            alias_method m, "indifferent_#{m}"
+          end
+
+          %w(include? member? has_key?).each do |key_alias|
+            alias_method key_alias, :indifferent_key?
+          end
+
+          class << self
+            def [](*)
+              super.convert!
+            end
+
+            def try_convert(*)
+              (hash = super) && self[hash]
+            end
+          end
+        end
+      end
+
+      def self.inject!(hash)
+        (class << hash; self; end).send :include, IndifferentAccess
+        hash.convert!
+      end
+
+      # Injects indifferent access into a duplicate of the hash
+      # provided. See #inject!
+      def self.inject(hash)
+        inject!(hash.dup)
+      end
+
+      def convert_key(key)
+        key.to_s
+      end
+
+      def convert!
+        keys.each do |k|
+          regular_writer convert_key(k), indifferent_value(regular_delete(k))
+        end
+        self
+      end
+
+      def indifferent_value(value)
+        if hash_lacking_indifference?(value)
+          IndifferentAccess.inject!(value)
+        elsif value.is_a?(::Array)
+          value.replace(value.map { |e| indifferent_value(e) })
+        else
+          value
+        end
+      end
+
+      def indifferent_default(key = nil)
+        return self[convert_key(key)] if key?(key)
+        regular_default(key)
+      end
+
+      def indifferent_update(other_hash)
+        return regular_update(other_hash) if hash_with_indifference?(other_hash)
+        other_hash.each_pair do |k, v|
+          self[k] = v
+        end
+      end
+
+      def indifferent_writer(key, value)
+        regular_writer convert_key(key), indifferent_value(value)
+      end
+
+      def indifferent_fetch(key, *args, &block)
+        regular_fetch convert_key(key), *args, &block
+      end
+
+      def indifferent_delete(key)
+        regular_delete convert_key(key)
+      end
+
+      def indifferent_key?(key)
+        regular_key? convert_key(key)
+      end
+
+      def indifferent_values_at(*indices)
+        indices.map { |i| self[i] }
+      end
+
+      def indifferent_access?
+        true
+      end
+
+      def indifferent_replace(other_hash)
+        (keys - other_hash.keys).each { |key| delete(key) }
+        other_hash.each { |key, value| self[key] = value }
+        self
+      end
+
+      protected
+
+      def hash_lacking_indifference?(other)
+        other.is_a?(::Hash) &&
+          !(other.respond_to?(:indifferent_access?) &&
+            other.indifferent_access?)
+      end
+
+      def hash_with_indifference?(other)
+        other.is_a?(::Hash) &&
+          other.respond_to?(:indifferent_access?) &&
+          other.indifferent_access?
+      end
+    end
+  end
+
   class DataSource < Hash
+    include Impl::IndifferentAccess
+
+    def initialize(hash={}, default=nil, &block)
+      if default.nil?
+        super(&block)
+      else
+        super(default)
+      end
+      hash.each {|key,value| self[key] = value }
+    end
+
     def param(key, type, options={})
       if self.has_key?(key)
         v = self[key]

data/lib/embulk/guess/csv.rb

@@ -68,7 +68,7 @@ module Embulk
         comment_line_marker, sample_records = guess_comment_line_marker(sample_records)
 
         first_types = SchemaGuess.types_from_array_records(sample_records[0, 1])
-        other_types = SchemaGuess.types_from_array_records(sample_records[1..-1])
+        other_types = SchemaGuess.types_from_array_records(sample_records[1..-1] || [])
 
         if first_types.size <= 1 || other_types.size <= 1
           # guess failed

data/lib/embulk/java/time_helper.rb

@@ -26,7 +26,7 @@ module Embulk
       def strptimeUsec(text)
         hash = Date._strptime(text, @format_string)
         unless hash
-          raise Java::TimestampParseException.new
+          raise Java::TimestampParseException.new("Failed to parse '" + text + "'")
         end
 
         if seconds = hash[:seconds]

data/lib/embulk/logger.rb

@@ -0,0 +1,152 @@
+
+module Embulk
+  require 'logger'
+
+  class Logger
+    def initialize(*args)
+      if args.length == 1
+        a = args[0]
+        if a.is_a?(Adapter)
+          @logger = a
+        elsif a.is_a?(::Logger)
+          @logger = StandardLoggerAdapter.new(a)
+        elsif Embulk.java? && (org.slf4j.Logger rescue nil) && a.is_a?(org.slf4j.Logger)
+          @logger = Slf4jAdapter.new(a)
+        else
+          @logger = StandardLoggerAdapter.new(*args)
+        end
+      else
+        @logger = StandardLoggerAdapter.new(*args)
+      end
+    end
+
+    module Adapter
+    end
+
+    def error(message=nil, &block) @logger.error(message, &block) end
+    def warn(message=nil, &block) @logger.warn(message, &block) end
+    def info(message=nil, &block) @logger.info(message, &block) end
+    def debug(message=nil, &block) @logger.debug(message, &block) end
+    def trace(message=nil, &block) @logger.trace(message, &block) end
+
+    def error?() @logger.error? end
+    def warn?() @logger.warn? end
+    def info?() @logger.info? end
+    def debug?() @logger.debug? end
+    def trace?() @logger.trace? end
+  end
+
+  class StandardLoggerAdapter < ::Logger
+    include Logger::Adapter
+
+    def initialize(*args)
+      super
+      if Embulk.java?
+        self.formatter = lambda do |severity,datetime,progname,message|
+          "#{datetime.strftime("%Y-%m-%d %H:%M:%S.%3N %z")} [#{severity}] (#{java.lang.Thread.currentThread.name}): #{message}\n"
+        end
+      else
+        self.formatter = lambda do |severity,datetime,progname,message|
+          "#{datetime.strftime("%Y-%m-%d %H:%M:%S.%3N %z")} [#{severity}]: #{message}\n"
+        end
+      end
+    end
+
+    def trace(message, &block)
+      debug(message, &block)
+    end
+
+    def trace?
+      debug?
+    end
+  end
+
+  class Slf4jAdapter
+    include Logger::Adapter
+
+    def initialize(logger)
+      @logger = logger
+    end
+
+    def error(message, &block)
+      if block
+        if @logger.isErrorEnabled
+          @logger.error(block.call)
+        end
+      else
+        @logger.error(message)
+      end
+    end
+
+    def warn(message, &block)
+      if block
+        if @logger.isWarnEnabled
+          @logger.warn(block.call)
+        end
+      else
+        @logger.warn(message)
+      end
+    end
+
+    def info(message, &block)
+      if block
+        if @logger.isInfoEnabled
+          @logger.info(block.call)
+        end
+      else
+        @logger.info(message)
+      end
+    end
+
+    def debug(message, &block)
+      if block
+        if @logger.isDebugEnabled
+          @logger.debug(block.call)
+        end
+      else
+        @logger.debug(message)
+      end
+    end
+
+    def trace(message, &block)
+      if block
+        if @logger.isTraceEnabled
+          @logger.trace(block.call)
+        end
+      else
+        @logger.trace(message)
+      end
+    end
+
+    def fatal?
+      @logger.isErrorEnabled()
+    end
+
+    def error?
+      @logger.isErrorEnabled()
+    end
+
+    def warn?
+      @logger.isWarnEnabled()
+    end
+
+    def debug?
+      @logger.isDebugEnabled()
+    end
+
+    def trace?
+      @logger.isTraceEnabled()
+    end
+  end
+
+  def self.logger
+    @@logger
+  end
+
+  def self.logger=(logger)
+    @@logger = logger
+  end
+
+  # default logger
+  @@logger = Logger.new(STDOUT)
+end

data/lib/embulk/plugin_registry.rb

@@ -1,6 +1,7 @@
 
 module Embulk
   require 'embulk/error'
+  require 'embulk/logger'
 
   class PluginRegistry
     def initialize(category, search_prefix)
@@ -80,10 +81,9 @@ module Embulk
     end
 
     def show_loaded_gems
-      # TODO use logger
       Gem.loaded_specs.each do |name,spec|
         if !@loaded_gems[name] && name =~ /^embulk/
-          puts "#{Time.now.strftime("%Y-%m-%d %H:%M:%S.%3N %z")}: Loaded plugin #{name} (#{spec.version})"
+          Embulk.logger.info "Loaded plugin #{name} (#{spec.version})"
           @loaded_gems[name] = true
         end
       end

data/lib/embulk/version.rb

@@ -1,3 +1,3 @@
 module Embulk
-  VERSION = '0.6.11'
+  VERSION = '0.6.12'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk
 version: !ruby/object:Gem::Version
-  version: 0.6.11
+  version: 0.6.12
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-05-31 00:00:00.000000000 Z
+date: 2015-06-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -232,6 +232,9 @@ files:
 - embulk-core/src/main/java/org/embulk/spi/util/Pages.java
 - embulk-core/src/main/java/org/embulk/spi/util/ResumableInputStream.java
 - embulk-core/src/main/java/org/embulk/spi/util/RetryExecutor.java
+- embulk-core/src/main/resources/embulk/logback-color.xml
+- embulk-core/src/main/resources/embulk/logback-console.xml
+- embulk-core/src/main/resources/embulk/logback-file.xml
 - embulk-core/src/test/java/org/embulk/EmbulkTestRuntime.java
 - embulk-core/src/test/java/org/embulk/GuiceBinder.java
 - embulk-core/src/test/java/org/embulk/RandomManager.java
@@ -297,6 +300,7 @@ files:
 - embulk-docs/src/release/release-0.6.1.rst
 - embulk-docs/src/release/release-0.6.10.rst
 - embulk-docs/src/release/release-0.6.11.rst
+- embulk-docs/src/release/release-0.6.12.rst
 - embulk-docs/src/release/release-0.6.2.rst
 - embulk-docs/src/release/release-0.6.3.rst
 - embulk-docs/src/release/release-0.6.4.rst
@@ -394,6 +398,7 @@ files:
 - lib/embulk/java/imports.rb
 - lib/embulk/java/time_helper.rb
 - lib/embulk/java_plugin.rb
+- lib/embulk/logger.rb
 - lib/embulk/output_plugin.rb
 - lib/embulk/page.rb
 - lib/embulk/page_builder.rb
@@ -412,8 +417,8 @@ files:
 - classpath/bval-jsr303-0.5.jar
 - classpath/commons-beanutils-core-1.8.3.jar
 - classpath/commons-lang3-3.1.jar
-- classpath/embulk-core-0.6.11.jar
-- classpath/embulk-standards-0.6.11.jar
+- classpath/embulk-core-0.6.12.jar
+- classpath/embulk-standards-0.6.12.jar
 - classpath/guava-18.0.jar
 - classpath/guice-4.0.jar
 - classpath/guice-multibindings-4.0.jar
@@ -424,14 +429,15 @@ files:
 - classpath/jackson-datatype-guava-2.5.3.jar
 - classpath/jackson-datatype-joda-2.5.3.jar
 - classpath/jackson-module-guice-2.5.3.jar
+- classpath/jansi-1.11.jar
 - classpath/javax.inject-1.jar
 - classpath/joda-time-2.7.jar
 - classpath/jruby-complete-1.7.19.jar
-- classpath/log4j-1.2.17.jar
+- classpath/logback-classic-1.1.3.jar
+- classpath/logback-core-1.1.3.jar
 - classpath/netty-buffer-5.0.0.Alpha1.jar
 - classpath/netty-common-5.0.0.Alpha1.jar
 - classpath/slf4j-api-1.7.10.jar
-- classpath/slf4j-log4j12-1.7.10.jar
 - classpath/slice-0.9.jar
 - classpath/snakeyaml-1.14.jar
 - classpath/validation-api-1.1.0.Final.jar