embulk 0.4.7 → 0.4.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2f4db59dbfae683eff9877c8794fab684f562224
-  data.tar.gz: 7969e968bae40633fb8c3d70a9ed2f80a0ee4a37
+  metadata.gz: ae306d06b0e181d289d3ff6d1117f0275d53d021
+  data.tar.gz: 5aefdda58d812bf753d41a4133e9b5500a88d9d0
 SHA512:
-  metadata.gz: 592576af399e8f184b1b28dfa32c298fb7ff236f9d956a37e5587e073ee64af0280e481a4a568e114b04391e0cea6977b364f4d9ac03f0a1a194a12b3a7cd19e
-  data.tar.gz: f2238b2e4c0bb06c1ad33754b86f9944845565c75ea0f36fb801fd2c1e5b1a87255a5ddbad763dc2f72b36724692cd11cd862cb05ee64827655a7ca797df408c
+  metadata.gz: 7ef6c84e52dadc846e6f69e0836aaaf8b18c763be94a178ed597e18d07e6583b4533b16366933a798a7391b40f88b459258b0be0255c43ae2e39e2b3666fdcee
+  data.tar.gz: 1808ce0b33520fc998f5bb023ab1a1d109d87152dcdc1994c32becb770178d749a7fe27dfdc27b956f61a6e9c664baeec188f1ebb9fe6e36e6a82d8f5840177a

data/.travis.yml

@@ -1,6 +1,9 @@
 language: java
 jdk:
   - oraclejdk8
+cache:
+  directories:  # run "travis cache --delete" to delete caches
+    - $HOME/.gradle
 sudo: false
 install:
   - pip install --user sphinx

data/README.md

@@ -28,7 +28,7 @@ The single-file package is the simplest way to try Embulk. You can download the
 Following 4 commands install embulk to your home directory:
 
 ```
-curl --create-dirs -o ~/.embulk/bin/embulk -L https://bintray.com/artifact/download/embulk/maven/embulk-0.4.7.jar
+curl --create-dirs -o ~/.embulk/bin/embulk -L https://bintray.com/artifact/download/embulk/maven/embulk-0.4.8.jar
 chmod +x ~/.embulk/bin/embulk
 echo 'export PATH="$HOME/.embulk/bin:$PATH"' >> ~/.bashrc
 source ~/.bashrc
@@ -39,7 +39,7 @@ source ~/.bashrc
 You can assume the jar file is a .bat file.
 
 ```
-curl -o embulk.bat -L https://bintray.com/artifact/download/embulk/maven/embulk-0.4.5.jar
+curl -o embulk.bat -L https://bintray.com/artifact/download/embulk/maven/embulk-0.4.8.jar
 ```
 
 ### Trying examples

data/bin/embulk

@@ -1,28 +1,47 @@
 #!/usr/bin/env ruby
 
-# TODO cygwin check
-cygwin = false
-
 java_args = []
-while i = ARGV.find_index {|arg| arg =~ /^\-D/ }
-  if ARGV[i] == "-D"
-    prop_path = ARGV[i + 1]
+jruby_args = []
+default_optimize = false
+overwrite_optimize = nil
+
+until ARGV.empty?
+  v = ARGV[0]
+  case v
+  when "-J+O"
+    overwrite_optimize = true
+    ARGV.shift
+  when "-J-O"
+    overwrite_optimize = false
+    ARGV.shift
+  when "-J"
+    prop_path = ARGV[1]
     unless prop_path
-      STDERR.puts "-D option requires an option"
+      STDERR.puts "-J option requires an option"
       exit 1
     end
-    ARGV.slice!(i, 2)
+    props = File.read(prop_path)
+    java_props = props.split("\n").reject {|prop| prop.strip.empty? }
+    java_args = java_props + java_args
+    ARGV.shift
+    ARGV.shift
+  when /-J(.*)/
+    java_args << v[2..-1]
+    ARGV.shift
+  when /-R(.*)/
+    jruby_args << v[2..-1]
+    ARGV.shift
+  when "run"
+    default_optimize = true
+    ARGV.shift
+    break
   else
-    java_args << ARGV[i][2..-1]
-    ARGV.slice!(i)
+    break
   end
 end
 
-if prop_path
-  props = File.read(prop_path)
-  java_props = props.split("\n").reject {|prop| prop.strip.empty? }
-  java_args = java_props + java_args
-end
+# TODO cygwin check
+cygwin = false
 
 java_cmd = ENV['JAVACMD']
 unless java_cmd
@@ -50,10 +69,17 @@ unless jruby_complete
   raise SystemExit.new(1)
 end
 
+if overwrite_optimize == true || (default_optimize == true && overwrite_optimize != false)
+    java_args = %w[-XX:+AggressiveOpts -XX:+UseConcMarkSweepGC] + java_args
+else
+    java_args = %w[-XX:+AggressiveOpts -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -Xverify:none] + java_args
+end
+
 # java ... -jar ruby-complete.jar $EMBULK_HOME/lib/embulk/command/embulk.rb "$@"
 cmdline = [java_cmd]
 cmdline.concat java_args
 cmdline << '-jar' << File.join(classpath_dir, jruby_complete)
+cmdline.concat jruby_args
 cmdline << File.join(lib_dir, 'embulk/command/embulk.rb')
 cmdline.concat ARGV
 exec *cmdline

data/build.gradle

@@ -12,7 +12,7 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards")]
 
 allprojects {
     group = 'org.embulk'
-    version = '0.4.7'
+    version = '0.4.8'
 
     apply plugin: 'java'
     apply plugin: 'maven-publish'
@@ -244,8 +244,14 @@ task set_version << {
     File ruby_ver = file('lib/embulk/version.rb')
     ruby_ver.write(ruby_ver.getText().replaceFirst("VERSION = '(\\d+)(\\.\\d+){2}'", "VERSION = '${to}'"))
 
-    File readme_ver = file('README.md')
-    readme_ver.write(readme_ver.getText().replaceFirst('embulk-(\\d+)(\\.\\d+){2}', "embulk-${to}"))
+    List<String> docs = [
+        'README.md',
+        'embulk-docs/src/recipe/scheduled-csv-load-to-elasticsearch-kibana4.rst'
+    ]
+    docs.each() { path ->
+        File doc = file(path)
+        doc.write(doc.getText().replaceAll('embulk-(\\d+)(\\.\\d+){2}', "embulk-${to}"))
+    }
 
     file("embulk-docs/src/release/release-${to}.rst").append("")
     "git add embulk-docs/src/release/release-${to}.rst".execute().waitFor()

data/embulk-core/src/main/java/org/embulk/command/Runner.java

@@ -158,7 +158,7 @@ public class Runner
         ConfigSource resumeConfig = loadYamlConfig(resumePath);
         ResumeState resume = resumeConfig.loadConfig(ResumeState.class);
 
-        ExecSession exec = newExecSession(config);
+        //ExecSession exec = newExecSession(config);  // not necessary
         LocalExecutor local = injector.getInstance(LocalExecutor.class);
         local.cleanup(config, resume);
 

data/embulk-core/src/main/java/org/embulk/command/VerticalPreviewPrinter.java

@@ -14,7 +14,7 @@ public class VerticalPreviewPrinter
     public VerticalPreviewPrinter(PrintStream out, ModelManager modelManager, Schema schema)
     {
         super(out, modelManager, schema);
-        this.format = "%" + maxColumnNameLength(schema) + "s (%" + maxColumnTypeNameLength(schema)+ "s) : %s\n";
+        this.format = "%" + maxColumnNameLength(schema) + "s (%" + maxColumnTypeNameLength(schema)+ "s) : %s%n";
     }
 
     private static int maxColumnNameLength(Schema schema)
@@ -39,7 +39,7 @@ public class VerticalPreviewPrinter
     protected void printRecord(String[] values) throws IOException
     {
         count++;
-        out.format("*************************** %d ***************************\n", count);
+        out.format("*************************** %d ***************************%n", count);
         for (int i=0; i < schema.getColumnCount(); i++) {
             out.format(format, schema.getColumnName(i), schema.getColumnType(i), values[i]);
         }

data/embulk-core/src/main/java/org/embulk/spi/time/JRubyTimeParserHelper.java

@@ -2,7 +2,7 @@ package org.embulk.spi.time;
 
 public interface JRubyTimeParserHelper
 {
-    public long strptime(String text) throws TimestampParseException;
+    public long strptimeUsec(String text) throws TimestampParseException;
 
     public String getZone();
 }

data/embulk-core/src/main/java/org/embulk/spi/time/TimestampFormatter.java

@@ -48,7 +48,7 @@ public class TimestampFormatter
     public String format(Timestamp value)
     {
         // TODO optimize by using reused StringBuilder
-        dateFormat.setDateTime(new DateTime(value.toEpochMilli(), timeZone));
+        dateFormat.setDateTime(new DateTime(value.getEpochSecond()*1000, timeZone));
         dateFormat.setNSec(value.getNano());
         return dateFormat.format(null);
     }

data/embulk-core/src/main/java/org/embulk/spi/time/TimestampParser.java

@@ -41,7 +41,7 @@ public class TimestampParser
 
     public Timestamp parse(String text) throws TimestampParseException
     {
-        long localMillis = helper.strptime(text);
+        long localUsec = helper.strptimeUsec(text);
         String zone = helper.getZone();
 
         DateTimeZone timeZone = defaultTimeZone;
@@ -53,8 +53,10 @@ public class TimestampParser
             }
         }
 
-        long milli = timeZone.convertLocalToUTC(localMillis, false);
+        long localSec = localUsec / 1000000;
+        long usec = localUsec % 1000000;
+        long sec = timeZone.convertLocalToUTC(localSec*1000, false) / 1000;
 
-        return Timestamp.ofEpochMilli(milli);
+        return Timestamp.ofEpochSecond(sec, usec * 1000);
     }
 }

data/embulk-core/src/test/java/org/embulk/spi/time/TestTimestampFormatterParser.java

@@ -32,21 +32,36 @@ public class TestTimestampFormatterParser
     public void testSimpleFormat() throws Exception
     {
         ConfigSource config = Exec.newConfigSource()
-            .set("time_format", "%Y-%m-%d %H:%M:%S %Z");
+            .set("time_format", "%Y-%m-%d %H:%M:%S.%6N %Z");
         FormatterTestTask task = config.loadConfig(FormatterTestTask.class);
 
         TimestampFormatter formatter = task.getTimeFormat().newFormatter(task);
-        assertEquals("2014-11-19 02:46:29 UTC", formatter.format(Timestamp.ofEpochSecond(1416365189)));
+        assertEquals("2014-11-19 02:46:29.123456 UTC", formatter.format(Timestamp.ofEpochSecond(1416365189, 123456*1000)));
     }
 
     @Test
     public void testSimpleParse() throws Exception
     {
         ConfigSource config = Exec.newConfigSource()
-            .set("time_format", "%Y-%m-%d %H:%M:%S %Z");
+            .set("time_format", "%Y-%m-%d %H:%M:%S.%N %Z");
         ParserTestTask task = config.loadConfig(ParserTestTask.class);
 
         TimestampParser parser = task.getTimeFormat().newParser(task);
-        assertEquals(Timestamp.ofEpochSecond(1416365189), parser.parse("2014-11-19 02:46:29 UTC"));
+        assertEquals(Timestamp.ofEpochSecond(1416365189, 123456*1000), parser.parse("2014-11-19 02:46:29.123456 UTC"));
+    }
+
+    @Test
+    public void testUnixtimeFormat() throws Exception
+    {
+        ConfigSource config = Exec.newConfigSource()
+            .set("time_format", "%s");
+
+        FormatterTestTask ftask = config.loadConfig(FormatterTestTask.class);
+        TimestampFormatter formatter = ftask.getTimeFormat().newFormatter(ftask);
+        assertEquals("1416365189", formatter.format(Timestamp.ofEpochSecond(1416365189)));
+
+        ParserTestTask ptask = config.loadConfig(ParserTestTask.class);
+        TimestampParser parser = ptask.getTimeFormat().newParser(ptask);
+        assertEquals(Timestamp.ofEpochSecond(1416365189), parser.parse("1416365189"));
     }
 }

data/embulk-docs/src/index.rst

@@ -11,6 +11,7 @@ https://github.com/embulk/embulk
 .. toctree::
    :maxdepth: 2
 
+   recipe
    release
 
 * `JavaDoc <javadoc/index.html>`_

data/embulk-docs/src/recipe.rst

@@ -0,0 +1,8 @@
+Recipes
+==================================
+
+.. toctree::
+    :maxdepth: 1
+
+    recipe/scheduled-csv-load-to-elasticsearch-kibana4
+

data/embulk-docs/src/recipe/scheduled-csv-load-to-elasticsearch-kibana4.rst

@@ -0,0 +1,153 @@
+Scheduled bulk data loading to Elasticsearch + Kibana 4 from CSV files
+==================================
+
+.. contents::
+   :local:
+   :depth: 2
+
+This article shows how to:
+
+* Bulk load CSV files to Elasticsearch.
+* Visualize the data with Kibana interactively.
+* Schedule the data loading every hour using cron.
+
+This guide assumes you are using Ubuntu 12.0 Precise or Mac OS X.
+
+Setup Elasticsearch and Kibana 4
+------------------
+
+Step 1. Download and start Elasticsearch.
+~~~~~~~~~~~~~~~~~~
+
+You can find releases from the `Elasticsearch website <http://www.elasticsearch.org/download/>`_.
+For the smallest setup, you can unzip the package and run `./bin/elasticsearch` command:
+
+.. code-block:: console
+
+    $ wget https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.4.4.zip
+    $ unzip elasticsearch-1.4.4.zip
+    $ cd elasticsearch-1.4.4
+    $ ./bin/elasticsearch
+
+Step 2. Download and unzip Kibana:
+~~~~~~~~~~~~~~~~~~
+
+You can find releases from the `Kibana website <http://www.elasticsearch.org/overview/kibana/installation/>`_. Open a new console and run following commands:
+
+.. code-block:: console
+
+    $ wget https://download.elasticsearch.org/kibana/kibana/kibana-4.0.0-linux-x64.tar.gz
+    $ tar zxvf kibana-4.0.0-linux-x64.tar.gz
+    $ cd kibana-4.0.0-linux-x64
+    $ ./bin/kibana
+
+Note: If you're using Mac OS X, https://download.elasticsearch.org/kibana/kibana/kibana-4.0.0-darwin-x64.tar.gz is the URL to download.
+
+Now Elasticsearch and Kibana started. Open http://localhost:5601/ using your browser to see the Kibana's graphical interface.
+
+
+Setup Embulk
+------------------
+
+Step 1. Download Embulk binary:
+~~~~~~~~~~~~~~~~~~
+
+You can find the latest embulk binary from the `releases <https://bintray.com/embulk/maven/embulk/view#files>`_. Because Embulk is a single executable binary, you can simply download it to /usr/local/bin directory and set executable flag as following:
+
+.. code-block:: console
+
+    $ sudo wget https://bintray.com/artifact/download/embulk/maven/embulk-0.4.8.jar -O /usr/local/bin/embulk
+    $ sudo chmod +x /usr/local/bin/embulk
+
+Step 2. Install Elasticsearch plugin
+~~~~~~~~~~~~~~~~~~
+
+You also need Elasticsearch plugin for Embulk. You can install the plugin with this command:
+
+.. code-block:: console
+
+    $ embulk gem install embulk-output-elasticsearch
+
+Embulk includes CSV file reader in itself. Now everything is ready to use.
+
+Loading a CSV file
+------------------
+
+Assuming you have a CSV files at ``./mydata/csv/`` directory. If you don't have CSV files, you can create ones using ``embulk example ./mydata`` command.
+
+Create this configuration file and save as ``config.yml``:
+
+.. code-block:: yaml
+
+    in:
+      type: file
+      path_prefix: ./mydata/csv/
+    out:
+      type: elasticsearch
+      index_name: embulk
+      index_type: embulk
+      nodes:
+        - host: localhost
+
+In fact, this configuration lacks some important information. However, embulk guesses the other information. So, next step is to order embulk to guess them:
+
+.. code-block:: console
+
+    $ embulk guess config.yml -o config-complete.yml
+
+The generated config-complete.yml file should include complete information as following:
+
+.. code-block:: yaml
+
+    in:
+      type: file
+      path_prefix: ./mydata/csv/
+      decoders:
+      - {type: gzip}
+      parser:
+        charset: UTF-8
+        newline: CRLF
+        type: csv
+        delimiter: ','
+        quote: '"'
+        escape: ''
+        null_string: 'NULL'
+        header_line: true
+        columns:
+        - {name: id, type: long}
+        - {name: account, type: long}
+        - {name: time, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
+        - {name: purchase, type: timestamp, format: '%Y%m%d'}
+        - {name: comment, type: string}
+    out:
+      type: elasticsearch
+      index_name: embulk
+      index_type: embulk
+      nodes:
+      - {host: localhost}
+
+Now, you can run the bulk loading:
+
+.. code-block:: console
+
+    $ embulk run config-complete.yml -o next-config.yml
+
+Scheduling loading by cron
+------------------
+
+At the last step, you ran embulk command with ``-o next-config.yml`` file. The ``next-config.yml`` file should include a parameter named ``last_path``:
+
+.. code-block:: yaml
+
+    last_path: mydata/csv/sample_01.csv.gz
+
+With this configuration, embulk loads the files newer than this file in alphabetical order.
+
+For example, if you create ``./mydata/csv/sample_02.csv.gz`` file, embulk skips ``sample_01.csv.gz`` file and loads ``sample_02.csv.gz`` only next time. And the next next-config.yml file has ``last_path: mydata/csv/sample_02.csv.gz`` for the next next execution.
+
+So, if you want to loads newly created files every day, you can setup this cron schedule:
+
+.. code-block:: cron
+
+    0 * * * * embulk run /path/to/next-config.yml -o /path/to/next-config.yml
+

data/embulk-docs/src/release.rst

@@ -18,4 +18,5 @@ Release Notes
     release/release-0.4.5
     release/release-0.4.6
     release/release-0.4.7
+    release/release-0.4.8
 

data/embulk-docs/src/release/release-0.4.8.rst

@@ -0,0 +1,15 @@
+Release 0.4.8
+==================================
+
+General Changes
+------------------
+
+* ``spi.time.TimestampParser`` supports microseconds.
+* ``spi.time.TimestampFormatter`` supports milliseconds, microseconds and nanoseconds.
+* Fixed a bug at ``spi.time.TimestampParser`` that returns wrong timestamp if format string is ``%s`` which means epoch seconds (@hiroyuki-sato++)
+* ``embulk`` command installed by ``gem install embulk`` sets JVM optimization flags. This behavior is same with embulk.jar updated since 0.4.6.
+* ``preview`` subcommand with ``-G, --vertical`` option uses CRLF on Windows
+
+Release Date
+------------------
+2015-02-24

data/embulk-standards/src/main/java/org/embulk/standards/LocalFileOutputPlugin.java

@@ -7,7 +7,10 @@ import java.io.IOException;
 import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Locale;
+import java.util.IllegalFormatException;
 import org.embulk.config.Config;
+import org.embulk.config.ConfigException;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.ConfigDiff;
@@ -46,7 +49,11 @@ public class LocalFileOutputPlugin
         PluginTask task = config.loadConfig(PluginTask.class);
 
         // validate sequence_format
-        String.format(task.getSequenceFormat(), 0, 0);
+        try {
+            String dontCare = String.format(Locale.ENGLISH, task.getSequenceFormat(), 0, 0);
+        } catch (IllegalFormatException ex) {
+            throw new ConfigException("Invalid sequence_format: parameter for file output plugin", ex);
+        }
 
         return resume(task.dump(), taskCount, control);
     }

data/lib/embulk/java/time_helper.rb

@@ -23,7 +23,7 @@ module Embulk
       end
 
       # Override
-      def strptime(text)
+      def strptimeUsec(text)
         hash = Date._strptime(text, @format_string)
         unless hash
           raise Java::TimestampParseException.new
@@ -64,7 +64,7 @@ module Embulk
 
           @zone = zone
           time = Time.utc(year, mon, day, hour, min, sec, usec)
-          return time.tv_sec * 1000 + time.tv_usec / 1000
+          return time.tv_sec * 1_000_000 + time.tv_usec
         end
       end
 

data/lib/embulk/version.rb

@@ -1,3 +1,3 @@
 module Embulk
-  VERSION = '0.4.7'
+  VERSION = '0.4.8'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk
 version: !ruby/object:Gem::Version
-  version: 0.4.7
+  version: 0.4.8
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-02-24 00:00:00.000000000 Z
+date: 2015-02-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -248,6 +248,8 @@ files:
 - embulk-docs/push-gh-pages.sh
 - embulk-docs/src/conf.py
 - embulk-docs/src/index.rst
+- embulk-docs/src/recipe.rst
+- embulk-docs/src/recipe/scheduled-csv-load-to-elasticsearch-kibana4.rst
 - embulk-docs/src/release.rst
 - embulk-docs/src/release/release-0.1.0.rst
 - embulk-docs/src/release/release-0.2.0.rst
@@ -263,6 +265,7 @@ files:
 - embulk-docs/src/release/release-0.4.5.rst
 - embulk-docs/src/release/release-0.4.6.rst
 - embulk-docs/src/release/release-0.4.7.rst
+- embulk-docs/src/release/release-0.4.8.rst
 - embulk-standards/build.gradle
 - embulk-standards/src/main/java/org/embulk/standards/CsvFormatterPlugin.java
 - embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java
@@ -363,8 +366,8 @@ files:
 - classpath/bval-jsr303-0.5.jar
 - classpath/commons-beanutils-core-1.8.3.jar
 - classpath/commons-lang3-3.1.jar
-- classpath/embulk-core-0.4.7.jar
-- classpath/embulk-standards-0.4.7.jar
+- classpath/embulk-core-0.4.8.jar
+- classpath/embulk-standards-0.4.8.jar
 - classpath/guava-18.0.jar
 - classpath/guice-3.0.jar
 - classpath/guice-multibindings-3.0.jar