embulk 0.8.18 → 0.8.19

data/embulk-core/src/main/java/org/embulk/spi/TempFileSpace.java

@@ -27,7 +27,8 @@ public class TempFileSpace
 
     public File createTempFile(String fileExt)
     {
-        return createTempFile(Thread.currentThread().getName()+"_", fileExt);
+        // Thread names contain ':' which is not valid as file names in Windows.
+        return createTempFile(Thread.currentThread().getName().replaceAll(":", "_") + "_", fileExt);
     }
 
     public File createTempFile(String prefix, String fileExt)

data/embulk-core/src/test/java/org/embulk/spi/TestPageBuilderReader.java

@@ -327,6 +327,68 @@ public class TestPageBuilderReader
                 ).size());
     }
 
+    @Test
+    public void testDoubleWriteStringsToRow()
+    {
+        MockPageOutput output = new MockPageOutput();
+        Schema schema = Schema.builder()
+                .add("col0", STRING)
+                .add("col1", STRING)
+                .add("col2", STRING)
+                .build();
+
+        builder = new PageBuilder(bufferAllocator, schema, output);
+        builder.setString(0, "v0");
+        builder.setString(1, "v1");
+        builder.setNull(2);
+        builder.setString(0, "v2"); // stored to page for col0
+        builder.setNull(1);         // null is stored to page for col1
+        builder.setString(2, "v3"); // stored to page for col2
+        builder.addRecord();
+        builder.finish();
+        builder.close();
+
+        reader = new PageReader(schema);
+        reader.setPage(output.pages.get(0));
+        assertTrue(reader.nextRecord());
+        assertEquals(reader.getString(0), "v2");
+        assertTrue(reader.isNull(1));
+        assertEquals(reader.getString(2), "v3");
+        assertFalse(reader.nextRecord());
+        reader.close();
+    }
+
+    @Test
+    public void testDoubleWriteJsonsToRow()
+    {
+        MockPageOutput output = new MockPageOutput();
+        Schema schema = Schema.builder()
+                .add("col0", JSON)
+                .add("col1", JSON)
+                .add("col2", JSON)
+                .build();
+
+        builder = new PageBuilder(bufferAllocator, schema, output);
+        builder.setJson(0, newString("v0"));
+        builder.setJson(1, newString("v1"));
+        builder.setNull(2);
+        builder.setJson(0, newString("v2")); // store to page for col0
+        builder.setNull(1);                  // null is stored to page for col1
+        builder.setJson(2, newString("v3")); // store to page for col2
+        builder.addRecord();
+        builder.finish();
+        builder.close();
+
+        reader = new PageReader(schema);
+        reader.setPage(output.pages.get(0));
+        assertTrue(reader.nextRecord());
+        assertEquals(reader.getJson(0), newString("v2"));
+        assertTrue(reader.isNull(1));
+        assertEquals(reader.getJson(2), newString("v3"));
+        assertFalse(reader.nextRecord());
+        reader.close();
+    }
+
     @Test
     public void testRepeatableClose()
     {

data/embulk-docs/src/built-in.rst

@@ -48,7 +48,7 @@ A configuration file consists of following sections:
 
 * **out:** Output plugin options. An output plugin is either record-based (`Oracle <https://github.com/embulk/embulk-output-jdbc>`_, `Elasticsearch <https://github.com/muga/embulk-output-elasticsearch>`_, etc) or file-based (`Google Cloud Storage <https://github.com/hakobera/embulk-output-gcs>`_, `Command <https://github.com/embulk/embulk-output-command>`_, etc)
 
-  * **formatter:** If the output is file-based, formatter plugin formats a file format (such as built-in csv, `JSON <https://github.com/takei-yuya/embulk-formatter-jsonl>`_)
+  * **formatter:** If the output is file-based, formatter plugin formats a file format (such as built-in csv, `jsonl <https://github.com/takei-yuya/embulk-formatter-jsonl>`_)
 
   * **encoder:** If the output is file-based, encoder plugin encodes compression or encryption (such as built-in gzip or bzip2)
 
@@ -95,7 +95,7 @@ Configuration file can include another configuration file. To use it, configurat
 
 File will be searched from the relative path of the input configuration file. And file name will be ``_<name>.yml.liquid``. For example, if you add ``{% include 'subdir/inc' %}`` tag to ``myconfig/config.yml.liquid`` file, it includes ``myconfig/subdir/_inc.yml.liquid`` file.
 
-.. code-block:: yaml
+.. code-block:: liquid
 
     # config.yml.liquid
     {% include 'in_mysql' %}
@@ -128,17 +128,19 @@ The ``file`` input plugin reads files from local file system.
 Options
 ~~~~~~~~
 
-+----------------+----------+------------------------------------------------+-----------+
-| name           | type     | description                                    | required? |
-+================+==========+================================================+===========+
-| path\_prefix   | string   | Path prefix of input files                     | required  |
-+----------------+----------+------------------------------------------------+-----------+
-| parsers        | hash     | Parsers configurations (see below)             | required  |
-+----------------+----------+------------------------------------------------+-----------+
-| decoders       | array    | Decoder configuration (see below)              |           |
-+----------------+----------+------------------------------------------------+-----------+
-| last\_path     | string   | Name of last read file in previous operation   |           |
-+----------------+----------+------------------------------------------------+-----------+
++------------------+----------+------------------------------------------------+-----------------------+
+| name             | type     | description                                    | required?             |
++==================+==========+================================================+=======================+
+| path\_prefix     | string   | Path prefix of input files                     | required              |
++------------------+----------+------------------------------------------------+-----------------------+
+| parsers          | hash     | Parsers configurations (see below)             | required              |
++------------------+----------+------------------------------------------------+-----------------------+
+| decoders         | array    | Decoder configuration (see below)              |                       |
++------------------+----------+------------------------------------------------+-----------------------+
+| last\_path       | string   | Name of last read file in previous operation   |                       |
++------------------+----------+------------------------------------------------+-----------------------+
+| follow\_symlinks | boolean  | If `true`, follow symbolic link directories    | ``false`` by default  |
++------------------+----------+------------------------------------------------+-----------------------+
 
 The ``path_prefix`` option is required. If you have files as following, you may set ``path_prefix: /path/to/files/sample_``:
 
@@ -322,7 +324,7 @@ JSON parser plugin
 
 The ``json`` parser plugin parses a JSON file that contains a sequence of JSON objects. Example:
 
-.. code-block:: json
+.. code-block:: javascript
 
     {"time":1455829282,"ip":"93.184.216.34","name":frsyuki}
     {"time":1455829282,"ip":"172.36.8.109":sadayuki}
@@ -913,18 +915,22 @@ The guess executor is called by ``guess`` command. It executes default guess plu
 Options
 ~~~~~~~~
 
-+-----------------------+----------+----------------------------------------------------------------------+--------------------------------------+
-| name                  | type     | description                                                          | required?                            |
-+=======================+==========+======================================================================+======================================+
-| guess_plugins         | array    | ``guess`` command uses specified guess plugins.                      | ``[]`` by default                    |
-+-----------------------+----------+----------------------------------------------------------------------+--------------------------------------+
-| exclude_guess_plugins | array    | ``guess`` command doesn't use specified plugins.                     | ``[]`` by default                    |
-+-----------------------+----------+----------------------------------------------------------------------+--------------------------------------+
++---------------------------+----------+----------------------------------------------------------------------+--------------------------------------+
+| name                      | type     | description                                                          | required?                            |
++===========================+==========+======================================================================+======================================+
+| guess_plugins             | array    | ``guess`` command uses specified guess plugins.                      | ``[]`` by default                    |
++---------------------------+----------+----------------------------------------------------------------------+--------------------------------------+
+| exclude_guess_plugins     | array    | ``guess`` command doesn't use specified plugins.                     | ``[]`` by default                    |
++---------------------------+----------+----------------------------------------------------------------------+--------------------------------------+
+| guess_sample_buffer_bytes | int      | Bytes of sample buffer that it tries to read from input source.      | 32768 (32KB) by default              |
++-------------------------------+----------+----------------------------------------------------------------------+----------------------------------+
 
 The ``guess_plugins`` option includes specified guess plugin in the bottom of the list of default guess plugins.
 
 The ``exclude_guess_plugins`` option exclude specified guess plugins from the list of default guess plugins that the guess executor uses.
 
+The ``guess_sample_buffer_bytes`` option controls the bytes of sample buffer that GuessExecutor tries to read from specified input source.
+
 This example shows how to use ``csv_all_strings`` guess plugin, which suggests column types within CSV files as string types. It needs to be explicitly specified by users when it's used instead of ``csv`` guess plugin because the plugin is not included in default guess plugins. We also can exclude default ``csv`` guess plugin.
 
 Example
@@ -941,3 +947,35 @@ Example
     out:
       type: ...
       ...
+
+Preview executor
+---------------
+
+The preview executor is called by ``preview`` command. It tries to read sample buffer from a specified input source and writes them to Page objects. ``preview`` outputs the Page objects to console.
+
+Options
+~~~~~~~~
+
++-------------------------------+----------+----------------------------------------------------------------------+--------------------------------------+
+| name                          | type     | description                                                          | required?                            |
++===============================+==========+======================================================================+======================================+
+| preview_sample_buffer_bytes   | int      | Bytes of sample buffer that it tries to read from input source.      | 32768 (32KB) by default              |
++-------------------------------+----------+----------------------------------------------------------------------+--------------------------------------+
+
+The ``preview_sample_buffer_bytes`` option controls the bytes of sample buffer that PreviewExecutor tries to read from specified input source.
+
+This example shows how to change the bytes of sample buffer.
+
+Example
+~~~~~~~~
+
+.. code-block:: yaml
+
+    exec:
+      preview_sample_buffer_bytes: 65536 # 64KB
+    in:
+      type: ...
+      ...
+    out:
+      type: ...
+      ...

data/embulk-docs/src/customization.rst

@@ -19,7 +19,7 @@ Creating a new plugin is 4 steps:
 This article describes how to create a plugin step by step.
 
 Step 1: Creating a new project
-~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Embulk comes with a number of templates that generates a new project so that you can start development instantly. Because the generated project contains completely working code without additional code, you can focus on the necessary coding.
 
@@ -59,7 +59,7 @@ For example, if you want to parse a new file format using Java, type this comman
 This will create a Java-based parser plugin called ``myformat`` in ``embulk-parser-myformat`` directory.
 
 Step 2: Building the project
-~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 If the plugin is Java-based, you need to build the project. To build, type this command:
 
@@ -71,7 +71,7 @@ If the plugin is Java-based, you need to build the project. To build, type this
 Now, the plugin is ready to use.
 
 Step 3: Confirm it works
-~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~
 
 The next step is to actually use the plugin.
 
@@ -82,7 +82,7 @@ Let's suppose you have a configuration file named ``your-config.yml``. You can u
     $ embulk run -L ./embulk-parser-myformat/ your-config.yml
 
 Step 4: Modifying the code
-~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 The final step is to modify code as you want!
 
@@ -104,7 +104,7 @@ Releasing plugins
 You can release publicly so that all people can use your awesome plugins.
 
 Checking plugin description
-~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 To prepare the plugin ready to release, you need to include some additional information. The plugin information is written in this file:
 
@@ -136,14 +136,14 @@ You will find following section in the file.
 The items in above example are important. Please make sure that they are good.
 
 Creating account on RubyGems.org
-~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Embulk uses `RubyGems.org <https://rubygems.org/>`_ as a package distribution service. Please create an account there to release plugins at `Sign Up <https://rubygems.org/sign_up>`_ page.
 
 Don't forget the password! It will be necessary at the next step.
 
 Releasing the plugin to RubyGems.org
-~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 Now, you're ready to release the plugin. To release, type following command:
 
@@ -158,7 +158,7 @@ Now, you're ready to release the plugin. To release, type following command:
 If everything is good, you can find your plugin at https://rubygems.org/. Congratulations!
 
 Installing your plugin
-~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~
 
 Usage of plugin installer is:
 

data/embulk-docs/src/developers/index.rst

@@ -0,0 +1,45 @@
+For core/plugin developers
+===========================
+
+Plugin API documents
+---------------------
+
+* `JavaDoc <javadoc/index.html>`_
+* `RDoc <rdoc/_index.html>`_
+
+Developer guidelines
+---------------------
+
+* Guidelines for core developers (TBA)
+* Guidelines for plugin developers (TBA)
+
+"Embulk v2"
+-----------------------
+
+The Embulk project plans a set of big updates in the Embulk core so that Embulk, its community and the eco-system keep sustainable. Backward compatibility in plugins and configurations is considered especially carefully. Other parts, for example command line options, may have some incompatibility. Join discussions if you have concerns or requests!
+
+JFYI: "Embulk v2" is just a code name. The next versions will be still ``0.?``.
+
+Summary for plugin developers
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+(TBA)
+
+Design docs for core developers
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+* `Summary <https://docs.google.com/document/d/1-8jDAisymt9SwnJpQiEnACIKC-OGMt7Ygv5FpkCeApU/edit>`_
+
+  * `Un-JRuby in the boot process <https://docs.google.com/document/d/1f-ziuHY4fXXmSLNHCXP3AAXUajEyPMLQAvH4jgTjpGg/edit>`_
+
+  * [TBD] Update plugin loading: non-Gem plugins, namespaces, and versions
+
+  * [TBD] Un-JRuby in the plugin load process
+
+  * [TBD] Provide better OAuth support in plugins
+
+  * [TBD] Buffer in output plugins
+
+  * [TBD] Structure documents: configs and errors
+
+  * [TBD] Deliver errors and notifications more flexibly

data/embulk-docs/src/index.rst

@@ -29,7 +29,7 @@ You can define a bulk data loading using combination of input and output plugins
    :width: 640px
    :target: https://github.com/embulk/embulk#quick-start
 
-For example, `this tutorial <recipe/scheduled-csv-load-to-elasticsearch-kibana4.html>`_ describes how to use **file** input plugin with **csv** parser plugin and **gzip** decoder plugin to read CSV files, and **elasticsearch** output plugin to load the records to Elasticsearch.
+For example, `this tutorial <recipe/scheduled-csv-load-to-elasticsearch-kibana5.html>`_ describes how to use **file** input plugin with **csv** parser plugin and **gzip** decoder plugin to read CSV files, and **elasticsearch** output plugin to load the records to Elasticsearch.
 
 Documents
 ------------------
@@ -71,18 +71,22 @@ Documents
    built-in
 
 .. toctree::
-   :maxdepth: 3
+   :maxdepth: 2
 
-   customization
+   release
+
+For developers
+---------------
+
+* `Github <https://github.com/embulk/embulk>`_
 
 * `JavaDoc <javadoc/index.html>`_
 
 * `RDoc <rdoc/_index.html>`_
 
-* `Github <https://github.com/embulk/embulk>`_
+* `Other resources for developers <developers/index.html>`_
 
 .. toctree::
-   :maxdepth: 2
-
-   release
+   :maxdepth: 3
 
+   customization

data/embulk-docs/src/recipe.rst

@@ -4,5 +4,5 @@ Recipes
 .. toctree::
     :maxdepth: 1
 
-    recipe/scheduled-csv-load-to-elasticsearch-kibana4
+    recipe/scheduled-csv-load-to-elasticsearch-kibana5
 

data/embulk-docs/src/recipe/{scheduled-csv-load-to-elasticsearch-kibana4.rst → scheduled-csv-load-to-elasticsearch-kibana5.rst}

@@ -1,5 +1,5 @@
-Scheduled bulk data loading to Elasticsearch + Kibana 4 from CSV files
-==================================
+Scheduled bulk data loading to Elasticsearch + Kibana 5 from CSV files
+======================================================================
 
 .. contents::
    :local:
@@ -11,56 +11,58 @@ This article shows how to:
 * Visualize the data with Kibana interactively.
 * Schedule the data loading every hour using cron.
 
-This guide assumes you are using Ubuntu 12.04 Precise or Mac OS X.
+This guide assumes you are using Ubuntu 16.10 Precise or macOS.
 
-Setup Elasticsearch and Kibana 4
-------------------
+Setup Elasticsearch and Kibana 5
+--------------------------------
 
 Step 1. Download and start Elasticsearch.
-~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-You can find releases from the `Elasticsearch website <http://www.elasticsearch.org/download/>`_.
+You can find releases from the `Elasticsearch website <https://www.elastic.co/downloads/elasticsearch>`_.
 For the smallest setup, you can unzip the package and run `./bin/elasticsearch` command:
 
 .. code-block:: console
 
-    $ wget https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/zip/elasticsearch/2.2.0/elasticsearch-2.2.0.zip
-    $ unzip elasticsearch-2.2.0.zip
-    $ cd elasticsearch-2.2.0
+    $ wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-5.3.0.zip
+    $ unzip elasticsearch-5.3.0.zip
+    $ cd elasticsearch-5.3.0
     $ ./bin/elasticsearch
 
 Step 2. Download and unzip Kibana:
-~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-You can find releases from the `Kibana website <http://www.elasticsearch.org/overview/kibana/installation/>`_. Open a new console and run following commands:
+You can find releases from the `Kibana website <https://www.elastic.co/downloads/kibana>`_. Open a new console and run following commands:
 
 .. code-block:: console
 
-    $ wget https://download.elastic.co/kibana/kibana/kibana-4.4.0-linux-x64.tar.gz
-    $ tar zxvf kibana-4.4.0-linux-x64.tar.gz
-    $ cd kibana-4.4.0-linux-x64
+    $ wget https://artifacts.elastic.co/downloads/kibana/kibana-5.3.0-linux-x86_64.tar.gz
+    $ tar zxvf kibana-5.3.0-linux-x86_64.tar.gz
+    $ cd kibana-5.3.0-linux-x86_64
     $ ./bin/kibana
 
-Note: If you're using Mac OS X, https://download.elastic.co/kibana/kibana/kibana-4.4.0-darwin-x64.tar.gz is the URL to download.
+Note: If you're using macOS, https://artifacts.elastic.co/downloads/kibana/kibana-5.3.0-darwin-x86_64.tar.gz is the URL to download.
 
 Now Elasticsearch and Kibana started. Open http://localhost:5601/ using your browser to see the Kibana's graphical interface.
 
 
 Setup Embulk
-------------------
+------------
 
 Step 1. Download Embulk binary:
-~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-You can find the latest embulk binary from the `releases <https://bintray.com/embulk/maven/embulk/view#files>`_. Because Embulk is a single executable binary, you can simply download it to /usr/local/bin directory and set executable flag as following:
+You can find the latest embulk binary from the `releases <https://bintray.com/embulk/maven/embulk/view#files>`_. Because Embulk is a single executable binary, you can simply download it to ~/.embulk/bin directory and set executable flag as following:
 
 .. code-block:: console
 
-    $ sudo wget http://dl.embulk.org/embulk-latest.jar -O /usr/local/bin/embulk
-    $ sudo chmod +x /usr/local/bin/embulk
+    $ curl --create-dirs -o ~/.embulk/bin/embulk -L "https://dl.embulk.org/embulk-latest.jar"
+    $ chmod +x ~/.embulk/bin/embulk
+    $ echo 'export PATH="$HOME/.embulk/bin:$PATH"' >> ~/.bashrc
+    $ source ~/.bashrc
 
 Step 2. Install Elasticsearch plugin
-~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 You also need Elasticsearch plugin for Embulk. You can install the plugin with this command:
 
@@ -140,7 +142,7 @@ Now, you can run the bulk loading:
     $ embulk run config.yml -c diff.yml
 
 Scheduling loading by cron
-------------------
+--------------------------
 
 At the last step, you ran embulk command with ``-c diff.yml`` file. The ``diff.yml`` file should include a parameter named ``last_path``:
 
@@ -155,7 +157,7 @@ For example, if you create ``./mydata/csv/sample_02.csv.gz`` file, embulk skips
 
 So, if you want to loads newly created files every day, you can setup this cron schedule:
 
-.. code-block:: cron
+.. code-block:: text
 
     0 * * * * embulk run /path/to/config.yml -c /path/to/diff.yml