embulk 0.8.14-java → 0.8.15-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 353b653ad68c876426f467ef64c82698598e9a59
-  data.tar.gz: 78a19ab05e47c9334267c02e4b4ea7ce2a0a3467
+  metadata.gz: c47e67c0b46d1a8dcd84337dee0d3479fd37f45f
+  data.tar.gz: edcd6bafc9095768db0798dcae9c0d53532f5054
 SHA512:
-  metadata.gz: 527b918b1809e5e04cbe5eaf51efd589d753589aa1569168dd8801cc89a680388aa2ef13c4b3041a99ced2637a9ff2751db3248eca05ae34a5ec3e6228af46b7
-  data.tar.gz: cf8ffc06d1e43b2b85c68d9f815b99265c2b48e65460ff97b05a8f6e73297d197390f5e7a850a01ff5482415d725a31bcca3ffd769c38ffac84492c7b1afd60a
+  metadata.gz: 1946063a900859720ba33919583f1bcfabb543af948b02260f3f91664445c2754aa2014e5b9c3c41cddd800ecb238b2450c39a0520d825d4f03d9d693f7d04f3
+  data.tar.gz: 8348fd5001511a3b718668eccb8b2686fd6cc2cf9635b38f2c941f79b20aa185c83bc01bb535559782895ead8a52b5cdb823c341a177afed7af1dff568e57001

data/README.md

@@ -204,7 +204,7 @@ Run following commands and follow its instruction:
 
 ```
 ./gradlew releaseCheck
-./gradlew release
+./gradlew clean cli gem && ./gradlew release
 git commit -am v$VERSION
 git tag v$VERSION
 ```

data/build.gradle

@@ -11,12 +11,12 @@ apply plugin: "com.github.jruby-gradle.jar"
 apply plugin: 'com.jfrog.bintray'
 apply plugin: 'com.github.johnrengelman.shadow'
 
-def java_projects = [project(":embulk-core"), project(":embulk-standards"), project(":embulk-cli")]
-def release_projects = [project(":embulk-core"), project(":embulk-standards")]
+def java_projects = [project(":embulk-core"), project(":embulk-standards"), project(":embulk-cli"), project(":embulk-test")]
+def release_projects = [project(":embulk-core"), project(":embulk-standards"), project(":embulk-test")]
 
 allprojects {
     group = 'org.embulk'
-    version = '0.8.14'
+    version = '0.8.15'
 
     ext {
         jrubyVersion = '9.1.5.0'

data/embulk-core/src/main/java/org/embulk/EmbulkEmbed.java

@@ -126,7 +126,7 @@ public class EmbulkEmbed
     {
         this.injector = injector;
         injector.getInstance(org.slf4j.ILoggerFactory.class);
-        this.bulkLoader = new BulkLoader(injector, systemConfig);
+        this.bulkLoader = injector.getInstance(BulkLoader.class);
         this.guessExecutor = injector.getInstance(GuessExecutor.class);
         this.previewExecutor = injector.getInstance(PreviewExecutor.class);
     }

data/embulk-core/src/main/java/org/embulk/exec/BulkLoader.java

@@ -62,7 +62,7 @@ public class BulkLoader
         this.injector = injector;
     }
 
-    private static class LoaderState
+    protected static class LoaderState
             implements ProcessState
     {
         private final Logger logger;
@@ -343,7 +343,17 @@ public class BulkLoader
                 ignoredExceptions.add(ex);
             }
 
-            return new ExecutionResult(configDiff, ignoredExceptions.build());
+            return new ExecutionResult(configDiff, false, ignoredExceptions.build());
+        }
+
+        public ExecutionResult buildExecuteResultOfSkippedExecution(ConfigDiff configDiff)
+        {
+            ImmutableList.Builder<Throwable> ignoredExceptions = ImmutableList.builder();
+            for (Throwable e : getExceptions()) {
+                ignoredExceptions.add(e);
+            }
+
+            return new ExecutionResult(configDiff, true, ignoredExceptions.build());
         }
 
         public ResumeState buildResumeState(ExecSession exec)
@@ -364,6 +374,11 @@ public class BulkLoader
         }
     }
 
+    protected LoaderState newLoaderState(Logger logger, ProcessPluginSet plugins)
+    {
+        return new LoaderState(logger, plugins);
+    }
+
     public ExecutionResult run(ExecSession exec, final ConfigSource config)
     {
         try {
@@ -418,7 +433,7 @@ public class BulkLoader
         }
     }
 
-    private static class ProcessPluginSet
+    protected static class ProcessPluginSet
     {
         private final PluginType inputPluginType;
         private final PluginType outputPluginType;
@@ -507,7 +522,7 @@ public class BulkLoader
         final ExecutorPlugin exec = newExecutorPlugin(task);
         final ProcessPluginSet plugins = new ProcessPluginSet(task);
 
-        final LoaderState state = new LoaderState(Exec.getLogger(BulkLoader.class), plugins);
+        final LoaderState state = newLoaderState(Exec.getLogger(BulkLoader.class), plugins);
         state.setTransactionStage(TransactionStage.INPUT_BEGIN);
         try {
             ConfigDiff inputConfigDiff = plugins.getInputPlugin().transaction(task.getInputConfig(), new InputPlugin.Control() {
@@ -565,7 +580,11 @@ public class BulkLoader
             return state.buildExecuteResult();
 
         } catch (Throwable ex) {
-            if (state.isAllTasksCommitted() && state.isAllTransactionsCommitted()) {
+            if (isSkippedTransaction(ex)) {
+                ConfigDiff configDiff = ((SkipTransactionException) ex).getConfigDiff();
+                return state.buildExecuteResultOfSkippedExecution(configDiff);
+            }
+            else if (state.isAllTasksCommitted() && state.isAllTransactionsCommitted()) {
                 // ignore the exception
                 return state.buildExecuteResultWithWarningException(ex);
             }
@@ -580,7 +599,7 @@ public class BulkLoader
         final ExecutorPlugin exec = newExecutorPlugin(task);
         final ProcessPluginSet plugins = new ProcessPluginSet(task);
 
-        final LoaderState state = new LoaderState(Exec.getLogger(BulkLoader.class), plugins);
+        final LoaderState state = newLoaderState(Exec.getLogger(BulkLoader.class), plugins);
         state.setTransactionStage(TransactionStage.INPUT_BEGIN);
         try {
             ConfigDiff inputConfigDiff = plugins.getInputPlugin().resume(resume.getInputTaskSource(), resume.getInputSchema(), resume.getInputTaskReports().size(), new InputPlugin.Control() {
@@ -642,7 +661,11 @@ public class BulkLoader
             return state.buildExecuteResult();
 
         } catch (Throwable ex) {
-            if (state.isAllTasksCommitted() && state.isAllTransactionsCommitted()) {
+            if (isSkippedTransaction(ex)) {
+                ConfigDiff configDiff = ((SkipTransactionException) ex).getConfigDiff();
+                return state.buildExecuteResultOfSkippedExecution(configDiff);
+            }
+            else if (state.isAllTasksCommitted() && state.isAllTransactionsCommitted()) {
                 // ignore the exception
                 return state.buildExecuteResultWithWarningException(ex);
             }
@@ -650,6 +673,11 @@ public class BulkLoader
         }
     }
 
+    private static boolean isSkippedTransaction(Throwable ex)
+    {
+        return ex instanceof SkipTransactionException;
+    }
+
     private static void restoreResumedTaskReports(ResumeState resume, LoaderState state)
     {
         int inputTaskCount = resume.getInputTaskReports().size();

data/embulk-core/src/main/java/org/embulk/exec/ExecModule.java

@@ -26,6 +26,8 @@ public class ExecModule
     {
         Preconditions.checkNotNull(binder, "binder is null.");
 
+        binder.bind(BulkLoader.class);
+
         binder.bind(ILoggerFactory.class).toProvider(LoggerProvider.class).in(Scopes.SINGLETON);
         binder.bind(ModelManager.class).in(Scopes.SINGLETON);
         binder.bind(BufferAllocator.class).to(PooledBufferAllocator.class).in(Scopes.SINGLETON);

data/embulk-core/src/main/java/org/embulk/exec/ExecutionResult.java

@@ -6,11 +6,13 @@ import org.embulk.config.ConfigDiff;
 public class ExecutionResult
 {
     private final ConfigDiff configDiff;
+    private final boolean skipped;
     private final List<Throwable> ignoredExceptions;
 
-    public ExecutionResult(ConfigDiff configDiff, List<Throwable> ignoredExceptions)
+    public ExecutionResult(ConfigDiff configDiff, boolean skipped, List<Throwable> ignoredExceptions)
     {
         this.configDiff = configDiff;
+        this.skipped = skipped;
         this.ignoredExceptions = ignoredExceptions;
     }
 
@@ -19,6 +21,11 @@ public class ExecutionResult
         return configDiff;
     }
 
+    public boolean isSkipped()
+    {
+        return skipped;
+    }
+
     public List<Throwable> getIgnoredExceptions()
     {
         return ignoredExceptions;

data/embulk-core/src/main/java/org/embulk/exec/SkipTransactionException.java

@@ -0,0 +1,23 @@
+package org.embulk.exec;
+
+import org.embulk.config.ConfigDiff;
+
+// Input/output plugins might need to stop Embulk before the transaction starts by depending
+// on the conditions of input/output data sources/destinations. They can throw this exception
+// if they want to do that. Embulk handles it and then stops the transaction.
+public class SkipTransactionException
+        extends RuntimeException
+{
+    private final ConfigDiff configDiff;
+
+    public SkipTransactionException(ConfigDiff configDiff)
+    {
+        super();
+        this.configDiff = configDiff;
+    }
+
+    public ConfigDiff getConfigDiff()
+    {
+        return configDiff;
+    }
+}

data/embulk-docs/src/built-in.rst

@@ -1,12 +1,12 @@
 Configuration
-==================================
+==============
 
 .. contents::
    :local:
    :depth: 2
 
 Embulk configuration file format
-------------------
+---------------------------------
 
 Embulk uses a YAML file to define a bulk data loading. Here is an example of the file:
 
@@ -60,7 +60,7 @@ In many cases, what you need to write is **in:**, **out**: and **formatter** sec
 
 
 Using variables
-~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~
 
 You can embed environment variables in configuration file using `Liquid template engine <http://liquidmarkup.org/>`_ (This is experimental feature. Behavior might change or be removed in future releases).
 
@@ -89,7 +89,7 @@ Environment variables are set to ``env`` variable.
 
 
 Including files
-~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~
 
 Configuration file can include another configuration file. To use it, configuration file name must end with ``.yml.liquid``.
 
@@ -121,12 +121,12 @@ With above 2 files, actual configuration file will be:
 
 
 Local file input plugin
-------------------
+------------------------
 
 The ``file`` input plugin reads files from local file system.
 
 Options
-~~~~~~~~~~~~~~~~~~
+~~~~~~~~
 
 +----------------+----------+------------------------------------------------+-----------+
 | name           | type     | description                                    | required? |
@@ -168,7 +168,7 @@ For example, if you set ``last_path: /path/to/files/sample_02.csv``, Embulk read
                 |-- sample_04.csv   -> read
 
 Example
-~~~~~~~~~~~~~~~~~~
+~~~~~~~~
 
 .. code-block:: yaml
 
@@ -187,7 +187,7 @@ CSV parser plugin
 The ``csv`` parser plugin parses CSV and TSV files.
 
 Options
-~~~~~~~~~~~~~~~~~~
+~~~~~~~~
 
 +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+---------------------------+
 | name                       | type     | description                                                                                                    |              required?    |
@@ -271,7 +271,7 @@ The ``null_string`` option converts certain values to NULL. Values will be conve
 You can use ``guess`` to automatically generate the column settings. See also `Quick Start <https://github.com/embulk/embulk#quick-start>`_.
 
 Example
-~~~~~~~~~~~~~~~~~~
+~~~~~~~~
 
 .. code-block:: yaml
 
@@ -295,8 +295,30 @@ Example
         - {name: comment, type: string}
 
 
+.. note::
+
+    CSV parser supports ``format: '%s'`` to parse UNIX timestamp in seconds (e.g. 1470148959) as timestamp.
+
+    However, CSV parser itself can't parse UNIX timestamp in millisecond (e.g. 1470148959542) as timestamp. You can still parse the column as ``long`` type first, then apply `timestamp_format <https://github.com/sonots/embulk-filter-timestamp_format>`_ filter plugin to convert long to timestamp. Here is an example:
+
+    .. code-block:: yaml
+
+       in:
+         type: file
+         path_prefix: /my_csv_files
+         parser:
+           ...
+           columns:
+           - {name: timestamp_in_seconds, type: timestamp, format: '%s'}
+           - {name: timestamp_in_millis, type: long}
+       filters:
+         - type: timestamp_format
+           columns:
+             - {name: timestamp_in_millis, from_unit: ms}
+
+
 JSON parser plugin
-------------------
+-------------------
 
 The ``json`` parser plugin parses a JSON file that contains a sequence of JSON objects. Example:
 
@@ -310,7 +332,7 @@ The ``json`` parser plugin parses a JSON file that contains a sequence of JSON o
 ``json`` parser plugin outputs a single record named "record" (type is json).
 
 Options
-~~~~~~~~~~~~~~~~~~
+~~~~~~~~
 
 +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
 | name                       | type     | description                                                                                                    |              required? |
@@ -320,7 +342,7 @@ Options
 
 
 Example
-~~~~~~~~~~~~~~~~~~
+~~~~~~~~
 
 .. code-block:: yaml
 
@@ -329,17 +351,17 @@ Example
         type: json
 
 Gzip decoder plugin
-------------------
+--------------------
 
 The ``gzip`` decoder plugin decompresses gzip files before input plugins read them.
 
 Options
-~~~~~~~~~~~~~~~~~~
+~~~~~~~~
 
 This plugin doesn't have any options.
 
 Example
-~~~~~~~~~~~~~~~~~~
+~~~~~~~~
 
 .. code-block:: yaml
 
@@ -350,17 +372,17 @@ Example
 
 
 BZip2 decoder plugin
-------------------
+---------------------
 
 The ``bzip2`` decoder plugin decompresses bzip2 files before input plugins read them.
 
 Options
-~~~~~~~~~~~~~~~~~~
+~~~~~~~~
 
 This plugin doesn't have any options.
 
 Example
-~~~~~~~~~~~~~~~~~~
+~~~~~~~~
 
 .. code-block:: yaml
 
@@ -371,12 +393,12 @@ Example
 
 
 File output plugin
-------------------
+-------------------
 
 The ``file`` output plugin writes records to local file system.
 
 Options
-~~~~~~~~~~~~~~~~~~
+~~~~~~~~
 
 +--------------------+----------+---------------------------------------------------+----------------------------+
 | name               | type     | description                                       | required?                  |
@@ -404,7 +426,7 @@ For example, if you set ``path_prefix: /path/to/output/sample_``, ``sequence_for
 ``sequence_format`` formats task index and sequence number in a task.
 
 Example
-~~~~~~~~~~~~~~~~~~
+~~~~~~~~
 
 .. code-block:: yaml
 
@@ -416,12 +438,12 @@ Example
         ...
 
 CSV formatter plugin
-------------------
+---------------------
 
 The ``csv`` formatter plugin formats records using CSV or TSV format.
 
 Options
-~~~~~~~~~~~~~~~~~~
+~~~~~~~~
 
 +----------------------+---------+-------------------------------------------------------------------------------------------------------+-------------------------------+
 | name                 | type    | description                                                                                           | required?                     |
@@ -474,7 +496,7 @@ The ``column_options`` option is a map whose keys are name of columns, and value
 +----------------------+---------+-------------------------------------------------------------------------------------------------------+-----------------------------------------+
 
 Example
-~~~~~~~~~~~~~~~~~~
+~~~~~~~~
 
 .. code-block:: yaml
 
@@ -496,12 +518,12 @@ Example
           mycol2: {format: '%Y-%m-%d %H:%M:%S', timezone: 'America/Los_Angeles'}
 
 Gzip encoder plugin
-------------------
+--------------------
 
 The ``gzip`` encoder plugin compresses output files using gzip.
 
 Options
-~~~~~~~~~~~~~~~~~~
+~~~~~~~~
 
 +---------+----------+----------------------------------------------------------------------+--------------------+
 | name    | type     | description                                                          | required?          |
@@ -510,7 +532,7 @@ Options
 +---------+----------+----------------------------------------------------------------------+--------------------+
 
 Example
-~~~~~~~~~~~~~~~~~~
+~~~~~~~~
 
 .. code-block:: yaml
 
@@ -521,73 +543,285 @@ Example
         level: 1
 
 
-Gzip encoder plugin
-------------------
+BZip2 encoder plugin
+---------------------
 
-The ``gzip`` encoder plugin compresses output files using gzip.
+The ``bzip2`` encoder plugin compresses output files using bzip2.
 
 Options
-~~~~~~~~~~~~~~~~~~
+~~~~~~~~
 
 +---------+----------+----------------------------------------------------------------------+--------------------+
 | name    | type     | description                                                          | required?          |
 +=========+==========+======================================================================+====================+
-| level   | integer  | Compression level. From 0 (no compression) to 9 (best compression).  | ``6`` by default   |
+| level   | integer  | Compression level. From 1 to 9 (best compression).                   | ``9`` by default   |
 +---------+----------+----------------------------------------------------------------------+--------------------+
 
 Example
-~~~~~~~~~~~~~~~~~~
+~~~~~~~~
 
 .. code-block:: yaml
 
     out:
       ...
       encoders:
-      - type: gzip
-        level: 1
+      - type: bzip2
+        level: 6
 
-BZip2 encoder plugin
-------------------
 
-The ``bzip2`` encoder plugin compresses output files using bzip2.
+Rename filter plugin
+---------------------
+
+The ``rename`` filter plugin changes column names. This plugin has no impact on performance.
 
 Options
-~~~~~~~~~~~~~~~~~~
+~~~~~~~~
 
 +---------+----------+----------------------------------------------------------------------+--------------------+
 | name    | type     | description                                                          | required?          |
 +=========+==========+======================================================================+====================+
-| level   | integer  | Compression level. From 1 to 9 (best compression).                   | ``9`` by default   |
+| rules   | array    | An array of rule-based renaming operations. (See below for rules.)   | ``[]`` by default  |
++---------+----------+----------------------------------------------------------------------+--------------------+
+| columns | hash     | A map whose keys are existing column names. values are new names.    | ``{}`` by default  |
 +---------+----------+----------------------------------------------------------------------+--------------------+
 
+Renaming rules
+~~~~~~~~~~~~~~~
+
+The ``rules`` is an array of rules as below applied top-down for all the columns.
+
++-------------------------+----------------------------------------------------------------------------------------+
+| rule                    | description                                                                            |
++=========================+========================================================================================+
+| character\_types        | Restrict characters by types. Replace restricted characteres.                          |
++-------------------------+----------------------------------------------------------------------------------------+
+| first\_character\_types | Restrict the first character by types. Prefix or replace first restricted characters.  |
++-------------------------+----------------------------------------------------------------------------------------+
+| lower\_to\_upper        | Convert lower-case alphabets to upper-case.                                            |
++-------------------------+----------------------------------------------------------------------------------------+
+| regex\_replace          | Replace with a regular expressions.                                                    |
++-------------------------+----------------------------------------------------------------------------------------+
+| truncate                | Truncate.                                                                              |
++-------------------------+----------------------------------------------------------------------------------------+
+| upper\_to\_lower        | Convert upper-case alphabets to lower-case                                             |
++-------------------------+----------------------------------------------------------------------------------------+
+| unique\_number\_suffix  | Make column names unique in the schema.                                                |
++-------------------------+----------------------------------------------------------------------------------------+
+
+Renaming rule: character\_types
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The rule ``character_types`` replaces restricted characters.
+
++-------------------+--------------------------------------------------------------------------------------------------------------------------------------------+--------------------+
+| option            | description                                                                                                                                | required?          |
++===================+============================================================================================================================================+====================+
+| pass\_characteres | Characters to be allowed.                                                                                                                  | ``""`` by default  |
++-------------------+--------------------------------------------------------------------------------------------------------------------------------------------+--------------------+
+| pass\_types       | Sets of characters to be allowed. The array must consist of "a-z" (lower-case alphabets), "A-Z" (upper-case alphabets), or "0-9" (digits). | ``[]`` by default  |
++-------------------+--------------------------------------------------------------------------------------------------------------------------------------------+--------------------+
+| replace           | A character that disallowed characters are replaced with. It must consist of just 1 character.                                             | ``"_"`` by default |
++-------------------+--------------------------------------------------------------------------------------------------------------------------------------------+--------------------+
+
 Example
-~~~~~~~~~~~~~~~~~~
+""""""""
 
 .. code-block:: yaml
 
-    out:
+    # This configuration replaces characters into "_" except for "_", lower-case alphabets, and digits.
+    filters:
       ...
-      encoders:
-      - type: bzip2
-        level: 6
+      - type: rename
+        rules:
+        - rule: character_types
+          pass_characters: "_"
+          pass_types: [ "a-z", "0-9" ]
 
 
-Rename filter plugin
-------------------
+Renaming rule: first\_character\_types
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-The ``rename`` filter plugin changes column names. This plugin has no impact on performance.
+The rule ``first_character_types`` prefixes or replaces a restricted character at the beginning.
 
-Options
-~~~~~~~~~~~~~~~~~~
++-------------------+--------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------+
+| option            | description                                                                                                                                | required?                                    |
++===================+============================================================================================================================================+==============================================+
+| pass\_characteres | Characters to be allowed.                                                                                                                  | ``""`` by default                            |
++-------------------+--------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------+
+| pass\_types       | Sets of characters to be allowed. The array must consist of "a-z" (lower-case alphabets), "A-Z" (upper-case alphabets), or "0-9" (digits). | ``[]`` by default                            |
++-------------------+--------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------+
+| prefix            | A character that a disallowed first character is replaced with.                                                                            | one of ``prefix`` or ``replace`` is required |
++-------------------+--------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------+
+| replace           | A character that a disallowed first character is prefixed with.                                                                            | one of ``prefix`` or ``replace`` is required |
++-------------------+--------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------+
 
-+---------+----------+----------------------------------------------------------------------+--------------------+
-| name    | type     | description                                                          | required?          |
-+=========+==========+======================================================================+====================+
-| columns | hash     | A map whose keys are existing column names. values are new names.    | ``{}`` by default  |
-+---------+----------+----------------------------------------------------------------------+--------------------+
+Example
+""""""""
+
+.. code-block:: yaml
+
+    # This configuration prefixes a column name with "_" unless the name starts from "_" or a lower-case alphabet.
+    filters:
+      ...
+      - type: rename
+        rules:
+        - rule: first_character_types
+          pass_characters: "_"
+          pass_types: [ "a-z" ]
+          prefix: "_"
+
+Renaming rule: lower\_to\_upper
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The rule ``lower_to_upper`` converts lower-case alphabets to upper-case.
+
+Example
+""""""""
+
+.. code-block:: yaml
+
+    # This configuration converts all lower-case alphabets to upper-case.
+    filters:
+      ...
+      - type: rename
+        rules:
+        - rule: lower_to_upper
+
+
+Renaming rule: regex\_replace
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The rule ``regex_replace`` replaces column names based on a regular expression.
+
++---------+--------------------------------------------------------------------------------------------------------------------------------------+-----------+
+| option  | description                                                                                                                          | required? |
++=========+======================================================================================================================================+===========+
+| match   | A `Java-style regular expression <https://docs.oracle.com/javase/tutorial/essential/regex/>`_ to which this string is to be matched. | required  |
++---------+--------------------------------------------------------------------------------------------------------------------------------------+-----------+
+| replace | A string to be substibuted for each match in Java-style.                                                                             | required  |
++---------+--------------------------------------------------------------------------------------------------------------------------------------+-----------+
+
+Example
+""""""""
+
+.. code-block:: yaml
+
+    # This configuration replaces all patterns
+    filters:
+      ...
+      - type: rename
+        rules:
+        - rule: regex_replace
+          match: "([0-9]+)_dollars"
+          replace: "USD$1"
+
+
+Renaming rule: truncate
+^^^^^^^^^^^^^^^^^^^^^^^^
+
+The rule ``truncate`` truncates column names.
+
++------------+-----------------------------------------------------+--------------------+
+| option     | description                                         | required?          |
++============+=====================================================+====================+
+| max_length | The length to which the column names are truncated. | ``128`` by default |
++------------+-----------------------------------------------------+--------------------+
+
+Example
+""""""""
+
+.. code-block:: yaml
+
+    # This configuration drops all characters after the 20th character.
+    filters:
+      ...
+      - type: rename
+        rules:
+        - rule: truncate
+          max_length: 20
+
+Renaming rule: upper\_to\_lower
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The rule ``upper_to_lower`` converts upper-case alphabets to lower-case.
+
+Example
+""""""""
+
+.. code-block:: yaml
+
+    # This configuration converts all upper-case alphabets to lower-case.
+    filters:
+      ...
+      - type: rename
+        rules:
+        - rule: upper_to_lower
+
+Renaming rule: unique\_number\_suffix
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+The rule ``unique_number_suffix`` makes column names unique in the schema by suffixing numbers.
+
++------------+-----------------------------------------------------------------------------------------------------------------------------+--------------------+
+| option     | description                                                                                                                 | required?          |
++============+=============================================================================================================================+====================+
+| delimiter  | A delimiter character inserted before a suffix number. It must be just 1 non-digit character.                               | ``"_"`` by default |
++------------+-----------------------------------------------------------------------------------------------------------------------------+--------------------+
+| digits     | An integer that specifies the number of zero-filled digits of a suffix number. The suffix number zero-filled to the digits. | optional           |
++------------+-----------------------------------------------------------------------------------------------------------------------------+--------------------+
+| max_length | The length to which the column names are truncated. The column name is truncated before the suffix number.                  | optional           |
++------------+-----------------------------------------------------------------------------------------------------------------------------+--------------------+
+| offset     | An integer where the suffix number starts. The first duplicative column name is suffixed by (```offset``` + 1).             | ``1`` by default   |
++------------+-----------------------------------------------------------------------------------------------------------------------------+--------------------+
+
+.. hint::
+   The procedure to make column names unique is not very trivial. There are many feasible ways. This renaming rule works as follows:
+
+   Basic policies:
+
+   * Suffix numbers are counted per original column name.
+   * Column names are fixed from the first column to the last column.
+
+   Actual procedure applied from the first (leftmost) column to the last (rightmost) column:
+
+   1. Fix the column name as-is with truncating if the truncated name is not duplicated with left columns.
+   2. Suffix the column name otherwise.
+
+      a. Try to append the suffix number for the original column name with truncating.
+      b. Fix it if the suffixed name is not duplicated with left columns nor original columns.
+      c. Retry (a) with the suffix number increased otherwise.
 
 Example
-~~~~~~~~~~~~~~~~~~
+""""""""
+
+.. code-block:: yaml
+
+    # This configuration suffixes numbers to duplicative column names. (Ex. ["column", "column", "column"] goes to ["column", "column_2", "column_3"].)
+    filters:
+      ...
+      - type: rename
+        rules:
+        - rule: unique_number_suffix
+
+Example of renaming rules
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code-block:: yaml
+
+    filters:
+      ...
+      - type: rename
+        rules:
+        - rule: upper_to_lower        # All upper-case are converted to lower-case.
+        - rule: character_types       # Only lower-case, digits and "_" are allowed. (No upper-case by the rule ahove.)
+          pass_types: [ "a-z", "0-9" ]
+          pass_characters: "_"
+        - rule: unique_number_suffix  # Ensure all column names are unique.
+
+Columns: not recommended
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+``columns`` is not recommended to use anymore. Consider using ``rules`` instead.
 
 .. code-block:: yaml
 
@@ -598,13 +832,16 @@ Example
           my_existing_column1: new_column1
           my_existing_column2: new_column2
 
+.. hint::
+   ``columns`` are applied before ``rules`` if ``columns`` and ``rules`` are specified together. (It is discouraged to specify them together, though.)
+
 Local executor plugin
-------------------
+----------------------
 
 The ``local`` executor plugin runs tasks using local threads. This is the only built-in executor plugin.
 
 Options
-~~~~~~~~~~~~~~~~~~
+~~~~~~~~
 
 +------------------+----------+----------------------------------------------------------------------+--------------------------------------+
 | name             | type     | description                                                          | required?                            |
@@ -620,7 +857,7 @@ The ``max_threads`` option controls maximum concurrency. Setting smaller number
 The ``min_output_tasks`` option enables "page scattering". The feature is enabled if number of input tasks is less than ``min_output_tasks``. It uses multiple filter & output threads for each input task so that one input task can use multiple threads. Setting larger number here is useful if embulk doesn't use multi-threading with enough concurrency due to too few number of input tasks. Setting 1 here disables page scattering completely.
 
 Example
-~~~~~~~~~~~~~~~~~~
+~~~~~~~~
 
 .. code-block:: yaml
 
@@ -635,12 +872,12 @@ Example
       ...
 
 Guess executor
-------------------
+---------------
 
 The guess executor is called by ``guess`` command. It executes default guess plugins in a sequential order and suggests Embulk config by appropriate guess plugin. The default guess plugins and the order are ``gzip``, ``'bzip2``, ``json`` and ``csv``.
 
 Options
-~~~~~~~~~~~~~~~~~~
+~~~~~~~~
 
 +-----------------------+----------+----------------------------------------------------------------------+--------------------------------------+
 | name                  | type     | description                                                          | required?                            |
@@ -657,7 +894,7 @@ The ``exclude_guess_plugins`` option exclude specified guess plugins from the li
 This example shows how to use ``csv_all_strings`` guess plugin, which suggests column types within CSV files as string types. It needs to be explicitly specified by users when it's used instead of ``csv`` guess plugin because the plugin is not included in default guess plugins. We also can exclude default ``csv`` guess plugin.
 
 Example
-~~~~~~~~~~~~~~~~~~
+~~~~~~~~
 
 .. code-block:: yaml