embulk 0.8.15 → 0.8.16

data/embulk-docs/src/built-in.rst

@@ -835,6 +835,40 @@ Columns: not recommended
 .. hint::
    ``columns`` are applied before ``rules`` if ``columns`` and ``rules`` are specified together. (It is discouraged to specify them together, though.)
 
+
+Remove columns filter plugin
+-----------------------------
+
+The ``remove_columns`` filter plugin removes columns from schema.
+
+Options
+~~~~~~~~
+
++--------------------------+----------+------------------------------------------------------------+-----------------------+
+| name                     | type     | description                                                | required?             |
++==========================+==========+============================================================+=======================+
+| remove                   | array    | An array of names of columns that it removes from schema.  | ``[]`` by default     |
++--------------------------+----------+------------------------------------------------------------+-----------------------+
+| keep                     | array    | An array of names of columns that it keeps in schema.      | ``[]`` by default     |
++--------------------------+----------+------------------------------------------------------------+-----------------------+
+| accept_unmatched_columns | boolean  | If true, skip columns that aren't included in schemas.     | ``false`` by default  |
++--------------------------+----------+------------------------------------------------------------+-----------------------+
+
+
+remove: and keep: options are not multi-select.
+
+Example
+~~~~~~~~
+
+.. code-block:: yaml
+
+    # This configuration removes "_c0" and "_c1" named columns from schema.
+    filters:
+      ...
+      - type: remove_columns
+        remove: ["_c0", "_c1"]
+
+
 Local executor plugin
 ----------------------
 

data/embulk-docs/src/release.rst

@@ -4,6 +4,7 @@ Release Notes
 .. toctree::
     :maxdepth: 1
 
+    release/release-0.8.16
     release/release-0.8.15
     release/release-0.8.14
     release/release-0.8.13

data/embulk-docs/src/release/release-0.8.16.rst

@@ -0,0 +1,43 @@
+Release 0.8.16
+==================================
+
+General Changes
+------------------
+
+* Added remove_columns filter plugin [#530]
+
+  * http://www.embulk.org/docs/built-in.html#remove_columns-filter-plugin
+
+* Supported timestamp format "%Q". (@hiroyuki-sato) [#468, #531]
+
+* Improved csv guess plugin:
+
+  * Added semicolon as delimiter suggest candidate in csv guess plugin. [#527]
+
+  * Enabled suggesting for a few rows [#533]
+
+  * Enabled suggesting for a single column [#540]
+
+* Changed and removed limitation of minimum 40 bytes size limit of guessing. [#518]
+
+* Refactored and introduced TestingEmbulk#{Input,Parser,Output}Builder to embulk-test. [#513, #514, #526]
+
+* Fixed PageBuilder to avoid NullPointerException. [#535]
+
+* Fixed ResumableInputStream to avoid NullPointerException. [#472]
+
+* Fixed TaskValidationException to inherit ConfigException. [#520]
+
+* Fixed build.gradle to use Task.doLast instead of Task.leftShift. [#536]
+
+* Fixed build failure on AppVeyor by FileNotFoundException. [#537]
+
+* Added updateJRuby task to make it easy to upgrade version of JRuby. [#538]
+
+* Upgraded gradle v3.2.1. [#528]
+
+  * Release notes: https://docs.gradle.org/3.2.1/release-notes
+
+Release Date
+------------------
+2017-01-27

data/embulk-standards/build.gradle

@@ -3,4 +3,5 @@ dependencies {
     compile 'org.apache.commons:commons-compress:1.10'
 
     testCompile project(':embulk-core').sourceSets.test.output
+    testCompile project(':embulk-test')
 }

data/embulk-standards/src/main/java/org/embulk/standards/RemoveColumnsFilterPlugin.java

@@ -0,0 +1,268 @@
+package org.embulk.standards;
+
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.inject.Inject;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.Column;
+import org.embulk.spi.ColumnVisitor;
+import org.embulk.spi.Exec;
+import org.embulk.spi.FilterPlugin;
+import org.embulk.spi.Page;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.PageReader;
+import org.embulk.spi.Schema;
+import org.embulk.spi.SchemaConfigException;
+import org.slf4j.Logger;
+
+import java.util.List;
+import java.util.Map;
+import java.util.HashMap;
+
+import static java.util.Locale.ENGLISH;
+import static org.embulk.spi.Exec.getBufferAllocator;
+
+public class RemoveColumnsFilterPlugin
+        implements FilterPlugin
+{
+    public interface PluginTask
+            extends Task
+    {
+        @Config("remove")
+        @ConfigDefault("null")
+        public Optional<List<String>> getRemove();
+
+        // TODO remove_pattern option
+
+        @Config("keep")
+        @ConfigDefault("null")
+        public Optional<List<String>> getKeep();
+
+        // TODO keep_pattern option
+
+        @Config("accept_unmatched_columns")
+        @ConfigDefault("false")
+        public boolean getAcceptUnmatchedColumns();
+
+        public void setIndexMapping(int[] mapping);
+        public int[] getIndexMapping();
+    }
+
+    private final Logger LOG;
+
+    @Inject
+    public RemoveColumnsFilterPlugin()
+    {
+        LOG = Exec.getLogger(getClass());
+    }
+
+    @Override
+    public void transaction(ConfigSource config, Schema inputSchema, FilterPlugin.Control control)
+    {
+        PluginTask task = config.loadConfig(PluginTask.class);
+
+        // validate remove: and keep:
+        if (task.getRemove().isPresent() && task.getKeep().isPresent()) {
+            throw new ConfigException("remove: and keep: must not be multi-select");
+        }
+        if (!task.getRemove().isPresent() && !task.getKeep().isPresent()) {
+            throw new ConfigException("Must require remove: or keep:");
+        }
+
+        boolean acceptUnmatchedColumns = task.getAcceptUnmatchedColumns();
+
+        ImmutableList.Builder<Column> outputColumns = ImmutableList.builder();
+        int index = 0;
+        int[] indexMapping = new int[inputSchema.size()];
+        for (int i = 0; i < indexMapping.length; i++) {
+            indexMapping[i] = -1;
+        }
+        if (task.getRemove().isPresent()) { // specify remove:
+            List<String> removeColumns = getExistentColumns(inputSchema, task.getRemove().get(), acceptUnmatchedColumns);
+            for (Column column : inputSchema.getColumns()) {
+                if (!removeColumns.contains(column.getName())) {
+                    outputColumns.add(new Column(index, column.getName(), column.getType()));
+                    indexMapping[column.getIndex()] = index;
+                    index++;
+                }
+            }
+        }
+        else { // specify keep:
+            List<String> keepColumns = getExistentColumns(inputSchema, task.getKeep().get(), acceptUnmatchedColumns);
+            for (Column column : inputSchema.getColumns()) {
+                if (keepColumns.contains(column.getName())) {
+                    outputColumns.add(new Column(index, column.getName(), column.getType()));
+                    indexMapping[column.getIndex()] = index;
+                    index++;
+                }
+            }
+        }
+
+        task.setIndexMapping(indexMapping);
+        control.run(task.dump(), new Schema(outputColumns.build()));
+    }
+
+    private List<String> getExistentColumns(Schema schema, List<String> specifiedColumns, boolean acceptUnmatch)
+    {
+        ImmutableList.Builder<String> existentColumns = ImmutableList.builder();
+        for (String column : specifiedColumns) {
+            try {
+                schema.lookupColumn(column);
+                existentColumns.add(column);
+            }
+            catch (SchemaConfigException e) {
+                if (!acceptUnmatch) {
+                    throw new ConfigException(String.format(ENGLISH, "Column '%s' doesn't exist in the schema", column));
+                }
+            }
+        }
+        return existentColumns.build();
+    }
+
+    @Override
+    public PageOutput open(TaskSource taskSource, Schema inputSchema,
+            Schema outputSchema, PageOutput output)
+    {
+        PluginTask task = taskSource.loadTask(PluginTask.class);
+        PageReader pageReader = new PageReader(inputSchema);
+        PageBuilder pageBuilder = new PageBuilder(getBufferAllocator(), outputSchema, output);
+        return new PageConverter(pageReader, pageBuilder, task.getIndexMapping());
+    }
+
+    static class PageConverter
+            implements PageOutput
+    {
+        private final PageReader pageReader;
+        private final PageBuilder pageBuilder;
+        private final int[] indexMapping;
+
+        PageConverter(PageReader pageReader, PageBuilder pageBuilder, int[] indexMapping)
+        {
+            this.pageReader = pageReader;
+            this.pageBuilder = pageBuilder;
+            this.indexMapping = indexMapping;
+        }
+
+        @Override
+        public void add(Page page)
+        {
+            pageReader.setPage(page);
+            while (pageReader.nextRecord()) {
+                pageReader.getSchema().visitColumns(new ColumnVisitor() {
+                    @Override
+                    public void booleanColumn(Column inputColumn)
+                    {
+                        int index = indexMapping[inputColumn.getIndex()];
+                        if (index >= 0) {
+                            if (pageReader.isNull(inputColumn)) {
+                                pageBuilder.setNull(index);
+                            }
+                            else {
+                                pageBuilder.setBoolean(index, pageReader.getBoolean(inputColumn));
+                            }
+                        }
+                    }
+
+                    @Override
+                    public void longColumn(Column inputColumn)
+                    {
+                        int index = indexMapping[inputColumn.getIndex()];
+                        if (index >= 0) {
+                            if (pageReader.isNull(inputColumn)) {
+                                pageBuilder.setNull(index);
+                            }
+                            else {
+                                pageBuilder.setLong(index, pageReader.getLong(inputColumn));
+                            }
+                        }
+                    }
+
+                    @Override
+                    public void doubleColumn(Column inputColumn)
+                    {
+                        int index = indexMapping[inputColumn.getIndex()];
+                        if (index >= 0) {
+                            if (pageReader.isNull(inputColumn)) {
+                                pageBuilder.setNull(index);
+                            }
+                            else {
+                                pageBuilder.setDouble(index, pageReader.getDouble(inputColumn));
+                            }
+                        }
+                    }
+
+                    @Override
+                    public void stringColumn(Column inputColumn)
+                    {
+                        int index = indexMapping[inputColumn.getIndex()];
+                        if (index >= 0) {
+                            if (pageReader.isNull(inputColumn)) {
+                                pageBuilder.setNull(index);
+                            }
+                            else {
+                                pageBuilder.setString(index, pageReader.getString(inputColumn));
+                            }
+                        }
+                    }
+
+                    @Override
+                    public void timestampColumn(Column inputColumn)
+                    {
+                        int index = indexMapping[inputColumn.getIndex()];
+                        if (index >= 0) {
+                            if (pageReader.isNull(inputColumn)) {
+                                pageBuilder.setNull(index);
+                            }
+                            else {
+                                pageBuilder.setTimestamp(index, pageReader.getTimestamp(inputColumn));
+                            }
+                        }
+                    }
+
+                    @Override
+                    public void jsonColumn(Column inputColumn)
+                    {
+                        int index = indexMapping[inputColumn.getIndex()];
+                        if (index >= 0) {
+                            if (pageReader.isNull(inputColumn)) {
+                                pageBuilder.setNull(index);
+                            }
+                            else {
+                                pageBuilder.setJson(index, pageReader.getJson(inputColumn));
+                            }
+                        }
+                    }
+                });
+                pageBuilder.addRecord();
+            }
+        }
+
+        private Map<String, Integer> newColumnIndex(Schema schema)
+        {
+            ImmutableMap.Builder<String, Integer> builder = ImmutableMap.builder();
+            for (Column column : schema.getColumns()) {
+                builder.put(column.getName(), column.getIndex());
+            }
+            return builder.build();
+        }
+
+        @Override
+        public void finish()
+        {
+            pageBuilder.finish();
+        }
+
+        @Override
+        public void close()
+        {
+            pageBuilder.close();
+        }
+    }
+}

data/embulk-standards/src/main/java/org/embulk/standards/RenameFilterPlugin.java

@@ -30,6 +30,19 @@ import java.util.regex.PatternSyntaxException;
 import javax.validation.constraints.Min;
 import javax.validation.constraints.Size;
 
+/**
+ * |RenameFilterPlugin| renames column names.
+ *
+ * NOTE: This filter should bahave always in the same way for the same configuration.
+ * Changes in its behavior confuse users who are working with the same configuration.
+ *
+ * Even when a buggy behavior is found, fix it by:
+ * 1) Adding a new option, and
+ * 2) Implementing a new behavior in the new option.
+ *
+ * Keep the buggy behavior with the old configuration except for fatal failures so
+ * that users are not confused.
+ */
 public class RenameFilterPlugin
         implements FilterPlugin
 {

data/embulk-standards/src/main/java/org/embulk/standards/StandardPluginModule.java

@@ -47,6 +47,7 @@ public class StandardPluginModule
 
         // filter plugins
         registerPluginTo(binder, FilterPlugin.class, "rename", RenameFilterPlugin.class);
+        registerPluginTo(binder, FilterPlugin.class, "remove_columns", RemoveColumnsFilterPlugin.class);
 
         // default guess plugins
         registerDefaultGuessPluginTo(binder, new PluginType("gzip"));

data/embulk-standards/src/test/java/org/embulk/standards/TestRemoveColumnsFilterPlugin.java

@@ -0,0 +1,121 @@
+package org.embulk.standards;
+
+import com.google.common.collect.ImmutableList;
+import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigSource;
+import org.embulk.exec.PartialExecutionException;
+import org.embulk.test.TestingEmbulk;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.file.Path;
+
+import static org.embulk.test.EmbulkTests.copyResource;
+import static org.embulk.test.EmbulkTests.readResource;
+import static org.embulk.test.EmbulkTests.readSortedFile;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class TestRemoveColumnsFilterPlugin
+{
+    private static final String RESOURCE_NAME_PREFIX = "org/embulk/standards/remove_columns/test/";
+
+    @Rule
+    public TestingEmbulk embulk = TestingEmbulk.builder().build();
+
+    @Test
+    public void useKeepOption()
+            throws Exception
+    {
+        assertRecordsByResource(embulk, "test_keep_in.yml", "test_keep_filter.yml",
+                "test_keep.csv", "test_keep_expected.csv");
+    }
+
+    @Test
+    public void useKeepWithAcceptUnmatched()
+            throws Exception
+    {
+        assertRecordsByResource(embulk, "test_keep_in.yml", "test_keep_with_unmatched_filter.yml",
+                "test_keep.csv", "test_keep_expected.csv");
+    }
+
+    @Test
+    public void useKeepWithoutAcceptUnmatched()
+            throws Exception
+    {
+        try {
+            assertRecordsByResource(embulk, "test_keep_in.yml", "test_keep_without_unmatched_filter.yml",
+                    "test_keep.csv", "test_keep_expected.csv");
+            fail();
+        }
+        catch (PartialExecutionException ex) {
+            assertTrue(ex.getCause() instanceof ConfigException);
+        }
+    }
+
+    @Test
+    public void useKeepWithDuplicatedColumnNames()
+            throws Exception
+    {
+        assertRecordsByResource(embulk, "test_keep_with_duplicated_column_names_in.yml", "test_keep_with_duplicated_column_names.yml",
+                "test_keep_with_duplicated_column_names.csv", "test_keep_with_duplicated_column_names_expected.csv");
+    }
+
+    @Test
+    public void useRemove()
+            throws Exception
+    {
+        assertRecordsByResource(embulk, "test_remove_in.yml", "test_remove_filter.yml",
+                "test_remove.csv", "test_remove_expected.csv");
+    }
+
+    @Test
+    public void useRemoveWithAcceptUnmatched()
+            throws Exception
+    {
+        assertRecordsByResource(embulk, "test_remove_in.yml", "test_remove_with_unmatched_filter.yml",
+                "test_remove.csv", "test_remove_expected.csv");
+    }
+
+    @Test
+    public void useRemoveWithoutAcceptUnmatched()
+            throws Exception
+    {
+        try {
+            assertRecordsByResource(embulk, "test_remove_in.yml", "test_remove_without_unmatched_filter.yml",
+                    "test_remove.csv", "test_remove_expected.csv");
+            fail();
+        }
+        catch (PartialExecutionException ex) {
+            assertTrue(ex.getCause() instanceof ConfigException);
+        }
+    }
+
+    static void assertRecordsByResource(TestingEmbulk embulk,
+            String inConfigYamlResourceName, String filterConfigYamlResourceName,
+            String sourceCsvResourceName, String resultCsvResourceName)
+            throws IOException
+    {
+        Path inputPath = embulk.createTempFile("csv");
+        Path outputPath = embulk.createTempFile("csv");
+
+        // in: config
+        copyResource(RESOURCE_NAME_PREFIX + sourceCsvResourceName, inputPath);
+        ConfigSource inConfig = embulk.loadYamlResource(RESOURCE_NAME_PREFIX + inConfigYamlResourceName)
+                .set("path_prefix", inputPath.toAbsolutePath().toString());
+
+        // remove_columns filter config
+        ConfigSource filterConfig = embulk.loadYamlResource(RESOURCE_NAME_PREFIX + filterConfigYamlResourceName);
+
+        TestingEmbulk.RunResult result = embulk.inputBuilder()
+                .in(inConfig)
+                .filters(ImmutableList.of(filterConfig))
+                .outputPath(outputPath)
+                .run();
+
+        assertThat(readSortedFile(outputPath), is(readResource(RESOURCE_NAME_PREFIX + resultCsvResourceName)));
+    }
+}