embulk-filter-column 0.4.0 → 0.5.0.pre1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 572e638f806833f390196e60aa5791c0f23aff76
-  data.tar.gz: db719c3a30d31e86bea2f75afaa09849c92ec631
+  metadata.gz: 1e88bb8b22f8f2030982764303175dd77b97a42a
+  data.tar.gz: e7b65b4cc79b92b3aa89596fc3e1253fa7897ace
 SHA512:
-  metadata.gz: 6a4c06e967fc14763e386326166ae4b0ae9247fca8a158dc41e25a9bb430e26430d70fb2603abd29fedfb54f822a2fac152078628f62eb3fe816bb31a7882d45
-  data.tar.gz: edb08b9d1b7fc6e53a41d86ab82a61a7f848737bf37f6baa0ec6048c5475c5040db4f165fb5306a21f4fef6e69b7afdce4f78855381c1c5b48e5aaec4fa3b303
+  metadata.gz: 7cc74b699dc85ec17ff45ab2d991f77219b7fa93ef03d2d444b21d36b83c47689b4d08ebe81dfc0e428134f9a380262367fd8d3cd33987b831d8628e64a79713
+  data.tar.gz: e2cb3d98ec5b678f67d94c8b70854f2b46c5db7544cfbe1f2713653229195c1c57ee2b28c8fe423ab345a01f16b4b7d68337ebfd4c93719512e68c7c35e7a1e2

data/.gitignore

@@ -10,3 +10,4 @@ build/
 .tags
 .ruby-version
 *.iml
+.DS_Store

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# 0.5.0.pre1 (2016-05-24)
+
+Enhancements:
+
+* Support JSONPath (like) name
+
 # 0.4.0 (2016-02-01)
 
 Enhancements:

data/README.md

@@ -109,6 +109,28 @@ VmjbjAA0tOoSEPv_vKAGMtD_0aXZji0abGe7_VXHmUQ,3962
 C40P5H1WcBx-aWFDJCI8th6QPEI2DOUgupt_gB8UutE,7323
 ```
 
+## JSONPath (like) name
+
+For type: json column, you can specify [JSONPath](http://goessner.net/articles/JsonPath/) for column's name as:
+
+```
+$.payload.key1
+$.payload.array[0]
+$.payload.array[*]
+```
+
+EXAMPLE:
+
+* [example/json_columns.yml](example/json_columns.yml)
+* [example/json_add_columns.yml](example/json_add_columns.yml)
+* [example/json_drop_columns.yml](example/json_drop_columns.yml)
+
+NOTE:
+
+* JSONPath syntax is not fully supported
+* Embulk's type: json cannot have timestamp column, so `type: timesatmp` for `add_columns` or `columns` with default is not available
+* `src` for `add_columns` or `columns` is not supported yet
+
 ## ToDo
 
 * Write test
@@ -119,7 +141,7 @@ Run example:
 
 ```
 $ ./gradlew classpath
-$ embulk run -I lib example.yml
+$ embulk preview -I lib example/example.yml
 ```
 
 Run test:

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.4.0"
+version = "0.5.0.pre1"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 

data/example/columns.yml

@@ -29,6 +29,7 @@ filters:
       - {name: foo,  default: 1, type: long}
       - {name: id}
       - {name: copy_score, src: score}
-      - {name: json, default: "{\"foo\": \"FOO\"}"}
+      - {name: json, default: "{\"foo\":\"FOO\"}"}
+      - {name: $.json.foo}
 out:
   type: stdout

data/example/example.yml

@@ -0,0 +1,35 @@
+# in:
+#   type: random
+#   rows: 100
+#   schema:
+#     id: primary_key
+#     name: string
+#     score: integer
+in:
+  type: file
+  path_prefix: example/example.csv
+  parser:
+    type: csv
+    charset: UTF-8
+    newline: CRLF
+    null_string: 'NULL'
+    skip_header_lines: 1
+    comment_line_marker: '#'
+    columns:
+      - {name: time,  type: timestamp, format: "%Y-%m-%d"}
+      - {name: id, type: long}
+      - {name: name, type: string}
+      - {name: score, type: double}
+      - {name: json, type: json}
+filters:
+  - type: column
+    columns:
+      - {name: time, default: "2015-07-13", format: "%Y-%m-%d"}
+      - {name: name, default: "foo"}
+      - {name: foo,  default: 1, type: long}
+      - {name: id}
+      - {name: copy_score, src: score}
+      - {name: json, default: "{\"foo\":\"FOO\"}"}
+      - {name: $.json.foo}
+out:
+  type: stdout

data/example/json_add_columns.yml

@@ -0,0 +1,31 @@
+in:
+  type: file
+  path_prefix: example/example.csv
+  parser:
+    type: csv
+    charset: UTF-8
+    newline: CRLF
+    null_string: 'NULL'
+    skip_header_lines: 1
+    comment_line_marker: '#'
+    columns:
+      - {name: time,  type: timestamp, format: "%Y-%m-%d"}
+      - {name: id, type: long}
+      - {name: name, type: string}
+      - {name: score, type: double}
+      - {name: json, type: json}
+filters:
+  - type: column
+    default_timezone: "Asia/Tokyo"
+    default_timestamp_format: "%Y-%m-%d"
+    columns:
+      - {name: time}
+      - {name: id}
+      - {name: name}
+      - {name: score}
+      - {name: json, default: "{}"}
+    add_columns:
+      - {name: $.json.foo, type: long, default: 1}
+      - {name: $.json.d, type: string, default: "2015-07-13"}
+out:
+  type: stdout

data/example/json_columns.yml

@@ -0,0 +1,23 @@
+in:
+  type: file
+  path_prefix: example/example.csv
+  parser:
+    type: csv
+    charset: UTF-8
+    newline: CRLF
+    null_string: 'NULL'
+    skip_header_lines: 1
+    comment_line_marker: '#'
+    columns:
+      - {name: time,  type: timestamp, format: "%Y-%m-%d"}
+      - {name: id, type: long}
+      - {name: name, type: string}
+      - {name: score, type: double}
+      - {name: json, type: json}
+filters:
+  - type: column
+    columns:
+      - {name: json, default: "{\"foo\":\"FOO\"}"}
+      - {name: $.json.foo}
+out:
+  type: stdout

data/example/json_drop_columns.yml

@@ -0,0 +1,22 @@
+in:
+  type: file
+  path_prefix: example/example.csv
+  parser:
+    type: csv
+    charset: UTF-8
+    newline: CRLF
+    null_string: 'NULL'
+    skip_header_lines: 1
+    comment_line_marker: '#'
+    columns:
+      - {name: time,  type: timestamp, format: "%Y-%m-%d"}
+      - {name: id, type: long}
+      - {name: name, type: string}
+      - {name: score, type: double}
+      - {name: json, type: json}
+filters:
+  - type: column
+    drop_columns:
+      - {name: $.json.foo }
+out:
+  type: stdout

data/gradlew

@@ -112,8 +112,8 @@ fi
 
 # For Cygwin, switch paths to Windows format before running java
 if $cygwin ; then
-    APP_HOME=`cygpath --path --mixed "$APP_HOME"`
-    CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
+    APP_HOME=`cygpath --name --mixed "$APP_HOME"`
+    CLASSPATH=`cygpath --name --mixed "$CLASSPATH"`
 
     # We build the pattern for arguments to be converted via cygpath
     ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
@@ -134,7 +134,7 @@ if $cygwin ; then
         CHECK2=`echo "$arg"|egrep -c "^-"`                                 ### Determine if an option
 
         if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then                    ### Added a condition
-            eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
+            eval `echo args$i`=`cygpath --name --ignore --mixed "$arg"`
         else
             eval `echo args$i`="\"$arg\""
         fi

data/lib/embulk/filter/column.rb

@@ -1,3 +1,3 @@
 Embulk::JavaPlugin.register_filter(
-  "column", "org.embulk.filter.ColumnFilterPlugin",
+  "column", "org.embulk.filter.column.ColumnFilterPlugin",
   File.expand_path('../../../../classpath', __FILE__))

data/settings.gradle

@@ -0,0 +1 @@
+rootProject.name = 'embulk-filter-column'

data/src/main/java/org/embulk/filter/column/ColumnFilterPlugin.java

@@ -0,0 +1,260 @@
+package org.embulk.filter.column;
+
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableList;
+
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
+import org.embulk.config.TaskSource;
+
+import org.embulk.spi.Column;
+import org.embulk.spi.Exec;
+import org.embulk.spi.FilterPlugin;
+import org.embulk.spi.Page;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.PageReader;
+import org.embulk.spi.Schema;
+import org.embulk.spi.SchemaConfigException;
+import org.embulk.spi.time.TimestampParser;
+import org.embulk.spi.type.Type;
+
+import org.joda.time.DateTimeZone;
+import org.slf4j.Logger;
+
+import java.util.List;
+
+public class ColumnFilterPlugin implements FilterPlugin
+{
+    private static final Logger logger = Exec.getLogger(ColumnFilterPlugin.class);
+
+    public ColumnFilterPlugin()
+    {
+    }
+
+    // NOTE: This is not spi.ColumnConfig
+    interface ColumnConfig extends Task
+    {
+        @Config("name")
+        public String getName();
+
+        @Config("type")
+        @ConfigDefault("null")
+        public Optional<Type> getType(); // required only for addColumns
+
+        @Config("default")
+        @ConfigDefault("null")
+        public Optional<Object> getDefault();
+
+        @Config("format")
+        @ConfigDefault("null")
+        public Optional<String> getFormat();
+
+        @Config("timezone")
+        @ConfigDefault("null")
+        public Optional<DateTimeZone> getTimeZone();
+
+        @Config("src")
+        @ConfigDefault("null")
+        public Optional<String> getSrc();
+    }
+
+    interface PluginTask extends Task, TimestampParser.Task
+    {
+        @Config("columns")
+        @ConfigDefault("[]")
+        public List<ColumnConfig> getColumns();
+
+        @Config("add_columns")
+        @ConfigDefault("[]")
+        public List<ColumnConfig> getAddColumns();
+
+        @Config("drop_columns")
+        @ConfigDefault("[]")
+        public List<ColumnConfig> getDropColumns();
+
+        // See TimestampParser for default_timestamp_format, and default_timezone
+    }
+
+    @Override
+    public void transaction(final ConfigSource config, final Schema inputSchema,
+            final FilterPlugin.Control control)
+    {
+        PluginTask task = config.loadConfig(PluginTask.class);
+
+        configure(task);
+        Schema outputSchema = buildOutputSchema(task, inputSchema);
+
+        control.run(task.dump(), outputSchema);
+    }
+
+    private void configure(PluginTask task)
+    {
+        List<ColumnConfig> columns = task.getColumns();
+        List<ColumnConfig> addColumns = task.getAddColumns();
+        List<ColumnConfig> dropColumns = task.getDropColumns();
+
+        if (columns.size() == 0 && addColumns.size() == 0 && dropColumns.size() == 0) {
+            throw new ConfigException("One of \"columns\", \"add_columns\", \"drop_columns\" must be specified.");
+        }
+
+        if (columns.size() > 0 && dropColumns.size() > 0) {
+            throw new ConfigException("Either of \"columns\", \"drop_columns\" can be specified.");
+        }
+    }
+
+    private Schema buildOutputSchema(PluginTask task, Schema inputSchema)
+    {
+        List<ColumnConfig> columns = task.getColumns();
+        List<ColumnConfig> addColumns = task.getAddColumns();
+        List<ColumnConfig> dropColumns = task.getDropColumns();
+
+        // Automatically get column type from inputSchema for columns and dropColumns
+        ImmutableList.Builder<Column> builder = ImmutableList.builder();
+        int i = 0;
+        if (dropColumns.size() > 0) {
+            for (Column inputColumn : inputSchema.getColumns()) {
+                String name = inputColumn.getName();
+                boolean matched = false;
+                for (ColumnConfig dropColumn : dropColumns) {
+                    // skip json path notation to build outputSchema
+                    if (dropColumn.getName().startsWith("$.")) {
+                        continue;
+                    }
+                    if (dropColumn.getName().equals(name)) {
+                        matched = true;
+                        break;
+                    }
+                }
+                if (! matched) {
+                    Column outputColumn = new Column(i++, name, inputColumn.getType());
+                    builder.add(outputColumn);
+                }
+            }
+        }
+        else if (columns.size() > 0) {
+            for (ColumnConfig column : columns) {
+                // skip json path notation to build output schema
+                if (column.getName().startsWith("$.")) {
+                    continue;
+                }
+                if (column.getSrc().isPresent() && column.getSrc().get().startsWith("$.")) {
+                    continue;
+                }
+
+                String name                   = column.getName();
+                Optional<Type>   type         = column.getType();
+                Optional<Object> defaultValue = column.getDefault();
+                Optional<String> src          = column.getSrc();
+
+                String srcName = src.isPresent() ? src.get() : name;
+                Column inputColumn;
+                try {
+                    inputColumn = inputSchema.lookupColumn(srcName);
+                }
+                catch (SchemaConfigException ex) {
+                    inputColumn = null;
+                }
+                if (inputColumn != null) { // filter or copy column
+                    Column outputColumn = new Column(i++, name, inputColumn.getType());
+                    builder.add(outputColumn);
+                }
+                else if (type.isPresent() && defaultValue.isPresent()) { // add column
+                    Column outputColumn = new Column(i++, name, type.get());
+                    builder.add(outputColumn);
+                }
+                else {
+                    throw new SchemaConfigException(String.format("columns: Column src '%s' is not found in inputSchema. Column '%s' does not have \"type\" and \"default\"", srcName, name));
+                }
+            }
+        }
+        else {
+            for (Column column : inputSchema.getColumns()) {
+                Column outputColumn = new Column(i++, column.getName(), column.getType());
+                builder.add(outputColumn);
+            }
+        }
+
+        // Add columns to last. If you want to add to head or middle, you can use `columns` option
+        if (addColumns.size() > 0) {
+            for (ColumnConfig column : addColumns) {
+                // skip json path notation to build output schema
+                if (column.getName().startsWith("$.")) {
+                    continue;
+                }
+                if (column.getSrc().isPresent() && column.getSrc().get().startsWith("$.")) {
+                    continue;
+                }
+
+                String name                   = column.getName();
+                Optional<Type> type           = column.getType();
+                Optional<Object> defaultValue = column.getDefault();
+                Optional<String> src          = column.getSrc();
+
+                String srcName = null;
+                Column inputColumn = null;
+                if (src.isPresent()) {
+                    srcName = src.get();
+                    try {
+                        inputColumn = inputSchema.lookupColumn(srcName);
+                    }
+                    catch (SchemaConfigException ex) {
+                        inputColumn = null;
+                    }
+                }
+                if (inputColumn != null) { // copy column
+                    Column outputColumn = new Column(i++, name, inputColumn.getType());
+                    builder.add(outputColumn);
+                }
+                else if (type.isPresent() && defaultValue.isPresent()) { // add column
+                    Column outputColumn = new Column(i++, name, type.get());
+                    builder.add(outputColumn);
+                }
+                else {
+                    throw new SchemaConfigException(String.format("add_columns: Column src '%s' is not found in inputSchema, Column '%s' does not have \"type\" and \"default\"", srcName, name));
+                }
+            }
+        }
+
+        return new Schema(builder.build());
+    }
+
+    @Override
+    public PageOutput open(final TaskSource taskSource, final Schema inputSchema,
+            final Schema outputSchema, final PageOutput output)
+    {
+        final PluginTask task = taskSource.loadTask(PluginTask.class);
+
+        return new PageOutput() {
+            private PageReader pageReader = new PageReader(inputSchema);
+            private PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), outputSchema, output);
+            private ColumnVisitorImpl visitor = new ColumnVisitorImpl(task, inputSchema, outputSchema, pageReader, pageBuilder);
+
+            @Override
+            public void finish()
+            {
+                pageBuilder.finish();
+            }
+
+            @Override
+            public void close()
+            {
+                pageBuilder.close();
+            }
+
+            @Override
+            public void add(Page page)
+            {
+                pageReader.setPage(page);
+
+                while (pageReader.nextRecord()) {
+                    outputSchema.visitColumns(visitor);
+                    pageBuilder.addRecord();
+                }
+            }
+        };
+    }
+}