embulk-parser-jsonl 0.1.2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7f4944036ca4cacbe9f2b8a0943503f77c92aca1
-  data.tar.gz: a11616b53812123915ccdd8e88938471f81b2aea
+  metadata.gz: e169a403e6e9d48e00b0a5d4b5aed92774e556a1
+  data.tar.gz: 0b762ec444862c9fc3f575d1b7ddd87207632d08
 SHA512:
-  metadata.gz: bf717db902192c17124a393b063972067e2e9d46b66dfc7247cf427f03bcec551d10c799c2ec6923ca486ba44ec0017d72598280106d356eb3f9464fdbf94d0f
-  data.tar.gz: be23fbef9cba986f15fecebd91ba15059f1fea6fe13aaebd62373488aa092e6f42b7fb13ab471bf125c38f8ab587a7a0bfe62dc6b465dece554a80e6e0792e25
+  metadata.gz: 6a16d1e617e0bcf7ccd1231704374f3ba30c49012c21b7b9b03744b245120e1fe174a2c42d9cc8b481836485c0c9cafa54a8fecfab9417c063fac161b6f650da
+  data.tar.gz: 44cdc4bb41d368162b6b58a537c3c1eecf2f0b3685dd87643ecd394e2666cbd58aa55c0db09f82be9c61449b17b47ea5d164284f92e4cc2b3890e1d7b7f852bb

data/CHANGELOG.md

@@ -1,7 +1,11 @@
+## 0.2.0 - 2016-05-28
+
+[new feature] Support typecast option [#9](https://github.com/shun0102/embulk-parser-jsonl/pull/9)
+
 ## 0.1.2 - 2016-03-27
 
 [new feature] Support column_options option [#4](https://github.com/shun0102/embulk-parser-jsonl/pull/4)
-[maintenance] Upgrade Embulk v0.8.8 [#6](https://github.com/shun0102/embulk-parser-jsonl/pull/6)
+[maintenance] Upgrade Embulk v0.8.8 [#6](https://github.com/shun0102/embulk-parser-jsonl/pull/6)## 0.1.2 - 2016-03-27
 
 ## 0.1.1 - 2016-03-17
 

data/README.md

@@ -1,6 +1,6 @@
 # Jsonl parser plugin for Embulk
 
-TODO: Write short description here and embulk-parser-jsonl.gemspec file.
+[JSONL (JSON Lines)](http://jsonlines.org/) parser plugin for Embulk
 
 ## Overview
 
@@ -9,21 +9,24 @@ TODO: Write short description here and embulk-parser-jsonl.gemspec file.
 
 ## Configuration
 
-- **type**: specify this parser as jsonl
-- **columns**: specify column name and type (array, required)
+- **type**: Specify this parser as jsonl
+- **columns**: Specify column name and type. See below (array, required)
+* **stop_on_invalid_record**: Stop bulk load transaction if a file includes invalid record (such as invalid timestamp) (boolean, default: false)
+* **default_timezone**: Default timezone of the timestamp (string, default: UTC)
+* **default_timestamp_format**: Default timestamp format of the timestamp (string, default: `%Y-%m-%d %H:%M:%S.%N %z`)
+* **newline**: Newline character (CRLF, LF or CR) (string, default: CRLF)
+* **charset**: Character encoding (eg. ISO-8859-1, UTF-8) (string, default: UTF-8)
+* **default_typecast**: Specify whether to cast values automatically to the specified types or not (boolean, default: true)
 
-## Example
+### columns
 
-```yaml
-in:
-  type: any file input plugin type
-  parser:
-    type: jsonl
-    columns:
-      - {name: first_name, type: string}
-      - {name: last_name, type: string}
-      - {name: age, type: long}
-```
+* **name**: Name of the column (string, required)
+* **type**: Type of the column (string, required)
+* **timezone**: Timezone of the timestamp if type is timestamp (string, default: default_timestamp)
+* **format**: Format of the timestamp if type is timestamp (string, default: default_format)
+* **typecast**: Whether cast values or not (boolean, default: default_typecast)
+
+## Guess
 
 (If guess supported) you don't have to write `parser:` section in the configuration file. After writing `in:` section, you can let embulk guess `parser:` section using this command:
 

data/bench/gen_dummy.rb

@@ -0,0 +1,5 @@
+File::open('bench/sample.jsonl', 'w') { |f|
+  (1..1000000).each {
+    f.puts(%Q[{"foo":"foo","bool":true,"bool_str":"true","int":10,"int_str":"20","double":1.5,"double_str":"2.5","array":[1,2,3]}])
+  }
+}

data/bench/typecast.yml

@@ -0,0 +1,17 @@
+in:
+  type: file
+  path_prefix: "bench/sample"
+  parser:
+    type: jsonl
+    default_typecast: true
+    columns:
+      - {name: "foo", type: "string"}
+      - {name: "bool", type: "boolean"}
+      - {name: "bool_str", type: "boolean"}
+      - {name: "int", type: "long"}
+      - {name: "int_str", type: "long"}
+      - {name: "double", type: "double"}
+      - {name: "double_str", type: "double"}
+      - {name: "array", type: "json"}
+out:
+  type: "null"

data/bench/without_typecast.yml

@@ -0,0 +1,17 @@
+in:
+  type: file
+  path_prefix: "bench/sample"
+  parser:
+    type: jsonl
+    default_typecast: false
+    columns:
+      - {name: "foo", type: "string"}
+      - {name: "bool", type: "boolean"}
+      - {name: "bool_str", type: "string"}
+      - {name: "int", type: "long"}
+      - {name: "int_str", type: "string"}
+      - {name: "double", type: "double"}
+      - {name: "double_str", type: "string"}
+      - {name: "array", type: "json"}
+out:
+  type: "null"

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.1.2"
+version = "0.2.0"
 
 compileJava.options.encoding = 'UTF-8' // source encoding
 sourceCompatibility = 1.7

data/embulk-parser-jsonl.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-parser-jsonl"
-  spec.version       = "0.1.2"
+  spec.version       = "0.2.0"
   spec.authors       = ["Shunsuke Mikami"]
   spec.summary       = "Jsonl parser plugin for Embulk"
   spec.description   = "Parses Jsonl files read by other file input plugins."

data/example/compat.yml

@@ -0,0 +1,21 @@
+in:
+  type: file
+  path_prefix: "example/sample"
+  parser:
+    type: jsonl
+    columns:
+      - {name: "foo", type: "string"}
+      - {name: "bool", type: "boolean"}
+      - {name: "bool_str", type: "boolean"}
+      - {name: "int", type: "string"}
+      - {name: "int_str", type: "string"}
+      - {name: "time", type: "timestamp", format: '%Y-%m-%d %H:%M:%S'}
+      - {name: "double", type: "double"}
+      - {name: "double_str", type: "string"}
+      - {name: "array", type: "json"}
+    column_options:
+      bool_str: {type: "boolean"}
+      int_str: {type: "long"}
+      double_str: {type: "double"}
+out:
+  type: stdout

data/example/example.yml

@@ -1,23 +1,18 @@
 in:
   type: file
-  path_prefix: "sample"
+  path_prefix: "example/sample"
   parser:
     type: jsonl
+    # default_typecast: true # default: true
     columns:
       - {name: "foo", type: "string"}
       - {name: "bool", type: "boolean"}
-      - {name: "bool_str", type: "string"}
+      - {name: "bool_str", type: "boolean"}
       - {name: "int", type: "long"}
-      - {name: "int_str", type: "string"}
+      - {name: "int_str", type: "long"}
       - {name: "time", type: "timestamp", format: '%Y-%m-%d %H:%M:%S'}
       - {name: "double", type: "double"}
-      - {name: "double_str", type: "string"}
+      - {name: "double_str", type: "double"}
       - {name: "array", type: "json"}
-    column_options:
-      bool_str: {type: "boolean"}
-      int_str: {type: "long"}
-      double_str: {type: "double"}
-
-
 out:
   type: stdout

data/example/example_without_typecast.yml

@@ -0,0 +1,18 @@
+in:
+  type: file
+  path_prefix: "example/sample"
+  parser:
+    type: jsonl
+    default_typecast: false
+    columns:
+      - {name: "foo", type: "string"}
+      - {name: "bool", type: "boolean"}
+      - {name: "bool_str", type: "string"}
+      - {name: "int", type: "long"}
+      - {name: "int_str", type: "string"}
+      - {name: "time", type: "timestamp", format: '%Y-%m-%d %H:%M:%S'}
+      - {name: "double", type: "double"}
+      - {name: "double_str", type: "string"}
+      - {name: "array", type: "json"}
+out:
+  type: stdout

data/example/sample.json

@@ -1,2 +1,2 @@
-{"foo": "bar", "bool": true, "bool_str": "true", "int": 1, "int_str": "42", "time": "2016-3-2 00:39:18", "double": 1.2, "double_str": "2.4", "array": [1, 2, 3]}
-{"foo": null, "bool": false, "bool_str": "false", "int": 1, "int_str": "42", "time": "2016-3-2 00:39:18", "double": 1.2, "double_str": "2.4", "array": [{"inner": "hoge"}, {"inner": 1.5}]}
+{"foo": "bar", "bool": true, "bool_str": "true", "int": 42, "int_str": "42", "time": "2016-3-2 00:39:18", "double": 2.4, "double_str": "2.4", "array": [1, 2, 3]}
+{"foo": null, "bool": false, "bool_str": "false", "int": 42, "int_str": "42", "time": "2016-3-2 00:39:18", "double": 2.4, "double_str": "2.4", "array": [{"inner": "hoge"}, {"inner": 1.5}]}

data/src/main/java/org/embulk/parser/jsonl/ColumnCaster.java

@@ -0,0 +1,97 @@
+package org.embulk.parser.jsonl;
+
+import org.embulk.parser.jsonl.cast.BooleanCast;
+import org.embulk.parser.jsonl.cast.DoubleCast;
+import org.embulk.parser.jsonl.cast.JsonCast;
+import org.embulk.parser.jsonl.cast.LongCast;
+import org.embulk.parser.jsonl.cast.StringCast;
+import org.embulk.spi.DataException;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.time.TimestampParser;
+import org.msgpack.value.Value;
+
+class ColumnCaster
+{
+    ColumnCaster() {}
+
+    public static boolean asBoolean(Value value) throws DataException
+    {
+        if (value.isBooleanValue()) {
+            return value.asBooleanValue().getBoolean();
+        }
+        else if (value.isIntegerValue()) {
+            return LongCast.asBoolean(value.asIntegerValue().asLong());
+        }
+        else if (value.isFloatValue()) {
+            return DoubleCast.asBoolean(value.asFloatValue().toDouble());
+        }
+        else if (value.isStringValue()) {
+            return StringCast.asBoolean(value.asStringValue().asString());
+        }
+        else {
+            return JsonCast.asBoolean(value);
+        }
+    }
+
+    public static long asLong(Value value) throws DataException
+    {
+        if (value.isBooleanValue()) {
+            return BooleanCast.asLong(value.asBooleanValue().getBoolean());
+        }
+        else if (value.isIntegerValue()) {
+            return value.asIntegerValue().asLong();
+        }
+        else if (value.isFloatValue()) {
+            return DoubleCast.asLong(value.asFloatValue().toDouble());
+        }
+        else if (value.isStringValue()) {
+            return StringCast.asLong(value.asStringValue().asString());
+        }
+        else {
+            return JsonCast.asLong(value);
+        }
+    }
+
+    public static double asDouble(Value value) throws DataException
+    {
+        if (value.isBooleanValue()) {
+            return BooleanCast.asDouble(value.asBooleanValue().getBoolean());
+        }
+        else if (value.isIntegerValue()) {
+            return LongCast.asDouble(value.asIntegerValue().asLong());
+        }
+        else if (value.isFloatValue()) {
+            return value.asFloatValue().toDouble();
+        }
+        else if (value.isStringValue()) {
+            return StringCast.asDouble(value.asStringValue().asString());
+        }
+        else {
+            return JsonCast.asDouble(value);
+        }
+    }
+
+    public static String asString(Value value) throws DataException
+    {
+        return value.toString();
+    }
+
+    public static Timestamp asTimestamp(Value value, TimestampParser parser) throws DataException
+    {
+        if (value.isBooleanValue()) {
+            return BooleanCast.asTimestamp(value.asBooleanValue().getBoolean());
+        }
+        else if (value.isIntegerValue()) {
+            return LongCast.asTimestamp(value.asIntegerValue().asLong());
+        }
+        else if (value.isFloatValue()) {
+            return DoubleCast.asTimestamp(value.asFloatValue().toDouble());
+        }
+        else if (value.isStringValue()) {
+            return StringCast.asTimestamp(value.asStringValue().asString(), parser);
+        }
+        else {
+            return JsonCast.asTimestamp(value);
+        }
+    }
+}

data/src/main/java/org/embulk/parser/jsonl/ColumnVisitorImpl.java

@@ -0,0 +1,164 @@
+package org.embulk.parser.jsonl;
+
+import com.google.common.base.Optional;
+import org.embulk.parser.jsonl.JsonlParserPlugin.PluginTask;
+import org.embulk.parser.jsonl.JsonlParserPlugin.TypecastColumnOption;
+
+import org.embulk.spi.Column;
+import org.embulk.spi.ColumnConfig;
+import org.embulk.spi.ColumnVisitor;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.Schema;
+import org.embulk.spi.SchemaConfig;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.time.TimestampParser;
+import org.msgpack.core.MessageTypeException;
+import org.msgpack.value.Value;
+
+public class ColumnVisitorImpl implements ColumnVisitor {
+    protected final PluginTask task;
+    protected final Schema schema;
+    protected final PageBuilder pageBuilder;
+    protected final TimestampParser[] timestampParsers;
+    protected final Boolean autoTypecasts[];
+
+    protected Value value;
+
+    public ColumnVisitorImpl(PluginTask task, Schema schema, PageBuilder pageBuilder, TimestampParser[] timestampParsers)
+    {
+        this.task = task;
+        this.schema = schema;
+        this.pageBuilder = pageBuilder;
+        this.timestampParsers = timestampParsers;
+        this.autoTypecasts = new Boolean[schema.size()];
+        buildAutoTypecasts();
+    }
+
+    private void buildAutoTypecasts()
+    {
+        for (Column column : schema.getColumns()) {
+            this.autoTypecasts[column.getIndex()] = task.getDefaultTypecast();
+        }
+
+        Optional<SchemaConfig> schemaConfig = task.getSchemaConfig();
+        if (schemaConfig.isPresent()) {
+            for (ColumnConfig columnConfig : schemaConfig.get().getColumns()) {
+                TypecastColumnOption columnOption = columnConfig.getOption().loadConfig(TypecastColumnOption.class);
+                Boolean autoTypecast = columnOption.getTypecast().or(task.getDefaultTypecast());
+                Column column = schema.lookupColumn(columnConfig.getName());
+                this.autoTypecasts[column.getIndex()] = autoTypecast;
+            }
+        }
+    }
+
+    public void setValue(Value value)
+    {
+        this.value = value;
+    }
+
+    @Override
+    public void booleanColumn(Column column)
+    {
+        if (isNil(value)) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            try {
+                boolean booleanValue = autoTypecasts[column.getIndex()] ? ColumnCaster.asBoolean(value) : value.asBooleanValue().getBoolean();
+                pageBuilder.setBoolean(column, booleanValue);
+            }
+            catch (MessageTypeException e) {
+                throw new JsonRecordValidateException(String.format("failed to get \"%s\" as Boolean", value), e);
+            }
+        }
+    }
+
+    @Override
+    public void longColumn(Column column)
+    {
+        if (isNil(value)) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            try {
+                long longValue = autoTypecasts[column.getIndex()] ? ColumnCaster.asLong(value) : value.asIntegerValue().toLong();
+                pageBuilder.setLong(column, longValue);
+            }
+            catch (MessageTypeException e) {
+                throw new JsonRecordValidateException(String.format("failed to get \"%s\" as Long", value), e);
+            }
+        }
+    }
+
+    @Override
+    public void doubleColumn(Column column)
+    {
+        if (isNil(value)) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            try {
+                double doubleValue = autoTypecasts[column.getIndex()] ? ColumnCaster.asDouble(value) : value.asFloatValue().toDouble();
+                pageBuilder.setDouble(column, doubleValue);
+            }
+            catch (MessageTypeException e) {
+                throw new JsonRecordValidateException(String.format("failed get \"%s\" as Double", value), e);
+            }
+        }
+    }
+
+    @Override
+    public void stringColumn(Column column)
+    {
+        if (isNil(value)) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            try {
+                String string = autoTypecasts[column.getIndex()] ? ColumnCaster.asString(value) : value.asStringValue().toString();
+                pageBuilder.setString(column, string);
+            }
+            catch (MessageTypeException e) {
+                throw new JsonRecordValidateException(String.format("failed to get \"%s\" as String", value), e);
+            }
+        }
+    }
+
+    @Override
+    public void timestampColumn(Column column)
+    {
+        if (isNil(value)) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            try {
+                Timestamp timestamp = ColumnCaster.asTimestamp(value, timestampParsers[column.getIndex()]);
+                pageBuilder.setTimestamp(column, timestamp);
+            }
+            catch (MessageTypeException e) {
+                throw new JsonRecordValidateException(String.format("failed to get \"%s\" as Timestamp", value), e);
+            }
+        }
+    }
+
+    @Override
+    public void jsonColumn(Column column)
+    {
+        if (isNil(value)) {
+            pageBuilder.setNull(column);
+        }
+        else {
+            try {
+                pageBuilder.setJson(column, value);
+            }
+            catch (MessageTypeException e) {
+                throw new JsonRecordValidateException(String.format("failed to get \"%s\" as Json", value), e);
+            }
+        }
+    }
+
+    protected boolean isNil(Value v)
+    {
+        return v == null || v.isNilValue();
+    }
+}