embulk-filter-expand_json 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1eb2c8defd0d9ac7920f13cdde2f95877f7219e2
-  data.tar.gz: 5cd7424ed6d9863c5572a4b07e3fa83097986ed2
+  metadata.gz: ab391ec7a08e2c564a88ab1984b2331df266d360
+  data.tar.gz: 34c7c68b0ebb842a0b7e50866ba07f05a3565036
 SHA512:
-  metadata.gz: 46c382f407a962967f65d08a33758380307f2f2403991ad77dded4a68df3e0e88e15a67e0b71b67e7b5c4d76a5a2dc2aa0833ae76dcf078006106b99b6dc0734
-  data.tar.gz: db059276a090fdbddf43ddec272c7cea372aeed6c60cb0b6f00f48084fdc820c370f585bc4abeeedd14662b1a90f1d7c51209510eaa3d246ea21cd484162108e
+  metadata.gz: 50363ce3c2f5ba54554f4e8aa31f7d9a7ac7a1f58588e0004e6210d4cd3dddf2c8197517924dfb7de99d19a9704c969cad75246a791304c2d2a72fac75d6a974
+  data.tar.gz: 668b2b73ba967ca41b27902456290ead55fd4678ddfc2ebe4758448674b214a78a5beae3886fd46e07691720c0beff26a7165404713ca8b2ed9fb0676cc2e589

data/.travis.yml

@@ -1,3 +1,4 @@
+dist: precise
 language: java
 jdk:
   - openjdk7
@@ -6,4 +7,8 @@ jdk:
 script:
   - ./gradlew test
 after_success:
-  - ./gradlew jacocoTestReport coveralls
+  - ./gradlew jacocoTestReport coveralls
+addons:
+  hosts:
+    - me
+  hostname: me

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+0.2.1 (2017-09-12)
+==================
+- [Enhancement] Support type conversion from floating point numbers to integers
+  - https://github.com/civitaspo/embulk-filter-expand_json/pull/33
+
 0.2.0 (2017-07-14)
 ==================
 - [Incompatible Change]: Remove `time_zone` option, use `default_timezone` instead and column-based timezone.

data/README.md

@@ -17,6 +17,7 @@ expand columns having json into multiple columns
   - **name**: name of the column. you can define [JsonPath](http://goessner.net/articles/JsonPath/) style.
   - **type**: type of the column (see below)
   - **format**: format of the timestamp if type is timestamp
+  - **timezone**: Time zone of each timestamp columns if values don’t include time zone description (`UTC` by default)
 - **keep_expanding_json_column**: Not remove the expanding json column from input schema if it's true (false by default)
 - **default_timezone**: Time zone of timestamp columns if values don’t include time zone description (`UTC` by default)
 - **stop_on_invalid_record**: Stop bulk load transaction if an invalid record is included (false by default)

data/build.gradle

@@ -15,7 +15,7 @@ configurations {
     provided
 }
 
-version = "0.2.0"
+version = "0.2.1"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 

data/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
-#Wed Jan 13 12:41:02 JST 2016
+#Sun Jan 08 00:35:58 PST 2017
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-2.10-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-3.2.1-bin.zip

data/src/main/java/org/embulk/filter/expand_json/FilteredPageOutput.java

@@ -329,7 +329,13 @@ public class FilteredPageOutput
                     pageBuilder.setLong(expandedJsonColumn.getColumn(), Long.parseLong(finalValue));
                 }
                 catch (NumberFormatException e) {
-                    throw new JsonValueInvalidException(String.format("Failed to parse '%s' as long", finalValue), e);
+                    // ad-hoc workaround for exponential notation
+                    try {
+                        pageBuilder.setLong(expandedJsonColumn.getColumn(), (long) Double.parseDouble(finalValue));
+                    }
+                    catch (NumberFormatException e2) {
+                        throw new JsonValueInvalidException(String.format("Failed to parse '%s' as long", finalValue), e);
+                    }
                 }
             }
             else if (Types.TIMESTAMP.equals(expandedJsonColumn.getColumn().getType())) {

data/src/test/java/org/embulk/filter/expand_json/TestExpandJsonFilterPlugin.java

@@ -966,6 +966,55 @@ public class TestExpandJsonFilterPlugin
         });
     }
 
+    @Test
+    public void testParseNumbersInExponentialNotation()
+    {
+        final String configYaml = "" +
+                "type: expand_json\n" +
+                "json_column_name: _c1\n" +
+                "root: $.\n" +
+                "expanded_columns:\n" +
+                "  - {name: _j0, type: double}\n" +
+                "  - {name: _j1, type: long}\n";
+        ConfigSource config = getConfigFromYaml(configYaml);
+        final Schema schema = schema("_c1", STRING);
+
+        expandJsonFilterPlugin.transaction(config, schema, new Control()
+        {
+            @Override
+            public void run(TaskSource taskSource, Schema outputSchema)
+            {
+                MockPageOutput mockPageOutput = new MockPageOutput();
+
+                String doubleFloatingPoint = "-1.234e-5";
+                double doubleFixedPoint = -0.00001234; // Use in Asserting.
+                String longFloatingPoint = "12345e3";
+                long longFixedPoint = 12_345_000L; // Use in Asserting.
+
+                String data = String.format(
+                        "{\"_j0\":%s, \"_j1\":%s}",
+                        doubleFloatingPoint,
+                        longFloatingPoint);
+
+                try (PageOutput pageOutput = expandJsonFilterPlugin.open(taskSource, schema, outputSchema, mockPageOutput)) {
+                    for (Page page : PageTestUtils.buildPage(runtime.getBufferAllocator(), schema, data, c1Data)) {
+                        pageOutput.add(page);
+                    }
+
+                    pageOutput.finish();
+                }
+
+                PageReader pageReader = new PageReader(outputSchema);
+
+                for (Page page : mockPageOutput.pages) {
+                    pageReader.setPage(page);
+                    assertEquals(doubleFixedPoint, pageReader.getDouble(outputSchema.getColumn(0)), 0.0);
+                    assertEquals(longFixedPoint, pageReader.getLong(outputSchema.getColumn(1)));
+                }
+            }
+        });
+    }
+
     private static Schema schema(Object... nameAndTypes)
     {
         Schema.Builder builder = Schema.builder();

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-expand_json
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Civitaspo
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-07-14 00:00:00.000000000 Z
+date: 2017-09-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -65,7 +65,7 @@ files:
 - src/test/java/org/embulk/filter/expand_json/TestExpandJsonFilterPlugin.java
 - classpath/accessors-smart-1.2.jar
 - classpath/asm-5.0.4.jar
-- classpath/embulk-filter-expand_json-0.2.0.jar
+- classpath/embulk-filter-expand_json-0.2.1.jar
 - classpath/json-path-2.4.0.jar
 - classpath/json-smart-2.3.jar
 - classpath/slf4j-api-1.7.25.jar