RubyGems - embulk-filter-json_csv2arrayofobjects - Versions diffs - 0.1.0 - Mend

embulk-filter-json_csv2arrayofobjects 0.1.0

Files changed (22) hide show

checksums.yaml +7 -0
data/.gitignore +12 -0
data/LICENSE.txt +21 -0
data/README.md +91 -0
data/build.gradle +98 -0
data/config/checkstyle/checkstyle.xml +128 -0
data/config/checkstyle/default.xml +108 -0
data/example/example.tsv +2 -0
data/example/example.yml +31 -0
data/gradle/wrapper/gradle-wrapper.jar +0 -0
data/gradle/wrapper/gradle-wrapper.properties +6 -0
data/gradlew +169 -0
data/gradlew.bat +84 -0
data/lib/embulk/filter/json_csv2arrayofobjects.rb +3 -0
data/src/main/java/org/embulk/filter/json_csv2arrayofobjects/ColumnVisitorImpl.java +118 -0
data/src/main/java/org/embulk/filter/json_csv2arrayofobjects/Filter.java +68 -0
data/src/main/java/org/embulk/filter/json_csv2arrayofobjects/JsonCsv2arrayofobjectsFilterPlugin.java +137 -0
data/src/main/java/org/embulk/filter/json_csv2arrayofobjects/StringCast.java +101 -0
data/src/test/java/org/embulk/filter/json_csv2arrayofobjects/TestFilter.java +238 -0
data/src/test/java/org/embulk/filter/json_csv2arrayofobjects/TestJsonCsv2arrayofobjectsFilterPlugin.java +67 -0
data/src/test/java/org/embulk/filter/json_csv2arrayofobjects/TestStringCast.java +99 -0
metadata +98 -0

checksums.yaml ADDED

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: dcbe1c2f3ff858ef3f0f26e0e2633a13d03a7e45
+  data.tar.gz: 0998ad3bac1582ae7d8dcabd70b20c7aad17b520
+SHA512:
+  metadata.gz: 3add00c7a490cfa7ff320e492ac2b0b7290750c86d4f16495c906b1a6ac5f99b35a6b38630e190d29f6490397ec3d3c62e9832570b3b44ebf0a99c8a2324b983
+  data.tar.gz: b8c7130648264f71966abb9444898cfcd5ab746bceea97182963107c31ab55249abcf84f64d6d48ca4940c6a585d358c7d248bb547812a533dfe55f7d128f366

data/.gitignore ADDED

@@ -0,0 +1,12 @@
+*~
+/pkg/
+/tmp/
+*.gemspec
+.gradle/
+/classpath/
+build/
+.idea
+/.settings/
+/.metadata/
+.classpath
+.project

data/LICENSE.txt ADDED

@@ -0,0 +1,21 @@
+MIT License
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md ADDED

@@ -0,0 +1,91 @@
+# Json Csv2arrayofobjects filter plugin for Embulk
+An Embulk filter plugin to convert csv in JSON to array of objects.
+## Overview
+* **Plugin type**: filter
+## Configuration
+* **column**: input column name (string, required)
+* **key**: json path (string, required)
+* **delimiter**: delimiter (string, default: `,`)
+* **sub_delimiter**: sub-delimiter (string, default: `-`)
+* **sequence_name**: key name for sequence of the elements (string, default: `null`)
+* **output_keys**:
+  - **name**: name (required)
+  - **type**: embulk type (required)
+## Example
+* a simple example
+input:
+```json
+{"csv1": "1-a,2-b,3-c"}
+{"csv1": "10-o,11-p,12-q"}
+```
+yaml:
+```yaml
+filters:
+  - type: json_csv2arrayofobjects
+    column: json1
+    key: csv1
+    output_keys:
+      - {name: number, type: long}
+      - {name: code, type: string}
+```
+output:
+```json
+{"csv1":[{"code":"a","number":1},{"code":"b","number":2},{"code":"c","number":3}]}
+{"csv1":[{"code":"o","number":10},{"code":"p","number":11},{"code":"q","number":12}]}
+```
+* an another example
+input:
+```json
+{"key": {"csv2": "10.0:x:f-11.5:y:t"}}
+{"key": {"csv2": "50.0:A:t-51.5:B:t-52.0:C:t"}}
+```
+yaml:
+```yaml
+filters
+  - type: json_csv2arrayofobjects
+    column: json2
+    key: key.csv2
+    delimiter: "-"
+    sub_delimiter: ":"
+    sequence_name: "seq"
+    output_keys:
+      - {name: decimal, type: double}
+      - {name: code, type: string}
+      - {name: bool, type: boolean}
+```
+output:
+```json
+{"key":{"csv2":[{"code":"x","bool":false,"decimal":10.0,"seq":0},{"code":"y","bool":true,"decimal":11.5,"seq":1}]}}
+{"key":{"csv2":[{"code":"A","bool":true,"decimal":50.0,"seq":0},{"code":"B","bool":true,"decimal":51.5,"seq":1},{"code":"C","bool":true,"decimal":52.0,"seq":2}]}}
+```
+## Thanks
+Implementation of [StringCast.java](https://github.com/ysk24ok/embulk-filter-json_csv2arrayofobjects/blob/master/src/main/java/org/embulk/filter/json_csv2arrayofobjects/StringCast.java) use that of [embulk-filter-typecast](https://github.com/sonots/embulk-filter-typecast) as a reference.
+## Build
+```
+$ ./gradlew gem  # -t to watch change of files and rebuild continuously
+```

data/build.gradle ADDED

@@ -0,0 +1,98 @@
+plugins {
+    id "com.jfrog.bintray" version "1.1"
+    id "com.github.jruby-gradle.base" version "0.1.5"
+    id "java"
+    id "checkstyle"
+}
+import com.github.jrubygradle.JRubyExec
+repositories {
+    mavenCentral()
+    jcenter()
+}
+configurations {
+    provided
+}
+version = "0.1.0"
+sourceCompatibility = 1.7
+targetCompatibility = 1.7
+dependencies {
+    compile  "org.embulk:embulk-core:0.8.27"
+    provided "org.embulk:embulk-core:0.8.27"
+    compile "com.jayway.jsonpath:json-path:2.+"
+    // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
+    testCompile "junit:junit:4.+"
+    testCompile "org.embulk:embulk-core:0.8.+:tests"
+}
+task classpath(type: Copy, dependsOn: ["jar"]) {
+    doFirst { file("classpath").deleteDir() }
+    from (configurations.runtime - configurations.provided + files(jar.archivePath))
+    into "classpath"
+}
+clean { delete "classpath" }
+checkstyle {
+    configFile = file("${project.rootDir}/config/checkstyle/checkstyle.xml")
+    toolVersion = '6.14.1'
+}
+checkstyleMain {
+    configFile = file("${project.rootDir}/config/checkstyle/default.xml")
+    ignoreFailures = true
+}
+checkstyleTest {
+    configFile = file("${project.rootDir}/config/checkstyle/default.xml")
+    ignoreFailures = true
+}
+task checkstyle(type: Checkstyle) {
+    classpath = sourceSets.main.output + sourceSets.test.output
+    source = sourceSets.main.allJava + sourceSets.test.allJava
+}
+task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) {
+    jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
+    script "${project.name}.gemspec"
+    doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
+}
+task gemPush(type: JRubyExec, dependsOn: ["gem"]) {
+    jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "push"
+    script "pkg/${project.name}-${project.version}.gem"
+}
+task "package"(dependsOn: ["gemspec", "classpath"]) {
+    doLast {
+        println "> Build succeeded."
+        println "> You can run embulk with '-L ${file(".").absolutePath}' argument."
+    }
+}
+task gemspec {
+    ext.gemspecFile = file("${project.name}.gemspec")
+    inputs.file "build.gradle"
+    outputs.file gemspecFile
+    doLast { gemspecFile.write($/
+Gem::Specification.new do |spec|
+  spec.name          = "${project.name}"
+  spec.version       = "${project.version}"
+  spec.authors       = ["Yusuke NISHIOKA"]
+  spec.summary       = %[Json Csv2arrayofobjects filter plugin for Embulk]
+  spec.description   = %[Json Csv2arrayofobjects]
+  spec.email         = ["yusuke.nishioka.0713@gmail.com"]
+  spec.licenses      = ["MIT"]
+  # TODO set this: spec.homepage      = "https://github.com/yusuke.nishioka.0713/embulk-filter-json_csv2arrayofobjects"
+  spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
+  spec.test_files    = spec.files.grep(%r"^(test|spec)/")
+  spec.require_paths = ["lib"]
+  #spec.add_dependency 'YOUR_GEM_DEPENDENCY', ['~> YOUR_GEM_DEPENDENCY_VERSION']
+  spec.add_development_dependency 'bundler', ['~> 1.0']
+  spec.add_development_dependency 'rake', ['>= 10.0']
+end
+/$)
+    }
+}
+clean { delete "${project.name}.gemspec" }

data/config/checkstyle/checkstyle.xml ADDED

@@ -0,0 +1,128 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE module PUBLIC
+        "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
+        "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
+<module name="Checker">
+    <!-- https://github.com/facebook/presto/blob/master/src/checkstyle/checks.xml -->
+    <module name="FileTabCharacter"/>
+    <module name="NewlineAtEndOfFile">
+        <property name="lineSeparator" value="lf"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\r"/>
+        <property name="message" value="Line contains carriage return"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value=" \n"/>
+        <property name="message" value="Line has trailing whitespace"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\{\n\n"/>
+        <property name="message" value="Blank line after opening brace"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\n\n\s*\}"/>
+        <property name="message" value="Blank line before closing brace"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\n\n\n"/>
+        <property name="message" value="Multiple consecutive blank lines"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\n\n\Z"/>
+        <property name="message" value="Blank line before end of file"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="Preconditions\.checkNotNull"/>
+        <property name="message" value="Use of checkNotNull"/>
+    </module>
+    <module name="TreeWalker">
+        <module name="EmptyBlock">
+            <property name="option" value="text"/>
+            <property name="tokens" value="
+                LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
+                LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
+        </module>
+        <module name="EmptyStatement"/>
+        <module name="EmptyForInitializerPad"/>
+        <module name="EmptyForIteratorPad">
+            <property name="option" value="space"/>
+        </module>
+        <module name="MethodParamPad">
+            <property name="allowLineBreaks" value="true"/>
+            <property name="option" value="nospace"/>
+        </module>
+        <module name="ParenPad"/>
+        <module name="TypecastParenPad"/>
+        <module name="NeedBraces"/>
+        <module name="LeftCurly">
+            <property name="option" value="nl"/>
+            <property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
+        </module>
+        <module name="LeftCurly">
+            <property name="option" value="eol"/>
+            <property name="tokens" value="
+                 LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
+                 LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
+        </module>
+        <module name="RightCurly">
+            <property name="option" value="alone"/>
+        </module>
+        <module name="GenericWhitespace"/>
+        <module name="WhitespaceAfter"/>
+        <module name="NoWhitespaceBefore"/>
+        <module name="UpperEll"/>
+        <module name="DefaultComesLast"/>
+        <module name="ArrayTypeStyle"/>
+        <module name="MultipleVariableDeclarations"/>
+        <module name="ModifierOrder"/>
+        <module name="OneStatementPerLine"/>
+        <module name="StringLiteralEquality"/>
+        <module name="MutableException"/>
+        <module name="EqualsHashCode"/>
+        <module name="InnerAssignment"/>
+        <module name="InterfaceIsType"/>
+        <module name="HideUtilityClassConstructor"/>
+        <module name="MemberName"/>
+        <module name="LocalVariableName"/>
+        <module name="LocalFinalVariableName"/>
+        <module name="TypeName"/>
+        <module name="PackageName"/>
+        <module name="ParameterName"/>
+        <module name="StaticVariableName"/>
+        <module name="ClassTypeParameterName">
+            <property name="format" value="^[A-Z][0-9]?$"/>
+        </module>
+        <module name="MethodTypeParameterName">
+            <property name="format" value="^[A-Z][0-9]?$"/>
+        </module>
+        <module name="AvoidStarImport"/>
+        <module name="RedundantImport"/>
+        <module name="UnusedImports"/>
+        <module name="ImportOrder">
+            <property name="groups" value="*,javax,java"/>
+            <property name="separated" value="true"/>
+            <property name="option" value="bottom"/>
+            <property name="sortStaticImportsAlphabetically" value="true"/>
+        </module>
+        <module name="WhitespaceAround">
+            <property name="allowEmptyConstructors" value="true"/>
+            <property name="allowEmptyMethods" value="true"/>
+            <property name="ignoreEnhancedForColon" value="false"/>
+            <property name="tokens" value="
+                ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
+                BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
+                LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
+                LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
+                LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
+                LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
+                PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
+                STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
+        </module>
+    </module>
+</module>

data/config/checkstyle/default.xml ADDED

@@ -0,0 +1,108 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE module PUBLIC
+        "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
+        "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
+<!--
+     This is a subset of ./checkstyle.xml which allows some loose styles
+-->
+<module name="Checker">
+    <module name="FileTabCharacter"/>
+    <module name="NewlineAtEndOfFile">
+        <property name="lineSeparator" value="lf"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\r"/>
+        <property name="message" value="Line contains carriage return"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value=" \n"/>
+        <property name="message" value="Line has trailing whitespace"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\n\n\n"/>
+        <property name="message" value="Multiple consecutive blank lines"/>
+    </module>
+    <module name="RegexpMultiline">
+        <property name="format" value="\n\n\Z"/>
+        <property name="message" value="Blank line before end of file"/>
+    </module>
+    <module name="TreeWalker">
+        <module name="EmptyBlock">
+            <property name="option" value="text"/>
+            <property name="tokens" value="
+                LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
+                LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
+        </module>
+        <module name="EmptyStatement"/>
+        <module name="EmptyForInitializerPad"/>
+        <module name="EmptyForIteratorPad">
+            <property name="option" value="space"/>
+        </module>
+        <module name="MethodParamPad">
+            <property name="allowLineBreaks" value="true"/>
+            <property name="option" value="nospace"/>
+        </module>
+        <module name="ParenPad"/>
+        <module name="TypecastParenPad"/>
+        <module name="NeedBraces"/>
+        <module name="LeftCurly">
+            <property name="option" value="nl"/>
+            <property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
+        </module>
+        <module name="LeftCurly">
+            <property name="option" value="eol"/>
+            <property name="tokens" value="
+                 LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
+                 LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
+        </module>
+        <module name="RightCurly">
+            <property name="option" value="alone"/>
+        </module>
+        <module name="GenericWhitespace"/>
+        <module name="WhitespaceAfter"/>
+        <module name="NoWhitespaceBefore"/>
+        <module name="UpperEll"/>
+        <module name="DefaultComesLast"/>
+        <module name="ArrayTypeStyle"/>
+        <module name="MultipleVariableDeclarations"/>
+        <module name="ModifierOrder"/>
+        <module name="OneStatementPerLine"/>
+        <module name="StringLiteralEquality"/>
+        <module name="MutableException"/>
+        <module name="EqualsHashCode"/>
+        <module name="InnerAssignment"/>
+        <module name="InterfaceIsType"/>
+        <module name="HideUtilityClassConstructor"/>
+        <module name="MemberName"/>
+        <module name="LocalVariableName"/>
+        <module name="LocalFinalVariableName"/>
+        <module name="TypeName"/>
+        <module name="PackageName"/>
+        <module name="ParameterName"/>
+        <module name="StaticVariableName"/>
+        <module name="ClassTypeParameterName">
+            <property name="format" value="^[A-Z][0-9]?$"/>
+        </module>
+        <module name="MethodTypeParameterName">
+            <property name="format" value="^[A-Z][0-9]?$"/>
+        </module>
+        <module name="WhitespaceAround">
+            <property name="allowEmptyConstructors" value="true"/>
+            <property name="allowEmptyMethods" value="true"/>
+            <property name="ignoreEnhancedForColon" value="false"/>
+            <property name="tokens" value="
+                ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
+                BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
+                LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
+                LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
+                LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
+                LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
+                PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
+                STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
+        </module>
+    </module>
+</module>