embulk-parser-csv_guessable 0.1.5 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 07a5d30689afe074b0db47f61c04651e74b76024
-  data.tar.gz: b16fda4794ed77f2111d4e047210c8966f0fed2e
+  metadata.gz: f277752d08cd0133b915c5b00bc17d42cc87537d
+  data.tar.gz: 705c7bba540b5c58e2be6dea18f204b51cbd50a2
 SHA512:
-  metadata.gz: ca548cf755f02e6be06bef5664754271ac1f5041e4baa0dd015e546ec4e00396051a6fc8d532603d01a778069baf9398ae132b1770b70c6eb1790954cf6eafb3
-  data.tar.gz: 368cb17eb071324704ff2ac131511c4f96e67f3a8eb109fbe9e7dec549757dc0b10cbed1176246d1db18f43f48ad98a4b4213f8f482d70f12a2ee2f5e7283473
+  metadata.gz: 24ef39e21fbf84eb370411796857eff54c4c891097be3bdb9202459074d2af6ee0fd98082702f7a1db00836534876db7c5fc9bc0701b4d33a01c984509e86b2c
+  data.tar.gz: c35874678bb4f097ebd59c76d7dd16037ffbd12768c3fda61066331661cd9549b39bc4f6cb26be9f76a3aaf931fe29f24008441fff0dedfb402e724c5443c920

data/README.md

@@ -8,13 +8,17 @@ Csv file sometimes has a schema in the header.
 **embulk-parser-csv_guessable** parses such a csv by using their header as column name.
 This plugin is useful in case of target csv schema changes frequently.
 
-It behaves as original csv parser when **embulk-parser-csv_guessable** conifgs(`schema_file` and `schema_line`) is not defined.
+It behaves as original csv parser when **embulk-parser-csv_guessable** configs(`schema_file` and `schema_line`) is not defined.
 
 ## Overview
 
 * **Plugin type**: parser
 * **Guess supported**: no
 
+## Prerequisites
+ - java: 1.8+ 
+ - embulk: 0.9+
+
 ## Configuration
 
 - **schema_file**: filename which has schema.(string, default: `null`)

data/build.gradle

@@ -1,6 +1,6 @@
 plugins {
     id "com.jfrog.bintray" version "1.1"
-    id "com.github.jruby-gradle.base" version "0.1.5"
+    id "com.github.jruby-gradle.base" version "1.5.0"
     id "java"
     id "checkstyle"
 }
@@ -13,19 +13,19 @@ configurations {
     provided
 }
 
-version = "0.1.5"
+version = "0.2.0"
 
-sourceCompatibility = 1.7
-targetCompatibility = 1.7
+sourceCompatibility = 1.8
+targetCompatibility = 1.8
 
 dependencies {
-    compile "org.embulk:embulk-core:0.8.+"
-    compile "org.embulk:embulk-standards:0.8.+"
-    compile "com.opencsv:opencsv:3.9"
-    provided "org.embulk:embulk-core:0.8.+"
+    compile "org.embulk:embulk-core:0.9.+"
+    compile "org.embulk:embulk-standards:0.9.+"
+    compile "com.opencsv:opencsv:4.2"
+    provided "org.embulk:embulk-core:0.9.+"
     testCompile "junit:junit:4.+"
-    testCompile "org.embulk:embulk-core:0.8.+:tests"
-    testCompile "org.embulk:embulk-standards:0.8.+:tests"
+    testCompile "org.embulk:embulk-core:0.9.+:tests"
+    testCompile "org.embulk:embulk-standards:0.9.+:tests"
 }
 
 task classpath(type: Copy, dependsOn: ["jar"]) {
@@ -53,20 +53,22 @@ task checkstyle(type: Checkstyle) {
 }
 
 task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) {
-    jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
-    script "${project.name}.gemspec"
+    jrubyArgs "-S"
+    script "gem"
+    scriptArgs "build", "${project.name}.gemspec"
     doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
 }
 
 task gemPush(type: JRubyExec, dependsOn: ["gem"]) {
-    jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "push"
-    script "pkg/${project.name}-${project.version}.gem"
+    jrubyArgs "-S"
+    script "gem"
+    scriptArgs "push", "pkg/${project.name}-${project.version}.gem"
 }
 
 task "package"(dependsOn: ["gemspec", "classpath"]) {
     doLast {
         println "> Build succeeded."
-        println "> You can run embulk with '-L ${file(".").absolutePath}' argument."
+        println "> You can run embulk with ${argumentToRunEmbulkJava} argument."
     }
 }
 

data/gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
-distributionUrl=https\://services.gradle.org/distributions/gradle-4.2.1-bin.zip
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-zipStorePath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.9-bin.zip
 zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists

data/src/main/java/org/embulk/parser/csv_guessable/CsvGuessableParserPlugin.java

@@ -141,7 +141,9 @@ public class CsvGuessableParserPlugin
 
             String header = readHeader(task.getSchemaFile().get().getPath(), schemaLine, task.getCharset());
             log.debug(header);
-            ArrayList<ColumnConfig> schema = newColumns(header, config);
+            String delimiter = task.getDelimiter();
+            ArrayList<ColumnConfig> schema = newColumns(header, config, delimiter);
+
 
             /* alias and set type */
             if (task.getSchemaConfig().isPresent()) {
@@ -396,12 +398,12 @@ public class CsvGuessableParserPlugin
         return line;
     }
 
-    private ArrayList<ColumnConfig> newColumns(String header, ConfigSource config)
+    private ArrayList<ColumnConfig> newColumns(String header, ConfigSource config, String delimiter)
     {
         ArrayList<ColumnConfig> columns = new ArrayList<ColumnConfig>();
         PluginTask task = config.loadConfig(PluginTask.class);
 
-        try (CSVReader reader = new CSVReader(new StringReader(header))) {
+        try (CSVReader reader = new CSVReader(new StringReader(header), delimiter.charAt(0))) {
             String[] csv = reader.readNext();
             for (String column : csv) {
                 columns.add(new ColumnConfig(column, Types.STRING, config));

data/src/test/java/org/embulk/parser/csv_guessable/TestCsvGuessableParserPlugin.java

@@ -120,6 +120,35 @@ public class TestCsvGuessableParserPlugin
         }
     }
 
+    @Test
+    public void guessableTsv()
+            throws Exception
+    {
+        String configYaml = "" +
+                "type: csv_guessable\n" +
+                "schema_file: src/test/resources/org/embulk/parser/csv_guessable/data/test.tsv\n" + // TODO: FIX PATH
+                "schema_line: 1\n" +
+                "delimiter: \"\\t\"";
+        ConfigSource config = getConfigFromYaml(configYaml);
+        transaction(config, fileInput("data/test.tsv"));
+        List<Object[]> records = Pages.toObjects(schema, output.pages);
+        assertEquals(2, records.size());
+
+        Object[] record;
+        {
+            record = records.get(0);
+            assertEquals("100", record[0]);
+            assertEquals("test-title", record[1]);
+            assertEquals("ok", record[2]);
+        }
+        {
+            record = records.get(1);
+            assertEquals("191", record[0]);
+            assertEquals("title2", record[1]);
+            assertEquals("ng", record[2]);
+        }
+    }
+
     @Test
     public void specifyType()
             throws Exception

data/src/test/resources/org/embulk/parser/csv_guessable/data/test.tsv

@@ -0,0 +1,3 @@
+"id"	"title"	"status"
+"100"	"test-title"	"ok"
+"191"	"title2"	"ng"

metadata

@@ -1,43 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: embulk-parser-csv_guessable
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.2.0
 platform: ruby
 authors:
 - koooge
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-10-15 00:00:00.000000000 Z
+date: 2018-08-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: bundler
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '1.0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
+  name: bundler
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: rake
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '1.0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '10.0'
+  name: rake
   prerelease: false
   type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10.0'
 description: Parses Guessable Csv files read by other file input plugins.
 email:
 - koooooge@gmail.com
@@ -45,11 +45,21 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .circleci/config.yml
-- .gitignore
+- ".circleci/config.yml"
+- ".gitignore"
 - LICENSE.txt
 - README.md
 - build.gradle
+- classpath/commons-beanutils-1.9.3.jar
+- classpath/commons-collections-3.2.2.jar
+- classpath/commons-collections4-4.1.jar
+- classpath/commons-compress-1.10.jar
+- classpath/commons-lang3-3.7.jar
+- classpath/commons-logging-1.2.jar
+- classpath/commons-text-1.3.jar
+- classpath/embulk-parser-csv_guessable-0.2.0.jar
+- classpath/embulk-standards-0.9.7.jar
+- classpath/opencsv-4.2.jar
 - config/checkstyle/checkstyle.xml
 - config/checkstyle/default.xml
 - gradle/wrapper/gradle-wrapper.jar
@@ -67,19 +77,12 @@ files:
 - src/main/java/org/embulk/parser/csv_guessable/CsvTokenizer.java
 - src/test/java/org/embulk/parser/csv_guessable/TestCsvGuessableParserPlugin.java
 - src/test/resources/org/embulk/parser/csv_guessable/data/test.csv
+- src/test/resources/org/embulk/parser/csv_guessable/data/test.tsv
 - src/test/resources/org/embulk/parser/csv_guessable/data/test_alias.csv
 - src/test/resources/org/embulk/parser/csv_guessable/yml/guess_and_set_type.yml
 - src/test/resources/org/embulk/parser/csv_guessable/yml/guess_from_header.yml
 - src/test/resources/org/embulk/parser/csv_guessable/yml/original-csv.yml
 - src/test/resources/org/embulk/parser/csv_guessable/yml/replace_column_name.yml
-- classpath/commons-lang3-3.5.jar
-- classpath/embulk-parser-csv_guessable-0.1.5.jar
-- classpath/embulk-standards-0.8.35.jar
-- classpath/opencsv-3.9.jar
-- classpath/commons-beanutils-1.9.3.jar
-- classpath/commons-compress-1.10.jar
-- classpath/commons-collections-3.2.2.jar
-- classpath/commons-logging-1.2.jar
 homepage: https://github.com/koooge/embulk-parser-csv_guessable
 licenses:
 - MIT
@@ -90,17 +93,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.1.9
+rubygems_version: 2.6.8
 signing_key:
 specification_version: 4
 summary: Guessable Csv parser plugin for Embulk