embulk-parser-csv_guessable 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 90dc39f04076979425a69d11b3177e1e4b1d5e7a
-  data.tar.gz: 5dca8965baaeb7fbe51f5f9df63f385b09d9bdb1
+  metadata.gz: 1bd20694daee60a0018828d263039d4fcbdc28bf
+  data.tar.gz: 34b6b8c95c8e8e375059d7ff87dd46314fb3ef55
 SHA512:
-  metadata.gz: 61bd54ee36352ab6667654f6dfdfcccd77de37efdd49f3325924538ba3d921737a1e9139691cc8aa8617f47893cef8219f0f50bff454a456df01309ed4668617
-  data.tar.gz: d6ab6e9d35ae8932ee5aa1a035c8f1440e6c4d1cf41565010278ca88d13a91cd85e2444476106a29cae6bd4312c5b2987d1211dfb45c26e7b6ae9bf511e12b75
+  metadata.gz: 844084c1b76193789c12821bf9eb76a3fef969b33e42e2ea9ff6b9935848276412c3fee3af558d04aef3414ba1e93e72bfbab35393cc0a935d994b9f98ff8e4e
+  data.tar.gz: 57c246a6c2d551a55de98e8f420354da9c51970ef1e53184d87c0f6a22dd1dde47e9f49e3df4e1a6987a76fd71aac7ee84590e2c06be50a5a1bf2296bcd26398

data/README.md

@@ -1,8 +1,11 @@
-# Csv Guessable parser plugin for Embulk
-**embulk-parser-csv_gussable** (runtime) guesses and parses csv which has schema in header.
+# Guessable csv parser plugin for Embulk
+**embulk-parser-csv_guessable** (runtime)guesses and parses csv which has schema in header.
+
+Csv file sometimes has a schema in the header.
+**embulk-parser-csv_guessable** parses such a csv by using their header as column name.
 This plugin is useful in case of target csv schema changes frequently.
 
-Also it can behave as original csv parser without **embulk-parser-csv_guessable** specified configs.
+It behaves as original csv parser when **embulk-parser-csv_guessable** conifgs(`schema_file` and `schema_line`) is not defined.
 
 ## Overview
 
@@ -12,12 +15,19 @@ Also it can behave as original csv parser without **embulk-parser-csv_guessable*
 ## Configuration
 
 - **schema_file**: filename which has schema.(string, default: `null`)
-- **schema_line**: schema line in header. (integer default: `"1"`)
-- **(TODO)columns**: Columns attributes for parse. `embulk-parser-csv_guessable` use this config only when `"schema_file"` is set. If `"schema_file"` isn't set, this is same as original csv parser's `"columns"`. (hash, default: `null`)
+- **schema_line**: schema line in header. (integer default: `1`)
+- **columns**: Columns attributes for parse. `embulk-parser-csv_guessable` use this config only when `schema_file` is set. If `"schema_file"` isn't set, this is same as the original csv parser's `columns`. (hash, default: `null`)
+    - **value_name**: Name of the column in the header. rename to `name`
+    - **name**: Name of the column
+    - **type**: Type of the column
+    - **format**: Format of the timestamp if type is timestamp
+    - **date**: Set date part if the format doesn't include date part
 - any other csv configs: see [www.embulk.org](http://www.embulk.org/docs/built-in.html#csv-parser-plugin)
 
+The `columns` 
+
 ## Example
-test.csv
+test.csv (There is a schema at the first line.)
 
 ```csv
 id, title, description
@@ -32,11 +42,11 @@ in:
   type: any file input plugin type
   parser:
     type: csv_guessable
-    schema_file: data/test.csv
+    schema_file: test.csv
     schema_line: 1
 ```
 
-(To explain)
+(For explain)
 In case original csv parser 
 config.yml
 ```yaml
@@ -51,6 +61,22 @@ in:
     - {name: description, type: string}
 ```
 
+## Example2
+rename column name and set type Example
+
+```yaml
+in:
+  type: any file input plugin type
+  parser:
+    type: csv_guessable
+    schema_file test.csv
+    schema_line: 1
+    columns:
+    - {value_name: 'id', name: 'number', type: long}
+    - {value_name: 'title', name: 'description', type: string}
+    - {value_name: 'status', name: 'ok?', type: string}
+```
+
 <!--
 (If guess supported) you don't have to write `parser:` section in the configuration file. After writing `in:` section, you can let embulk guess `parser:` section using this command:
 -->

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.1.2"
+version = "0.1.3"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
@@ -25,6 +25,7 @@ dependencies {
     provided "org.embulk:embulk-core:0.8.22"
     testCompile "junit:junit:4.+"
     testCompile "org.embulk:embulk-core:0.8.+:tests"
+    testCompile "org.embulk:embulk-standards:0.8.+:tests"
 }
 
 task classpath(type: Copy, dependsOn: ["jar"]) {
@@ -78,8 +79,8 @@ Gem::Specification.new do |spec|
   spec.name          = "${project.name}"
   spec.version       = "${project.version}"
   spec.authors       = ["koooge"]
-  spec.summary       = %[Csv Guessable parser plugin for Embulk]
-  spec.description   = %[Parses Csv Guessable files read by other file input plugins.]
+  spec.summary       = %[Guessable Csv parser plugin for Embulk]
+  spec.description   = %[Parses Guessable Csv files read by other file input plugins.]
   spec.email         = ["koooooge@gmail.com"]
   spec.licenses      = ["MIT"]
   spec.homepage      = "https://github.com/koooge/embulk-parser-csv_guessable"

data/src/main/java/org/embulk/parser/csv_guessable/CsvGuessableParserPlugin.java

@@ -40,6 +40,7 @@ import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.ArrayList;
+import java.util.List;
 
 public class CsvGuessableParserPlugin
         extends CsvParserPlugin
@@ -132,23 +133,38 @@ public class CsvGuessableParserPlugin
         PluginTask task = config.loadConfig(PluginTask.class);
         SchemaConfig schemaConfig = null;
 
-        if (task.getSchemaFile().isPresent()) { /* embulk-parser-csv_guessable */
-            if (task.getHeaderLine().isPresent()) {
-                // TODO: use 'columns' as hints for guess
-                throw new ConfigException("embulk-parsre-csv_gussable will use 'columnes' as hints for guess as hints for guess. Please delete 'columnes' now.");
-            }
-            else { /* guess from header */
-                int schemaLine = task.getSchemaLine();
-                task.setSkipHeaderLines(schemaLine); // TODO: use 'skip_header_line'
-
-                String header = readHeader(task.getSchemaFile().get().getPath(), schemaLine);
-                log.debug(header);
-                ArrayList<ColumnConfig> columns = newColumns(header, config);
-                log.debug(columns.toString());
-                schemaConfig = new SchemaConfig(columns);
+        if (task.getSchemaFile().isPresent()) {
+            int schemaLine = task.getSchemaLine();
+            task.setSkipHeaderLines(schemaLine); // TODO: use 'skip_header_line'
+
+            String header = readHeader(task.getSchemaFile().get().getPath(), schemaLine);
+            log.debug(header);
+            ArrayList<ColumnConfig> schema = newColumns(header, config);
+
+            /* alias and set type */
+            if (task.getSchemaConfig().isPresent()) {
+                List<ColumnConfig> columns = task.getSchemaConfig().get().getColumns();
+                for (ColumnConfig column : columns) {
+                    String name = column.getName();
+                    try {
+                        name = column.getConfigSource().get(String.class, "value_name");
+                    }
+                    catch (ConfigException e) {
+                        /* only setType */
+                    }
+                    for (int i = 0; i < schema.size(); ++i) {
+                        ColumnConfig c = schema.get(i);
+                        if (c.getName().equals(name)) {
+                            schema.set(i, new ColumnConfig(name, column.getType(), column.getOption()));
+                        }
+                    }
+                }
             }
+
+            log.debug(schema.toString());
+            schemaConfig = new SchemaConfig(schema);
         }
-        else { /* embulk-parser-csv embulk */
+        else if (task.getSchemaConfig().isPresent()) { /* original CsvParserPlugin */
             // backward compatibility
             if (task.getHeaderLine().isPresent()) {
                 if (task.getSkipHeaderLines() > 0) {
@@ -163,6 +179,9 @@ public class CsvGuessableParserPlugin
             }
             schemaConfig = task.getSchemaConfig().get();
         }
+        else {
+            throw new ConfigException("Field 'columns' or 'schema_file' is required but not set");
+        }
 
         control.run(task.dump(), schemaConfig.toSchema());
     }
@@ -377,7 +396,7 @@ public class CsvGuessableParserPlugin
 
     private ArrayList<ColumnConfig> newColumns(String header, ConfigSource config)
     {
-        ArrayList columns = new ArrayList<ArrayList>();
+        ArrayList<ColumnConfig> columns = new ArrayList<ColumnConfig>();
         PluginTask task = config.loadConfig(PluginTask.class);
 
         try (CSVReader reader = new CSVReader(new StringReader(header))) {

data/src/test/java/org/embulk/parser/csv_guessable/TestCsvGuessableParserPlugin.java

@@ -1,18 +1,34 @@
 package org.embulk.parser.csv_guessable;
 
+import com.google.common.collect.ImmutableList;
 import org.embulk.EmbulkTestRuntime;
 import org.embulk.config.ConfigException;
 import org.embulk.config.ConfigLoader;
 import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskSource;
 import org.embulk.spi.Exec;
+import org.embulk.spi.FileInput;
+import org.embulk.spi.ParserPlugin;
+import org.embulk.spi.Schema;
+import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
+import org.embulk.spi.util.InputStreamFileInput;
+import org.embulk.standards.TestCsvParserPlugin;
+import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
 
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
 import static org.embulk.parser.csv_guessable.CsvGuessableParserPlugin.PluginTask;
-import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
 
 public class TestCsvGuessableParserPlugin
+        extends TestCsvParserPlugin
 {
     @Rule
     public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
@@ -20,61 +36,118 @@ public class TestCsvGuessableParserPlugin
     @Rule
     public ExpectedException exception = ExpectedException.none();
 
+    private CsvGuessableParserPlugin plugin;
+    private MockPageOutput output;
+
+    @Before
+    public void createResouce()
+    {
+        plugin = new CsvGuessableParserPlugin();
+        output = new MockPageOutput();
+    }
+
     private ConfigSource getConfigFromYaml(String yaml)
     {
         ConfigLoader loader = new ConfigLoader(Exec.getModelManager());
         return loader.fromYamlString(yaml);
     }
 
-    /*
-    @Test
-    public void throwExceptionWithoutRequisite()
+    @Test(expected = ConfigException.class)
+    public void checkColumnsRequired()
     {
         String configYaml = "" +
-                "type: csv_with_header";
-
+                "type: csv_guessable";
         ConfigSource config = getConfigFromYaml(configYaml);
+        PluginTask task = config.loadConfig(PluginTask.class);
 
-        exception.expect(ConfigException.class);
-        exception.expectMessage("Field either 'columns' or 'schema_file' is required but not set");
+        if (!task.getSchemaConfig().isPresent() && !task.getSchemaFile().isPresent()) {
+            throw new ConfigException("Field 'columns' or 'schema_line' is required but not set");
+        }
     }
-    */
 
     @Test
     public void defaultValue()
     {
         String configYaml = "" +
-                "type: csv_with_header\n" +
-                "schema_line: 2";
+                "type: csv_guessable";
         ConfigSource config = getConfigFromYaml(configYaml);
         PluginTask task = config.loadConfig(PluginTask.class);
 
-        assertFalse(task.getSchemaConfig().isPresent());
-        assertFalse(task.getSchemaFile().isPresent());
+        assertNull(task.getSchemaConfig().orNull());
+        assertNull(task.getSchemaFile().orNull());
+        assertEquals(1, task.getSchemaLine());
     }
 
-    /*
     @Test
     public void originalCsvParserPlugin()
     {
         String configYaml = "" +
-                "type: csv_with_header\n" +
+                "type: csv_guessable\n" +
                 "columns:\n" +
                 "  - {name: id, type: long}\n" +
                 "  - {name: title, type: string}\n" +
                 "  - {name: status, type: string}";
         ConfigSource config = getConfigFromYaml(configYaml);
         PluginTask task = config.loadConfig(PluginTask.class);
+
+        // TODO: impl or extends
     }
 
     @Test
-    public void csvGuessable()
+    public void guessableCsv()
+            throws Exception
     {
+        String configYaml = "" +
+                "type: csv_guessable\n" +
+                "schema_file: src/test/resources/org/embulk/parser/csv_guessable/data/test.csv\n" + // TODO: FIX PATH
+                "schema_line: 1";
+        ConfigSource config = getConfigFromYaml(configYaml);
+        PluginTask task = config.loadConfig(PluginTask.class);
+//        transaction(config, fileInput(new File(this.getClass().getResource("data/test.csv").getPath())));
+
+        // TODO: impl
     }
 
     @Test
-    public void replaceColumnsName()
+    public void replaceColumnsMetadata()
+            throws Exception
+    {
+        String configYaml = "" +
+                "type: csv_guessable\n" +
+                "schema_file: src/test/resources/org/embulk/parser/csv_guessable/data/test.csv\n" + // TODO: FIX PATH
+                "schema_line: 1\n" +
+                "columns:\n" +
+                "- {value_name: '#', name: 'number', type: long}\n" +
+                "- {value_name: 'title', name: 'description', type: string}\n" +
+                "- {value_name: 'status', name: 'ok?', type: string}";
+        ConfigSource config = getConfigFromYaml(configYaml);
+
+        // TODO: impl
+    }
+
+    private void transaction(ConfigSource config, final FileInput input)
+    {
+        plugin.transaction(config, new ParserPlugin.Control()
+        {
+            @Override
+            public void run(TaskSource taskSource, Schema schema)
+            {
+                plugin.run(taskSource, schema, input, output);
+            }
+        });
+    }
+
+    private FileInput fileInput(File file)
+            throws Exception
+    {
+        FileInputStream in = new FileInputStream(file);
+        return new InputStreamFileInput(runtime.getBufferAllocator(), provider(in));
+    }
+
+    private InputStreamFileInput.IteratorProvider provider(InputStream... inputStreams)
+            throws IOException
     {
+        return new InputStreamFileInput.IteratorProvider(
+                ImmutableList.copyOf(inputStreams));
     }
-    */
 }

data/src/test/resources/org/embulk/parser/csv_guessable/yml/guess_and_set_type.yml

@@ -0,0 +1,13 @@
+in:
+  type: file
+  path_prefix: data/test
+  parser:
+    type: csv_guessable
+    schema_file: data/test.csv
+    schema_line: 1
+    columns:
+    - {name: 'id', type: long}
+    - {name: 'title', type: string}
+    - {name: 'status', type: string}
+out:
+  type: stdout

data/src/test/resources/org/embulk/parser/csv_guessable/yml/original-csv.yml

@@ -0,0 +1,12 @@
+in:
+  type: file
+  path_prefix: data/test
+  parser:
+    type: csv_guessable
+    skip_header_lines: 1
+    columns:
+    - {name: id, type: long}
+    - {name: title, type: string}
+    - {name: status, type: string}
+out:
+  type: stdout

data/src/test/resources/org/embulk/parser/csv_guessable/yml/replace_column_name.yml

@@ -0,0 +1,13 @@
+in:
+  type: file
+  path_prefix: data/test
+  parser:
+    type: csv_guessable
+    schema_file: data/test.csv
+    schema_line: 1
+    columns:
+    - {value_name: 'id', name: 'number', type: long}
+    - {value_name: 'title', name: 'description', type: string}
+    - {value_name: 'status', name: 'ok?', type: string}
+out:
+  type: stdout

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-parser-csv_guessable
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - koooge
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-06-02 00:00:00.000000000 Z
+date: 2017-06-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -38,7 +38,7 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '10.0'
-description: Parses Csv Guessable files read by other file input plugins.
+description: Parses Guessable Csv files read by other file input plugins.
 email:
 - koooooge@gmail.com
 executables: []
@@ -60,16 +60,17 @@ files:
 - src/main/java/org/embulk/parser/csv_guessable/CsvGuessableParserPlugin.java
 - src/main/java/org/embulk/parser/csv_guessable/CsvTokenizer.java
 - src/test/java/org/embulk/parser/csv_guessable/TestCsvGuessableParserPlugin.java
-- src/test/resources/data/test.csv
-- src/test/resources/data/test_alias.yml
-- src/test/resources/yml/guess_from_header.yml
-- src/test/resources/yml/original-csv.yml
-- src/test/resources/yml/replace_column_name.yml
+- src/test/resources/org/embulk/parser/csv_guessable/data/test.csv
+- src/test/resources/org/embulk/parser/csv_guessable/data/test_alias.yml
+- src/test/resources/org/embulk/parser/csv_guessable/yml/guess_and_set_type.yml
+- src/test/resources/org/embulk/parser/csv_guessable/yml/guess_from_header.yml
+- src/test/resources/org/embulk/parser/csv_guessable/yml/original-csv.yml
+- src/test/resources/org/embulk/parser/csv_guessable/yml/replace_column_name.yml
 - classpath/commons-lang3-3.5.jar
+- classpath/embulk-parser-csv_guessable-0.1.3.jar
 - classpath/opencsv-3.9.jar
 - classpath/commons-beanutils-1.9.3.jar
 - classpath/commons-compress-1.10.jar
-- classpath/embulk-parser-csv_guessable-0.1.2.jar
 - classpath/embulk-standards-0.8.22.jar
 - classpath/commons-collections-3.2.2.jar
 - classpath/commons-logging-1.2.jar
@@ -96,5 +97,5 @@ rubyforge_project:
 rubygems_version: 2.1.9
 signing_key:
 specification_version: 4
-summary: Csv Guessable parser plugin for Embulk
+summary: Guessable Csv parser plugin for Embulk
 test_files: []

data/src/test/resources/yml/original-csv.yml

@@ -1,12 +0,0 @@
-in:
-  type: file
-  path_prefix: data/test
-  parser:
-    type: csv_guessable
-    skip_header_line: 1
-    columns:
-      - {name: id, type: long}
-      - {name: title, type: string}
-      - {name: status, type: string}
-out:
-  type: stdout

data/src/test/resources/yml/replace_column_name.yml

@@ -1,13 +0,0 @@
-in:
-  type: file
-  path_prefix: data/test
-  parser:
-    type: csv_guessable
-    schema_file: data/test.csv
-    schema_line: 1
-    columns:
-      - {value_name: '#', name: number, type: long}
-      - {value_name: 'title', name: description, type: string}
-      - {value_name: 'status', name: ok?, type: string}
-out:
-  type: stdout