embulk 0.8.15 → 0.8.16

data/embulk-standards/src/test/java/org/embulk/standards/TestRenameFilterPlugin.java

@@ -31,6 +31,14 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 
+/**
+ * Testing |RenameFilterPlugin|.
+ *
+ * NOTE: DO NOT CHANGE THE EXISTING TESTS.
+ *
+ * As written in the comment of |RenameFilterPlugin|, the existing behaviors
+ * should not change so that users are not confused.
+ */
 public class TestRenameFilterPlugin
 {
     @Rule

data/embulk-standards/src/test/java/org/embulk/standards/guess/TestCsvAllStringsGuessPlugin.java

@@ -0,0 +1,38 @@
+package org.embulk.standards.guess;
+
+import com.google.common.collect.ImmutableList;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.DataSource;
+import org.embulk.test.TestingEmbulk;
+import org.junit.Rule;
+import org.junit.Test;
+
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+
+public class TestCsvAllStringsGuessPlugin
+{
+    private static final String RESOURCE_NAME_PREFIX = "org/embulk/standards/guess/csv_all_strings/test/";
+
+    @Rule
+    public TestingEmbulk embulk = TestingEmbulk.builder()
+            .build();
+
+    @Test
+    public void testSimple()
+            throws Exception
+    {
+        ConfigSource exec = embulk.newConfig()
+                .set("guess_plugins", ImmutableList.of("csv_all_strings"))
+                .set("exclude_guess_plugins", ImmutableList.of("csv"));
+
+        ConfigDiff guessed =
+            embulk.parserBuilder()
+            .exec(exec)
+            .inputResource(RESOURCE_NAME_PREFIX + "test_simple.csv")
+            .guess();
+
+        assertThat(guessed, is((DataSource) embulk.loadYamlResource(RESOURCE_NAME_PREFIX + "test_simple_guessed.yml")));
+    }
+}

data/embulk-standards/src/test/java/org/embulk/standards/guess/TestCsvGuessPlugin.java

@@ -0,0 +1,229 @@
+package org.embulk.standards.guess;
+
+import com.google.common.collect.ImmutableList;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.DataSource;
+import org.embulk.test.TestingEmbulk;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+
+public class TestCsvGuessPlugin
+{
+    private static final String RESOURCE_NAME_PREFIX = "org/embulk/standards/guess/csv/test/";
+
+    @Rule
+    public TestingEmbulk embulk = TestingEmbulk.builder()
+            .build();
+
+    @Test
+    public void testSimple()
+            throws Exception
+    {
+        assertGuessByResource(embulk,
+                "test_simple_seed.yml", "test_simple.csv",
+                "test_simple_guessed.yml");
+    }
+
+    @Test
+    public void testFor1Rows()
+            throws Exception
+    {
+        assertGuessByResource(embulk,
+                "test_1_rows_seed.yml", "test_1_rows.csv",
+                "test_1_rows_guessed.yml");
+    }
+
+    @Test
+    public void testFor1RowsWithTrimNeeded()
+            throws Exception
+    {
+        assertGuessByResource(embulk,
+                "test_1_rows_with_trim_needed_seed.yml", "test_1_rows_with_trim_needed.csv",
+                "test_1_rows_with_trim_needed_guessed.yml");
+    }
+
+    @Test
+    public void testFor1RowsAndHeader()
+            throws Exception
+    {
+        assertGuessByResource(embulk,
+                "test_1_rows_and_header_seed.yml", "test_1_rows_and_header.csv",
+                "test_1_rows_and_header_guessed.yml");
+    }
+
+    @Test
+    public void testFor1RowsAndHeaderWithTrimNeeded()
+            throws Exception
+    {
+        assertGuessByResource(embulk,
+                "test_1_rows_and_header_with_trim_needed_seed.yml", "test_1_rows_and_header_with_trim_needed.csv",
+                "test_1_rows_and_header_with_trim_needed_guessed.yml");
+    }
+
+    @Test
+    public void testFor2Rows()
+            throws Exception
+    {
+        assertGuessByResource(embulk,
+                "test_2_rows_seed.yml", "test_2_rows.csv",
+                "test_2_rows_guessed.yml");
+    }
+
+    @Test
+    public void testFor2RowsAndHeader()
+            throws Exception
+    {
+        assertGuessByResource(embulk,
+                "test_2_rows_and_header_seed.yml", "test_2_rows_and_header.csv",
+                "test_2_rows_and_header_guessed.yml");
+    }
+
+    @Test
+    public void testFor1IntSingleColumnRow()
+            throws Exception
+    {
+        assertGuessByResource(embulk,
+                "test_1_int_single_column_row_seed.yml", "test_1_int_single_column_row.csv",
+                "test_1_int_single_column_row_guessed.yml");
+    }
+
+    @Test
+    public void testFor1StringSingleColumnRow()
+            throws Exception
+    {
+        assertGuessByResource(embulk,
+                "test_1_string_single_column_row_seed.yml", "test_1_string_single_column_row.csv",
+                "test_1_string_single_column_row_guessed.yml");
+    }
+
+    @Test
+    public void testFor2StringSingleColumnRows()
+            throws Exception
+    {
+        assertGuessByResource(embulk,
+                "test_2_string_single_column_rows_seed.yml", "test_2_string_single_column_rows.csv",
+                "test_2_string_single_column_rows_guessed.yml");
+    }
+
+    @Test
+    public void testFor1StringSingleColumnAndHeader()
+            throws Exception
+    {
+        assertGuessByResource(embulk,
+                "test_1_string_single_column_row_and_header_seed.yml", "test_1_string_single_column_row_and_header.csv",
+                "test_1_string_single_column_row_and_header_guessed.yml");
+    }
+
+    @Test
+    public void testFor2IntSingleColumnRows()
+            throws Exception
+    {
+        assertGuessByResource(embulk,
+                "test_2_int_single_column_rows_seed.yml", "test_2_int_single_column_rows.csv",
+                "test_2_int_single_column_rows_guessed.yml");
+    }
+
+    @Test
+    public void testFor1IntSingleColumnAndHeader()
+            throws Exception
+    {
+        assertGuessByResource(embulk,
+                "test_1_int_single_column_row_and_header_seed.yml", "test_1_int_single_column_row_and_header.csv",
+                "test_1_int_single_column_row_and_header_guessed.yml");
+    }
+
+    @Test
+    public void testIntSingleColumnWithHeader()
+            throws Exception
+    {
+        assertGuessByResource(embulk,
+                "test_int_single_column_with_header_seed.yml", "test_int_single_column_with_header.csv",
+                "test_int_single_column_with_header_guessed.yml");
+    }
+
+    @Test
+    public void testIntSingleColumn()
+            throws Exception
+    {
+        assertGuessByResource(embulk,
+                "test_int_single_column_seed.yml", "test_int_single_column.csv",
+                "test_int_single_column_guessed.yml");
+    }
+
+    @Test
+    public void testStringSingleColumnWithHeader()
+            throws Exception
+    {
+        assertGuessByResource(embulk,
+                "test_string_single_column_with_header_seed.yml", "test_string_single_column_with_header.csv",
+                "test_string_single_column_with_header_guessed.yml");
+    }
+
+    @Test
+    public void testStringSingleColumn()
+            throws Exception
+    {
+        assertGuessByResource(embulk,
+                "test_string_single_column_seed.yml", "test_string_single_column.csv",
+                "test_string_single_column_guessed.yml");
+    }
+
+    @Test
+    public void suggestTabAsDelimiter()
+            throws Exception
+    {
+        assertGuessByResource(embulk,
+                "test_tab_delimiter_seed.yml", "test_tab_delimiter.csv",
+                "test_tab_delimiter_guessed.yml");
+    }
+
+    @Test
+    public void suggestSemicolonAsDelimiter()
+            throws Exception
+    {
+        assertGuessByResource(embulk,
+                "test_semicolon_delimiter_seed.yml", "test_semicolon_delimiter.csv",
+                "test_semicolon_delimiter_guessed.yml");
+    }
+
+    @Test
+    public void suggestSingleQuoteAsQuote()
+            throws Exception
+    {
+        assertGuessByResource(embulk,
+                "test_single_quote_seed.yml", "test_single_quote.csv",
+                "test_single_quote_guessed.yml");
+    }
+
+    @Test
+    public void suggestBackslashAsEscape()
+            throws Exception
+    {
+        assertGuessByResource(embulk,
+                "test_backslash_escape_seed.yml", "test_backslash_escape.csv",
+                "test_backslash_escape_guessed.yml");
+    }
+
+    static void assertGuessByResource(TestingEmbulk embulk, String seedYamlResourceName, String sourceCsvResourceName,
+            String resultResourceName)
+            throws IOException
+    {
+        ConfigSource seed = embulk.loadYamlResource(RESOURCE_NAME_PREFIX + seedYamlResourceName);
+
+        ConfigDiff guessed =
+            embulk.parserBuilder()
+            .parser(seed)
+            .exec(embulk.newConfig().set("exclude_guess_plugins", ImmutableList.of("json")))
+            .inputResource(RESOURCE_NAME_PREFIX + sourceCsvResourceName)
+            .guess();
+
+        assertThat(guessed, is((DataSource) embulk.loadYamlResource(RESOURCE_NAME_PREFIX + resultResourceName)));
+    }
+}
+

data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_int_single_column_row.csv

@@ -0,0 +1 @@
+1

data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_int_single_column_row_and_header.csv

@@ -0,0 +1,2 @@
+id
+1

data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_int_single_column_row_and_header_guessed.yml

@@ -0,0 +1,12 @@
+charset: UTF-8
+newline: LF
+type: csv
+delimiter: ','
+quote: '"'
+escape: '"'
+trim_if_not_quoted: false
+skip_header_lines: 1
+allow_extra_columns: false
+allow_optional_columns: false
+columns:
+- {name: id, type: long}

data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_int_single_column_row_and_header_seed.yml

@@ -0,0 +1 @@
+{}

data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_int_single_column_row_guessed.yml

@@ -0,0 +1,12 @@
+charset: UTF-8
+newline: LF
+type: csv
+delimiter: ','
+quote: '"'
+escape: '"'
+trim_if_not_quoted: false
+skip_header_lines: 0
+allow_extra_columns: false
+allow_optional_columns: false
+columns:
+- {name: c0, type: long}

data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_int_single_column_row_seed.yml

@@ -0,0 +1 @@
+{}

data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_rows.csv

@@ -0,0 +1 @@
+1,32864,2015-01-27 19:23:49,20150127,embulk

data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_rows_and_header.csv

@@ -0,0 +1,2 @@
+id,account,time,purchase,comment
+1,32864,2015-01-27 19:23:49,20150127,embulk

data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_rows_and_header_guessed.yml

@@ -0,0 +1,16 @@
+charset: UTF-8
+newline: LF
+type: csv
+delimiter: ','
+quote: '"'
+escape: '"'
+trim_if_not_quoted: false
+skip_header_lines: 1
+allow_extra_columns: false
+allow_optional_columns: false
+columns:
+- {name: id, type: long}
+- {name: account, type: long}
+- {name: time, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
+- {name: purchase, type: timestamp, format: '%Y%m%d'}
+- {name: comment, type: string}

data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_rows_and_header_seed.yml

@@ -0,0 +1 @@
+{}

data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_rows_and_header_with_trim_needed.csv

@@ -0,0 +1,2 @@
+ id , account ,time, purchase , comment
+ 1 , 32864 ,2015-01-27 19:23:49, 20150127 , embulk

data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_rows_and_header_with_trim_needed_guessed.yml

@@ -0,0 +1,16 @@
+charset: UTF-8
+newline: LF
+type: csv
+delimiter: ','
+quote: '"'
+escape: '"'
+trim_if_not_quoted: true
+skip_header_lines: 1
+allow_extra_columns: false
+allow_optional_columns: false
+columns:
+- {name: id, type: long}
+- {name: account, type: long}
+- {name: time, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
+- {name: purchase, type: timestamp, format: '%Y%m%d'}
+- {name: comment, type: string}

data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_rows_and_header_with_trim_needed_seed.yml

@@ -0,0 +1 @@
+{}

data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_rows_guessed.yml

@@ -0,0 +1,16 @@
+charset: UTF-8
+newline: LF
+type: csv
+delimiter: ','
+quote: '"'
+escape: '"'
+trim_if_not_quoted: false
+skip_header_lines: 0
+allow_extra_columns: false
+allow_optional_columns: false
+columns:
+- {name: c0, type: long}
+- {name: c1, type: long}
+- {name: c2, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
+- {name: c3, type: timestamp, format: '%Y%m%d'}
+- {name: c4, type: string}

data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_rows_seed.yml

@@ -0,0 +1 @@
+{}

data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_rows_with_trim_needed.csv

@@ -0,0 +1 @@
+ 1 , 32864 ,2015-01-27 19:23:49, 20150127 , embulk

data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_rows_with_trim_needed_guessed.yml

@@ -0,0 +1,16 @@
+charset: UTF-8
+newline: LF
+type: csv
+delimiter: ','
+quote: '"'
+escape: '"'
+trim_if_not_quoted: true
+skip_header_lines: 0
+allow_extra_columns: false
+allow_optional_columns: false
+columns:
+- {name: c0, type: long}
+- {name: c1, type: long}
+- {name: c2, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
+- {name: c3, type: timestamp, format: '%Y%m%d'}
+- {name: c4, type: string}

data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_rows_with_trim_needed_seed.yml

@@ -0,0 +1 @@
+{}

data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_string_single_column_row.csv

@@ -0,0 +1 @@
+ac97447f-146c-4d6c-914e-c283bb165804

data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_string_single_column_row_and_header.csv

@@ -0,0 +1,2 @@
+id
+ac97447f-146c-4d6c-914e-c283bb165804

data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_string_single_column_row_and_header_guessed.yml

@@ -0,0 +1,12 @@
+charset: UTF-8
+newline: LF
+type: csv
+delimiter: ','
+quote: '"'
+escape: '"'
+trim_if_not_quoted: false
+skip_header_lines: 1
+allow_extra_columns: false
+allow_optional_columns: false
+columns:
+- {name: id, type: string}

data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_string_single_column_row_and_header_seed.yml

@@ -0,0 +1 @@
+{}

data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_string_single_column_row_guessed.yml

@@ -0,0 +1,12 @@
+charset: UTF-8
+newline: LF
+type: csv
+delimiter: ','
+quote: '"'
+escape: '"'
+trim_if_not_quoted: false
+skip_header_lines: 0
+allow_extra_columns: false
+allow_optional_columns: false
+columns:
+- {name: c0, type: string}