embulk 0.8.37-java → 0.10.26-java

data/embulk-standards/src/test/java/org/embulk/standards/guess/TestCsvAllStringsGuessPlugin.java

@@ -1,38 +0,0 @@
-package org.embulk.standards.guess;
-
-import com.google.common.collect.ImmutableList;
-import org.embulk.config.ConfigDiff;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.DataSource;
-import org.embulk.test.TestingEmbulk;
-import org.junit.Rule;
-import org.junit.Test;
-
-import static org.hamcrest.Matchers.is;
-import static org.junit.Assert.assertThat;
-
-public class TestCsvAllStringsGuessPlugin
-{
-    private static final String RESOURCE_NAME_PREFIX = "org/embulk/standards/guess/csv_all_strings/test/";
-
-    @Rule
-    public TestingEmbulk embulk = TestingEmbulk.builder()
-            .build();
-
-    @Test
-    public void testSimple()
-            throws Exception
-    {
-        ConfigSource exec = embulk.newConfig()
-                .set("guess_plugins", ImmutableList.of("csv_all_strings"))
-                .set("exclude_guess_plugins", ImmutableList.of("csv"));
-
-        ConfigDiff guessed =
-            embulk.parserBuilder()
-            .exec(exec)
-            .inputResource(RESOURCE_NAME_PREFIX + "test_simple.csv")
-            .guess();
-
-        assertThat(guessed, is((DataSource) embulk.loadYamlResource(RESOURCE_NAME_PREFIX + "test_simple_guessed.yml")));
-    }
-}

data/embulk-standards/src/test/java/org/embulk/standards/guess/TestCsvGuessPlugin.java

@@ -1,248 +0,0 @@
-package org.embulk.standards.guess;
-
-import com.google.common.collect.ImmutableList;
-import org.embulk.config.ConfigDiff;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.DataSource;
-import org.embulk.test.TestingEmbulk;
-import org.junit.Rule;
-import org.junit.Test;
-
-import java.io.IOException;
-
-import static org.hamcrest.Matchers.is;
-import static org.junit.Assert.assertThat;
-
-public class TestCsvGuessPlugin
-{
-    private static final String RESOURCE_NAME_PREFIX = "org/embulk/standards/guess/csv/test/";
-
-    @Rule
-    public TestingEmbulk embulk = TestingEmbulk.builder()
-            .build();
-
-    @Test
-    public void testSimple()
-            throws Exception
-    {
-        assertGuessByResource(embulk,
-                "test_simple_seed.yml", "test_simple.csv",
-                "test_simple_guessed.yml");
-    }
-
-    @Test
-    public void testFor1Rows()
-            throws Exception
-    {
-        assertGuessByResource(embulk,
-                "test_1_rows_seed.yml", "test_1_rows.csv",
-                "test_1_rows_guessed.yml");
-    }
-
-    @Test
-    public void testFor1RowsWithTrimNeeded()
-            throws Exception
-    {
-        assertGuessByResource(embulk,
-                "test_1_rows_with_trim_needed_seed.yml", "test_1_rows_with_trim_needed.csv",
-                "test_1_rows_with_trim_needed_guessed.yml");
-    }
-
-    @Test
-    public void testFor1RowsAndHeader()
-            throws Exception
-    {
-        assertGuessByResource(embulk,
-                "test_1_rows_and_header_seed.yml", "test_1_rows_and_header.csv",
-                "test_1_rows_and_header_guessed.yml");
-    }
-
-    @Test
-    public void testFor1RowsAndHeaderWithTrimNeeded()
-            throws Exception
-    {
-        assertGuessByResource(embulk,
-                "test_1_rows_and_header_with_trim_needed_seed.yml", "test_1_rows_and_header_with_trim_needed.csv",
-                "test_1_rows_and_header_with_trim_needed_guessed.yml");
-    }
-
-    @Test
-    public void testFor2Rows()
-            throws Exception
-    {
-        assertGuessByResource(embulk,
-                "test_2_rows_seed.yml", "test_2_rows.csv",
-                "test_2_rows_guessed.yml");
-    }
-
-    @Test
-    public void testFor2RowsAndHeader()
-            throws Exception
-    {
-        assertGuessByResource(embulk,
-                "test_2_rows_and_header_seed.yml", "test_2_rows_and_header.csv",
-                "test_2_rows_and_header_guessed.yml");
-    }
-
-    @Test
-    public void testFor1IntSingleColumnRow()
-            throws Exception
-    {
-        assertGuessByResource(embulk,
-                "test_1_int_single_column_row_seed.yml", "test_1_int_single_column_row.csv",
-                "test_1_int_single_column_row_guessed.yml");
-    }
-
-    @Test
-    public void testFor1StringSingleColumnRow()
-            throws Exception
-    {
-        assertGuessByResource(embulk,
-                "test_1_string_single_column_row_seed.yml", "test_1_string_single_column_row.csv",
-                "test_1_string_single_column_row_guessed.yml");
-    }
-
-    @Test
-    public void testFor2StringSingleColumnRows()
-            throws Exception
-    {
-        assertGuessByResource(embulk,
-                "test_2_string_single_column_rows_seed.yml", "test_2_string_single_column_rows.csv",
-                "test_2_string_single_column_rows_guessed.yml");
-    }
-
-    @Test
-    public void testFor1StringSingleColumnAndHeader()
-            throws Exception
-    {
-        assertGuessByResource(embulk,
-                "test_1_string_single_column_row_and_header_seed.yml", "test_1_string_single_column_row_and_header.csv",
-                "test_1_string_single_column_row_and_header_guessed.yml");
-    }
-
-    @Test
-    public void testFor2IntSingleColumnRows()
-            throws Exception
-    {
-        assertGuessByResource(embulk,
-                "test_2_int_single_column_rows_seed.yml", "test_2_int_single_column_rows.csv",
-                "test_2_int_single_column_rows_guessed.yml");
-    }
-
-    @Test
-    public void testFor1IntSingleColumnAndHeader()
-            throws Exception
-    {
-        assertGuessByResource(embulk,
-                "test_1_int_single_column_row_and_header_seed.yml", "test_1_int_single_column_row_and_header.csv",
-                "test_1_int_single_column_row_and_header_guessed.yml");
-    }
-
-    @Test
-    public void testIntSingleColumnWithHeader()
-            throws Exception
-    {
-        assertGuessByResource(embulk,
-                "test_int_single_column_with_header_seed.yml", "test_int_single_column_with_header.csv",
-                "test_int_single_column_with_header_guessed.yml");
-    }
-
-    @Test
-    public void testIntSingleColumn()
-            throws Exception
-    {
-        assertGuessByResource(embulk,
-                "test_int_single_column_seed.yml", "test_int_single_column.csv",
-                "test_int_single_column_guessed.yml");
-    }
-
-    @Test
-    public void testDoubleSingleColumn()
-            throws Exception
-    {
-        assertGuessByResource(embulk,
-                "test_double_single_column_seed.yml", "test_double_single_column.csv",
-                "test_double_single_column_guessed.yml");
-    }
-
-    @Test
-    public void testStringSingleColumnWithHeader()
-            throws Exception
-    {
-        assertGuessByResource(embulk,
-                "test_string_single_column_with_header_seed.yml", "test_string_single_column_with_header.csv",
-                "test_string_single_column_with_header_guessed.yml");
-    }
-
-    @Test
-    public void testStringSingleColumn()
-            throws Exception
-    {
-        assertGuessByResource(embulk,
-                "test_string_single_column_seed.yml", "test_string_single_column.csv",
-                "test_string_single_column_guessed.yml");
-    }
-
-    @Test
-    public void suggestTabAsDelimiter()
-            throws Exception
-    {
-        assertGuessByResource(embulk,
-                "test_tab_delimiter_seed.yml", "test_tab_delimiter.csv",
-                "test_tab_delimiter_guessed.yml");
-    }
-
-    @Test
-    public void suggestSemicolonAsDelimiter()
-            throws Exception
-    {
-        assertGuessByResource(embulk,
-                "test_semicolon_delimiter_seed.yml", "test_semicolon_delimiter.csv",
-                "test_semicolon_delimiter_guessed.yml");
-    }
-
-    @Test
-    public void suggestSingleQuoteAsQuote()
-            throws Exception
-    {
-        assertGuessByResource(embulk,
-                "test_single_quote_seed.yml", "test_single_quote.csv",
-                "test_single_quote_guessed.yml");
-    }
-
-    @Test
-    public void suggestBackslashAsEscape()
-            throws Exception
-    {
-        assertGuessByResource(embulk,
-                "test_backslash_escape_seed.yml", "test_backslash_escape.csv",
-                "test_backslash_escape_guessed.yml");
-    }
-
-    @Test
-    public void skipSuggestIfEmptySampleRecords()
-            throws Exception
-    {
-        // This test checks that the CSV guess doesn't suggest anything by invalid formatted CSV file.
-        assertGuessByResource(embulk,
-                "test_skip_suggest_if_empty_sample_records_seed.yml", "test_skip_suggest_if_empty_sample_records.csv",
-                "test_skip_suggest_if_empty_sample_records_guessed.yml");
-    }
-
-    static void assertGuessByResource(TestingEmbulk embulk, String seedYamlResourceName, String sourceCsvResourceName,
-            String resultResourceName)
-            throws IOException
-    {
-        ConfigSource seed = embulk.loadYamlResource(RESOURCE_NAME_PREFIX + seedYamlResourceName);
-
-        ConfigDiff guessed =
-            embulk.parserBuilder()
-            .parser(seed)
-            .exec(embulk.newConfig().set("exclude_guess_plugins", ImmutableList.of("json")))
-            .inputResource(RESOURCE_NAME_PREFIX + sourceCsvResourceName)
-            .guess();
-
-        assertThat(guessed, is((DataSource) embulk.loadYamlResource(RESOURCE_NAME_PREFIX + resultResourceName)));
-    }
-}
-

data/embulk-standards/src/test/java/org/embulk/standards/preview/TestFilePreview.java

@@ -1,73 +0,0 @@
-package org.embulk.standards.preview;
-
-import org.embulk.config.ConfigSource;
-import org.embulk.exec.PreviewResult;
-import org.embulk.test.TestingEmbulk;
-import org.junit.Rule;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.nio.file.Path;
-
-import static org.embulk.test.EmbulkTests.copyResource;
-import static org.embulk.test.EmbulkTests.readFile;
-import static org.embulk.test.EmbulkTests.readResource;
-import static org.hamcrest.Matchers.is;
-import static org.junit.Assert.assertThat;
-
-public class TestFilePreview
-{
-    private static final String RESOURCE_NAME_PREFIX = "org/embulk/standards/preview/file/test/";
-
-    @Rule
-    public TestingEmbulk embulk = TestingEmbulk.builder()
-            .build();
-
-    @Test
-    public void testSimple()
-            throws Exception
-    {
-        assertPreviewedRecords(embulk, "test_simple_load.yml", "test_simple.csv", "test_simple_previewed.csv");
-    }
-
-    @Test
-    public void changePreviewSampleBufferBytes()
-            throws Exception
-    {
-        assertPreviewedRecords(embulk, "test_sample_buffer_bytes_load.yml", "test_sample_buffer_bytes_exec.yml",
-                "test_sample_buffer_bytes.csv", "test_sample_buffer_bytes_previewed.csv");
-    }
-
-    private static void assertPreviewedRecords(TestingEmbulk embulk,
-            String loadYamlResourceName, String sourceCsvResourceName, String resultCsvResourceName)
-            throws IOException
-    {
-        assertPreviewedRecords(embulk, loadYamlResourceName, null, sourceCsvResourceName, resultCsvResourceName);
-    }
-
-    private static void assertPreviewedRecords(TestingEmbulk embulk,
-            String loadYamlResourceName, String execYamlResourceName, String sourceCsvResourceName, String resultCsvResourceName)
-            throws IOException
-    {
-        Path inputPath = embulk.createTempFile("csv");
-        Path outputPath = embulk.createTempFile("csv");
-
-        // in: config
-        copyResource(RESOURCE_NAME_PREFIX + sourceCsvResourceName, inputPath);
-        ConfigSource load = embulk.loadYamlResource(RESOURCE_NAME_PREFIX + loadYamlResourceName)
-                .set("path_prefix", inputPath.toAbsolutePath().toString());
-
-        // exec: config
-        final TestingEmbulk.InputBuilder builder = embulk.inputBuilder();
-        if (execYamlResourceName != null) {
-            final ConfigSource execConfig = embulk.loadYamlResource(RESOURCE_NAME_PREFIX + execYamlResourceName);
-            builder.exec(execConfig);
-        }
-
-        // execute preview
-        final PreviewResult result = builder.in(load).outputPath(outputPath).preview();
-
-        assertThat(readFile(outputPath), is(readResource(RESOURCE_NAME_PREFIX + resultCsvResourceName)));
-    }
-}
-

data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_int_single_column_row.csv

@@ -1 +0,0 @@
-1

data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_int_single_column_row_and_header.csv

@@ -1,2 +0,0 @@
-id
-1

data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_int_single_column_row_and_header_guessed.yml

@@ -1,12 +0,0 @@
-charset: UTF-8
-newline: LF
-type: csv
-delimiter: ','
-quote: '"'
-escape: '"'
-trim_if_not_quoted: false
-skip_header_lines: 1
-allow_extra_columns: false
-allow_optional_columns: false
-columns:
-- {name: id, type: long}

data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_int_single_column_row_and_header_seed.yml

@@ -1 +0,0 @@
-{}

data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_int_single_column_row_guessed.yml

@@ -1,12 +0,0 @@
-charset: UTF-8
-newline: LF
-type: csv
-delimiter: ','
-quote: '"'
-escape: '"'
-trim_if_not_quoted: false
-skip_header_lines: 0
-allow_extra_columns: false
-allow_optional_columns: false
-columns:
-- {name: c0, type: long}

data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_int_single_column_row_seed.yml

@@ -1 +0,0 @@
-{}

data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_rows.csv

@@ -1 +0,0 @@
-1,32864,2015-01-27 19:23:49,20150127,embulk

data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_rows_and_header.csv

@@ -1,2 +0,0 @@
-id,account,time,purchase,comment
-1,32864,2015-01-27 19:23:49,20150127,embulk

data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_rows_and_header_guessed.yml

@@ -1,16 +0,0 @@
-charset: UTF-8
-newline: LF
-type: csv
-delimiter: ','
-quote: '"'
-escape: '"'
-trim_if_not_quoted: false
-skip_header_lines: 1
-allow_extra_columns: false
-allow_optional_columns: false
-columns:
-- {name: id, type: long}
-- {name: account, type: long}
-- {name: time, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
-- {name: purchase, type: timestamp, format: '%Y%m%d'}
-- {name: comment, type: string}

data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_rows_and_header_seed.yml

@@ -1 +0,0 @@
-{}

data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_rows_and_header_with_trim_needed.csv

@@ -1,2 +0,0 @@
- id , account ,time, purchase , comment
- 1 , 32864 ,2015-01-27 19:23:49, 20150127 , embulk

data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_1_rows_and_header_with_trim_needed_guessed.yml

@@ -1,16 +0,0 @@
-charset: UTF-8
-newline: LF
-type: csv
-delimiter: ','
-quote: '"'
-escape: '"'
-trim_if_not_quoted: true
-skip_header_lines: 1
-allow_extra_columns: false
-allow_optional_columns: false
-columns:
-- {name: id, type: long}
-- {name: account, type: long}
-- {name: time, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
-- {name: purchase, type: timestamp, format: '%Y%m%d'}
-- {name: comment, type: string}