embulk 0.8.36-java → 0.10.25-java

data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv/test/test_tab_delimiter_seed.yml

@@ -1 +0,0 @@
-{}

data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv_all_strings/test/test_simple.csv

@@ -1,5 +0,0 @@
-id,account,time,purchase,comment
-1,32864,2015-01-27 19:23:49,20150127,embulk
-2,14824,2015-01-27 19:01:23,20150127,embulk jruby
-3,27559,2015-01-28 02:20:02,20150128,"Embulk ""csv"" parser plugin"
-4,11270,2015-01-29 11:54:36,20150129,NULL

data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv_all_strings/test/test_simple_guessed.yml

@@ -1,17 +0,0 @@
-charset: UTF-8
-newline: LF
-type: csv
-delimiter: ','
-quote: '"'
-escape: '"'
-null_string: "NULL"
-trim_if_not_quoted: false
-skip_header_lines: 1
-allow_extra_columns: false
-allow_optional_columns: false
-columns:
-- {name: id, type: string}
-- {name: account, type: string}
-- {name: time, type: string}
-- {name: purchase, type: string}
-- {name: comment, type: string}

data/embulk-standards/src/test/resources/org/embulk/standards/guess/csv_all_strings/test/test_simple_seed.yml

@@ -1 +0,0 @@
-{}

data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes.csv

@@ -1,5 +0,0 @@
-id,account,time,purchase,comment
-1,32864,2015-01-27 19:23:49,20150127,embulk
-2,14824,2015-01-27 19:01:23,20150127,embulk jruby
-3,27559,2015-01-28 02:20:02,20150128,"Embulk ""csv"" parser plugin"
-4,11270,2015-01-29 11:54:36,20150129,NULL

data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes_exec.yml

@@ -1 +0,0 @@
-preview_sample_buffer_bytes: 96

data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes_load.yml

@@ -1,19 +0,0 @@
-type: file
-parser:
-  charset: UTF-8
-  newline: LF
-  type: csv
-  delimiter: ','
-  quote: '"'
-  escape: '"'
-  null_string: "NULL"
-  trim_if_not_quoted: false
-  skip_header_lines: 1
-  allow_extra_columns: false
-  allow_optional_columns: false
-  columns:
-  - {name: id, type: long}
-  - {name: account, type: long}
-  - {name: time, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
-  - {name: purchase, type: timestamp, format: '%Y%m%d'}
-  - {name: comment, type: string}

data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_sample_buffer_bytes_previewed.csv

@@ -1 +0,0 @@
-1,32864,2015-01-27 19:23:49.000000 +0000,2015-01-27 00:00:00.000000 +0000,embulk

data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_simple.csv

@@ -1,5 +0,0 @@
-id,account,time,purchase,comment
-1,32864,2015-01-27 19:23:49,20150127,embulk
-2,14824,2015-01-27 19:01:23,20150127,embulk jruby
-3,27559,2015-01-28 02:20:02,20150128,"Embulk ""csv"" parser plugin"
-4,11270,2015-01-29 11:54:36,20150129,NULL

data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_simple_load.yml

@@ -1,19 +0,0 @@
-type: file
-parser:
-  charset: UTF-8
-  newline: LF
-  type: csv
-  delimiter: ','
-  quote: '"'
-  escape: '"'
-  null_string: "NULL"
-  trim_if_not_quoted: false
-  skip_header_lines: 1
-  allow_extra_columns: false
-  allow_optional_columns: false
-  columns:
-  - {name: id, type: long}
-  - {name: account, type: long}
-  - {name: time, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
-  - {name: purchase, type: timestamp, format: '%Y%m%d'}
-  - {name: comment, type: string}

data/embulk-standards/src/test/resources/org/embulk/standards/preview/file/test/test_simple_previewed.csv

@@ -1,4 +0,0 @@
-1,32864,2015-01-27 19:23:49.000000 +0000,2015-01-27 00:00:00.000000 +0000,embulk
-2,14824,2015-01-27 19:01:23.000000 +0000,2015-01-27 00:00:00.000000 +0000,embulk jruby
-3,27559,2015-01-28 02:20:02.000000 +0000,2015-01-28 00:00:00.000000 +0000,"Embulk ""csv"" parser plugin"
-4,11270,2015-01-29 11:54:36.000000 +0000,2015-01-29 00:00:00.000000 +0000,

data/embulk-standards/src/test/resources/org/embulk/standards/remove_columns/test/test_keep.csv

@@ -1,5 +0,0 @@
-id,account,time,purchase,comment
-1,32864,2015-01-27 19:23:49,20150127,embulk
-2,14824,2015-01-27 19:01:23,20150127,embulk jruby
-3,27559,2015-01-28 02:20:02,20150128,"Embulk ""csv"" parser plugin"
-4,11270,2015-01-29 11:54:36,20150129,NULL

data/embulk-standards/src/test/resources/org/embulk/standards/remove_columns/test/test_keep_expected.csv

@@ -1,4 +0,0 @@
-1,32864,20150127
-2,14824,20150127
-3,27559,20150128
-4,11270,20150129

data/embulk-standards/src/test/resources/org/embulk/standards/remove_columns/test/test_keep_filter.yml

@@ -1,2 +0,0 @@
-type: remove_columns
-keep: [id, account, purchase]

data/embulk-standards/src/test/resources/org/embulk/standards/remove_columns/test/test_keep_in.yml

@@ -1,18 +0,0 @@
-type: file
-parser:
-  charset: UTF-8
-  newline: CRLF
-  type: csv
-  delimiter: ','
-  quote: '"'
-  escape: '"'
-  trim_if_not_quoted: false
-  skip_header_lines: 1
-  allow_extra_columns: false
-  allow_optional_columns: false
-  columns:
-  - {name: id, type: long}
-  - {name: account, type: long}
-  - {name: time, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
-  - {name: purchase, type: string}
-  - {name: comment, type: string}

data/embulk-standards/src/test/resources/org/embulk/standards/remove_columns/test/test_keep_with_duplicated_column_names.csv

@@ -1,5 +0,0 @@
-id,id,comment,id
-1,32864,a,k1
-2,14824,b,k2
-3,27559,c,k3
-4,11270,d,k4

data/embulk-standards/src/test/resources/org/embulk/standards/remove_columns/test/test_keep_with_duplicated_column_names.yml

@@ -1,2 +0,0 @@
-type: remove_columns
-keep: [id]

data/embulk-standards/src/test/resources/org/embulk/standards/remove_columns/test/test_keep_with_duplicated_column_names_expected.csv

@@ -1,4 +0,0 @@
-1,32864,k1
-2,14824,k2
-3,27559,k3
-4,11270,k4

data/embulk-standards/src/test/resources/org/embulk/standards/remove_columns/test/test_keep_with_duplicated_column_names_in.yml

@@ -1,17 +0,0 @@
-type: file
-parser:
-  charset: UTF-8
-  newline: CRLF
-  type: csv
-  delimiter: ','
-  quote: '"'
-  escape: '"'
-  trim_if_not_quoted: false
-  skip_header_lines: 1
-  allow_extra_columns: false
-  allow_optional_columns: false
-  columns:
-  - {name: id, type: long}
-  - {name: id, type: long}
-  - {name: comment, type: string}
-  - {name: id, type: string}

data/embulk-standards/src/test/resources/org/embulk/standards/remove_columns/test/test_keep_with_unmatched_filter.yml

@@ -1,3 +0,0 @@
-type: remove_columns
-keep: [invalid, id, account, purchase]
-accept_unmatched_columns: true

data/embulk-standards/src/test/resources/org/embulk/standards/remove_columns/test/test_keep_without_unmatched_filter.yml

@@ -1,2 +0,0 @@
-type: remove_columns
-keep: [invalid, id, account, purchase]

data/embulk-standards/src/test/resources/org/embulk/standards/remove_columns/test/test_remove.csv

@@ -1,5 +0,0 @@
-id,account,time,purchase,comment
-1,32864,2015-01-27 19:23:49,20150127,embulk
-2,14824,2015-01-27 19:01:23,20150127,embulk jruby
-3,27559,2015-01-28 02:20:02,20150128,"Embulk ""csv"" parser plugin"
-4,11270,2015-01-29 11:54:36,20150129,NULL

data/embulk-standards/src/test/resources/org/embulk/standards/remove_columns/test/test_remove_expected.csv

@@ -1,4 +0,0 @@
-1,32864,20150127
-2,14824,20150127
-3,27559,20150128
-4,11270,20150129

data/embulk-standards/src/test/resources/org/embulk/standards/remove_columns/test/test_remove_filter.yml

@@ -1,2 +0,0 @@
-type: remove_columns
-remove: [time,comment]

data/embulk-standards/src/test/resources/org/embulk/standards/remove_columns/test/test_remove_in.yml

@@ -1,18 +0,0 @@
-type: file
-parser:
-  charset: UTF-8
-  newline: CRLF
-  type: csv
-  delimiter: ','
-  quote: '"'
-  escape: '"'
-  trim_if_not_quoted: false
-  skip_header_lines: 1
-  allow_extra_columns: false
-  allow_optional_columns: false
-  columns:
-  - {name: id, type: long}
-  - {name: account, type: long}
-  - {name: time, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
-  - {name: purchase, type: string}
-  - {name: comment, type: string}

data/embulk-standards/src/test/resources/org/embulk/standards/remove_columns/test/test_remove_with_unmatched_filter.yml

@@ -1,3 +0,0 @@
-type: remove_columns
-remove: [invalid, time, comment]
-accept_unmatched_columns: true

data/embulk-standards/src/test/resources/org/embulk/standards/remove_columns/test/test_remove_without_unmatched_filter.yml

@@ -1,2 +0,0 @@
-type: remove_columns
-remove: [invalid, time, comment]

data/embulk-test/build.gradle

@@ -1,6 +0,0 @@
-dependencies {
-    compile 'junit:junit:4.12'
-    compile 'org.hamcrest:hamcrest-library:1.3'
-
-    compile project(':embulk-core')
-}

data/embulk-test/src/main/java/org/embulk/test/EmbulkTests.java

@@ -1,75 +0,0 @@
-package org.embulk.test;
-
-import com.google.common.base.Throwables;
-import com.google.common.io.CharStreams;
-import com.google.common.io.Resources;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.Collections;
-import java.util.List;
-import org.embulk.EmbulkEmbed;
-import org.embulk.config.ConfigSource;
-import static com.google.common.base.Strings.isNullOrEmpty;
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
-import static org.hamcrest.Matchers.is;
-import static org.junit.Assume.assumeThat;
-
-public class EmbulkTests
-{
-    private EmbulkTests()
-    { }
-
-    public static ConfigSource config(String envName)
-    {
-        String path = System.getenv(envName);
-        assumeThat(isNullOrEmpty(path), is(false));
-        try {
-            return EmbulkEmbed.newSystemConfigLoader().fromYamlFile(new File(path));
-        } catch (IOException ex) {
-            throw Throwables.propagate(ex);
-        }
-    }
-
-    public static String readResource(String name)
-    {
-        try (InputStream in = Resources.getResource(name).openStream()) {
-            return CharStreams.toString(new InputStreamReader(in, UTF_8));
-        }
-        catch (IOException ex) {
-            throw new RuntimeException(ex);
-        }
-    }
-
-    public static void copyResource(String resource, Path dest)
-        throws IOException
-    {
-        Files.createDirectories(dest.getParent());
-        try (InputStream input = Resources.getResource(resource).openStream()) {
-            Files.copy(input, dest, REPLACE_EXISTING);
-        }
-    }
-
-    public static String readFile(Path path) throws IOException
-    {
-        try (InputStream in = Files.newInputStream(path)) {
-            return CharStreams.toString(new InputStreamReader(in, UTF_8));
-        }
-    }
-
-    public static String readSortedFile(Path path) throws IOException
-    {
-        List<String> lines = Files.readAllLines(path, UTF_8);
-        Collections.sort(lines);
-        StringBuilder sb = new StringBuilder();
-        for (String line : lines) {
-            sb.append(line);
-            sb.append("\n");
-        }
-        return sb.toString();
-    }
-}

data/embulk-test/src/main/java/org/embulk/test/PreviewResultInputPlugin.java

@@ -1,65 +0,0 @@
-package org.embulk.test;
-
-import org.embulk.config.ConfigDiff;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.Task;
-import org.embulk.config.TaskReport;
-import org.embulk.config.TaskSource;
-import org.embulk.exec.PreviewResult;
-import org.embulk.spi.Exec;
-import org.embulk.spi.InputPlugin;
-import org.embulk.spi.Page;
-import org.embulk.spi.PageOutput;
-import org.embulk.spi.Schema;
-
-import java.util.List;
-
-import static com.google.common.base.Preconditions.checkState;
-
-/**
- * This plugin is used for TestingEmbulk.InputBuilder.preview().
- */
-public final class PreviewResultInputPlugin
-        implements InputPlugin
-{
-    private static PreviewResult previewResult;
-
-    public static void setPreviewResult(PreviewResult result)
-    {
-        previewResult = result;
-    }
-
-    @Override
-    public ConfigDiff transaction(ConfigSource config, Control control)
-    {
-        checkState(previewResult != null, "PreviewResult object must be set");
-        return resume(config.loadConfig(Task.class).dump(), previewResult.getSchema(), 1, control);
-    }
-
-    @Override
-    public ConfigDiff resume(TaskSource taskSource, Schema schema, int taskCount, Control control)
-    {
-        control.run(taskSource, schema, taskCount);
-        return Exec.newConfigDiff();
-    }
-
-    @Override
-    public void cleanup(TaskSource taskSource, Schema schema, int taskCount, List<TaskReport> successTaskReports)
-    {
-    }
-
-    @Override
-    public TaskReport run(TaskSource taskSource, Schema schema, int taskIndex, PageOutput output)
-    {
-        for (Page page : previewResult.getPages()) {
-            output.add(page);
-        }
-        return Exec.newTaskReport();
-    }
-
-    @Override
-    public ConfigDiff guess(ConfigSource config)
-    {
-        return Exec.newConfigDiff();
-    }
-}

data/embulk-test/src/main/java/org/embulk/test/TestingBulkLoader.java

@@ -1,129 +0,0 @@
-package org.embulk.test;
-
-import com.google.common.base.Function;
-import com.google.common.base.Optional;
-import com.google.common.collect.ImmutableList;
-import com.google.inject.Binder;
-import com.google.inject.Inject;
-import com.google.inject.Injector;
-import com.google.inject.Module;
-import com.google.inject.util.Modules;
-import java.util.List;
-
-import org.embulk.config.ConfigSource;
-import org.embulk.config.TaskReport;
-import org.embulk.exec.BulkLoader;
-import org.embulk.exec.ExecutionResult;
-import org.embulk.exec.ForSystemConfig;
-import org.embulk.exec.ResumeState;
-import org.embulk.spi.Exec;
-import org.embulk.spi.ExecSession;
-import org.embulk.spi.InputPlugin;
-import org.embulk.spi.Schema;
-import org.slf4j.Logger;
-
-import static org.embulk.plugin.InjectedPluginSource.registerPluginTo;
-
-class TestingBulkLoader
-        extends BulkLoader
-{
-    static Function<List<Module>, List<Module>> override()
-    {
-        return new Function<List<Module>, List<Module>>() {
-            @Override
-            public List<Module> apply(List<Module> modules)
-            {
-                Module override = new Module() {
-                    public void configure(Binder binder)
-                    {
-                        binder.bind(BulkLoader.class).to(TestingBulkLoader.class);
-                        registerPluginTo(binder, InputPlugin.class, "preview_result", PreviewResultInputPlugin.class);
-                    }
-                };
-                return ImmutableList.of(Modules.override(modules).with(ImmutableList.of(override)));
-            }
-        };
-    }
-
-    @Inject
-    public TestingBulkLoader(Injector injector,
-            @ForSystemConfig ConfigSource systemConfig)
-    {
-        super(injector, systemConfig);
-    }
-
-    @Override
-    protected LoaderState newLoaderState(Logger logger, ProcessPluginSet plugins)
-    {
-        return new TestingLoaderState(logger, plugins);
-    }
-
-    protected static class TestingLoaderState
-            extends LoaderState
-    {
-        public TestingLoaderState(Logger logger, ProcessPluginSet plugins)
-        {
-            super(logger, plugins);
-        }
-
-        @Override
-        public ExecutionResult buildExecuteResultWithWarningException(Throwable ex)
-        {
-            ExecutionResult result = super.buildExecuteResultWithWarningException(ex);
-            return new TestingExecutionResult(result, buildResumeState(Exec.session()), Exec.session());
-        }
-    }
-
-    static class TestingExecutionResult
-            extends ExecutionResult
-            implements TestingEmbulk.RunResult
-    {
-        private final Schema inputSchema;
-        private final Schema outputSchema;
-        private final List<TaskReport> inputTaskReports;
-        private final List<TaskReport> outputTaskReports;
-
-        public TestingExecutionResult(ExecutionResult orig,
-                ResumeState resumeState, ExecSession session)
-        {
-            super(orig.getConfigDiff(), orig.isSkipped(), orig.getIgnoredExceptions());
-            this.inputSchema = resumeState.getInputSchema();
-            this.outputSchema = resumeState.getOutputSchema();
-            this.inputTaskReports = buildReports(resumeState.getInputTaskReports(), session);
-            this.outputTaskReports = buildReports(resumeState.getOutputTaskReports(), session);
-        }
-
-        private static List<TaskReport> buildReports(List<Optional<TaskReport>> optionalReports, ExecSession session)
-        {
-            ImmutableList.Builder<TaskReport> reports = ImmutableList.builder();
-            for (Optional<TaskReport> report : optionalReports) {
-                reports.add(report.or(session.newTaskReport()));
-            }
-            return reports.build();
-        }
-
-        @Override
-        public Schema getInputSchema()
-        {
-            return inputSchema;
-        }
-
-        @Override
-        public Schema getOutputSchema()
-        {
-            return outputSchema;
-        }
-
-        @Override
-        public List<TaskReport> getInputTaskReports()
-        {
-            return inputTaskReports;
-        }
-
-        @Override
-        public List<TaskReport> getOutputTaskReports()
-        {
-            return outputTaskReports;
-        }
-    }
-}