embulk 0.8.39-java → 0.10.24-java

data/embulk-standards/src/main/java/org/embulk/standards/StandardPluginExtension.java

@@ -1,16 +0,0 @@
-package org.embulk.standards;
-
-import java.util.List;
-import com.google.common.collect.ImmutableList;
-import com.google.inject.Module;
-import org.embulk.spi.Extension;
-import org.embulk.config.ConfigSource;
-
-public class StandardPluginExtension
-        implements Extension
-{
-    public List<Module> getModules(ConfigSource systemConfig)
-    {
-        return ImmutableList.<Module>of(new StandardPluginModule());
-    }
-}

data/embulk-standards/src/main/java/org/embulk/standards/StandardPluginModule.java

@@ -1,60 +0,0 @@
-package org.embulk.standards;
-
-import com.google.common.base.Preconditions;
-import com.google.inject.Binder;
-import com.google.inject.Module;
-import org.embulk.spi.FilterPlugin;
-import org.embulk.spi.FormatterPlugin;
-import org.embulk.spi.InputPlugin;
-import org.embulk.spi.OutputPlugin;
-import org.embulk.spi.ParserPlugin;
-import org.embulk.spi.DecoderPlugin;
-import org.embulk.spi.EncoderPlugin;
-import org.embulk.plugin.DefaultPluginType;
-import static org.embulk.plugin.InjectedPluginSource.registerPluginTo;
-import static org.embulk.exec.GuessExecutor.registerDefaultGuessPluginTo;
-
-public class StandardPluginModule
-        implements Module
-{
-    @Override
-    public void configure(Binder binder)
-    {
-        Preconditions.checkNotNull(binder, "binder is null.");
-
-        // input plugins
-        registerPluginTo(binder, InputPlugin.class, "config", ConfigInputPlugin.class);
-        registerPluginTo(binder, InputPlugin.class, "file", LocalFileInputPlugin.class);
-
-        // parser plugins
-        registerPluginTo(binder, ParserPlugin.class, "csv", CsvParserPlugin.class);
-        registerPluginTo(binder, ParserPlugin.class, "json", JsonParserPlugin.class);
-
-        // file decoder plugins
-        registerPluginTo(binder, DecoderPlugin.class, "gzip", GzipFileDecoderPlugin.class);
-        registerPluginTo(binder, DecoderPlugin.class, "bzip2", Bzip2FileDecoderPlugin.class);
-
-        // output plugins
-        registerPluginTo(binder, OutputPlugin.class, "file", LocalFileOutputPlugin.class);
-        registerPluginTo(binder, OutputPlugin.class, "null", NullOutputPlugin.class);
-        registerPluginTo(binder, OutputPlugin.class, "stdout", StdoutOutputPlugin.class);
-
-        // formatter plugins
-        registerPluginTo(binder, FormatterPlugin.class, "csv", CsvFormatterPlugin.class);
-
-        // file encoder plugins
-        registerPluginTo(binder, EncoderPlugin.class, "gzip", GzipFileEncoderPlugin.class);
-        registerPluginTo(binder, EncoderPlugin.class, "bzip2", Bzip2FileEncoderPlugin.class);
-
-        // filter plugins
-        registerPluginTo(binder, FilterPlugin.class, "rename", RenameFilterPlugin.class);
-        registerPluginTo(binder, FilterPlugin.class, "remove_columns", RemoveColumnsFilterPlugin.class);
-
-        // default guess plugins
-        registerDefaultGuessPluginTo(binder, DefaultPluginType.create("gzip"));
-        registerDefaultGuessPluginTo(binder, DefaultPluginType.create("bzip2"));
-        registerDefaultGuessPluginTo(binder, DefaultPluginType.create("json")); // should be registered before CsvGuessPlugin
-        registerDefaultGuessPluginTo(binder, DefaultPluginType.create("csv"));
-        // charset and newline guess plugins are loaded and invoked by CsvGuessPlugin
-    }
-}

data/embulk-standards/src/main/java/org/embulk/standards/StdoutOutputPlugin.java

@@ -1,91 +0,0 @@
-package org.embulk.standards;
-
-import java.util.List;
-import org.embulk.config.Config;
-import org.embulk.config.ConfigDefault;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.TaskSource;
-import org.embulk.config.ConfigDiff;
-import org.embulk.config.TaskReport;
-import org.embulk.config.Task;
-import org.embulk.spi.time.TimestampFormatter;
-import org.embulk.spi.Schema;
-import org.embulk.spi.Page;
-import org.embulk.spi.Exec;
-import org.embulk.spi.OutputPlugin;
-import org.embulk.spi.TransactionalPageOutput;
-import org.embulk.spi.PageReader;
-import org.embulk.spi.util.PagePrinter;
-import org.joda.time.DateTimeZone;
-
-public class StdoutOutputPlugin
-        implements OutputPlugin
-{
-    public interface PluginTask
-            extends Task
-    {
-        @Config("timezone")
-        @ConfigDefault("\"UTC\"")
-        public DateTimeZone getTimeZone();
-    }
-
-    @Override
-    public ConfigDiff transaction(ConfigSource config,
-            Schema schema, int taskCount,
-            OutputPlugin.Control control)
-    {
-        final PluginTask task = config.loadConfig(PluginTask.class);
-        return resume(task.dump(), schema, taskCount, control);
-    }
-
-    @Override
-    public ConfigDiff resume(TaskSource taskSource,
-            Schema schema, int taskCount,
-            OutputPlugin.Control control)
-    {
-        control.run(taskSource);
-        return Exec.newConfigDiff();
-    }
-
-    public void cleanup(TaskSource taskSource,
-            Schema schema, int taskCount,
-            List<TaskReport> successTaskReports)
-    { }
-
-    @Override
-    public TransactionalPageOutput open(TaskSource taskSource, final Schema schema,
-            int taskIndex)
-    {
-        final PluginTask task = taskSource.loadTask(PluginTask.class);
-
-        return new TransactionalPageOutput() {
-            private final PageReader reader = new PageReader(schema);
-            private final PagePrinter printer = new PagePrinter(schema, task.getTimeZone());
-
-            public void add(Page page)
-            {
-                reader.setPage(page);
-                while (reader.nextRecord()) {
-                    System.out.println(printer.printRecord(reader, ","));
-                }
-            }
-
-            public void finish()
-            {
-                System.out.flush();
-            }
-
-            public void close()
-            {
-                reader.close();
-            }
-
-            public void abort() { }
-
-            public TaskReport commit()
-            {
-                return Exec.newTaskReport();
-            }
-        };
-    }
-}

data/embulk-standards/src/main/resources/META-INF/services/org.embulk.spi.Extension

@@ -1 +0,0 @@
-org.embulk.standards.StandardPluginExtension

data/embulk-standards/src/test/java/org/embulk/standards/TestCsvFormatterPlugin.java

@@ -1,312 +0,0 @@
-package org.embulk.standards;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import org.joda.time.DateTimeZone;
-import org.junit.Rule;
-import org.junit.Test;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import static org.junit.Assert.assertEquals;
-import java.nio.charset.Charset;
-import org.embulk.EmbulkTestRuntime;
-import org.embulk.config.ConfigSource;
-import org.embulk.spi.Exec;
-import org.embulk.spi.util.Newline;
-
-public class TestCsvFormatterPlugin
-{
-    @Rule
-    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
-
-    @Test
-    public void checkDefaultValues()
-    {
-        ConfigSource config = Exec.newConfigSource();
-
-        CsvFormatterPlugin.PluginTask task = config.loadConfig(CsvFormatterPlugin.PluginTask.class);
-        assertEquals(Charset.forName("utf-8"), task.getCharset());
-        assertEquals(Newline.CRLF, task.getNewline());
-        assertEquals(true, task.getHeaderLine());
-        assertEquals(',', task.getDelimiterChar());
-        assertEquals('\"', task.getQuoteChar());
-        assertEquals(CsvFormatterPlugin.QuotePolicy.MINIMAL, task.getQuotePolicy());
-        assertEquals(false, task.getEscapeChar().isPresent());
-        assertEquals("", task.getNullString());
-        assertEquals(DateTimeZone.UTC, task.getDefaultTimeZone());
-        assertEquals("%Y-%m-%d %H:%M:%S.%6N %z", task.getDefaultTimestampFormat());
-        assertEquals(Newline.LF, task.getNewlineInField());
-    }
-
-    @Test
-    public void checkLoadConfig()
-    {
-        ConfigSource config = Exec.newConfigSource()
-                .set("charset", "utf-16")
-                .set("newline", "LF")
-                .set("header_line", false)
-                .set("delimiter", "\t")
-                .set("quote", "\\")
-                .set("quote_policy", "ALL")
-                .set("escape", "\"")
-                .set("null_string", "\\N")
-                .set("newline_in_field", "CRLF");
-
-        CsvFormatterPlugin.PluginTask task = config.loadConfig(CsvFormatterPlugin.PluginTask.class);
-        assertEquals(Charset.forName("utf-16"), task.getCharset());
-        assertEquals(Newline.LF, task.getNewline());
-        assertEquals(false, task.getHeaderLine());
-        assertEquals('\t', task.getDelimiterChar());
-        assertEquals('\\', task.getQuoteChar());
-        assertEquals(CsvFormatterPlugin.QuotePolicy.ALL, task.getQuotePolicy());
-        assertEquals('\"', (char) task.getEscapeChar().get());
-        assertEquals("\\N", task.getNullString());
-        assertEquals(Newline.CRLF, task.getNewlineInField());
-    }
-
-    @Test
-    public void testQuoteValue()
-            throws NoSuchMethodException, SecurityException, IllegalAccessException, IllegalArgumentException, InvocationTargetException
-    {
-        Method method = CsvFormatterPlugin.class.getDeclaredMethod("setQuoteValue", String.class, char.class);
-        method.setAccessible(true);
-        CsvFormatterPlugin formatter = new CsvFormatterPlugin();
-
-        assertEquals("\"ABCD\"", method.invoke(formatter, "ABCD", '"'));
-        assertEquals("\"\"", method.invoke(formatter, "", '"'));
-        assertEquals("'ABCD'", method.invoke(formatter, "ABCD", '\''));
-        assertEquals("''", method.invoke(formatter, "", '\''));
-    }
-
-    @Test
-    public void testEscapeQuote()
-            throws NoSuchMethodException, SecurityException, IllegalAccessException, IllegalArgumentException, InvocationTargetException
-    {
-        Method method = CsvFormatterPlugin.class.getDeclaredMethod("setEscapeAndQuoteValue", String.class, char.class,
-                CsvFormatterPlugin.QuotePolicy.class, char.class, char.class, String.class, String.class);
-        method.setAccessible(true);
-        CsvFormatterPlugin formatter = new CsvFormatterPlugin();
-
-        char delimiter = ',';
-        CsvFormatterPlugin.QuotePolicy policy = CsvFormatterPlugin.QuotePolicy.MINIMAL;
-        String newline = Newline.LF.getString();
-
-        assertEquals("\"AB\\\"CD\"", method.invoke(formatter, "AB\"CD", delimiter, policy, '"', '\\', newline, ""));
-        assertEquals("\"AB\"\"CD\"", method.invoke(formatter, "AB\"CD", delimiter, policy, '"', '"', newline, ""));
-    }
-
-    @Test
-    public void testQuotePolicyAll()
-            throws NoSuchMethodException, SecurityException, IllegalAccessException, IllegalArgumentException, InvocationTargetException
-    {
-        Method method = CsvFormatterPlugin.class.getDeclaredMethod("setEscapeAndQuoteValue", String.class, char.class,
-                CsvFormatterPlugin.QuotePolicy.class, char.class, char.class, String.class, String.class);
-        method.setAccessible(true);
-        CsvFormatterPlugin formatter = new CsvFormatterPlugin();
-
-        char delimiter = ',';
-        char quote = '"';
-        char escape = '"';
-        CsvFormatterPlugin.QuotePolicy policy = CsvFormatterPlugin.QuotePolicy.ALL;
-        String newline = Newline.LF.getString();
-        String nullString = "";
-
-        @SuppressWarnings("unchecked")
-        ImmutableList<ImmutableMap<String, String>> testCases = ImmutableList.of(
-                ImmutableMap.of("expected", "\"true\"", "actual", "true"),
-                ImmutableMap.of("expected", "\"false\"", "actual", "false"),
-                ImmutableMap.of("expected", "\"0\"", "actual", "0"),
-                ImmutableMap.of("expected", "\"1\"", "actual", "1"),
-                ImmutableMap.of("expected", "\"1234\"", "actual", "1234"),
-                ImmutableMap.of("expected", "\"-1234\"", "actual", "-1234"),
-                ImmutableMap.of("expected", "\"+1234\"", "actual", "+1234"),
-                ImmutableMap.of("expected", "\"0x4d2\"", "actual", "0x4d2"),
-                ImmutableMap.of("expected", "\"123L\"", "actual", "123L"),
-                ImmutableMap.of("expected", "\"3.141592\"", "actual", "3.141592"),
-                ImmutableMap.of("expected", "\"1,000\"", "actual", "1,000"),
-                ImmutableMap.of("expected", "\"ABC\"", "actual", "ABC"),
-                ImmutableMap.of("expected", "\"ABC\"\"DEF\"", "actual", "ABC\"DEF"),
-                ImmutableMap.of("expected", "\"ABC\nDEF\"", "actual", "ABC\nDEF"),
-                ImmutableMap.of("expected", "\"\"", "actual", ""),
-                ImmutableMap.of("expected", "\"NULL\"", "actual", "NULL"),
-                ImmutableMap.of("expected", "\"2015-01-01 12:01:01\"", "actual", "2015-01-01 12:01:01"),
-                ImmutableMap.of("expected", "\"20150101\"", "actual", "20150101"));
-
-        for (ImmutableMap testCase : testCases) {
-            String expected = (String) testCase.get("expected");
-            String actual = (String) testCase.get("actual");
-            assertEquals(expected, method.invoke(formatter, actual, delimiter, policy, quote, escape, newline, nullString));
-        }
-    }
-
-    @Test
-    public void testQuotePolicyMinimal()
-            throws NoSuchMethodException, SecurityException, IllegalAccessException, IllegalArgumentException, InvocationTargetException
-    {
-        Method method = CsvFormatterPlugin.class.getDeclaredMethod("setEscapeAndQuoteValue", String.class, char.class,
-                CsvFormatterPlugin.QuotePolicy.class, char.class, char.class, String.class, String.class);
-        method.setAccessible(true);
-        CsvFormatterPlugin formatter = new CsvFormatterPlugin();
-
-        char delimiter = ',';
-        char quote = '"';
-        char escape = '"';
-        CsvFormatterPlugin.QuotePolicy policy = CsvFormatterPlugin.QuotePolicy.MINIMAL;
-        String newline = Newline.LF.getString();
-        String nullString = "";
-
-        @SuppressWarnings("unchecked")
-        ImmutableList<ImmutableMap<String, String>> testCases = ImmutableList.of(
-                ImmutableMap.of("expected", "true", "actual", "true"),
-                ImmutableMap.of("expected", "false", "actual", "false"),
-                ImmutableMap.of("expected", "0", "actual", "0"),
-                ImmutableMap.of("expected", "1", "actual", "1"),
-                ImmutableMap.of("expected", "1234", "actual", "1234"),
-                ImmutableMap.of("expected", "-1234", "actual", "-1234"),
-                ImmutableMap.of("expected", "+1234", "actual", "+1234"),
-                ImmutableMap.of("expected", "0x4d2", "actual", "0x4d2"),
-                ImmutableMap.of("expected", "123L", "actual", "123L"),
-                ImmutableMap.of("expected", "3.141592", "actual", "3.141592"),
-                ImmutableMap.of("expected", "\"1,000\"", "actual", "1,000"),
-                ImmutableMap.of("expected", "ABC", "actual", "ABC"),
-                ImmutableMap.of("expected", "\"ABC\"\"DEF\"", "actual", "ABC\"DEF"),
-                ImmutableMap.of("expected", "\"ABC\nDEF\"", "actual", "ABC\nDEF"),
-                ImmutableMap.of("expected", "\"\"", "actual", ""),
-                ImmutableMap.of("expected", "NULL", "actual", "NULL"),
-                ImmutableMap.of("expected", "2015-01-01 12:01:01", "actual", "2015-01-01 12:01:01"),
-                ImmutableMap.of("expected", "20150101", "actual", "20150101"));
-
-        for (ImmutableMap testCase : testCases) {
-            String expected = (String) testCase.get("expected");
-            String actual = (String) testCase.get("actual");
-            assertEquals(expected, method.invoke(formatter, actual, delimiter, policy, quote, escape, newline, nullString));
-        }
-    }
-
-    @Test
-    public void testQuotePolicyNone()
-            throws NoSuchMethodException, SecurityException, IllegalAccessException, IllegalArgumentException, InvocationTargetException
-    {
-        Method method = CsvFormatterPlugin.class.getDeclaredMethod("setEscapeAndQuoteValue", String.class, char.class,
-                CsvFormatterPlugin.QuotePolicy.class, char.class, char.class, String.class, String.class);
-        method.setAccessible(true);
-        CsvFormatterPlugin formatter = new CsvFormatterPlugin();
-
-        char delimiter = ',';
-        char quote = '"';
-        char escape = '\\';
-        CsvFormatterPlugin.QuotePolicy policy = CsvFormatterPlugin.QuotePolicy.NONE;
-        String newline = Newline.LF.getString();
-        String nullString = "";
-
-        @SuppressWarnings("unchecked")
-        ImmutableList<ImmutableMap<String, String>> testCases = ImmutableList.of(
-                ImmutableMap.of("expected", "true", "actual", "true"),
-                ImmutableMap.of("expected", "false", "actual", "false"),
-                ImmutableMap.of("expected", "0", "actual", "0"),
-                ImmutableMap.of("expected", "1", "actual", "1"),
-                ImmutableMap.of("expected", "1234", "actual", "1234"),
-                ImmutableMap.of("expected", "-1234", "actual", "-1234"),
-                ImmutableMap.of("expected", "+1234", "actual", "+1234"),
-                ImmutableMap.of("expected", "0x4d2", "actual", "0x4d2"),
-                ImmutableMap.of("expected", "123L", "actual", "123L"),
-                ImmutableMap.of("expected", "3.141592", "actual", "3.141592"),
-                ImmutableMap.of("expected", "1\\,000", "actual", "1,000"),
-                ImmutableMap.of("expected", "ABC", "actual", "ABC"),
-                ImmutableMap.of("expected", "ABC\"DEF", "actual", "ABC\"DEF"),
-                ImmutableMap.of("expected", "ABC\\\nDEF", "actual", "ABC\nDEF"),
-                ImmutableMap.of("expected", "", "actual", ""),
-                ImmutableMap.of("expected", "NULL", "actual", "NULL"),
-                ImmutableMap.of("expected", "2015-01-01 12:01:01", "actual", "2015-01-01 12:01:01"),
-                ImmutableMap.of("expected", "20150101", "actual", "20150101"));
-
-        for (ImmutableMap testCase : testCases) {
-            String expected = (String) testCase.get("expected");
-            String actual = (String) testCase.get("actual");
-            assertEquals(expected, method.invoke(formatter, actual, delimiter, policy, quote, escape, newline, nullString));
-        }
-    }
-
-    @Test
-    public void testNewlineInField()
-            throws NoSuchMethodException, SecurityException, IllegalAccessException, IllegalArgumentException, InvocationTargetException
-    {
-        Method method = CsvFormatterPlugin.class.getDeclaredMethod("setEscapeAndQuoteValue", String.class, char.class,
-                CsvFormatterPlugin.QuotePolicy.class, char.class, char.class, String.class, String.class);
-        method.setAccessible(true);
-        CsvFormatterPlugin formatter = new CsvFormatterPlugin();
-
-        char delimiter = ',';
-        char quote = '"';
-        char escape = '"';
-        String newline;
-        CsvFormatterPlugin.QuotePolicy policy = CsvFormatterPlugin.QuotePolicy.MINIMAL;
-        String nullString = "";
-
-        ImmutableList<ImmutableMap<String, String>> testCases;
-
-        newline = Newline.LF.getString();
-        testCases = ImmutableList.of(
-                ImmutableMap.of("expected", "\"ABC\nDEF\"", "actual", "ABC\r\nDEF"),
-                ImmutableMap.of("expected", "\"ABC\nDEF\"", "actual", "ABC\rDEF"),
-                ImmutableMap.of("expected", "\"ABC\nDEF\"", "actual", "ABC\nDEF"));
-
-        for (ImmutableMap testCase : testCases) {
-            String expected = (String) testCase.get("expected");
-            String actual = (String) testCase.get("actual");
-            assertEquals(expected, method.invoke(formatter, actual, delimiter, policy, quote, escape, newline, nullString));
-        }
-
-
-        newline = Newline.CRLF.getString();
-        testCases = ImmutableList.of(
-                ImmutableMap.of("expected", "\"ABC\r\nDEF\"", "actual", "ABC\r\nDEF"),
-                ImmutableMap.of("expected", "\"ABC\r\nDEF\"", "actual", "ABC\rDEF"),
-                ImmutableMap.of("expected", "\"ABC\r\nDEF\"", "actual", "ABC\nDEF"));
-
-        for (ImmutableMap testCase : testCases) {
-            String expected = (String) testCase.get("expected");
-            String actual = (String) testCase.get("actual");
-            assertEquals(expected, method.invoke(formatter, actual, delimiter, policy, quote, escape, newline, nullString));
-        }
-
-
-        newline = Newline.CR.getString();
-        testCases = ImmutableList.of(
-                ImmutableMap.of("expected", "\"ABC\rDEF\"", "actual", "ABC\r\nDEF"),
-                ImmutableMap.of("expected", "\"ABC\rDEF\"", "actual", "ABC\rDEF"),
-                ImmutableMap.of("expected", "\"ABC\rDEF\"", "actual", "ABC\nDEF"));
-
-        for (ImmutableMap testCase : testCases) {
-            String expected = (String) testCase.get("expected");
-            String actual = (String) testCase.get("actual");
-            assertEquals(expected, method.invoke(formatter, actual, delimiter, policy, quote, escape, newline, nullString));
-        }
-    }
-
-    @Test
-    public void testNullString()
-            throws NoSuchMethodException, SecurityException, IllegalAccessException, IllegalArgumentException, InvocationTargetException
-    {
-        Method method = CsvFormatterPlugin.class.getDeclaredMethod("setEscapeAndQuoteValue", String.class, char.class,
-                CsvFormatterPlugin.QuotePolicy.class, char.class, char.class, String.class, String.class);
-        method.setAccessible(true);
-        CsvFormatterPlugin formatter = new CsvFormatterPlugin();
-
-        char delimiter = ',';
-        char quote = '"';
-        char escape = '"';
-        CsvFormatterPlugin.QuotePolicy policy = CsvFormatterPlugin.QuotePolicy.MINIMAL;
-        String newline = Newline.LF.getString();
-
-        assertEquals("\"\"", method.invoke(formatter, "", delimiter, CsvFormatterPlugin.QuotePolicy.MINIMAL, quote, escape, newline, ""));
-        assertEquals("N/A", method.invoke(formatter, "N/A", delimiter, CsvFormatterPlugin.QuotePolicy.MINIMAL, quote, escape, newline, ""));
-        assertEquals("", method.invoke(formatter, "", delimiter, CsvFormatterPlugin.QuotePolicy.NONE, quote, escape, newline, ""));
-        assertEquals("N/A", method.invoke(formatter, "N/A", delimiter, CsvFormatterPlugin.QuotePolicy.NONE, quote, escape, newline, ""));
-
-        assertEquals("", method.invoke(formatter, "", delimiter, CsvFormatterPlugin.QuotePolicy.MINIMAL, quote, escape, newline, "N/A"));
-        assertEquals("\"N/A\"", method.invoke(formatter, "N/A", delimiter, CsvFormatterPlugin.QuotePolicy.MINIMAL, quote, escape, newline, "N/A"));
-        assertEquals("", method.invoke(formatter, "", delimiter, CsvFormatterPlugin.QuotePolicy.NONE, quote, escape, newline, "N/A"));
-        assertEquals("N/A", method.invoke(formatter, "N/A", delimiter, CsvFormatterPlugin.QuotePolicy.NONE, quote, escape, newline, "N/A"));
-    }
-}