embulk 0.8.39-java → 0.10.24-java

data/embulk-standards/src/test/java/org/embulk/standards/TestJsonParserPlugin.java

@@ -1,351 +0,0 @@
-package org.embulk.standards;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.io.CharSource;
-import org.embulk.EmbulkTestRuntime;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.TaskSource;
-import org.embulk.spi.DataException;
-import org.embulk.spi.Exec;
-import org.embulk.spi.FileInput;
-import org.embulk.spi.ParserPlugin;
-import org.embulk.spi.Schema;
-import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
-import org.embulk.spi.util.InputStreamFileInput;
-import org.embulk.spi.util.Pages;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.msgpack.value.Value;
-
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.List;
-import java.util.Map;
-
-import static org.embulk.standards.JsonParserPlugin.InvalidEscapeStringPolicy.PASSTHROUGH;
-import static org.embulk.standards.JsonParserPlugin.InvalidEscapeStringPolicy.SKIP;
-import static org.embulk.standards.JsonParserPlugin.InvalidEscapeStringPolicy.UNESCAPE;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import static org.msgpack.value.ValueFactory.newArray;
-import static org.msgpack.value.ValueFactory.newBoolean;
-import static org.msgpack.value.ValueFactory.newInteger;
-import static org.msgpack.value.ValueFactory.newMap;
-import static org.msgpack.value.ValueFactory.newString;
-
-public class TestJsonParserPlugin
-{
-    @Rule
-    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
-
-    private ConfigSource config;
-    private JsonParserPlugin plugin;
-    private MockPageOutput output;
-
-    @Before
-    public void createResource()
-    {
-        config = config();
-        plugin = new JsonParserPlugin();
-        output = new MockPageOutput();
-    }
-
-    @Test
-    public void checkDefaultValues()
-    {
-        ConfigSource config = Exec.newConfigSource();
-
-        JsonParserPlugin.PluginTask task = config.loadConfig(JsonParserPlugin.PluginTask.class);
-        assertEquals(false, task.getStopOnInvalidRecord());
-        assertEquals(JsonParserPlugin.InvalidEscapeStringPolicy.PASSTHROUGH, task.getInvalidEscapeStringPolicy());
-    }
-
-
-    @Test
-    public void readNormalJson()
-            throws Exception
-    {
-        transaction(config, fileInput(
-                "{\"_c0\":true,\"_c1\":10,\"_c2\":\"embulk\",\"_c3\":{\"k\":\"v\"}}",
-                "{}",
-                "{\n" +
-                        "\"_c0\":false,\n" +
-                        "\"_c1\":-10,\n" +
-                        "\"_c2\":\"エンバルク\",\n" +
-                        "\"_c3\":[\"e0\",\"e1\"]\n" +
-                "}",
-                "[1, 2, 3]", // this line should be skipped.
-                "\"embulk\"", // this line should be skipped.
-                "10", // this line should be skipped.
-                "true", // this line should be skipped.
-                "false", // this line should be skipped.
-                "null" // this line should be skipped.
-        ));
-
-        List<Object[]> records = Pages.toObjects(plugin.newSchema(), output.pages);
-        assertEquals(3, records.size());
-
-        Object[] record;
-        Map<Value, Value> map;
-        { // "{\"_c0\":true,\"_c1\":10,\"_c2\":\"embulk\",\"_c3\":{\"k\":\"v\"}}"
-            record = records.get(0);
-            assertEquals(1, record.length);
-            map = ((Value)record[0]).asMapValue().map();
-
-            assertEquals(newBoolean(true), map.get(newString("_c0")));
-            assertEquals(newInteger(10L), map.get(newString("_c1")));
-            assertEquals(newString("embulk"), map.get(newString("_c2")));
-            assertEquals(newMap(newString("k"), newString("v")), map.get(newString("_c3")));
-        }
-        { // "{}"
-            record = records.get(1);
-            assertEquals(1, record.length);
-            assertTrue(((Value)record[0]).asMapValue().map().isEmpty());
-        }
-        {
-            record = records.get(2);
-            assertEquals(1, record.length);
-            map = ((Value)record[0]).asMapValue().map();
-
-            assertEquals(newBoolean(false), map.get(newString("_c0")));
-            assertEquals(newInteger(-10L), map.get(newString("_c1")));
-            assertEquals(newString("エンバルク"), map.get(newString("_c2")));
-            assertEquals(newArray(newString("e0"), newString("e1")), map.get(newString("_c3")));
-        }
-    }
-
-    @Test
-    public void useStopOnInvalidRecord()
-            throws Exception
-    {
-        ConfigSource config = this.config.deepCopy().set("stop_on_invalid_record", true);
-
-        try {
-            transaction(config, fileInput(
-                    "[1, 2, 3]" // throw DataException
-            ));
-            fail();
-        }
-        catch (Throwable t) {
-            assertTrue(t instanceof DataException);
-        }
-    }
-
-    @Test
-    public void readBrokenJson()
-    {
-        try {
-            transaction(config, fileInput(
-                    "{\"_c0\":true,\"_c1\":10," // throw DataException
-            ));
-            fail();
-        }
-        catch (Throwable t) {
-            assertTrue(t instanceof DataException);
-        }
-    }
-
-    @Test
-    public void useDefaultInvalidEscapeStringFunction()
-            throws Exception
-    {
-        try {
-            transaction(config, fileInput(
-                    "{\"\\a\":\"b\"}\\" // throw DataException
-            ));
-            fail();
-        }
-        catch (Throwable t) {
-            assertTrue(t instanceof DataException);
-        }
-    }
-
-    @Test
-    public void usePassthroughInvalidEscapeStringFunction()
-            throws Exception
-    {
-        try {
-            ConfigSource config = this.config.deepCopy().set("invalid_string_escapes", "PASSTHROUGH");
-            transaction(config, fileInput(
-                    "{\"\\a\":\"b\"}\\" // throw DataException
-            ));
-            fail();
-        }
-        catch (Throwable t) {
-            assertTrue(t instanceof DataException);
-        }
-    }
-
-    @Test
-    public void useSkipInvalidEscapeString()
-            throws Exception
-    {
-        ConfigSource config = this.config.deepCopy().set("invalid_string_escapes", "SKIP");
-        transaction(config, fileInput(
-                "{\"\\a\":\"b\"}\\"
-        ));
-
-        List<Object[]> records = Pages.toObjects(plugin.newSchema(), output.pages);
-        assertEquals(1, records.size());
-        Object[] record = records.get(0);
-        Map<Value, Value> map = ((Value)record[0]).asMapValue().map();
-        assertEquals(newString("b"), map.get(newString("")));
-    }
-
-    @Test
-    public void useUnEscapeInvalidEscapeString()
-            throws Exception
-    {
-        ConfigSource config = this.config.deepCopy().set("invalid_string_escapes", "UNESCAPE");
-        transaction(config, fileInput(
-                "{\"\\a\":\"b\"}\\"
-        ));
-
-        List<Object[]> records = Pages.toObjects(plugin.newSchema(), output.pages);
-        assertEquals(1, records.size());
-        Object[] record = records.get(0);
-        Map<Value, Value> map = ((Value)record[0]).asMapValue().map();
-        assertEquals(newString("b"), map.get(newString("a")));
-    }
-
-    @Test
-    public void checkInvalidEscapeStringFunction()
-            throws Exception
-    {
-        //PASSTHROUGH
-        {
-            String json = "{\\\"_c0\\\":true,\\\"_c1\\\":10,\\\"_c2\\\":\\\"embulk\\\",\\\"_c3\\\":{\\\"k\\\":\\\"v\\\"}}";
-            CharSource actual = plugin.invalidEscapeStringFunction(PASSTHROUGH).apply(json);
-            assertEquals(json , actual.read());
-        }
-
-        {
-            String json = "{\"abc\b\f\n\r\t\\\\u0001\":\"efg\"}\\";
-            CharSource actual = plugin.invalidEscapeStringFunction(PASSTHROUGH).apply(json);
-            assertEquals(json , actual.read());
-        }
-
-        {
-            String json = "{\"\\a\":\"b\"}\\";
-            CharSource actual = plugin.invalidEscapeStringFunction(PASSTHROUGH).apply(json);
-            assertEquals(json , actual.read());
-        }
-
-        //SKIP
-        {
-            String json = "{\\\"_c0\\\":true,\\\"_c1\\\":10,\\\"_c2\\\":\\\"embulk\\\",\\\"_c3\\\":{\\\"k\\\":\\\"v\\\"}}";
-            CharSource actual = plugin.invalidEscapeStringFunction(SKIP).apply(json);
-            assertEquals(json , actual.read());
-        }
-
-        {
-            // valid charset u0001
-            String json = "{\"abc\b\f\n\r\t\\\\u0001\":\"efg\"}\\";
-            CharSource actual = plugin.invalidEscapeStringFunction(SKIP).apply(json);
-            assertEquals("{\"abc\b\f\n\r\t\\\\u0001\":\"efg\"}" , actual.read());
-        }
-
-
-        {
-            // invalid charset \\u12xY remove forwarding backslash and u
-            String json = "{\"\\u12xY\":\"efg\"}\\";
-            CharSource actual = plugin.invalidEscapeStringFunction(SKIP).apply(json);
-            assertEquals("{\"12xY\":\"efg\"}" , actual.read());
-        }
-
-        {
-            String json = "{\"\\a\":\"b\"}\\";
-            CharSource actual = plugin.invalidEscapeStringFunction(SKIP).apply(json);
-            // backslash and `a` will removed.
-            assertEquals("{\"\":\"b\"}" , actual.read());
-        }
-
-        {
-            // end of lines backspash.
-            String json = "{\"\\a\":\"b\"}" +
-                    "\n" +
-                    "\\";
-            CharSource actual = plugin.invalidEscapeStringFunction(SKIP).apply(json);
-            // backslash and `a` will removed.
-            assertEquals("{\"\":\"b\"}\n" , actual.read());
-        }
-
-        //UNESCAPE
-        {
-            String json = "{\\\"_c0\\\":true,\\\"_c1\\\":10,\\\"_c2\\\":\\\"embulk\\\",\\\"_c3\\\":{\\\"k\\\":\\\"v\\\"}}";
-            CharSource actual = plugin.invalidEscapeStringFunction(UNESCAPE).apply(json);
-            assertEquals(json , actual.read());
-        }
-
-        {
-            String json = "{\"abc\b\f\n\r\t\\\\u0001\":\"efg\"}\\";
-            CharSource actual = plugin.invalidEscapeStringFunction(UNESCAPE).apply(json);
-            assertEquals("{\"abc\b\f\n\r\t\\\\u0001\":\"efg\"}" , actual.read());
-        }
-
-        {
-            // invalid charset u000x remove forwarding backslash
-            String json = "{\"\\u000x\":\"efg\"}\\";
-            CharSource actual = plugin.invalidEscapeStringFunction(UNESCAPE).apply(json);
-            assertEquals("{\"u000x\":\"efg\"}" , actual.read());
-        }
-
-
-        {
-            String json = "{\"\\a\":\"b\"}\\";
-            CharSource actual = plugin.invalidEscapeStringFunction(UNESCAPE).apply(json);
-            // backslash will removed.
-            assertEquals("{\"a\":\"b\"}" , actual.read());
-        }
-
-        {
-            // end of lines backspash.
-            String json = "{\"\\a\":\"b\"}" +
-                    "\n" +
-                    "\\";
-            CharSource actual = plugin.invalidEscapeStringFunction(SKIP).apply(json);
-            // backslash and `a` will removed.
-            assertEquals("{\"\":\"b\"}\n" , actual.read());
-        }
-
-
-    }
-
-    private ConfigSource config()
-    {
-        return runtime.getExec().newConfigSource();
-    }
-
-    private void transaction(ConfigSource config, final FileInput input)
-    {
-        plugin.transaction(config, new ParserPlugin.Control() {
-            @Override
-            public void run(TaskSource taskSource, Schema schema)
-            {
-                plugin.run(taskSource, schema, input, output);
-            }
-        });
-    }
-
-    private FileInput fileInput(String... lines)
-            throws Exception
-    {
-        StringBuilder sb = new StringBuilder();
-        for (String line : lines) {
-            sb.append(line).append("\n");
-        }
-
-        ByteArrayInputStream in = new ByteArrayInputStream(sb.toString().getBytes("UTF-8"));
-        return new InputStreamFileInput(runtime.getBufferAllocator(), provider(in));
-    }
-
-    private InputStreamFileInput.IteratorProvider provider(InputStream... inputStreams)
-            throws IOException
-    {
-        return new InputStreamFileInput.IteratorProvider(
-                ImmutableList.copyOf(inputStreams));
-    }
-}

data/embulk-standards/src/test/java/org/embulk/standards/TestRemoveColumnsFilterPlugin.java

@@ -1,121 +0,0 @@
-package org.embulk.standards;
-
-import com.google.common.collect.ImmutableList;
-import org.embulk.config.ConfigException;
-import org.embulk.config.ConfigSource;
-import org.embulk.exec.PartialExecutionException;
-import org.embulk.test.TestingEmbulk;
-import org.junit.Rule;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.nio.file.Path;
-
-import static org.embulk.test.EmbulkTests.copyResource;
-import static org.embulk.test.EmbulkTests.readResource;
-import static org.embulk.test.EmbulkTests.readSortedFile;
-import static org.hamcrest.Matchers.is;
-import static org.junit.Assert.assertThat;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-public class TestRemoveColumnsFilterPlugin
-{
-    private static final String RESOURCE_NAME_PREFIX = "org/embulk/standards/remove_columns/test/";
-
-    @Rule
-    public TestingEmbulk embulk = TestingEmbulk.builder().build();
-
-    @Test
-    public void useKeepOption()
-            throws Exception
-    {
-        assertRecordsByResource(embulk, "test_keep_in.yml", "test_keep_filter.yml",
-                "test_keep.csv", "test_keep_expected.csv");
-    }
-
-    @Test
-    public void useKeepWithAcceptUnmatched()
-            throws Exception
-    {
-        assertRecordsByResource(embulk, "test_keep_in.yml", "test_keep_with_unmatched_filter.yml",
-                "test_keep.csv", "test_keep_expected.csv");
-    }
-
-    @Test
-    public void useKeepWithoutAcceptUnmatched()
-            throws Exception
-    {
-        try {
-            assertRecordsByResource(embulk, "test_keep_in.yml", "test_keep_without_unmatched_filter.yml",
-                    "test_keep.csv", "test_keep_expected.csv");
-            fail();
-        }
-        catch (PartialExecutionException ex) {
-            assertTrue(ex.getCause() instanceof ConfigException);
-        }
-    }
-
-    @Test
-    public void useKeepWithDuplicatedColumnNames()
-            throws Exception
-    {
-        assertRecordsByResource(embulk, "test_keep_with_duplicated_column_names_in.yml", "test_keep_with_duplicated_column_names.yml",
-                "test_keep_with_duplicated_column_names.csv", "test_keep_with_duplicated_column_names_expected.csv");
-    }
-
-    @Test
-    public void useRemove()
-            throws Exception
-    {
-        assertRecordsByResource(embulk, "test_remove_in.yml", "test_remove_filter.yml",
-                "test_remove.csv", "test_remove_expected.csv");
-    }
-
-    @Test
-    public void useRemoveWithAcceptUnmatched()
-            throws Exception
-    {
-        assertRecordsByResource(embulk, "test_remove_in.yml", "test_remove_with_unmatched_filter.yml",
-                "test_remove.csv", "test_remove_expected.csv");
-    }
-
-    @Test
-    public void useRemoveWithoutAcceptUnmatched()
-            throws Exception
-    {
-        try {
-            assertRecordsByResource(embulk, "test_remove_in.yml", "test_remove_without_unmatched_filter.yml",
-                    "test_remove.csv", "test_remove_expected.csv");
-            fail();
-        }
-        catch (PartialExecutionException ex) {
-            assertTrue(ex.getCause() instanceof ConfigException);
-        }
-    }
-
-    static void assertRecordsByResource(TestingEmbulk embulk,
-            String inConfigYamlResourceName, String filterConfigYamlResourceName,
-            String sourceCsvResourceName, String resultCsvResourceName)
-            throws IOException
-    {
-        Path inputPath = embulk.createTempFile("csv");
-        Path outputPath = embulk.createTempFile("csv");
-
-        // in: config
-        copyResource(RESOURCE_NAME_PREFIX + sourceCsvResourceName, inputPath);
-        ConfigSource inConfig = embulk.loadYamlResource(RESOURCE_NAME_PREFIX + inConfigYamlResourceName)
-                .set("path_prefix", inputPath.toAbsolutePath().toString());
-
-        // remove_columns filter config
-        ConfigSource filterConfig = embulk.loadYamlResource(RESOURCE_NAME_PREFIX + filterConfigYamlResourceName);
-
-        TestingEmbulk.RunResult result = embulk.inputBuilder()
-                .in(inConfig)
-                .filters(ImmutableList.of(filterConfig))
-                .outputPath(outputPath)
-                .run();
-
-        assertThat(readSortedFile(outputPath), is(readResource(RESOURCE_NAME_PREFIX + resultCsvResourceName)));
-    }
-}