embulk 0.8.25-java → 0.8.26-java

data/embulk-core/src/test/java/org/embulk/plugin/TestPluginType.java

@@ -2,6 +2,7 @@ package org.embulk.plugin;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
 import java.util.HashMap;
@@ -54,6 +55,29 @@ public class TestPluginType
         assertEquals("e", mavenType.getName());
         assertEquals("org.embulk.foobar", mavenType.getGroup());
         assertEquals("0.1.2", mavenType.getVersion());
+        assertNull(mavenType.getClassifier());
         assertEquals("maven:org.embulk.foobar:e:0.1.2", mavenType.getFullName());
     }
+
+    @Test
+    public void testMappingMavenWithClassifier()
+    {
+        HashMap<String, String> mapping = new HashMap<String, String>();
+        mapping.put("source", "maven");
+        mapping.put("name", "e");
+        mapping.put("group", "org.embulk.foobar");
+        mapping.put("version", "0.1.2");
+        mapping.put("classifier", "bar");
+
+        PluginType type = PluginType.createFromStringMapForTesting(mapping);
+        assertTrue(type instanceof MavenPluginType);
+        assertEquals(PluginSource.Type.MAVEN, type.getSourceType());
+        MavenPluginType mavenType = (MavenPluginType) type;
+        assertTrue(mavenType.equals(mavenType));
+        assertEquals("e", mavenType.getName());
+        assertEquals("org.embulk.foobar", mavenType.getGroup());
+        assertEquals("0.1.2", mavenType.getVersion());
+        assertEquals("bar", mavenType.getClassifier());
+        assertEquals("maven:org.embulk.foobar:e:0.1.2:bar", mavenType.getFullName());
+    }
 }

data/embulk-core/src/test/java/org/embulk/plugin/TestPluginTypeSerDe.java

@@ -1,6 +1,7 @@
 package org.embulk.plugin;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
 import org.embulk.EmbulkTestRuntime;
@@ -46,5 +47,21 @@ public class TestPluginTypeSerDe
         assertEquals(mavenPluginType.getName(), "foo");
         assertEquals(mavenPluginType.getGroup(), "org.embulk.bar");
         assertEquals(mavenPluginType.getVersion(), "0.1.2");
+        assertNull(mavenPluginType.getClassifier());
+    }
+
+    @Test
+    public void testParseTypeMavenWithClassifier()
+    {
+        PluginType pluginType = testRuntime.getModelManager().readObjectWithConfigSerDe(
+            PluginType.class,
+            "{ \"name\": \"foo\", \"source\": \"maven\", \"group\": \"org.embulk.bar\", \"version\": \"0.1.2\", \"classifier\": \"foo\" }");
+        assertTrue(pluginType instanceof MavenPluginType);
+        assertEquals(PluginSource.Type.MAVEN, pluginType.getSourceType());
+        MavenPluginType mavenPluginType = (MavenPluginType) pluginType;
+        assertEquals(mavenPluginType.getName(), "foo");
+        assertEquals(mavenPluginType.getGroup(), "org.embulk.bar");
+        assertEquals(mavenPluginType.getVersion(), "0.1.2");
+        assertEquals(mavenPluginType.getClassifier(), "foo");
     }
 }

data/embulk-core/src/test/java/org/embulk/plugin/jar/ExampleJarSpiV0.java

@@ -0,0 +1,9 @@
+package org.embulk.plugin.jar;
+
+public class ExampleJarSpiV0
+{
+    public String getTestString()
+    {
+        return "foobar";
+    }
+}

data/embulk-core/src/test/java/org/embulk/plugin/jar/JarBuilder.java

@@ -0,0 +1,101 @@
+package org.embulk.plugin.jar;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.TreeSet;
+import java.util.jar.Attributes;
+import java.util.jar.JarEntry;
+import java.util.jar.JarOutputStream;
+import java.util.jar.Manifest;
+
+public class JarBuilder
+{
+    public JarBuilder()
+    {
+        this.manifest = new Manifest();
+        this.entries = new HashMap<String, Path>();
+    }
+
+    public void build(final Path pathToExistingFile)
+            throws Exception
+    {
+        try (final JarOutputStream output = buildPluginJar(pathToExistingFile, this.manifest)) {
+            for (String entryName : new TreeSet<String>(this.entries.keySet())) {
+                final Path pathToRealFile = this.entries.get(entryName);
+                if (pathToRealFile == null) {
+                    final JarEntry entry = new JarEntry(entryName + "/");
+                    entry.setMethod(JarEntry.STORED);
+                    entry.setSize(0);
+                    entry.setCrc(0);
+                    output.putNextEntry(entry);
+                    output.closeEntry();
+                }
+                else {
+                    final JarEntry entry = new JarEntry(entryName);
+                    output.putNextEntry(entry);
+                    Files.copy(pathToRealFile, output);
+                    output.closeEntry();
+                }
+            }
+        }
+    }
+
+    public void addClass(final Class<?> klass)
+            throws Exception
+    {
+        final Path classFileRelativePath = getClassFileRelativePath(klass);
+        final Path classFileFullPath = getClassFileFullPath(klass);
+        this.addFile(classFileRelativePath.toString(), classFileFullPath);
+
+        Path directoryPath = classFileRelativePath.getParent();
+        while (directoryPath != null) {
+            this.addDirectoryIfAbsent(directoryPath.toString());
+            directoryPath = directoryPath.getParent();
+        }
+    }
+
+    public void addManifestV0(final String embulkPluginMainClass)
+    {
+        final Attributes attributes = this.manifest.getMainAttributes();
+        attributes.put(Attributes.Name.MANIFEST_VERSION, "1.0");
+        attributes.putValue(MANIFEST_PLUGIN_SPI_VERSION, "0");
+        attributes.putValue(MANIFEST_PLUGIN_MAIN_CLASS, embulkPluginMainClass);
+    }
+
+    private void addDirectoryIfAbsent(final String name)
+    {
+        if (!(this.entries.containsKey(name))) {
+            this.entries.put(name, null);
+        }
+    }
+
+    private void addFile(final String name, final Path pathToRealFile)
+    {
+        this.entries.put(name, pathToRealFile);
+    }
+
+    private JarOutputStream buildPluginJar(final Path pathToExistingFile, final Manifest embeddedManifest)
+            throws Exception
+    {
+        return new JarOutputStream(Files.newOutputStream(pathToExistingFile), embeddedManifest);
+    }
+
+    private Path getClassFileRelativePath(final Class<?> klass)
+    {
+        return Paths.get(klass.getName().replace('.', '/') + ".class");
+    }
+
+    private Path getClassFileFullPath(final Class<?> klass)
+            throws Exception
+    {
+        return Paths.get(klass.getClassLoader().getResource(klass.getName().replace('.', '/') + ".class").toURI());
+    }
+
+    private final Manifest manifest;
+    private final HashMap<String, Path> entries;
+
+    private static final String MANIFEST_PLUGIN_SPI_VERSION = "Embulk-Plugin-Spi-Version";
+    private static final String MANIFEST_PLUGIN_MAIN_CLASS = "Embulk-Plugin-Main-Class";
+}

data/embulk-core/src/test/java/org/embulk/plugin/jar/TestJarPluginLoader.java

@@ -0,0 +1,60 @@
+package org.embulk.plugin.jar;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import org.embulk.EmbulkTestRuntime;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+public class TestJarPluginLoader
+{
+    @Rule
+    public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+    @Rule
+    public EmbulkTestRuntime testRuntime = new EmbulkTestRuntime();
+
+    @Test
+    public void test() throws Exception
+    {
+        final Path jarPath = createTemporaryJarFile();
+
+        final JarBuilder jarBuilder = new JarBuilder();
+        jarBuilder.addManifestV0(ExampleJarSpiV0.class.getName());
+        jarBuilder.addClass(ExampleJarSpiV0.class);
+        jarBuilder.build(jarPath);
+
+        final Class<?> loadedClass;
+        try (final JarPluginLoader loader = JarPluginLoader.load(jarPath, testRuntime.getPluginClassLoaderFactory())) {
+            assertEquals(0, loader.getPluginSpiVersion());
+            loadedClass = loader.getPluginMainClass();
+        }
+
+        final Object instanceObject = loadedClass.newInstance();
+        assertTrue(instanceObject instanceof ExampleJarSpiV0);
+
+        final ExampleJarSpiV0 instance = (ExampleJarSpiV0) instanceObject;
+        assertEquals("foobar", instance.getTestString());
+    }
+
+    private Path createTemporaryJarFile() throws Exception
+    {
+        final String temporaryDirectoryString =
+            System.getProperty("org.embulk.plugin.jar.TestJarPluginLoader.temporaryDirectory");
+
+        final Path temporaryDirectoryPath;
+        if (temporaryDirectoryString == null) {
+            temporaryDirectoryPath = temporaryFolder.getRoot().toPath();
+        }
+        else {
+            temporaryDirectoryPath = Paths.get(temporaryDirectoryString);
+        }
+
+        return Files.createTempFile(temporaryDirectoryPath, "testplugin", ".jar");
+    }
+}

data/embulk-core/src/test/java/org/embulk/plugin/maven/TestMavenArtifactFinder.java

@@ -0,0 +1,41 @@
+package org.embulk.plugin.maven;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.net.URL;
+import java.net.URISyntaxException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import org.junit.Test;
+
+public class TestMavenArtifactFinder
+{
+    @Test
+    public void testArtifacts() throws Exception
+    {
+        final Path basePath = getMavenPath();
+        final MavenArtifactFinder finder = MavenArtifactFinder.create(basePath);
+        assertEquals(buildExpectedPath(basePath, GROUP_DIRECTORY, "embulk-example-maven-artifact", "0.1.2"),
+                     finder.findMavenArtifactJar("org.embulk.example", "embulk-example-maven-artifact", null, "0.1.2"));
+    }
+
+    private Path getMavenPath() throws URISyntaxException
+    {
+        return Paths.get(this.getClass().getClassLoader().getResource("m2.test").toURI());
+    }
+
+    private Path buildExpectedPath(
+            final Path basePath, final Path groupDirectory, final String artifactId, final String version)
+    {
+        return basePath
+            .resolve(groupDirectory)
+            .resolve(artifactId)
+            .resolve(version)
+            .resolve(artifactId + "-" + version + ".jar");
+    }
+
+    private static final Path GROUP_DIRECTORY = Paths.get("org").resolve("embulk").resolve("example");
+}

data/embulk-core/src/test/resources/m2.test/.gitignore

@@ -0,0 +1 @@
+_remote.repositories

data/embulk-core/src/test/resources/m2.test/org/embulk/example/embulk-example-maven-artifact/0.1.2/embulk-example-maven-artifact-0.1.2.jar.sha1

@@ -0,0 +1 @@
+5a068856f7c40ef2cb376018c213f622f3a114d8

data/embulk-core/src/test/resources/m2.test/org/embulk/example/embulk-example-maven-artifact/0.1.2/embulk-example-maven-artifact-0.1.2.pom

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>org.embulk.example</groupId>
+  <artifactId>embulk-example-maven-artifact</artifactId>
+  <version>0.1.2</version>
+  <packaging>jar</packaging>
+  <name>embulk-example-maven-artifact</name>
+</project>

data/embulk-core/src/test/resources/m2.test/org/embulk/example/embulk-example-maven-artifact/0.1.2/embulk-example-maven-artifact-0.1.2.pom.sha1

@@ -0,0 +1 @@
+63a970e5288cd0452f0558f9dc8ea8498bc8e5b4

data/embulk-docs/src/built-in.rst

@@ -329,21 +329,38 @@ The ``json`` parser plugin parses a JSON file that contains a sequence of JSON o
 
 .. code-block:: javascript
 
-    {"time":1455829282,"ip":"93.184.216.34","name":frsyuki}
-    {"time":1455829282,"ip":"172.36.8.109":sadayuki}
-    {"time":1455829284,"ip":"example.com","name":Treasure Data}
-    {"time":1455829282,"ip":"10.98.43.1","name":MessagePack}
+    {"time":1455829282,"ip":"93.184.216.34","name":"frsyuki"}
+    {"time":1455829282,"ip":"172.36.8.109", "name":"sadayuki"}
+    {"time":1455829284,"ip":"example.com","name":"Treasure Data"}
+    {"time":1455829282,"ip":"10.98.43.1","name":"MessagePack"}
 
 ``json`` parser plugin outputs a single record named "record" (type is json).
 
 Options
 ~~~~~~~~
 
-+----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
-| name                       | type     | description                                                                                                    |              required? |
-+============================+==========+================================================================================================================+========================+
-| stop\_on\_invalid\_record  | boolean  | Stop bulk load transaction if a file includes invalid record (such as invalid json)                            | ``false`` by default   |
-+----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
++----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------------+
+| name                       | type     | description                                                                                                    |          required?           |
++============================+==========+================================================================================================================+==============================+
+| stop\_on\_invalid\_record  | boolean  | Stop bulk load transaction if a file includes invalid record (such as invalid json)                            | ``false`` by default         |
++----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------------+
+| invalid\_string\_escapes   | enum     | Escape strategy of invalid json string such as using invalid ``\`` like ``\a``. (PASSTHROUGH, SKIP, UNESCAPE)  | ``PASSTHROUGH`` by default   |
++----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------------+
+
+
+if you set invalid\_string\_escapes and appear invalid JSON string (such as ``\a``), it makes following the action.
+
++----------------------------+------------------+
+| invalid\_string\_escapes   | convert to       |
++============================+==================+
+| PASSTHROUGH *1             | ``\a``           |
++----------------------------+------------------+
+| SKIP                       | empty string     |
++----------------------------+------------------+
+| UNESCAPE                   | ``a``            |
++----------------------------+------------------+
+
+(\*1): Throwing an exception.
 
 
 Example

data/embulk-docs/src/release.rst

@@ -4,6 +4,7 @@ Release Notes
 .. toctree::
     :maxdepth: 1
 
+    release/release-0.8.26
     release/release-0.8.25
     release/release-0.8.24
     release/release-0.8.23

data/embulk-docs/src/release/release-0.8.26.rst

@@ -0,0 +1,16 @@
+Release 0.8.26
+==================================
+
+General Changes
+------------------
+
+* Add ConfigInputPlugin for easier testing [#678]
+* Add MavenPluginSource to load JAR-based plugins into Embulk [#686] [#687] [#684] [#701]
+* Fix the "migrate" subcommand to handle build.gradle correctly [#702] [#703]
+* Fix parent_first_{packages,resources}.properties to delegate msgpack class loading to parent [#694]
+* Fix documents [#683] [#685]
+
+
+Release Date
+------------------
+2017-07-01

data/embulk-standards/src/main/java/org/embulk/standards/ConfigInputPlugin.java

@@ -0,0 +1,170 @@
+package org.embulk.standards;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import java.util.List;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
+import org.embulk.config.TaskReport;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.Column;
+import org.embulk.spi.ColumnVisitor;
+import org.embulk.spi.DataException;
+import org.embulk.spi.Exec;
+import org.embulk.spi.InputPlugin;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.Schema;
+import org.embulk.spi.SchemaConfig;
+import org.embulk.spi.json.JsonParser;
+import org.embulk.spi.json.JsonParseException;
+import org.embulk.spi.time.TimestampParseException;
+import org.embulk.spi.time.TimestampParser;
+import org.embulk.spi.util.Timestamps;
+
+public class ConfigInputPlugin
+        implements InputPlugin
+{
+    private interface PluginTask
+            extends Task, TimestampParser.Task
+    {
+        @Config("columns")
+        SchemaConfig getSchemaConfig();
+
+        @Config("values")
+        List<List<List<JsonNode>>> getValues();
+    }
+
+    @Override
+    public ConfigDiff transaction(ConfigSource config,
+            InputPlugin.Control control)
+    {
+        final PluginTask task = config.loadConfig(PluginTask.class);
+        final Schema schema = task.getSchemaConfig().toSchema();
+        final List<List<List<JsonNode>>> values = task.getValues();
+        final int taskCount = values.size();
+
+        return resume(task.dump(), schema, taskCount, control);
+    }
+
+    @Override
+    public ConfigDiff resume(TaskSource taskSource,
+            Schema schema, int taskCount,
+            InputPlugin.Control control)
+    {
+        control.run(taskSource, schema, taskCount);
+        return Exec.newConfigDiff();
+    }
+
+    @Override
+    public void cleanup(TaskSource taskSource,
+            Schema schema, int taskCount,
+            List<TaskReport> successTaskReports)
+    {
+    }
+
+    @Override
+    public TaskReport run(TaskSource taskSource,
+            Schema schema, int taskIndex,
+            PageOutput output)
+    {
+        final PluginTask task = taskSource.loadTask(PluginTask.class);
+        final List<List<JsonNode>> taskValues = task.getValues().get(taskIndex);
+        final TimestampParser[] timestampParsers = Timestamps.newTimestampColumnParsers(task, task.getSchemaConfig());
+        final JsonParser jsonParser = new JsonParser();
+
+        try (final PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), schema, output)) {
+            for (final List<JsonNode> rowValues : taskValues) {
+                schema.visitColumns(new ColumnVisitor() {
+                        public void booleanColumn(Column column)
+                        {
+                            final JsonNode value = rowValues.get(column.getIndex());
+                            if (value == null || value.isNull()) {
+                                pageBuilder.setNull(column);
+                            }
+                            else {
+                                pageBuilder.setBoolean(column, value.asBoolean());
+                            }
+                        }
+
+                        public void longColumn(Column column)
+                        {
+                            final JsonNode value = rowValues.get(column.getIndex());
+                            if (value == null || value.isNull()) {
+                                pageBuilder.setNull(column);
+                            }
+                            else {
+                                pageBuilder.setLong(column, value.asLong());
+                            }
+                        }
+
+                        public void doubleColumn(Column column)
+                        {
+                            final JsonNode value = rowValues.get(column.getIndex());
+                            if (value == null || value.isNull()) {
+                                pageBuilder.setNull(column);
+                            }
+                            else {
+                                pageBuilder.setDouble(column, value.asDouble());
+                            }
+                        }
+
+                        public void stringColumn(Column column)
+                        {
+                            final JsonNode value = rowValues.get(column.getIndex());
+                            if (value == null || value.isNull()) {
+                                pageBuilder.setNull(column);
+                            }
+                            else {
+                                pageBuilder.setString(column, value.asText());
+                            }
+                        }
+
+                        public void timestampColumn(Column column)
+                        {
+                            final JsonNode value = rowValues.get(column.getIndex());
+                            if (value == null || value.isNull()) {
+                                pageBuilder.setNull(column);
+                            }
+                            else {
+                                try {
+                                    pageBuilder.setTimestamp(column,
+                                                             timestampParsers[column.getIndex()].parse(value.asText()));
+                                }
+                                catch (TimestampParseException ex) {
+                                    throw new DataException(ex);
+                                }
+                            }
+                        }
+
+                        public void jsonColumn(Column column)
+                        {
+                            final JsonNode value = rowValues.get(column.getIndex());
+                            if (value == null || value.isNull()) {
+                                pageBuilder.setNull(column);
+                            }
+                            else {
+                                try {
+                                    pageBuilder.setJson(column, jsonParser.parse(value.toString()));
+                                }
+                                catch (JsonParseException ex) {
+                                    throw new DataException(ex);
+                                }
+                            }
+                        }
+                    });
+                pageBuilder.addRecord();
+            }
+            pageBuilder.finish();
+        }
+
+        return Exec.newTaskReport();
+    }
+
+    @Override
+    public ConfigDiff guess(ConfigSource config)
+    {
+        return Exec.newConfigDiff();
+    }
+}