embulk 0.6.17 → 0.6.18

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c667cedd2e533e4a3404c81baa92618dfd826644
-  data.tar.gz: fc3c840b3c855395c02b3d2ba3e84cf3428a8a55
+  metadata.gz: ff085042dbecce04ca4a8095c59802bc4336a67e
+  data.tar.gz: aab28c9c0d9ae11a758ba6148baa12a8973e7903
 SHA512:
-  metadata.gz: e30b36e74c05cc9a3f63565691fd4366803be573f9909c1cfcce1292a65a1c7b70a1a6d324990555f89b774c786a8dc1316012f9f9a4330f52c3963573cedaa9
-  data.tar.gz: 8299b1e6c264a0031e2de1c557fd1a91eae1d24bf7badfc671e81fdc007421c4cc77dbc40cb8e6c15f66d54bd259be44b925772104055105988a704401f52418
+  metadata.gz: bace1db62e73980869cbf11118be290cdcacda974473bb6e334fbb0075bbbdcf8dec7096cff6f35c8de185baa67b6f5efee498e13d974183a5ec15a7ab9bf0db
+  data.tar.gz: 2d1e67648bc4eec548433c69f147e70647fe958d53c15234af3df6289e562a2aaa2e1d49bc9342204f83f6718f2965fa9b7713e5a415bb3e733d3a32e9fcdeac

data/build.gradle

@@ -11,7 +11,7 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards")]
 
 allprojects {
     group = 'org.embulk'
-    version = '0.6.17'
+    version = '0.6.18'
 
     ext {
         jrubyVersion = '1.7.19'

data/embulk-core/build.gradle

@@ -22,14 +22,14 @@ dependencies {
     compile 'com.fasterxml.jackson.datatype:jackson-datatype-joda:2.5.3'
     compile 'com.fasterxml.jackson.module:jackson-module-guice:2.5.3'
     compile 'ch.qos.logback:logback-classic:1.1.3'
-    compile 'org.slf4j:slf4j-api:1.7.10'
+    compile 'org.slf4j:slf4j-api:1.7.12'
     compile 'org.jruby:jruby-complete:' + project.jrubyVersion
     compile 'com.google.code.findbugs:annotations:3.0.0'
     compile 'org.yaml:snakeyaml:1.14'
     compile 'javax.validation:validation-api:1.1.0.Final'
     compile 'org.apache.bval:bval-jsr303:0.5'
     compile 'io.airlift:slice:0.9'
-    compile 'joda-time:joda-time:2.7'
+    compile 'joda-time:joda-time:2.8.1'
     compile 'io.netty:netty-buffer:5.0.0.Alpha1'
     compile 'org.fusesource.jansi:jansi:1.11'
 

data/embulk-core/src/main/java/org/embulk/config/ConfigLoader.java

@@ -2,12 +2,14 @@ package org.embulk.config;
 
 import java.io.File;
 import java.io.FileInputStream;
+import java.io.InputStream;
 import java.io.IOException;
 import java.util.Properties;
 import com.google.inject.Inject;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.core.JsonParser;
 import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.RuntimeJsonMappingException;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.fasterxml.jackson.databind.node.JsonNodeFactory;
 import org.yaml.snakeyaml.Yaml;
@@ -22,47 +24,77 @@ public class ConfigLoader
         this.model = model;
     }
 
-    public ConfigSource fromJson(JsonParser parser) throws IOException
+    public ConfigSource fromJsonFile(File file) throws IOException
     {
-        // TODO check parsed.isObject()
-        ObjectNode source = (ObjectNode) new ObjectMapper().readTree(parser);
-        return new DataSourceImpl(model, source);
+        try (FileInputStream is = new FileInputStream(file)) {
+            return fromJson(is);
+        }
+    }
+
+    public ConfigSource fromJson(InputStream stream) throws IOException
+    {
+        JsonNode node = new ObjectMapper().readTree(stream);
+        if (!node.isObject()) {
+            throw new RuntimeJsonMappingException("Expected object to load ConfigSource but got: "+node.getNodeType());
+        }
+        return new DataSourceImpl(model, (ObjectNode) node);
+    }
+
+    public ConfigSource fromYamlFile(File file) throws IOException
+    {
+        try (FileInputStream stream = new FileInputStream(file)) {
+            return fromYaml(stream);
+        }
     }
 
-    public ConfigSource fromYamlFile(File path) throws IOException
+    public ConfigSource fromYaml(InputStream stream) throws IOException
     {
         Yaml yaml = new Yaml();
-        Object parsedYaml;
-        try (FileInputStream is = new FileInputStream(path)) {
-            parsedYaml = yaml.load(is);
+        Object object = yaml.load(stream);
+        JsonNode node = objectToJson(object);
+        if (!node.isObject()) {
+            throw new RuntimeJsonMappingException("Expected object to load ConfigSource but got "+node);
         }
-        ObjectNode source = objectToJsonObject(parsedYaml);
+        return new DataSourceImpl(model, (ObjectNode) node);
+    }
+
+    @Deprecated
+    public ConfigSource fromJson(JsonParser parser) throws IOException
+    {
+        // TODO check parsed.isObject()
+        ObjectNode source = (ObjectNode) new ObjectMapper().readTree(parser);
         return new DataSourceImpl(model, source);
     }
 
     public ConfigSource fromPropertiesYamlLiteral(Properties props, String keyPrefix)
     {
-        // TODO exception handling
         ObjectNode source = new ObjectNode(JsonNodeFactory.instance);
+        DataSource ds = new DataSourceImpl(model, source);
         Yaml yaml = new Yaml();
-        for (String key : props.stringPropertyNames()) {
-            // TODO handle "." and "[...]" as map and array acccessor for example:
-            //      in.parser.type=csv => {"in": {"parser": {"type": "csv"}}}
-            if (!key.startsWith(keyPrefix)) {
+        for (String propName : props.stringPropertyNames()) {
+            if (!propName.startsWith(keyPrefix)) {
                 continue;
             }
-            String yamlValue = props.getProperty(key);
-            String keyName = key.substring(keyPrefix.length());
-            Object parsedValue = yaml.load(yamlValue);
-            JsonNode typedValue = objectToJson(parsedValue);
-            source.set(keyName, typedValue);
+            String keyName = propName.substring(keyPrefix.length());
+            String yamlValue = props.getProperty(propName);
+            Object parsedValue = yaml.load(yamlValue);  // TODO exception handling
+            JsonNode node = objectToJson(parsedValue);
+
+            // handle "." as a map acccessor. for example:
+            // in.parser.type=csv => {"in": {"parser": {"type": "csv"}}}
+            // TODO handle "[]" as array index
+            String[] fragments = keyName.split("\\.");
+            DataSource key = ds;
+            for (int i=0; i < fragments.length - 1; i++) {
+                key = key.getNestedOrSetEmpty(fragments[i]);  // TODO exception handling
+            }
+            key.set(fragments[fragments.length - 1], node);
         }
         return new DataSourceImpl(model, source);
     }
 
     private JsonNode objectToJson(Object object)
     {
-        // TODO exception
         ObjectMapper objectMapper = new ObjectMapper();
         try {
             return objectMapper.readTree(objectMapper.writeValueAsString(object));
@@ -70,14 +102,4 @@ public class ConfigLoader
             throw new RuntimeException(ex);
         }
     }
-
-    private ObjectNode objectToJsonObject(Object object)
-    {
-        // TODO exception
-        JsonNode json = objectToJson(object);
-        if (!json.isObject()) {
-            throw new RuntimeException("Expected object to deserialize ConfigSource but got "+json);
-        }
-        return (ObjectNode) json;
-    }
 }

data/embulk-core/src/main/java/org/embulk/spi/util/Filters.java

@@ -54,10 +54,10 @@ public abstract class Filters
             List<Schema> filterSchemas, PageOutput output)
     {
         PageOutput out = output;
-        int pos = 0;
-        while (pos < plugins.size()) {
+        int pos = plugins.size() - 1;
+        while (pos >= 0) {
             out = plugins.get(pos).open(taskSources.get(pos), filterSchemas.get(pos), filterSchemas.get(pos + 1), out);
-            pos++;
+            pos--;
         }
         return out;
     }

data/embulk-core/src/test/java/org/embulk/config/TestConfigLoader.java

@@ -0,0 +1,66 @@
+package org.embulk.config;
+
+import java.util.Properties;
+import java.io.InputStream;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import com.google.inject.Inject;
+import org.embulk.spi.Exec;
+import org.embulk.EmbulkTestRuntime;
+
+public class TestConfigLoader
+{
+    private ConfigLoader loader;
+
+    @Before
+    public void setup() throws Exception
+    {
+        this.loader = new ConfigLoader(new ModelManager(null, new ObjectMapper()));
+    }
+
+    @Test
+    public void testFromEmptyJson() throws IOException
+    {
+        ConfigSource config = loader.fromJson(newInputStream("{\"type\":\"test\",\"data\":1}"));
+        assertEquals("test", config.get(String.class, "type"));
+        assertEquals(1, (int) config.get(Integer.class, "data"));
+    }
+
+    @Test
+    public void testFromYamlProperties() throws IOException
+    {
+        Properties props = new Properties();
+        props.setProperty("type", "test");
+        props.setProperty("data", "1");
+
+        ConfigSource config = loader.fromPropertiesYamlLiteral(props, "");
+        assertEquals("test", config.get(String.class, "type"));
+        assertEquals(1, (int) config.get(Integer.class, "data"));
+    }
+
+    @Test
+    public void testFromYamlPropertiesNested() throws IOException
+    {
+        Properties props = new Properties();
+        props.setProperty("type", "test");
+        props.setProperty("columns.k1", "1");
+        props.setProperty("values.myval.data", "2");
+
+        ConfigSource config = loader.fromPropertiesYamlLiteral(props, "");
+        System.out.println("config: "+config);
+        assertEquals("test", config.get(String.class, "type"));
+        assertEquals(1, (int) config.getNested("columns").get(Integer.class, "k1"));
+        assertEquals(2, (int) config.getNested("values").getNested("myval").get(Integer.class, "data"));
+    }
+
+    private static InputStream newInputStream(String string)
+    {
+        byte[] bytes = string.getBytes(StandardCharsets.UTF_8);
+        return new ByteArrayInputStream(bytes);
+    }
+}

data/embulk-docs/src/release.rst

@@ -4,6 +4,7 @@ Release Notes
 .. toctree::
     :maxdepth: 1
 
+    release/release-0.6.18
     release/release-0.6.17
     release/release-0.6.16
     release/release-0.6.15

data/embulk-docs/src/release/release-0.6.18.rst

@@ -0,0 +1,14 @@
+Release 0.6.18
+==================================
+
+General Changes
+------------------
+
+* Upgraded slf4j-api from 1.7.10 to 1.7.12
+* Upgraded joda-time from 2.7 to 2.8.1
+* Fixed broken plugin template generation of java-filter and ruby (@sakama++)
+* Fixed stacking order of filter plugins. This fixes broken behavior when we use multiple filter plugins.
+
+Release Date
+------------------
+2015-07-22

data/lib/embulk/data/new/README.md.erb

@@ -2,7 +2,7 @@
 
 %case language
 %when :ruby
-TODO: Write short description here and <%= project_name %>.gemspec file.
+TODO: Write short description here and <%= full_project_name %>.gemspec file.
 %when :java
 TODO: Write short description here and build.gradle file.
 %else

data/lib/embulk/data/new/java/filter.java.erb

@@ -1,4 +1,4 @@
-package org.embulk.<%= embulk_category %>;
+package <%= java_package_name %>;
 
 import com.google.common.base.Optional;
 import org.embulk.config.Config;

data/lib/embulk/version.rb

@@ -1,3 +1,3 @@
 module Embulk
-  VERSION = '0.6.17'
+  VERSION = '0.6.18'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk
 version: !ruby/object:Gem::Version
-  version: 0.6.17
+  version: 0.6.18
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-07-17 00:00:00.000000000 Z
+date: 2015-07-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -265,6 +265,7 @@ files:
 - embulk-core/src/test/java/org/embulk/RandomManager.java
 - embulk-core/src/test/java/org/embulk/TestPluginSourceModule.java
 - embulk-core/src/test/java/org/embulk/TestUtilityModule.java
+- embulk-core/src/test/java/org/embulk/config/TestConfigLoader.java
 - embulk-core/src/test/java/org/embulk/config/TestConfigSource.java
 - embulk-core/src/test/java/org/embulk/config/TestTaskSource.java
 - embulk-core/src/test/java/org/embulk/plugin/MockPluginSource.java
@@ -333,6 +334,7 @@ files:
 - embulk-docs/src/release/release-0.6.15.rst
 - embulk-docs/src/release/release-0.6.16.rst
 - embulk-docs/src/release/release-0.6.17.rst
+- embulk-docs/src/release/release-0.6.18.rst
 - embulk-docs/src/release/release-0.6.2.rst
 - embulk-docs/src/release/release-0.6.3.rst
 - embulk-docs/src/release/release-0.6.4.rst
@@ -449,8 +451,8 @@ files:
 - classpath/bval-jsr303-0.5.jar
 - classpath/commons-beanutils-core-1.8.3.jar
 - classpath/commons-lang3-3.1.jar
-- classpath/embulk-core-0.6.17.jar
-- classpath/embulk-standards-0.6.17.jar
+- classpath/embulk-core-0.6.18.jar
+- classpath/embulk-standards-0.6.18.jar
 - classpath/guava-18.0.jar
 - classpath/guice-4.0.jar
 - classpath/guice-multibindings-4.0.jar
@@ -463,13 +465,13 @@ files:
 - classpath/jackson-module-guice-2.5.3.jar
 - classpath/jansi-1.11.jar
 - classpath/javax.inject-1.jar
-- classpath/joda-time-2.7.jar
+- classpath/joda-time-2.8.1.jar
 - classpath/jruby-complete-1.7.19.jar
 - classpath/logback-classic-1.1.3.jar
 - classpath/logback-core-1.1.3.jar
 - classpath/netty-buffer-5.0.0.Alpha1.jar
 - classpath/netty-common-5.0.0.Alpha1.jar
-- classpath/slf4j-api-1.7.10.jar
+- classpath/slf4j-api-1.7.12.jar
 - classpath/slice-0.9.jar
 - classpath/snakeyaml-1.14.jar
 - classpath/validation-api-1.1.0.Final.jar