embulk 0.6.13 → 0.6.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 98a09da0f4c6425b69f3094d7c5b105eea0e7d36
-  data.tar.gz: 26383243a036baa447c9bf3ab28d554e2a5fc9e5
+  metadata.gz: abf8ae432c5b7f3817ac7acd72aee485a9b4f616
+  data.tar.gz: 8454af492f563890e26f11cc9f6fbf7fe8a7b66a
 SHA512:
-  metadata.gz: acc2663c2fc200aeb2aeffc5e3156cdffbfbd8321a87bfbac1fa125e46f5c2006bdc9cdf63484b0157c622c32cf8be81bec336a029bd6c655e866537bc689578
-  data.tar.gz: 41696a15d2d1cd12efcd1d35574b19ddeeaa931f386f8c7093ab123328d460d5b1d82a94880f38c679b8f690edd43c7d5fcf679cee08b9ba019236b36460a9c7
+  metadata.gz: 49bb503770d751d426dcb2f4331faabff654bc906cd91610980bd435fde69c78e85043aea0a2f34f9906a293d3987d725493def95a769132993849cfcc14fafb
+  data.tar.gz: 50b97e0ffc2d76581eabcb174cbebf638cc1c8eae34fbaf139e711a3856a4eb6f4b197637213d13cbc17a9d36f9256f77f65682413e0727cae39c2c0c392882b

data/README.md

@@ -206,5 +206,5 @@ git tag v$VERSION
 
 See also:
 * [Bintray](https://bintray.com)
-* [How to acquire bintray API Keys](https://bintray.com/docs/usermanual/interacting/interacting_apikeys.html)
+* [How to acquire bintray API Keys](https://bintray.com/docs/usermanual/interacting/interacting_editingyouruserprofile.html#anchorAPIKEY)
 

data/build.gradle

@@ -11,7 +11,7 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards")]
 
 allprojects {
     group = 'org.embulk'
-    version = '0.6.13'
+    version = '0.6.14'
 
     ext {
         jrubyVersion = '1.7.19'

data/embulk-core/src/main/java/org/embulk/config/CommitReport.java

@@ -18,6 +18,9 @@ public interface CommitReport
     @Override
     public CommitReport setAll(DataSource other);
 
+    @Override
+    public CommitReport remove(String attrName);
+
     @Override
     public CommitReport deepCopy();
 

data/embulk-core/src/main/java/org/embulk/config/ConfigDiff.java

@@ -18,6 +18,9 @@ public interface ConfigDiff
     @Override
     public ConfigDiff setAll(DataSource other);
 
+    @Override
+    public ConfigDiff remove(String attrName);
+
     @Override
     public ConfigDiff deepCopy();
 

data/embulk-core/src/main/java/org/embulk/config/ConfigSource.java

@@ -20,6 +20,9 @@ public interface ConfigSource
     @Override
     public ConfigSource setAll(DataSource other);
 
+    @Override
+    public ConfigSource remove(String attrName);
+
     @Override
     public ConfigSource deepCopy();
 

data/embulk-core/src/main/java/org/embulk/config/DataSource.java

@@ -27,6 +27,8 @@ public interface DataSource
 
     public DataSource setAll(DataSource other);
 
+    public DataSource remove(String attrName);
+
     public DataSource deepCopy();
 
     public DataSource merge(DataSource other);

data/embulk-core/src/main/java/org/embulk/config/DataSourceImpl.java

@@ -115,7 +115,7 @@ public class DataSourceImpl
     public DataSourceImpl set(String attrName, Object v)
     {
         if (v == null) {
-            data.remove(attrName);
+            remove(attrName);
         } else {
             data.put(attrName, model.writeObjectAsJsonNode(v));
         }
@@ -138,6 +138,13 @@ public class DataSourceImpl
         return this;
     }
 
+    @Override
+    public DataSourceImpl remove(String attrName)
+    {
+        data.remove(attrName);
+        return this;
+    }
+
     @Override
     public DataSourceImpl deepCopy()
     {

data/embulk-core/src/main/java/org/embulk/config/TaskSource.java

@@ -20,6 +20,9 @@ public interface TaskSource
     @Override
     public TaskSource setAll(DataSource other);
 
+    @Override
+    public TaskSource remove(String attrName);
+
     @Override
     public TaskSource deepCopy();
 

data/embulk-core/src/main/java/org/embulk/spi/ColumnConfig.java

@@ -3,24 +3,37 @@ package org.embulk.spi;
 import java.util.Objects;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import org.embulk.config.ConfigSource;
 import org.embulk.spi.type.Type;
 import org.embulk.spi.type.TimestampType;
+import org.embulk.spi.Exec;
 
 public class ColumnConfig
 {
     private final String name;
     private final Type type;
-    private final String format;
+    private final ConfigSource option;
+
+    @Deprecated
+    public ColumnConfig(String name, Type type, String format)
+    {
+        this.name = name;
+        this.type = type;
+        this.option = Exec.newConfigSource();  // only for backward compatibility
+        if (format != null) {
+            option.set("format", format);
+        }
+    }
 
     @JsonCreator
     public ColumnConfig(
             @JsonProperty("name") String name,
             @JsonProperty("type") Type type,
-            @JsonProperty("format") String format)
+            @JsonProperty("option") ConfigSource option)
     {
         this.name = name;
         this.type = type;
-        this.format = format;
+        this.option = option;
     }
 
     @JsonProperty("name")
@@ -35,15 +48,23 @@ public class ColumnConfig
         return type;
     }
 
-    @JsonProperty("format")
+    @JsonProperty("option")
+    public ConfigSource getOption()
+    {
+        return option;
+    }
+
+    @Deprecated
     public String getFormat()
     {
-        return format;
+        return option.get(String.class, "format", null);
     }
 
     public Column toColumn(int index)
     {
+        String format = option.get(String.class, "format", null);
         if (type instanceof TimestampType && format != null) {
+            // this behavior is only for backward compatibility. TimestampType#getFormat is @Deprecated
             return new Column(index, name, ((TimestampType) type).withFormat(format));
         } else {
             return new Column(index, name, type);
@@ -61,7 +82,8 @@ public class ColumnConfig
         }
         ColumnConfig other = (ColumnConfig) obj;
         return Objects.equals(this.name, other.name) &&
-            Objects.equals(type, other.type);
+            Objects.equals(type, other.type) &&
+            Objects.equals(option, other.option);
     }
 
     @Override

data/embulk-core/src/main/java/org/embulk/spi/PageFormat.java

@@ -18,7 +18,7 @@ abstract class PageFormat
 
     static int nullBitSetSize(Schema schema)
     {
-        return (schema.size() + 7) / 8;
+        return (schema.getColumnCount() + 7) / 8;
     }
 
     static int recordHeaderSize(Schema schema)
@@ -33,11 +33,11 @@ abstract class PageFormat
 
     static int[] columnOffsets(Schema schema)
     {
-        int[] offsets = new int[schema.size()];
+        int[] offsets = new int[schema.getColumnCount()];
 
         if (!schema.isEmpty()) {
             offsets[0] = recordHeaderSize(schema);
-            for (int i=0; i < schema.size()-1; i++) {
+            for (int i=0; i < schema.getColumnCount()-1; i++) {
                 offsets[i+1] = offsets[i] + schema.getColumnType(i).getFixedStorageSize();
             }
         }

data/embulk-core/src/main/java/org/embulk/spi/ProcessTask.java

@@ -25,16 +25,17 @@ public class ProcessTask
     private final Schema executorSchema;
     private TaskSource executorTaskSource;
 
+    @JsonCreator
     public ProcessTask(
-            PluginType inputPluginType,
-            PluginType outputPluginType,
-            List<PluginType> filterPluginTypes,
-            TaskSource inputTaskSource,
-            TaskSource outputTaskSource,
-            List<TaskSource> filterTaskSources,
-            List<Schema> schemas,
-            Schema executorSchema,
-            TaskSource executorTaskSource)
+            @JsonProperty("inputType") PluginType inputPluginType,
+            @JsonProperty("outputType") PluginType outputPluginType,
+            @JsonProperty("filterTypes") List<PluginType> filterPluginTypes,
+            @JsonProperty("inputTask") TaskSource inputTaskSource,
+            @JsonProperty("outputTask") TaskSource outputTaskSource,
+            @JsonProperty("filterTasks") List<TaskSource> filterTaskSources,
+            @JsonProperty("schemas") List<Schema> schemas,
+            @JsonProperty("executorSchema") Schema executorSchema,
+            @JsonProperty("executorTask") TaskSource executorTaskSource)
     {
         this.inputPluginType = inputPluginType;
         this.outputPluginType = outputPluginType;
@@ -47,36 +48,6 @@ public class ProcessTask
         this.executorTaskSource = executorTaskSource;
     }
 
-    // TODO Because TimestampType doesn't store timestamp_format, serializing and deserializing
-    // Schema loses timestamp_format information. Here uses SchemaConfig instead to preseve it.
-
-    @JsonCreator
-    ProcessTask(
-            @JsonProperty("inputType") PluginType inputPluginType,
-            @JsonProperty("outputType") PluginType outputPluginType,
-            @JsonProperty("filterTypes") List<PluginType> filterPluginTypes,
-            @JsonProperty("inputTask") TaskSource inputTaskSource,
-            @JsonProperty("outputTask") TaskSource outputTaskSource,
-            @JsonProperty("filterTasks") List<TaskSource> filterTaskSources,
-            @JsonProperty("schemas") List<SchemaConfig> schemas,
-            @JsonProperty("executorSchema") SchemaConfig executorSchema,
-            @JsonProperty("executorTask") TaskSource executorTaskSource)
-    {
-        this(inputPluginType, outputPluginType, filterPluginTypes,
-                inputTaskSource, outputTaskSource, filterTaskSources,
-                ImmutableList.copyOf(Lists.transform(schemas,
-                        new Function<SchemaConfig, Schema>()
-                        {
-                            public Schema apply(SchemaConfig s)
-                            {
-                                return s.toSchema();
-                            }
-                        }
-                    )),
-                executorSchema.toSchema(),
-                executorTaskSource);
-    }
-
     @JsonProperty("inputType")
     public PluginType getInputPluginType()
     {
@@ -113,54 +84,18 @@ public class ProcessTask
         return filterTaskSources;
     }
 
-    @JsonIgnore
+    @JsonProperty("schemas")
     public List<Schema> getFilterSchemas()
     {
         return schemas;
     }
 
-    @JsonProperty("schemas")
-    public List<SchemaConfig> getFilterSchemaConfigs()
-    {
-        return Lists.transform(schemas,
-                new Function<Schema, SchemaConfig>()
-                {
-                    public SchemaConfig apply(Schema schema)
-                    {
-                        return schemaToSchemaConfig(schema);
-                    }
-                });
-    }
-
-    @JsonIgnore
+    @JsonProperty("executorSchema")
     public Schema getExecutorSchema()
     {
         return executorSchema;
     }
 
-    @JsonProperty("executorSchema")
-    SchemaConfig getExecutorSchemaConfig()
-    {
-        return schemaToSchemaConfig(executorSchema);
-    }
-
-    private static SchemaConfig schemaToSchemaConfig(Schema s)
-    {
-        return new SchemaConfig(Lists.transform(s.getColumns(),
-                    new Function<Column, ColumnConfig>()
-                    {
-                        public ColumnConfig apply(Column c)
-                        {
-                            if (c.getType() instanceof TimestampType) {
-                                return new ColumnConfig(c.getName(), c.getType(), ((TimestampType) c.getType()).getFormat());
-                            } else {
-                                return new ColumnConfig(c.getName(), c.getType(), null);
-                            }
-                        }
-                    }
-                ));
-    }
-
     @JsonIgnore
     public Schema getInputSchema()
     {

data/embulk-core/src/main/java/org/embulk/spi/Schema.java

@@ -2,12 +2,35 @@ package org.embulk.spi;
 
 import java.util.List;
 import java.util.Objects;
+import com.google.common.collect.ImmutableList;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonValue;
 import org.embulk.spi.type.Type;
 
 public class Schema
 {
+    public static class Builder
+    {
+        private final ImmutableList.Builder<Column> columns = ImmutableList.builder();
+        private int index = 0;  // next version of Guava will have ImmutableList.Builder.size()
+
+        public synchronized Builder add(String name, Type type)
+        {
+            columns.add(new Column(index++, name, type));
+            return this;
+        }
+
+        public Schema build()
+        {
+            return new Schema(columns.build());
+        }
+    }
+
+    public static Builder builder()
+    {
+        return new Builder();
+    }
+
     private final List<Column> columns;
 
     @JsonCreator
@@ -22,6 +45,11 @@ public class Schema
         return columns;
     }
 
+    public int size()
+    {
+        return columns.size();
+    }
+
     public int getColumnCount()
     {
         return columns.size();
@@ -54,9 +82,14 @@ public class Schema
         return columns.isEmpty();
     }
 
-    public int size()
+    public Column lookupColumn(String name)
     {
-        return columns.size();
+        for (Column c : columns) {
+            if (c.getName().equals(name)) {
+                return c;
+            }
+        }
+        throw new SchemaConfigException(String.format("Column '%s' is not found", name));
     }
 
     public int getFixedStorageSize()

data/embulk-core/src/main/java/org/embulk/spi/SchemaConfig.java

@@ -2,9 +2,11 @@ package org.embulk.spi;
 
 import java.util.List;
 import java.util.Objects;
+import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableList;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonValue;
+import org.embulk.spi.type.Type;
 
 public class SchemaConfig
 {
@@ -22,6 +24,46 @@ public class SchemaConfig
         return columns;
     }
 
+    public int size()
+    {
+        return columns.size();
+    }
+
+    public int getColumnCount()
+    {
+        return columns.size();
+    }
+
+    public ColumnConfig getColumn(int index)
+    {
+        return columns.get(index);
+    }
+
+    public String getColumnName(int index)
+    {
+        return getColumn(index).getName();
+    }
+
+    public Type getColumnType(int index)
+    {
+        return getColumn(index).getType();
+    }
+
+    public boolean isEmpty()
+    {
+        return columns.isEmpty();
+    }
+
+    public ColumnConfig lookupColumn(String name)
+    {
+        for (ColumnConfig c : columns) {
+            if (c.getName().equals(name)) {
+                return c;
+            }
+        }
+        throw new SchemaConfigException(String.format("Column '%s' is not found", name));
+    }
+
     public Schema toSchema()
     {
         ImmutableList.Builder<Column> builder = ImmutableList.builder();