embulk 0.1.0

data/embulk-core/src/main/java/org/embulk/command/TablePrinter.java

@@ -0,0 +1,119 @@
+package org.embulk.command;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Locale;
+import java.io.PrintStream;
+import java.io.Flushable;
+import java.io.IOException;
+
+class TablePrinter
+{
+    private static final int SAMPLES = 10;
+
+    private PrintStream out;
+
+    private String[] header;
+
+    private List<Object[]> samples;
+    private String format;
+    private String border;
+
+    public TablePrinter(PrintStream out, String... header)
+    {
+        this.out = out;
+        this.header = header;
+        this.samples = new ArrayList<Object[]>(SAMPLES);
+        samples.add(header);
+    }
+
+    public void add(Object... values) throws IOException
+    {
+        int min = header.length < values.length ? header.length : values.length;
+        Object[] cols = new Object[header.length];
+        for(int i=0; i < min; i++) {
+            if(values[i] == null) {
+                cols[i] = "";
+            } else {
+                cols[i] = valueToString(values[i]);
+            }
+        }
+        for(int i=min; i < header.length; i++) {
+            cols[i] = "";
+        }
+
+        if(samples == null) {
+            out.format(format, cols);
+            return;
+        }
+
+        samples.add(cols);
+        if(samples.size() < SAMPLES) {
+            return;
+        }
+    }
+
+    protected String valueToString(Object obj)
+    {
+        return obj.toString();
+    }
+
+    private void flushSamples()
+    {
+        StringBuilder borderBuilder = new StringBuilder();
+
+        StringBuilder sb = new StringBuilder();
+        sb.append("| ");
+        borderBuilder.append("+-");
+        for(int i=0; i < header.length; i++) {
+            if(i != 0) {
+                sb.append(" | ");
+                borderBuilder.append("-+-");
+            }
+            int colLen = maxLengthInColumn(i);
+            sb.append("%"+colLen+"s");
+            for(int b=0; b < colLen; b++) {
+                borderBuilder.append("-");
+            }
+        }
+        sb.append(" |");
+        borderBuilder.append("-+");
+        sb.append("\n");
+
+        this.format = sb.toString();
+        this.border = borderBuilder.toString();
+
+        out.println(border);
+
+        for(int i=0; i < samples.size(); i++) {
+            out.format(format, samples.get(i));
+            if(i == 0) {
+                out.println(border);
+            }
+        }
+
+        this.samples = null;
+    }
+
+    private int maxLengthInColumn(int i)
+    {
+        int max = 0;
+        for(Object[] cols : samples) {
+            String s = (String) cols[i];
+            int len = (s == null) ? 0 : s.length();
+            if(max < len) {
+                max = len;
+            }
+        }
+        return max;
+    }
+
+    public void finish() throws IOException
+    {
+        if(!samples.isEmpty()) {
+            flushSamples();
+        }
+        out.println(border);
+    }
+}
+

data/embulk-core/src/main/java/org/embulk/config/CommitReport.java

@@ -0,0 +1,26 @@
+package org.embulk.config;
+
+public interface CommitReport
+        extends DataSource
+{
+    @Override
+    public CommitReport getNested(String attrName);
+
+    @Override
+    public CommitReport getNestedOrSetEmpty(String attrName);
+
+    @Override
+    public CommitReport set(String attrName, Object v);
+
+    @Override
+    public CommitReport setNested(String attrName, DataSource v);
+
+    @Override
+    public CommitReport setAll(DataSource other);
+
+    @Override
+    public CommitReport deepCopy();
+
+    @Override
+    public CommitReport merge(DataSource other);
+}

data/embulk-core/src/main/java/org/embulk/config/Config.java

@@ -0,0 +1,15 @@
+package org.embulk.config;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Documented
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.METHOD)
+public @interface Config
+{
+    String value();
+}

data/embulk-core/src/main/java/org/embulk/config/ConfigDefault.java

@@ -0,0 +1,15 @@
+package org.embulk.config;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+@Documented
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.METHOD)
+public @interface ConfigDefault
+{
+    String value();
+}

data/embulk-core/src/main/java/org/embulk/config/ConfigException.java

@@ -0,0 +1,20 @@
+package org.embulk.config;
+
+public class ConfigException
+        extends RuntimeException
+{
+    public ConfigException(String message)
+    {
+        super(message);
+    }
+
+    public ConfigException(Throwable cause)
+    {
+        super(cause);
+    }
+
+    public ConfigException(String message, Throwable cause)
+    {
+        super(message, cause);
+    }
+}

data/embulk-core/src/main/java/org/embulk/config/ConfigLoader.java

@@ -0,0 +1,83 @@
+package org.embulk.config;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.Properties;
+import com.google.inject.Inject;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import org.yaml.snakeyaml.Yaml;
+
+public class ConfigLoader
+{
+    private final ModelManager model;
+
+    @Inject
+    public ConfigLoader(ModelManager model)
+    {
+        this.model = model;
+    }
+
+    public ConfigSource fromJson(JsonParser parser) throws IOException
+    {
+        // TODO check parsed.isObject()
+        ObjectNode source = (ObjectNode) new ObjectMapper().readTree(parser);
+        return new DataSourceImpl(model, source);
+    }
+
+    public ConfigSource fromYamlFile(File path) throws IOException
+    {
+        Yaml yaml = new Yaml();
+        Object parsedYaml;
+        try (FileInputStream is = new FileInputStream(path)) {
+            parsedYaml = yaml.load(is);
+        }
+        ObjectNode source = objectToJsonObject(parsedYaml);
+        return new DataSourceImpl(model, source);
+    }
+
+    public ConfigSource fromPropertiesYamlLiteral(Properties props, String keyPrefix)
+    {
+        // TODO exception handling
+        ObjectNode source = new ObjectNode(JsonNodeFactory.instance);
+        Yaml yaml = new Yaml();
+        for (String key : props.stringPropertyNames()) {
+            // TODO handle "." and "[...]" as map and array acccessor for example:
+            //      in.parser.type=csv => {"in": {"parser": {"type": "csv"}}}
+            if (!key.startsWith(keyPrefix)) {
+                continue;
+            }
+            String yamlValue = props.getProperty(key);
+            String keyName = key.substring(keyPrefix.length());
+            Object parsedValue = yaml.load(yamlValue);
+            JsonNode typedValue = objectToJson(parsedValue);
+            source.set(keyName, typedValue);
+        }
+        return new DataSourceImpl(model, source);
+    }
+
+    private JsonNode objectToJson(Object object)
+    {
+        // TODO exception
+        ObjectMapper objectMapper = new ObjectMapper();
+        try {
+            return objectMapper.readTree(objectMapper.writeValueAsString(object));
+        } catch (IOException ex) {
+            throw new RuntimeException(ex);
+        }
+    }
+
+    private ObjectNode objectToJsonObject(Object object)
+    {
+        // TODO exception
+        JsonNode json = objectToJson(object);
+        if (!json.isObject()) {
+            throw new RuntimeException("Expected object to deserialize ConfigSource but got "+json);
+        }
+        return (ObjectNode) json;
+    }
+}

data/embulk-core/src/main/java/org/embulk/config/ConfigSource.java

@@ -0,0 +1,28 @@
+package org.embulk.config;
+
+public interface ConfigSource
+        extends DataSource
+{
+    public <T extends Task> T loadConfig(Class<T> taskType);
+
+    @Override
+    public ConfigSource getNested(String attrName);
+
+    @Override
+    public ConfigSource getNestedOrSetEmpty(String attrName);
+
+    @Override
+    public ConfigSource set(String attrName, Object v);
+
+    @Override
+    public ConfigSource setNested(String attrName, DataSource v);
+
+    @Override
+    public ConfigSource setAll(DataSource other);
+
+    @Override
+    public ConfigSource deepCopy();
+
+    @Override
+    public ConfigSource merge(DataSource other);
+}

data/embulk-core/src/main/java/org/embulk/config/DataSource.java

@@ -0,0 +1,35 @@
+package org.embulk.config;
+
+import java.util.List;
+import java.util.Map;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+public interface DataSource
+{
+    public List<String> getAttributeNames();
+
+    public Iterable<Map.Entry<String, JsonNode>> getAttributes();
+
+    public boolean isEmpty();
+
+    public <E> E get(Class<E> type, String attrName);
+
+    public <E> E get(Class<E> type, String attrName, E defaultValue);
+
+    public DataSource getNested(String attrName);
+
+    public DataSource getNestedOrSetEmpty(String attrName);
+
+    public DataSource set(String attrName, Object v);
+
+    public DataSource setNested(String attrName, DataSource v);
+
+    public DataSource setAll(DataSource other);
+
+    public DataSource deepCopy();
+
+    public DataSource merge(DataSource other);
+
+    public ObjectNode getObjectNode();
+}

data/embulk-core/src/main/java/org/embulk/config/DataSourceImpl.java

@@ -0,0 +1,208 @@
+package org.embulk.config;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Iterator;
+import com.google.common.collect.ImmutableList;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+
+public class DataSourceImpl
+        implements ConfigSource, TaskSource, CommitReport, NextConfig
+{
+    protected final ObjectNode data;
+    protected final ModelManager model;
+
+    public DataSourceImpl(ModelManager model)
+    {
+        this(model, new ObjectNode(JsonNodeFactory.instance));
+    }
+
+    // visible for DataSourceSerDe, ConfigSourceLoader and TaskInvocationHandler.dump
+    public DataSourceImpl(ModelManager model, ObjectNode data)
+    {
+        this.data = data;
+        this.model = model;
+    }
+
+    protected DataSourceImpl newInstance(ModelManager model, ObjectNode data)
+    {
+        return new DataSourceImpl(model, (ObjectNode) data);
+    }
+
+    // visible for DataSourceSerDe.DataSourceSerializer
+    @Override
+    public ObjectNode getObjectNode()
+    {
+        return data;
+    }
+
+    @Override
+    public List<String> getAttributeNames()
+    {
+        return ImmutableList.copyOf(data.fieldNames());
+    }
+
+    @Override
+    public Iterable<Map.Entry<String, JsonNode>> getAttributes()
+    {
+        return new Iterable() {
+            public Iterator<Map.Entry<String, JsonNode>> iterator()
+            {
+                return data.fields();
+            }
+        };
+    }
+
+    @Override
+    public boolean isEmpty()
+    {
+        return !data.fieldNames().hasNext();
+    }
+
+    @Override
+    public <E> E get(Class<E> type, String attrName)
+    {
+        JsonNode json = data.get(attrName);
+        if (json == null) {
+            throw new ConfigException("Attribute "+attrName+" is required but not set");
+        }
+        return model.readObject(type, json.traverse());
+    }
+
+    @Override
+    public <E> E get(Class<E> type, String attrName, E defaultValue)
+    {
+        JsonNode json = data.get(attrName);
+        if (json == null) {
+            return defaultValue;
+        }
+        return model.readObject(type, json.traverse());
+    }
+
+    @Override
+    public DataSourceImpl getNested(String attrName)
+    {
+        JsonNode json = data.get(attrName);
+        if (json == null) {
+            throw new ConfigException("Attribute "+attrName+" is required but not set");
+        }
+        if (!json.isObject()) {
+            throw new ConfigException("Attribute "+attrName+" must be an object");
+        }
+        return newInstance(model, (ObjectNode) json);
+    }
+
+    @Override
+    public DataSourceImpl getNestedOrSetEmpty(String attrName)
+    {
+        JsonNode json = data.get(attrName);
+        if (json == null) {
+            json = data.objectNode();
+            data.set(attrName, json);
+        } else if (!json.isObject()) {
+            throw new ConfigException("Attribute "+attrName+" must be an object");
+        }
+        return newInstance(model, (ObjectNode) json);
+    }
+
+    @Override
+    public DataSourceImpl set(String attrName, Object v)
+    {
+        if (v == null) {
+            data.remove(attrName);
+        } else {
+            data.put(attrName, model.writeObjectAsJsonNode(v));
+        }
+        return this;
+    }
+
+    @Override
+    public DataSourceImpl setNested(String attrName, DataSource v)
+    {
+        data.put(attrName, v.getObjectNode());
+        return this;
+    }
+
+    @Override
+    public DataSourceImpl setAll(DataSource other)
+    {
+        for (Map.Entry<String, JsonNode> field : other.getAttributes()) {
+            data.put(field.getKey(), field.getValue());
+        }
+        return this;
+    }
+
+    @Override
+    public DataSourceImpl deepCopy()
+    {
+        return newInstance(model, data.deepCopy());
+    }
+
+    @Override
+    public DataSourceImpl merge(DataSource other)
+    {
+        mergeJsonObject(data, other.deepCopy().getObjectNode());
+        return this;
+    }
+
+    private static void mergeJsonObject(ObjectNode src, ObjectNode other)
+    {
+        Iterator<Map.Entry<String, JsonNode>> ite = other.fields();
+        while (ite.hasNext()) {
+            Map.Entry<String, JsonNode> pair = ite.next();
+            JsonNode s = src.get(pair.getKey());
+            JsonNode v = pair.getValue();
+            if (v.isObject() && s != null && s.isObject()) {
+                mergeJsonObject((ObjectNode) s, (ObjectNode) v);
+            } else if (v.isArray() && s != null && s.isArray()) {
+                mergeJsonArray((ArrayNode) s, (ArrayNode) v);
+            } else {
+                src.replace(pair.getKey(), v);
+            }
+        }
+    }
+
+    private static void mergeJsonArray(ArrayNode src, ArrayNode other)
+    {
+        src.addAll(other);
+    }
+
+    @Override
+    public <T extends Task> T loadTask(Class<T> taskType)
+    {
+        return model.readObject(taskType, data.traverse());
+    }
+
+    @Override
+    public <T extends Task> T loadConfig(Class<T> taskType)
+    {
+        return model.readObjectWithConfigSerDe(taskType, data.traverse());
+    }
+
+    @Override
+    public String toString()
+    {
+        return data.toString();
+    }
+
+    @Override
+    public boolean equals(Object other)
+    {
+        if (!(other instanceof DataSource)) {
+            return false;
+        }
+        return data.equals(((DataSource) other).getObjectNode());
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return data.hashCode();
+    }
+}