embulk 0.7.0-java

data/embulk-core/src/main/java/org/embulk/spi/PageFormat.java

@@ -0,0 +1,47 @@
+package org.embulk.spi;
+
+abstract class PageFormat
+{
+    // PageHeader
+    // +---+
+    // | 4 |
+    // +---+
+    // count (number of records)
+
+    private PageFormat() { }
+
+    static final int PAGE_HEADER_SIZE = 4;
+
+    // PageBuilder.setVariableLengthData and PageReader.readVariableLengthData
+    // uses 4 bytes integer
+    static final int VARIABLE_LENGTH_COLUMN_SIZE = 4;
+
+    static int nullBitSetSize(Schema schema)
+    {
+        return (schema.getColumnCount() + 7) / 8;
+    }
+
+    static int recordHeaderSize(Schema schema)
+    {
+        return 4 + nullBitSetSize(schema);
+    }
+
+    static int totalColumnSize(Schema schema)
+    {
+        return recordHeaderSize(schema) + schema.getFixedStorageSize();
+    }
+
+    static int[] columnOffsets(Schema schema)
+    {
+        int[] offsets = new int[schema.getColumnCount()];
+
+        if (!schema.isEmpty()) {
+            offsets[0] = recordHeaderSize(schema);
+            for (int i=0; i < schema.getColumnCount()-1; i++) {
+                offsets[i+1] = offsets[i] + schema.getColumnType(i).getFixedStorageSize();
+            }
+        }
+
+        return offsets;
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/PageOutput.java

@@ -0,0 +1,11 @@
+package org.embulk.spi;
+
+public interface PageOutput
+        extends AutoCloseable
+{
+    void add(Page page);
+
+    void finish();
+
+    void close();
+}

data/embulk-core/src/main/java/org/embulk/spi/PageReader.java

@@ -0,0 +1,226 @@
+package org.embulk.spi;
+
+import io.airlift.slice.Slice;
+import io.airlift.slice.Slices;
+import org.embulk.spi.time.Timestamp;
+
+public class PageReader
+        implements AutoCloseable
+{
+    private final Schema schema;
+    private final int[] columnOffsets;
+
+    private Page page = SENTINEL;
+    private Slice pageSlice = null;
+    private int pageRecordCount = 0;
+
+    private int readCount = 0;
+    private int position;
+    private final byte[] nullBitSet;
+
+    private static final Page SENTINEL = Page.wrap(Buffer.wrap(new byte[4]));  // buffer().release() does nothing
+
+    public PageReader(Schema schema)
+    {
+        this.schema = schema;
+        this.columnOffsets = PageFormat.columnOffsets(schema);
+        this.nullBitSet = new byte[PageFormat.nullBitSetSize(schema)];
+    }
+
+    public static int getRecordCount(Page page)
+    {
+        Buffer pageBuffer = page.buffer();
+        Slice pageSlice = Slices.wrappedBuffer(pageBuffer.array(), pageBuffer.offset(), pageBuffer.limit());
+        return pageSlice.getInt(0);  // see page format
+    }
+
+    public void setPage(Page page)
+    {
+        this.page.buffer().release();
+        this.page = SENTINEL;
+
+        Buffer pageBuffer = page.buffer();
+        Slice pageSlice = Slices.wrappedBuffer(pageBuffer.array(), pageBuffer.offset(), pageBuffer.limit());
+
+        pageRecordCount = pageSlice.getInt(0);  // see page format
+        readCount = 0;
+        position = PageFormat.PAGE_HEADER_SIZE;
+
+        this.page = page;
+        this.pageSlice = pageSlice;
+    }
+
+    public Schema getSchema()
+    {
+        return schema;
+    }
+
+    public boolean isNull(Column column)
+    {
+        return isNull(column.getIndex());
+    }
+
+    public boolean isNull(int columnIndex)
+    {
+        return (nullBitSet[columnIndex >>> 3] & (1 << (columnIndex & 7))) != 0;
+    }
+
+    public boolean getBoolean(Column column)
+    {
+        // TODO check type?
+        return getBoolean(column.getIndex());
+    }
+
+    public boolean getBoolean(int columnIndex)
+    {
+        return pageSlice.getByte(getOffset(columnIndex)) != (byte) 0;
+    }
+
+    public long getLong(Column column)
+    {
+        // TODO check type?
+        return getLong(column.getIndex());
+    }
+
+    public long getLong(int columnIndex)
+    {
+        return pageSlice.getLong(getOffset(columnIndex));
+    }
+
+    public double getDouble(Column column)
+    {
+        // TODO check type?
+        return getDouble(column.getIndex());
+    }
+
+    public double getDouble(int columnIndex)
+    {
+        return pageSlice.getDouble(getOffset(columnIndex));
+    }
+
+    public String getString(Column column)
+    {
+        // TODO check type?
+        return getString(column.getIndex());
+    }
+
+    public String getString(int columnIndex)
+    {
+        int index = pageSlice.getInt(getOffset(columnIndex));
+        return page.getStringReference(index);
+    }
+
+    public Timestamp getTimestamp(Column column)
+    {
+        // TODO check type?
+        return getTimestamp(column.getIndex());
+    }
+
+    public Timestamp getTimestamp(int columnIndex)
+    {
+        int offset = getOffset(columnIndex);
+        long sec = pageSlice.getLong(offset);
+        int nsec = pageSlice.getInt(offset + 8);
+        return Timestamp.ofEpochSecond(sec, nsec);
+    }
+
+    private int getOffset(int columnIndex)
+    {
+        return position + columnOffsets[columnIndex];
+    }
+
+    public boolean nextRecord()
+    {
+        if (pageRecordCount <= readCount) {
+            return false;
+        }
+
+        if (readCount > 0) {
+            // advance position excepting the first record
+            int lastRecordSize = pageSlice.getInt(position);
+            position += lastRecordSize;
+        }
+
+        readCount++;
+        pageSlice.getBytes(position + 4, nullBitSet, 0, nullBitSet.length);
+
+        return true;
+    }
+
+    @Override
+    public void close()
+    {
+        page.buffer().release();
+        page = SENTINEL;
+    }
+
+    /* TODO for variable-length types
+    public VariableLengthDataReader getVariableLengthData(int columnIndex, int variableLengthDataOffset)
+    {
+        return new VariableLengthDataReader(variableLengthDataOffset);
+    }
+
+    public class VariableLengthDataReader
+    {
+        private int offsetFromPosition;
+
+        VariableLengthDataReader(int offsetFromPosition)
+        {
+            this.offsetFromPosition = offsetFromPosition;
+        }
+
+        public byte readByte()
+        {
+            byte value = page.getByte(position + offsetFromPosition);
+            offsetFromPosition += 1;
+            return value;
+        }
+
+        public short readShort()
+        {
+            short value = page.getShort(position + offsetFromPosition);
+            offsetFromPosition += 2;
+            return value;
+        }
+
+        public int readInt()
+        {
+            int value = page.getInt(position + offsetFromPosition);
+            offsetFromPosition += 4;
+            return value;
+        }
+
+        public long readLong()
+        {
+            long value = page.getLong(position + offsetFromPosition);
+            offsetFromPosition += 8;
+            return value;
+        }
+
+        public float readFloat()
+        {
+            float value = page.getFloat(position + offsetFromPosition);
+            offsetFromPosition += 4;
+            return value;
+        }
+
+        public double readDouble()
+        {
+            double value = page.getDouble(position + offsetFromPosition);
+            offsetFromPosition += 8;
+            return value;
+        }
+
+        public void readBytes(byte[] data)
+        {
+            readBytes(data, 0, data.length);
+        }
+
+        public void readBytes(byte[] data, int off, int len)
+        {
+            page.getBytes(position + offsetFromPosition, data, off, len);
+            offsetFromPosition += len;
+        }
+    }
+    */
+}

data/embulk-core/src/main/java/org/embulk/spi/ParserPlugin.java

@@ -0,0 +1,17 @@
+package org.embulk.spi;
+
+import org.embulk.config.TaskSource;
+import org.embulk.config.ConfigSource;
+
+public interface ParserPlugin
+{
+    interface Control
+    {
+        void run(TaskSource taskSource, Schema schema);
+    }
+
+    void transaction(ConfigSource config, ParserPlugin.Control control);
+
+    void run(TaskSource taskSource, Schema schema,
+            FileInput input, PageOutput output);
+}

data/embulk-core/src/main/java/org/embulk/spi/ProcessState.java

@@ -0,0 +1,10 @@
+package org.embulk.spi;
+
+public interface ProcessState
+{
+    void initialize(int inputTaskCount, int outputTaskCount);
+
+    TaskState getInputTaskState(int inputTaskIndex);
+
+    TaskState getOutputTaskState(int outputTaskIndex);
+}

data/embulk-core/src/main/java/org/embulk/spi/ProcessTask.java

@@ -0,0 +1,117 @@
+package org.embulk.spi;
+
+import java.util.List;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import org.embulk.plugin.PluginType;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.util.Executors;
+
+public class ProcessTask
+{
+    private final PluginType inputPluginType;
+    private final PluginType outputPluginType;
+    private final List<PluginType> filterPluginTypes;
+    private final TaskSource inputTaskSource;
+    private final TaskSource outputTaskSource;
+    private final List<TaskSource> filterTaskSources;
+    private final List<Schema> schemas;
+    private final Schema executorSchema;
+    private TaskSource executorTaskSource;
+
+    @JsonCreator
+    public ProcessTask(
+            @JsonProperty("inputType") PluginType inputPluginType,
+            @JsonProperty("outputType") PluginType outputPluginType,
+            @JsonProperty("filterTypes") List<PluginType> filterPluginTypes,
+            @JsonProperty("inputTask") TaskSource inputTaskSource,
+            @JsonProperty("outputTask") TaskSource outputTaskSource,
+            @JsonProperty("filterTasks") List<TaskSource> filterTaskSources,
+            @JsonProperty("schemas") List<Schema> schemas,
+            @JsonProperty("executorSchema") Schema executorSchema,
+            @JsonProperty("executorTask") TaskSource executorTaskSource)
+    {
+        this.inputPluginType = inputPluginType;
+        this.outputPluginType = outputPluginType;
+        this.filterPluginTypes = filterPluginTypes;
+        this.inputTaskSource = inputTaskSource;
+        this.outputTaskSource = outputTaskSource;
+        this.filterTaskSources = filterTaskSources;
+        this.schemas = schemas;
+        this.executorSchema = executorSchema;
+        this.executorTaskSource = executorTaskSource;
+    }
+
+    @JsonProperty("inputType")
+    public PluginType getInputPluginType()
+    {
+        return inputPluginType;
+    }
+
+    @JsonProperty("outputType")
+    public PluginType getOutputPluginType()
+    {
+        return outputPluginType;
+    }
+
+    @JsonProperty("filterTypes")
+    public List<PluginType> getFilterPluginTypes()
+    {
+        return filterPluginTypes;
+    }
+
+    @JsonProperty("inputTask")
+    public TaskSource getInputTaskSource()
+    {
+        return inputTaskSource;
+    }
+
+    @JsonProperty("outputTask")
+    public TaskSource getOutputTaskSource()
+    {
+        return outputTaskSource;
+    }
+
+    @JsonProperty("filterTasks")
+    public List<TaskSource> getFilterTaskSources()
+    {
+        return filterTaskSources;
+    }
+
+    @JsonProperty("schemas")
+    public List<Schema> getFilterSchemas()
+    {
+        return schemas;
+    }
+
+    @JsonProperty("executorSchema")
+    public Schema getExecutorSchema()
+    {
+        return executorSchema;
+    }
+
+    @JsonIgnore
+    public Schema getInputSchema()
+    {
+        return Executors.getInputSchema(schemas);
+    }
+
+    @JsonIgnore
+    public Schema getOutputSchema()
+    {
+        return Executors.getOutputSchema(schemas);
+    }
+
+    @JsonIgnore
+    public void setExecutorTaskSource(TaskSource executorTaskSource)
+    {
+        this.executorTaskSource = executorTaskSource;
+    }
+
+    @JsonProperty("executorTask")
+    public TaskSource getExecutorTaskSource()
+    {
+        return executorTaskSource;
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/Schema.java

@@ -0,0 +1,134 @@
+package org.embulk.spi;
+
+import java.util.List;
+import java.util.Objects;
+import com.google.common.collect.ImmutableList;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonValue;
+import org.embulk.spi.type.Type;
+
+public class Schema
+{
+    public static class Builder
+    {
+        private final ImmutableList.Builder<Column> columns = ImmutableList.builder();
+        private int index = 0;  // next version of Guava will have ImmutableList.Builder.size()
+
+        public synchronized Builder add(String name, Type type)
+        {
+            columns.add(new Column(index++, name, type));
+            return this;
+        }
+
+        public Schema build()
+        {
+            return new Schema(columns.build());
+        }
+    }
+
+    public static Builder builder()
+    {
+        return new Builder();
+    }
+
+    private final List<Column> columns;
+
+    @JsonCreator
+    public Schema(List<Column> columns)
+    {
+        this.columns = columns;
+    }
+
+    @JsonValue
+    public List<Column> getColumns()
+    {
+        return columns;
+    }
+
+    public int size()
+    {
+        return columns.size();
+    }
+
+    public int getColumnCount()
+    {
+        return columns.size();
+    }
+
+    public Column getColumn(int index)
+    {
+        return columns.get(index);
+    }
+
+    public String getColumnName(int index)
+    {
+        return getColumn(index).getName();
+    }
+
+    public Type getColumnType(int index)
+    {
+        return getColumn(index).getType();
+    }
+
+    public void visitColumns(ColumnVisitor visitor)
+    {
+        for (Column column : columns) {
+            column.visit(visitor);
+        }
+    }
+
+    public boolean isEmpty()
+    {
+        return columns.isEmpty();
+    }
+
+    public Column lookupColumn(String name)
+    {
+        for (Column c : columns) {
+            if (c.getName().equals(name)) {
+                return c;
+            }
+        }
+        throw new SchemaConfigException(String.format("Column '%s' is not found", name));
+    }
+
+    public int getFixedStorageSize()
+    {
+        int total = 0;
+        for (Column column : columns) {
+            total += column.getType().getFixedStorageSize();
+        }
+        return total;
+    }
+
+    @Override
+    public boolean equals(Object obj)
+    {
+        if (this == obj) {
+            return true;
+        }
+        if (!(obj instanceof Schema)) {
+            return false;
+        }
+        Schema other = (Schema) obj;
+        return Objects.equals(columns, other.columns);
+    }
+
+    @Override
+    public int hashCode()
+    {
+        return Objects.hashCode(columns);
+    }
+
+    @Override
+    public String toString()
+    {
+        StringBuilder sbuf = new StringBuilder();
+        sbuf.append("Schema{\n");
+        for (Column c : columns) {
+            sbuf.append(String.format(" %4d: %s %s%n", c.getIndex(), c.getName(), c.getType()));
+        }
+        sbuf.append("}");
+        return sbuf.toString();
+    }
+}