embulk 0.8.36-java → 0.10.25-java

data/embulk-core/src/main/java/org/embulk/spi/PageFormat.java

@@ -1,47 +0,0 @@
-package org.embulk.spi;
-
-abstract class PageFormat
-{
-    // PageHeader
-    // +---+
-    // | 4 |
-    // +---+
-    // count (number of records)
-
-    private PageFormat() { }
-
-    static final int PAGE_HEADER_SIZE = 4;
-
-    // PageBuilder.setVariableLengthData and PageReader.readVariableLengthData
-    // uses 4 bytes integer
-    static final int VARIABLE_LENGTH_COLUMN_SIZE = 4;
-
-    static int nullBitSetSize(Schema schema)
-    {
-        return (schema.getColumnCount() + 7) / 8;
-    }
-
-    static int recordHeaderSize(Schema schema)
-    {
-        return 4 + nullBitSetSize(schema);
-    }
-
-    static int totalColumnSize(Schema schema)
-    {
-        return recordHeaderSize(schema) + schema.getFixedStorageSize();
-    }
-
-    static int[] columnOffsets(Schema schema)
-    {
-        int[] offsets = new int[schema.getColumnCount()];
-
-        if (!schema.isEmpty()) {
-            offsets[0] = recordHeaderSize(schema);
-            for (int i=0; i < schema.getColumnCount()-1; i++) {
-                offsets[i+1] = offsets[i] + schema.getColumnType(i).getFixedStorageSize();
-            }
-        }
-
-        return offsets;
-    }
-}

data/embulk-core/src/main/java/org/embulk/spi/PageOutput.java

@@ -1,11 +0,0 @@
-package org.embulk.spi;
-
-public interface PageOutput
-        extends AutoCloseable
-{
-    void add(Page page);
-
-    void finish();
-
-    void close();
-}

data/embulk-core/src/main/java/org/embulk/spi/PageReader.java

@@ -1,248 +0,0 @@
-package org.embulk.spi;
-
-import io.airlift.slice.Slice;
-import io.airlift.slice.Slices;
-import org.msgpack.value.Value;
-import org.embulk.spi.time.Timestamp;
-
-public class PageReader
-        implements AutoCloseable
-{
-    private final Schema schema;
-    private final int[] columnOffsets;
-
-    private Page page = SENTINEL;
-    private Slice pageSlice = null;
-    private int pageRecordCount = 0;
-
-    private int readCount = 0;
-    private int position;
-    private final byte[] nullBitSet;
-
-    private static final Page SENTINEL = Page.wrap(Buffer.wrap(new byte[4]));  // buffer().release() does nothing
-
-    public PageReader(Schema schema)
-    {
-        this.schema = schema;
-        this.columnOffsets = PageFormat.columnOffsets(schema);
-        this.nullBitSet = new byte[PageFormat.nullBitSetSize(schema)];
-    }
-
-    public static int getRecordCount(Page page)
-    {
-        Buffer pageBuffer = page.buffer();
-        Slice pageSlice = Slices.wrappedBuffer(pageBuffer.array(), pageBuffer.offset(), pageBuffer.limit());
-        return pageSlice.getInt(0);  // see page format
-    }
-
-    public void setPage(Page page)
-    {
-        this.page.buffer().release();
-        this.page = SENTINEL;
-
-        Buffer pageBuffer = page.buffer();
-        Slice pageSlice = Slices.wrappedBuffer(pageBuffer.array(), pageBuffer.offset(), pageBuffer.limit());
-
-        pageRecordCount = pageSlice.getInt(0);  // see page format
-        readCount = 0;
-        position = PageFormat.PAGE_HEADER_SIZE;
-
-        this.page = page;
-        this.pageSlice = pageSlice;
-    }
-
-    public Schema getSchema()
-    {
-        return schema;
-    }
-
-    public boolean isNull(Column column)
-    {
-        return isNull(column.getIndex());
-    }
-
-    public boolean isNull(int columnIndex)
-    {
-        return (nullBitSet[columnIndex >>> 3] & (1 << (columnIndex & 7))) != 0;
-    }
-
-    public boolean getBoolean(Column column)
-    {
-        // TODO check type?
-        return getBoolean(column.getIndex());
-    }
-
-    public boolean getBoolean(int columnIndex)
-    {
-        return pageSlice.getByte(getOffset(columnIndex)) != (byte) 0;
-    }
-
-    public long getLong(Column column)
-    {
-        // TODO check type?
-        return getLong(column.getIndex());
-    }
-
-    public long getLong(int columnIndex)
-    {
-        return pageSlice.getLong(getOffset(columnIndex));
-    }
-
-    public double getDouble(Column column)
-    {
-        // TODO check type?
-        return getDouble(column.getIndex());
-    }
-
-    public double getDouble(int columnIndex)
-    {
-        return pageSlice.getDouble(getOffset(columnIndex));
-    }
-
-    public String getString(Column column)
-    {
-        // TODO check type?
-        return getString(column.getIndex());
-    }
-
-    public String getString(int columnIndex)
-    {
-        if (isNull(columnIndex)) {
-            return null;
-        }
-        int index = pageSlice.getInt(getOffset(columnIndex));
-        return page.getStringReference(index);
-    }
-
-    public Timestamp getTimestamp(Column column)
-    {
-        // TODO check type?
-        return getTimestamp(column.getIndex());
-    }
-
-    public Timestamp getTimestamp(int columnIndex)
-    {
-        if (isNull(columnIndex)) {
-            return null;
-        }
-        int offset = getOffset(columnIndex);
-        long sec = pageSlice.getLong(offset);
-        int nsec = pageSlice.getInt(offset + 8);
-        return Timestamp.ofEpochSecond(sec, nsec);
-    }
-
-    public Value getJson(Column column)
-    {
-        // TODO check type?
-        return getJson(column.getIndex());
-    }
-
-    public Value getJson(int columnIndex)
-    {
-        if (isNull(columnIndex)) {
-            return null;
-        }
-        int index = pageSlice.getInt(getOffset(columnIndex));
-        return page.getValueReference(index);
-    }
-
-    private int getOffset(int columnIndex)
-    {
-        return position + columnOffsets[columnIndex];
-    }
-
-    public boolean nextRecord()
-    {
-        if (pageRecordCount <= readCount) {
-            return false;
-        }
-
-        if (readCount > 0) {
-            // advance position excepting the first record
-            int lastRecordSize = pageSlice.getInt(position);
-            position += lastRecordSize;
-        }
-
-        readCount++;
-        pageSlice.getBytes(position + 4, nullBitSet, 0, nullBitSet.length);
-
-        return true;
-    }
-
-    @Override
-    public void close()
-    {
-        page.buffer().release();
-        page = SENTINEL;
-    }
-
-    /* TODO for variable-length types
-    public VariableLengthDataReader getVariableLengthData(int columnIndex, int variableLengthDataOffset)
-    {
-        return new VariableLengthDataReader(variableLengthDataOffset);
-    }
-
-    public class VariableLengthDataReader
-    {
-        private int offsetFromPosition;
-
-        VariableLengthDataReader(int offsetFromPosition)
-        {
-            this.offsetFromPosition = offsetFromPosition;
-        }
-
-        public byte readByte()
-        {
-            byte value = page.getByte(position + offsetFromPosition);
-            offsetFromPosition += 1;
-            return value;
-        }
-
-        public short readShort()
-        {
-            short value = page.getShort(position + offsetFromPosition);
-            offsetFromPosition += 2;
-            return value;
-        }
-
-        public int readInt()
-        {
-            int value = page.getInt(position + offsetFromPosition);
-            offsetFromPosition += 4;
-            return value;
-        }
-
-        public long readLong()
-        {
-            long value = page.getLong(position + offsetFromPosition);
-            offsetFromPosition += 8;
-            return value;
-        }
-
-        public float readFloat()
-        {
-            float value = page.getFloat(position + offsetFromPosition);
-            offsetFromPosition += 4;
-            return value;
-        }
-
-        public double readDouble()
-        {
-            double value = page.getDouble(position + offsetFromPosition);
-            offsetFromPosition += 8;
-            return value;
-        }
-
-        public void readBytes(byte[] data)
-        {
-            readBytes(data, 0, data.length);
-        }
-
-        public void readBytes(byte[] data, int off, int len)
-        {
-            page.getBytes(position + offsetFromPosition, data, off, len);
-            offsetFromPosition += len;
-        }
-    }
-    */
-}

data/embulk-core/src/main/java/org/embulk/spi/ParserPlugin.java

@@ -1,17 +0,0 @@
-package org.embulk.spi;
-
-import org.embulk.config.TaskSource;
-import org.embulk.config.ConfigSource;
-
-public interface ParserPlugin
-{
-    interface Control
-    {
-        void run(TaskSource taskSource, Schema schema);
-    }
-
-    void transaction(ConfigSource config, ParserPlugin.Control control);
-
-    void run(TaskSource taskSource, Schema schema,
-            FileInput input, PageOutput output);
-}

data/embulk-core/src/main/java/org/embulk/spi/ProcessState.java

@@ -1,10 +0,0 @@
-package org.embulk.spi;
-
-public interface ProcessState
-{
-    void initialize(int inputTaskCount, int outputTaskCount);
-
-    TaskState getInputTaskState(int inputTaskIndex);
-
-    TaskState getOutputTaskState(int outputTaskIndex);
-}

data/embulk-core/src/main/java/org/embulk/spi/ProcessTask.java

@@ -1,117 +0,0 @@
-package org.embulk.spi;
-
-import java.util.List;
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import org.embulk.plugin.PluginType;
-import org.embulk.config.TaskSource;
-import org.embulk.spi.util.Executors;
-
-public class ProcessTask
-{
-    private final PluginType inputPluginType;
-    private final PluginType outputPluginType;
-    private final List<PluginType> filterPluginTypes;
-    private final TaskSource inputTaskSource;
-    private final TaskSource outputTaskSource;
-    private final List<TaskSource> filterTaskSources;
-    private final List<Schema> schemas;
-    private final Schema executorSchema;
-    private TaskSource executorTaskSource;
-
-    @JsonCreator
-    public ProcessTask(
-            @JsonProperty("inputType") PluginType inputPluginType,
-            @JsonProperty("outputType") PluginType outputPluginType,
-            @JsonProperty("filterTypes") List<PluginType> filterPluginTypes,
-            @JsonProperty("inputTask") TaskSource inputTaskSource,
-            @JsonProperty("outputTask") TaskSource outputTaskSource,
-            @JsonProperty("filterTasks") List<TaskSource> filterTaskSources,
-            @JsonProperty("schemas") List<Schema> schemas,
-            @JsonProperty("executorSchema") Schema executorSchema,
-            @JsonProperty("executorTask") TaskSource executorTaskSource)
-    {
-        this.inputPluginType = inputPluginType;
-        this.outputPluginType = outputPluginType;
-        this.filterPluginTypes = filterPluginTypes;
-        this.inputTaskSource = inputTaskSource;
-        this.outputTaskSource = outputTaskSource;
-        this.filterTaskSources = filterTaskSources;
-        this.schemas = schemas;
-        this.executorSchema = executorSchema;
-        this.executorTaskSource = executorTaskSource;
-    }
-
-    @JsonProperty("inputType")
-    public PluginType getInputPluginType()
-    {
-        return inputPluginType;
-    }
-
-    @JsonProperty("outputType")
-    public PluginType getOutputPluginType()
-    {
-        return outputPluginType;
-    }
-
-    @JsonProperty("filterTypes")
-    public List<PluginType> getFilterPluginTypes()
-    {
-        return filterPluginTypes;
-    }
-
-    @JsonProperty("inputTask")
-    public TaskSource getInputTaskSource()
-    {
-        return inputTaskSource;
-    }
-
-    @JsonProperty("outputTask")
-    public TaskSource getOutputTaskSource()
-    {
-        return outputTaskSource;
-    }
-
-    @JsonProperty("filterTasks")
-    public List<TaskSource> getFilterTaskSources()
-    {
-        return filterTaskSources;
-    }
-
-    @JsonProperty("schemas")
-    public List<Schema> getFilterSchemas()
-    {
-        return schemas;
-    }
-
-    @JsonProperty("executorSchema")
-    public Schema getExecutorSchema()
-    {
-        return executorSchema;
-    }
-
-    @JsonIgnore
-    public Schema getInputSchema()
-    {
-        return Executors.getInputSchema(schemas);
-    }
-
-    @JsonIgnore
-    public Schema getOutputSchema()
-    {
-        return Executors.getOutputSchema(schemas);
-    }
-
-    @JsonIgnore
-    public void setExecutorTaskSource(TaskSource executorTaskSource)
-    {
-        this.executorTaskSource = executorTaskSource;
-    }
-
-    @JsonProperty("executorTask")
-    public TaskSource getExecutorTaskSource()
-    {
-        return executorTaskSource;
-    }
-}