embulk 0.8.18-java → 0.8.19-java

data/embulk-core/build.gradle

@@ -31,7 +31,7 @@ dependencies {
     compile 'org.slf4j:slf4j-api:1.7.12'
     compile 'org.jruby:jruby-complete:' + project.jrubyVersion
     compile 'com.google.code.findbugs:annotations:3.0.0'
-    compile 'org.yaml:snakeyaml:1.14'
+    compile 'org.yaml:snakeyaml:1.18'
     compile 'javax.validation:validation-api:1.1.0.Final'
     compile 'org.apache.bval:bval-jsr303:0.5'
     compile 'io.airlift:slice:0.9'
@@ -45,7 +45,7 @@ dependencies {
 
     gems 'rubygems:bundler:1.10.6'
     gems 'rubygems:msgpack:1.1.0'
-    gems 'rubygems:liquid:3.0.6'
+    gems 'rubygems:liquid:4.0.0'
 }
 
 task unpackGems(type: JRubyPrepare) {

data/embulk-core/src/main/java/org/embulk/EmbulkVersion.java

@@ -0,0 +1,109 @@
+package org.embulk;
+
+import java.io.IOException;
+import java.net.URL;
+import java.security.CodeSource;
+import java.security.ProtectionDomain;
+import java.util.jar.Attributes;
+import java.util.jar.JarFile;
+import java.util.jar.Manifest;
+
+public final class EmbulkVersion
+{
+    private EmbulkVersion()
+    {
+    }
+
+    // Expecting Embulk is always packaged in the embulk-cli jar whenever the Embulk version is checked in Java.
+    static {
+        VERSION = getImplementationVersion(getSelfJarManifest(), "[embulk-version-unavailable]");
+    }
+
+    private static Manifest getSelfJarManifest()
+    {
+        try {
+            final ProtectionDomain protectionDomain;
+            try {
+                protectionDomain = EmbulkVersion.class.getProtectionDomain();
+            }
+            catch (SecurityException ex) {
+                System.err.println("Embulk version unavailable due to ProtectionDomain inaccessible.");
+                ex.printStackTrace();
+                return null;
+            }
+
+            final CodeSource codeSource = protectionDomain.getCodeSource();
+            if (codeSource == null) {
+                System.err.println("Embulk version unavailable due to CodeSource unavailable.");
+                return null;
+            }
+
+            final URL selfJarUrl = codeSource.getLocation();
+            if (selfJarUrl == null) {
+                System.err.println("Embulk version unavailable due to the location of CodeSource unavailable.");
+                return null;
+            }
+            else if (!selfJarUrl.getProtocol().equals("file")) {
+                System.err.println("Embulk version unavailable as the location of CodeSource is not local.");
+                return null;
+            }
+
+            final String selfJarPathString = selfJarUrl.getPath();
+            if (selfJarPathString == null) {
+                System.err.println("Embulk version unavailable due to the path of CodeSource unavailable.");
+                return null;
+            }
+            else if (selfJarPathString.isEmpty()) {
+                System.err.println("Embulk version unavailable due to the path of CodeSource empty.");
+                return null;
+            }
+
+            try (final JarFile selfJarFile = new JarFile(selfJarPathString)) {
+                try {
+                    return selfJarFile.getManifest();
+                }
+                catch (IllegalStateException ex) {
+                    System.err.println("Embulk version unavailable due to the jar file closed unexpectedly.");
+                    ex.printStackTrace();
+                    return null;
+                }
+                catch (IOException ex) {
+                    System.err.println("Embulk version unavailable due to failure to get the manifst in the jar file.");
+                    ex.printStackTrace();
+                    return null;
+                }
+            }
+            catch (SecurityException ex) {
+                System.err.println("Embulk version unavailable due to the jar file inaccessible.");
+                ex.printStackTrace();
+                return null;
+            }
+            catch (IOException ex) {
+                System.err.println("Embulk version unavailable due to failure to access the jar file.");
+                ex.printStackTrace();
+                return null;
+            }
+        }
+        catch (Throwable ex) {
+            System.err.println("Embulk version unavailable due to an unknown exception.");
+            ex.printStackTrace();
+            return null;
+        }
+    }
+
+    private static String getImplementationVersion(final Manifest manifest, final String defaultVersion)
+    {
+        if (manifest == null) {
+            return defaultVersion;
+        }
+        final Attributes mainAttributes = manifest.getMainAttributes();
+        final String implementationVersion = mainAttributes.getValue(Attributes.Name.IMPLEMENTATION_VERSION);
+        if (implementationVersion == null) {
+            System.err.println("Embulk version unavailable due to the manifest not containing Implementation-Version.");
+            return defaultVersion;
+        }
+        return implementationVersion;
+    }
+
+    public static final String VERSION;
+}

data/embulk-core/src/main/java/org/embulk/exec/GuessExecutor.java

@@ -55,6 +55,10 @@ public class GuessExecutor
         @Config("exclude_guess_plugins")
         @ConfigDefault("[]")
         public List<PluginType> getExcludeGuessPlugins();
+
+        @Config("guess_sample_buffer_bytes")
+        @ConfigDefault("32768") // 32 * 1024
+        public int getSampleBufferBytes();
     }
 
     public static void registerDefaultGuessPluginTo(Binder binder, PluginType type)
@@ -63,6 +67,13 @@ public class GuessExecutor
         multibinder.addBinding().toInstance(type);
     }
 
+    // Used by FileInputRunner#guess(..)
+    public static ConfigSource createSampleBufferConfigFromExecConfig(ConfigSource execConfig)
+    {
+        final GuessExecutorTask execTask = execConfig.loadConfig(GuessExecutorTask.class);
+        return Exec.newConfigSource().set("sample_buffer_bytes", execTask.getSampleBufferBytes());
+    }
+
     @Inject
     public GuessExecutor(@ForSystemConfig ConfigSource systemConfig,
             @ForGuess Set<PluginType> defaultGuessPlugins)

data/embulk-core/src/main/java/org/embulk/exec/PreviewExecutor.java

@@ -12,9 +12,9 @@ import org.embulk.config.Task;
 import org.embulk.config.TaskSource;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.TaskReport;
+import org.embulk.exec.SamplingParserPlugin.SampleBufferTask;
 import org.embulk.plugin.PluginType;
 import org.embulk.spi.Buffer;
-import org.embulk.spi.FileInputPlugin;
 import org.embulk.spi.FileInputRunner;
 import org.embulk.spi.Schema;
 import org.embulk.spi.Page;
@@ -26,6 +26,7 @@ import org.embulk.spi.Exec;
 import org.embulk.spi.ExecSession;
 import org.embulk.spi.ExecAction;
 import org.embulk.spi.util.Filters;
+import org.slf4j.Logger;
 
 public class PreviewExecutor
 {
@@ -35,6 +36,10 @@ public class PreviewExecutor
     public interface PreviewTask
             extends Task
     {
+        @Config("exec")
+        @ConfigDefault("{}")
+        public ConfigSource getExecConfig();
+
         @Config("in")
         @NotNull
         public ConfigSource getInputConfig();
@@ -52,6 +57,14 @@ public class PreviewExecutor
         public void setInputTask(TaskSource taskSource);
     }
 
+    public interface PreviewExecutorTask
+            extends Task
+    {
+        @Config("preview_sample_buffer_bytes")
+        @ConfigDefault("32768") // 32 * 1024
+        public int getSampleBufferBytes();
+    }
+
     @Inject
     public PreviewExecutor(Injector injector,
             @ForSystemConfig ConfigSource systemConfig)
@@ -93,7 +106,7 @@ public class PreviewExecutor
         List<FilterPlugin> filterPlugins = newFilterPlugins(task);
 
         if (inputPlugin instanceof FileInputRunner) { // file input runner
-            Buffer sample = SamplingParserPlugin.runFileInputSampling((FileInputRunner)inputPlugin, config.getNested("in"));
+            Buffer sample = SamplingParserPlugin.runFileInputSampling((FileInputRunner)inputPlugin, config.getNested("in"), createSampleBufferConfigFromExecConfig(config.getNested("exec")));
             FileInputRunner previewRunner = new FileInputRunner(new BufferFileInputPlugin(sample));
             return doPreview(task, previewRunner, filterPlugins);
         }
@@ -102,6 +115,12 @@ public class PreviewExecutor
         }
     }
 
+    private static ConfigSource createSampleBufferConfigFromExecConfig(ConfigSource execConfig)
+    {
+        final PreviewExecutorTask execTask = execConfig.loadConfig(PreviewExecutorTask.class);
+        return Exec.newConfigSource().set("sample_buffer_bytes", execTask.getSampleBufferBytes());
+    }
+
     private PreviewResult doPreview(final PreviewTask task, final InputPlugin input, final List<FilterPlugin> filterPlugins)
     {
         try {
@@ -144,16 +163,19 @@ public class PreviewExecutor
     private static class SamplingPageOutput
             implements PageOutput
     {
+        private final Logger log = Exec.getLogger(this.getClass());
         private final int sampleRows;
         private final Schema schema;
         private List<Page> pages;
         private int recordCount;
+        private PreviewResult res;
 
         public SamplingPageOutput(int sampleRows, Schema schema)
         {
             this.sampleRows = sampleRows;
             this.schema = schema;
             this.pages = new ArrayList<Page>();
+            this.res = null;
         }
 
         public int getRecordCount()
@@ -174,10 +196,14 @@ public class PreviewExecutor
         @Override
         public void finish()
         {
+            if (res != null) {
+                log.error("PreviewResult recreation will cause a bug. The plugin must call PageOutput#finish() only once.");
+            }
+
             if (recordCount == 0) {
                 throw new NoSampleException("No input records to preview");
             }
-            PreviewResult res = new PreviewResult(schema, pages);
+            res = new PreviewResult(schema, pages);
             pages = null;
             throw new PreviewedNoticeError(res);
         }

data/embulk-core/src/main/java/org/embulk/exec/SamplingParserPlugin.java

@@ -1,8 +1,13 @@
 package org.embulk.exec;
 
+import java.text.NumberFormat;
 import java.util.List;
+
 import com.google.inject.Inject;
 import com.google.common.base.Preconditions;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.Task;
 import org.embulk.config.TaskSource;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.TaskReport;
@@ -15,6 +20,9 @@ import org.embulk.spi.ParserPlugin;
 import org.embulk.spi.FileInput;
 import org.embulk.spi.FileInputRunner;
 import org.embulk.spi.PageOutput;
+import org.slf4j.Logger;
+
+import static java.util.Locale.ENGLISH;
 import static org.embulk.spi.util.Inputs.each;
 
 /*
@@ -25,9 +33,18 @@ public class SamplingParserPlugin
 {
     public static Buffer runFileInputSampling(final FileInputRunner runner, ConfigSource inputConfig)
     {
+        return runFileInputSampling(runner, inputConfig, Exec.newConfigSource());
+    }
+
+    public static Buffer runFileInputSampling(final FileInputRunner runner, ConfigSource inputConfig, ConfigSource sampleBufferConfig)
+    {
+        final SampleBufferTask sampleBufferTask = sampleBufferConfig.loadConfig(SampleBufferTask.class);
+
         // override in.parser.type so that FileInputRunner creates SamplingParserPlugin
         ConfigSource samplingInputConfig = inputConfig.deepCopy();
-        samplingInputConfig.getNestedOrSetEmpty("parser").set("type", "system_sampling");
+        samplingInputConfig.getNestedOrSetEmpty("parser")
+                .set("type", "system_sampling")
+                .set("sample_buffer_bytes", sampleBufferTask.getSampleBufferBytes());
         samplingInputConfig.set("decoders", null);
 
         try {
@@ -120,42 +137,59 @@ public class SamplingParserPlugin
         }
     }
 
-    private final int minSampleSize;
-    private final int sampleSize;
+    private final NumberFormat numberFormat = NumberFormat.getNumberInstance(ENGLISH);
+    private final Logger log = Exec.getLogger(this.getClass());
+    private final int minSampleBufferBytes;
+
+    public interface PluginTask
+            extends Task, SampleBufferTask
+    {
+    }
+
+    public interface SampleBufferTask
+            extends Task
+    {
+        @Config("sample_buffer_bytes")
+        @ConfigDefault("32768") // 32 * 1024
+        public int getSampleBufferBytes();
+    }
 
     @Inject
     public SamplingParserPlugin(@ForSystemConfig ConfigSource systemConfig)
     {
-        this.minSampleSize = 40;  // empty gzip file is 33 bytes. // TODO get sample size from system config
-        this.sampleSize = 32*1024;  // TODO get sample size from system config. See also GuessExecutor.run.
-        Preconditions.checkArgument(minSampleSize < sampleSize, "minSampleSize must be smaller than sampleSize");
+        this.minSampleBufferBytes = 40;  // empty gzip file is 33 bytes. // TODO get sample size from system config
     }
 
     @Override
     public void transaction(ConfigSource config, ParserPlugin.Control control)
     {
-        control.run(Exec.newTaskSource(), null);
+        PluginTask task = config.loadConfig(PluginTask.class);
+        Preconditions.checkArgument(minSampleBufferBytes < task.getSampleBufferBytes(), "minSampleBufferBytes must be smaller than sample_buffer_bytes");
+
+        log.info("Try to read {} bytes from input source", numberFormat.format(task.getSampleBufferBytes()));
+        control.run(task.dump(), null);
     }
 
     @Override
     public void run(TaskSource taskSource, Schema schema,
             FileInput input, PageOutput output)
     {
-        Buffer buffer = readSample(input, sampleSize);
+        PluginTask task = taskSource.loadTask(PluginTask.class);
+        Buffer buffer = readSample(input, task.getSampleBufferBytes());
         if (!taskSource.get(boolean.class, "force", false)) {
-            if (buffer.limit() < minSampleSize) {
+            if (buffer.limit() < minSampleBufferBytes) {
                 throw new NotEnoughSampleError(buffer.limit());
             }
         }
         throw new SampledNoticeError(buffer);
     }
 
-    public static Buffer readSample(FileInput fileInput, int sampleSize)
+    public static Buffer readSample(FileInput fileInput, int sampleBufferBytes)
     {
-        return readSample(fileInput, Buffer.allocate(sampleSize), 0, sampleSize);
+        return readSample(fileInput, Buffer.allocate(sampleBufferBytes), 0, sampleBufferBytes);
     }
 
-    public static Buffer readSample(FileInput fileInput, Buffer sample, int offset, int sampleSize)
+    public static Buffer readSample(FileInput fileInput, Buffer sample, int offset, int sampleBufferBytes)
     {
         if (!fileInput.nextFile()) {
             // no input files
@@ -168,7 +202,7 @@ public class SamplingParserPlugin
                 sample.setBytes(offset, buffer, 0, size);
                 offset += size;
                 buffer.release();
-                if (offset >= sampleSize) {
+                if (offset >= sampleBufferBytes) {
                     break;
                 }
             }

data/embulk-core/src/main/java/org/embulk/spi/FileInputRunner.java

@@ -17,6 +17,8 @@ import org.embulk.exec.SamplingParserPlugin;
 import org.embulk.exec.ConfigurableGuessInputPlugin;
 import org.embulk.exec.NoSampleException;
 
+import static org.embulk.exec.GuessExecutor.createSampleBufferConfigFromExecConfig;
+
 public class FileInputRunner
         implements InputPlugin, ConfigurableGuessInputPlugin
 {
@@ -80,14 +82,15 @@ public class FileInputRunner
         return guess(Exec.newConfigSource(), config);
     }
 
-    public ConfigDiff guess(ConfigSource execConfig, ConfigSource config)
+    public ConfigDiff guess(ConfigSource execConfig, ConfigSource inputConfig)
     {
-        Buffer sample = SamplingParserPlugin.runFileInputSampling(this, config);
+        final ConfigSource sampleBufferConfig = createSampleBufferConfigFromExecConfig(execConfig);
+        final Buffer sample = SamplingParserPlugin.runFileInputSampling(this, inputConfig, sampleBufferConfig);
         // SamplingParserPlugin.runFileInputSampling throws NoSampleException if there're
         // no files or all files are smaller than minSampleSize (40 bytes).
 
         GuessExecutor guessExecutor = Exec.getInjector().getInstance(GuessExecutor.class);
-        return guessExecutor.guessParserConfig(sample, config, execConfig);
+        return guessExecutor.guessParserConfig(sample, inputConfig, execConfig);
     }
 
     private class RunnerControl

data/embulk-core/src/main/java/org/embulk/spi/PageBuilder.java

@@ -1,16 +1,12 @@
 package org.embulk.spi;
 
-import java.io.Serializable;
-import java.util.Map;
 import java.util.List;
 import java.util.Arrays;
 import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.Collections;
-import com.google.common.collect.BiMap;
-import com.google.common.collect.HashBiMap;
 import io.airlift.slice.Slice;
 import io.airlift.slice.Slices;
+import org.embulk.spi.type.Type;
+import org.embulk.spi.type.Types;
 import org.msgpack.value.Value;
 import org.msgpack.value.ImmutableValue;
 import org.embulk.spi.time.Timestamp;
@@ -30,7 +26,8 @@ public class PageBuilder
     private int count;
     private int position;
     private final byte[] nullBitSet;
-    private final BiMap<String, Integer> stringReferences = HashBiMap.create();
+    private final Row row;
+    private List<String> stringReferences = new ArrayList<>();
     private List<ImmutableValue> valueReferences = new ArrayList<>();
     private int referenceSize;
     private int nextVariableLengthDataOffset;
@@ -43,6 +40,7 @@ public class PageBuilder
         this.columnOffsets = PageFormat.columnOffsets(schema);
         this.nullBitSet = new byte[PageFormat.nullBitSetSize(schema)];
         Arrays.fill(nullBitSet, (byte) -1);
+        this.row = Row.newRow(schema);
         this.fixedRecordSize = PageFormat.recordHeaderSize(schema) + PageFormat.totalColumnSize(schema);
         this.nextVariableLengthDataOffset = fixedRecordSize;
         newBuffer();
@@ -54,7 +52,7 @@ public class PageBuilder
         this.bufferSlice = Slices.wrappedBuffer(buffer.array(), buffer.offset(), buffer.capacity());
         this.count = 0;
         this.position = PageFormat.PAGE_HEADER_SIZE;
-        this.stringReferences.clear();
+        this.stringReferences = new ArrayList<>();
         this.valueReferences = new ArrayList<>();
         this.referenceSize = 0;
     }
@@ -71,12 +69,8 @@ public class PageBuilder
 
     public void setNull(int columnIndex)
     {
-        nullBitSet[columnIndex >>> 3] |= (1 << (columnIndex & 7));
-    }
+        row.setNull(columnIndex);
 
-    private void clearNull(int columnIndex)
-    {
-        nullBitSet[columnIndex >>> 3] &= ~(1 << (columnIndex & 7));
     }
 
     public void setBoolean(Column column, boolean value)
@@ -87,8 +81,7 @@ public class PageBuilder
 
     public void setBoolean(int columnIndex, boolean value)
     {
-        bufferSlice.setByte(getOffset(columnIndex), value ? (byte) 1 : (byte) 0);
-        clearNull(columnIndex);
+        row.setBoolean(columnIndex, value);
     }
 
     public void setLong(Column column, long value)
@@ -99,8 +92,7 @@ public class PageBuilder
 
     public void setLong(int columnIndex, long value)
     {
-        bufferSlice.setLong(getOffset(columnIndex), value);
-        clearNull(columnIndex);
+        row.setLong(columnIndex, value);
     }
 
     public void setDouble(Column column, double value)
@@ -111,8 +103,7 @@ public class PageBuilder
 
     public void setDouble(int columnIndex, double value)
     {
-        bufferSlice.setDouble(getOffset(columnIndex), value);
-        clearNull(columnIndex);
+        row.setDouble(columnIndex, value);
     }
 
     public void setString(Column column, String value)
@@ -125,19 +116,10 @@ public class PageBuilder
     {
         if (value == null) {
             setNull(columnIndex);
-            return;
         }
-
-        Integer reuseIndex = stringReferences.get(value);
-        if (reuseIndex != null) {
-            bufferSlice.setInt(getOffset(columnIndex), reuseIndex);
-        } else {
-            int index = stringReferences.size();
-            stringReferences.put(value, index);
-            bufferSlice.setInt(getOffset(columnIndex), index);
-            referenceSize += value.length() * 2 + 4;  // assuming size of char = size of byte * 2 + length
+        else {
+            row.setString(columnIndex, value);
         }
-        clearNull(columnIndex);
     }
 
     public void setJson(Column column, Value value)
@@ -150,14 +132,10 @@ public class PageBuilder
     {
         if (value == null) {
             setNull(columnIndex);
-            return;
         }
-
-        int index = valueReferences.size();
-        valueReferences.add(value.immutableValue());
-        bufferSlice.setInt(getOffset(columnIndex), index);
-        referenceSize += 256;  // TODO how to estimate size of the value?
-        clearNull(columnIndex);
+        else {
+            row.setJson(columnIndex, value);
+        }
     }
 
     public void setTimestamp(Column column, Timestamp value)
@@ -170,49 +148,76 @@ public class PageBuilder
     {
         if (value == null) {
             setNull(columnIndex);
-            return;
         }
+        else {
+            row.setTimestamp(columnIndex, value);
+        }
+    }
 
-        int offset = getOffset(columnIndex);
-        bufferSlice.setLong(offset, value.getEpochSecond());
-        bufferSlice.setInt(offset + 8, value.getNano());
+    private void writeNull(int columnIndex)
+    {
+        nullBitSet[columnIndex >>> 3] |= (1 << (columnIndex & 7));
+    }
+
+    private void clearNull(int columnIndex)
+    {
+        nullBitSet[columnIndex >>> 3] &= ~(1 << (columnIndex & 7));
+    }
+
+    private void writeBoolean(int columnIndex, boolean value)
+    {
+        bufferSlice.setByte(getOffset(columnIndex), value ? (byte) 1 : (byte) 0);
         clearNull(columnIndex);
     }
 
-    private int getOffset(int columnIndex)
+    private void writeLong(int columnIndex, long value)
     {
-        return position + columnOffsets[columnIndex];
+        bufferSlice.setLong(getOffset(columnIndex), value);
+        clearNull(columnIndex);
     }
 
-    private static class StringReferenceSortComparator
-            implements Comparator<Map.Entry<String, Integer>>, Serializable
+    private void writeDouble(int columnIndex, double value)
     {
-        @Override
-        public int compare(Map.Entry<String, Integer> e1, Map.Entry<String, Integer> e2)
-        {
-            return e1.getValue().compareTo(e2.getValue());
-        }
+        bufferSlice.setDouble(getOffset(columnIndex), value);
+        clearNull(columnIndex);
+    }
 
-        @Override
-        public boolean equals(Object obj)
-        {
-            return obj instanceof StringReferenceSortComparator;
-        }
+    private void writeString(int columnIndex, String value)
+    {
+        int index = stringReferences.size();
+        stringReferences.add(value);
+        bufferSlice.setInt(getOffset(columnIndex), index);
+        referenceSize += value.length() * 2 + 4;  // assuming size of char = size of byte * 2 + length
+        clearNull(columnIndex);
     }
 
-    private List<String> getSortedStringReferences()
+    private void writeJson(int columnIndex, Value value)
     {
-        ArrayList<Map.Entry<String, Integer>> s = new ArrayList<>(stringReferences.entrySet());
-        Collections.sort(s, new StringReferenceSortComparator());
-        String[] array = new String[s.size()];
-        for (int i=0; i < array.length; i++) {
-            array[i] = s.get(i).getKey();
-        }
-        return Arrays.asList(array);
+        int index = valueReferences.size();
+        valueReferences.add(value.immutableValue());
+        bufferSlice.setInt(getOffset(columnIndex), index);
+        referenceSize += 256;  // TODO how to estimate size of the value?
+        clearNull(columnIndex);
+    }
+
+    private void writeTimestamp(int columnIndex, Timestamp value)
+    {
+        int offset = getOffset(columnIndex);
+        bufferSlice.setLong(offset, value.getEpochSecond());
+        bufferSlice.setInt(offset + 8, value.getNano());
+        clearNull(columnIndex);
+    }
+
+    private int getOffset(int columnIndex)
+    {
+        return position + columnOffsets[columnIndex];
     }
 
     public void addRecord()
     {
+        // record
+        row.write(this);
+
         // record header
         bufferSlice.setInt(position, nextVariableLengthDataOffset);  // nextVariableLengthDataOffset means record size
         bufferSlice.setBytes(position + 4, nullBitSet);
@@ -237,7 +242,7 @@ public class PageBuilder
 
             // flush page
             Page page = Page.wrap(buffer)
-                .setStringReferences(getSortedStringReferences())
+                .setStringReferences(stringReferences)
                 .setValueReferences(valueReferences);
             buffer = null;
             bufferSlice = null;
@@ -270,6 +275,322 @@ public class PageBuilder
         output.close();
     }
 
+    /**
+     * Row is a container to stage values before adding into reference lists such as |stringReferences|.
+     *
+     * |Row| works as a buffer against plugins that may add values incorrectly without |PageBuilder#addRecord|.
+     * It accepts just one value per column while |PageBuilder| can double-store values regardless of columns.
+     * Double-stored values are overwritten.
+     */
+    private static class Row
+    {
+        private static Row newRow(Schema schema)
+        {
+            ColumnValue[] values = new ColumnValue[schema.getColumnCount()];
+            for (Column column : schema.getColumns()) {
+                values[column.getIndex()] = newValue(column);
+            }
+            return new Row(values);
+        }
+
+        private static ColumnValue newValue(Column column)
+        {
+            Type type = column.getType();
+            if (type.equals(Types.BOOLEAN)) {
+                return new BooleanColumnValue(column);
+            }
+            else if (type.equals(Types.DOUBLE)) {
+                return new DoubleColumnValue(column);
+            }
+            else if (type.equals(Types.LONG)) {
+                return new LongColumnValue(column);
+            }
+            else if (type.equals(Types.STRING)) {
+                return new StringColumnValue(column);
+            }
+            else if (type.equals(Types.JSON)) {
+                return new JsonColumnValue(column);
+            }
+            else if (type.equals(Types.TIMESTAMP)) {
+                return new TimestampColumnValue(column);
+            }
+            else {
+                throw new IllegalStateException("Unsupported type " + type.getName());
+            }
+        }
+
+        private final ColumnValue[] values;
+
+        private Row(ColumnValue[] values)
+        {
+            this.values = values;
+        }
+
+        private void setNull(int columnIndex)
+        {
+            values[columnIndex].setNull();
+        }
+
+        private void setBoolean(int columnIndex, boolean value)
+        {
+            values[columnIndex].setBoolean(value);
+        }
+
+        private void setLong(int columnIndex, long value)
+        {
+            values[columnIndex].setLong(value);
+        }
+
+        private void setDouble(int columnIndex, double value)
+        {
+            values[columnIndex].setDouble(value);
+        }
+
+        private void setString(int columnIndex, String value)
+        {
+            values[columnIndex].setString(value);
+        }
+
+        private void setJson(int columnIndex, Value value)
+        {
+            values[columnIndex].setJson(value);
+        }
+
+        private void setTimestamp(int columnIndex, Timestamp value)
+        {
+            values[columnIndex].setTimestamp(value);
+        }
+
+        private void write(PageBuilder pageBuilder)
+        {
+            for (ColumnValue v : values) {
+                v.write(pageBuilder);
+            }
+        }
+    }
+
+    private interface ColumnValue
+    {
+        void setBoolean(boolean value);
+
+        void setLong(long value);
+
+        void setDouble(double value);
+
+        void setString(String value);
+
+        void setJson(Value value);
+
+        void setTimestamp(Timestamp value);
+
+        void setNull();
+
+        void write(PageBuilder pageBuilder);
+    }
+
+    private static abstract class AbstractColumnValue
+            implements ColumnValue
+    {
+        protected final Column column;
+        protected boolean isNull;
+
+        protected AbstractColumnValue(Column column)
+        {
+            this.column = column;
+        }
+
+        public void setBoolean(boolean value)
+        {
+            throw new IllegalStateException("Not reach here");
+        }
+
+        public void setLong(long value)
+        {
+            throw new IllegalStateException("Not reach here");
+        }
+
+        public void setDouble(double value)
+        {
+            throw new IllegalStateException("Not reach here");
+        }
+
+        public void setString(String value)
+        {
+            throw new IllegalStateException("Not reach here");
+        }
+
+        public void setJson(Value value)
+        {
+            throw new IllegalStateException("Not reach here");
+        }
+
+        public void setTimestamp(Timestamp value)
+        {
+            throw new IllegalStateException("Not reach here");
+        }
+
+        public void setNull()
+        {
+            isNull = true;
+        }
+
+        public void write(PageBuilder pageBuilder)
+        {
+            if (!isNull) {
+                writeNotNull(pageBuilder);
+            }
+            else {
+                pageBuilder.writeNull(column.getIndex());
+            }
+        }
+
+        protected abstract void writeNotNull(PageBuilder pageBuilder);
+    }
+
+    private static class BooleanColumnValue
+            extends AbstractColumnValue
+    {
+        private boolean value;
+
+        BooleanColumnValue(Column column)
+        {
+            super(column);
+        }
+
+        @Override
+        public void setBoolean(boolean value)
+        {
+            this.value = value;
+            this.isNull = false;
+        }
+
+        @Override
+        public void writeNotNull(PageBuilder pageBuilder)
+        {
+            pageBuilder.writeBoolean(column.getIndex(), value);
+        }
+    }
+
+    private static class LongColumnValue
+            extends AbstractColumnValue
+    {
+        private long value;
+
+        LongColumnValue(Column column)
+        {
+            super(column);
+        }
+
+        @Override
+        public void setLong(long value)
+        {
+            this.value = value;
+            this.isNull = false;
+        }
+
+        @Override
+        public void writeNotNull(PageBuilder pageBuilder)
+        {
+            pageBuilder.writeLong(column.getIndex(), value);
+        }
+    }
+
+    private static class DoubleColumnValue
+            extends AbstractColumnValue
+    {
+        private double value;
+
+        DoubleColumnValue(Column column)
+        {
+            super(column);
+        }
+
+        @Override
+        public void setDouble(double value)
+        {
+            this.value = value;
+            this.isNull = false;
+        }
+
+        @Override
+        public void writeNotNull(PageBuilder pageBuilder)
+        {
+            pageBuilder.writeDouble(column.getIndex(), value);
+        }
+    }
+
+    private static class StringColumnValue
+            extends AbstractColumnValue
+    {
+        private String value;
+
+        StringColumnValue(Column column)
+        {
+            super(column);
+        }
+
+        @Override
+        public void setString(String value)
+        {
+            this.value = value;
+            this.isNull = false;
+        }
+
+        @Override
+        public void writeNotNull(PageBuilder pageBuilder)
+        {
+            pageBuilder.writeString(column.getIndex(), value);
+        }
+    }
+
+    private static class JsonColumnValue
+            extends AbstractColumnValue
+    {
+        private Value value;
+
+        JsonColumnValue(Column column)
+        {
+            super(column);
+        }
+
+        @Override
+        public void setJson(Value value)
+        {
+            this.value = value;
+            this.isNull = false;
+        }
+
+        @Override
+        public void writeNotNull(PageBuilder pageBuilder)
+        {
+            pageBuilder.writeJson(column.getIndex(), value);
+        }
+    }
+
+    private static class TimestampColumnValue
+            extends AbstractColumnValue
+    {
+        private Timestamp value;
+
+        TimestampColumnValue(Column column)
+        {
+            super(column);
+        }
+
+        @Override
+        public void setTimestamp(Timestamp value)
+        {
+            this.value = value;
+            this.isNull = false;
+        }
+
+        @Override
+        public void writeNotNull(PageBuilder pageBuilder)
+        {
+            pageBuilder.writeTimestamp(column.getIndex(), value);
+        }
+    }
+
     /* TODO for variable-length types
     private void flushAndTakeOverRemaingData()
     {