embulk 0.7.0-java

data/embulk-core/src/main/java/org/embulk/exec/PreviewResult.java

@@ -0,0 +1,27 @@
+package org.embulk.exec;
+
+import java.util.List;
+import org.embulk.spi.Schema;
+import org.embulk.spi.Page;
+
+public class PreviewResult
+{
+    private final Schema schema;
+    private final List<Page> pages;
+
+    public PreviewResult(Schema schema, List<Page> pages)
+    {
+        this.schema = schema;
+        this.pages = pages;
+    }
+
+    public Schema getSchema()
+    {
+        return schema;
+    }
+
+    public List<Page> getPages()
+    {
+        return pages;
+    }
+}

data/embulk-core/src/main/java/org/embulk/exec/PreviewedNoticeError.java

@@ -0,0 +1,17 @@
+package org.embulk.exec;
+
+public class PreviewedNoticeError
+        extends Error
+{
+    private final PreviewResult previewResult;
+
+    public PreviewedNoticeError(PreviewResult previewResult)
+    {
+        this.previewResult = previewResult;
+    }
+
+    public PreviewResult getPreviewResult()
+    {
+        return previewResult;
+    }
+}

data/embulk-core/src/main/java/org/embulk/exec/ResumeState.java

@@ -0,0 +1,100 @@
+package org.embulk.exec;
+
+import java.util.List;
+import com.google.common.base.Optional;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import org.embulk.config.TaskSource;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskReport;
+import org.embulk.config.CommitReport;
+import org.embulk.spi.Schema;
+
+public class ResumeState
+{
+    private final ConfigSource execSessionConfigSource;
+    private final TaskSource inputTaskSource;
+    private final TaskSource outputTaskSource;
+    private final Schema inputSchema;
+    private final Schema outputSchema;
+    private final List<Optional<TaskReport>> inputTaskReports;
+    private final List<Optional<TaskReport>> outputTaskReports;
+
+    @JsonCreator
+    public ResumeState(
+            @JsonProperty("exec_task") ConfigSource execSessionConfigSource,
+            @JsonProperty("in_task") TaskSource inputTaskSource,
+            @JsonProperty("out_task") TaskSource outputTaskSource,
+            @JsonProperty("in_schema") Schema inputSchema,
+            @JsonProperty("out_schema") Schema outputSchema,
+            @JsonProperty("in_reports") List<Optional<TaskReport>> inputTaskReports,
+            @JsonProperty("out_reports") List<Optional<TaskReport>> outputTaskReports)
+    {
+        this.execSessionConfigSource = execSessionConfigSource;
+        this.inputTaskSource = inputTaskSource;
+        this.outputTaskSource = outputTaskSource;
+        this.inputSchema = inputSchema;
+        this.outputSchema = outputSchema;
+        this.inputTaskReports = inputTaskReports;
+        this.outputTaskReports = outputTaskReports;
+    }
+
+    @JsonProperty("exec_task")
+    public ConfigSource getExecSessionConfigSource()
+    {
+        return execSessionConfigSource;
+    }
+
+    @JsonProperty("in_task")
+    public TaskSource getInputTaskSource()
+    {
+        return inputTaskSource;
+    }
+
+    @JsonProperty("out_task")
+    public TaskSource getOutputTaskSource()
+    {
+        return outputTaskSource;
+    }
+
+    @JsonProperty("in_schema")
+    public Schema getInputSchema()
+    {
+        return inputSchema;
+    }
+
+    @JsonProperty("out_schema")
+    public Schema getOutputSchema()
+    {
+        return outputSchema;
+    }
+
+    @JsonProperty("in_reports")
+    public List<Optional<TaskReport>> getInputTaskReports()
+    {
+        return inputTaskReports;
+    }
+
+    @Deprecated
+    @JsonIgnore
+    @SuppressWarnings("unchecked")
+    public List<Optional<CommitReport>> getInputCommitReports()
+    {
+        return (List) inputTaskReports;  // the only implementation of TaskReport is DataSourceImpl which implements CommitReport
+    }
+
+    @JsonProperty("out_reports")
+    public List<Optional<TaskReport>> getOutputTaskReports()
+    {
+        return outputTaskReports;
+    }
+
+    @Deprecated
+    @JsonIgnore
+    @SuppressWarnings("unchecked")
+    public List<Optional<CommitReport>> getOutputCommitReports()
+    {
+        return (List) outputTaskReports;  // the only implementation of TaskReport is DataSourceImpl which implements CommitReport;
+    }
+}

data/embulk-core/src/main/java/org/embulk/exec/SamplingParserPlugin.java

@@ -0,0 +1,136 @@
+package org.embulk.exec;
+
+import java.util.List;
+import com.google.inject.Inject;
+import com.google.common.base.Preconditions;
+import org.embulk.config.TaskSource;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskReport;
+import org.embulk.spi.Schema;
+import org.embulk.spi.Exec;
+import org.embulk.spi.Page;
+import org.embulk.spi.Buffer;
+import org.embulk.spi.InputPlugin;
+import org.embulk.spi.ParserPlugin;
+import org.embulk.spi.FileInput;
+import org.embulk.spi.FileInputRunner;
+import org.embulk.spi.PageOutput;
+import static org.embulk.spi.util.Inputs.each;
+
+/*
+ * Used by FileInputRunner.guess
+ */
+public class SamplingParserPlugin
+        implements ParserPlugin
+{
+    public static Buffer runFileInputSampling(final FileInputRunner runner, ConfigSource inputConfig)
+    {
+        // override in.parser.type so that FileInputRunner creates SamplingParserPlugin
+        ConfigSource samplingInputConfig = inputConfig.deepCopy();
+        samplingInputConfig.getNestedOrSetEmpty("parser").set("type", "system_sampling");
+        samplingInputConfig.set("decoders", null);
+
+        try {
+            runner.transaction(samplingInputConfig, new InputPlugin.Control() {
+                public List<TaskReport> run(TaskSource taskSource, Schema schema, int taskCount)
+                {
+                    if (taskCount == 0) {
+                        throw new NoSampleException("No input files to read sample data");
+                    }
+                    for (int taskIndex=0; taskIndex < taskCount; taskIndex++) {
+                        try {
+                            runner.run(taskSource, schema, taskIndex, new PageOutput() {
+                                @Override
+                                public void add(Page page)
+                                {
+                                    throw new RuntimeException("Input plugin must be a FileInputPlugin to guess parser configuration");  // TODO exception class
+                                }
+
+                                public void finish() { }
+
+                                public void close() { }
+                            });
+                        } catch (NotEnoughSampleError ex) {
+                            continue;
+                        }
+                    }
+                    throw new NoSampleException("All input files are smaller than minimum sampling size");  // TODO include minSampleSize in message
+                }
+            });
+            throw new AssertionError("SamplingParserPlugin must throw SampledNoticeError");
+        } catch (SampledNoticeError error) {
+            return error.getSample();
+        }
+    }
+
+    public static class SampledNoticeError
+            extends Error
+    {
+        private final Buffer sample;
+
+        public SampledNoticeError(Buffer sample)
+        {
+            this.sample = sample;
+        }
+
+        public Buffer getSample()
+        {
+            return sample;
+        }
+    }
+
+    public static class NotEnoughSampleError
+            extends Error
+    { }
+
+    private final int minSampleSize;
+    private final int sampleSize;
+
+    @Inject
+    public SamplingParserPlugin(@ForSystemConfig ConfigSource systemConfig)
+    {
+        this.minSampleSize = 40;  // empty gzip file is 33 bytes. // TODO get sample size from system config
+        this.sampleSize = 32*1024;  // TODO get sample size from system config
+        Preconditions.checkArgument(minSampleSize < sampleSize, "minSampleSize must be smaller than sampleSize");
+    }
+
+    @Override
+    public void transaction(ConfigSource config, ParserPlugin.Control control)
+    {
+        control.run(Exec.newTaskSource(), null);
+    }
+
+    @Override
+    public void run(TaskSource taskSource, Schema schema,
+            FileInput input, PageOutput output)
+    {
+        Buffer buffer = readSample(input, sampleSize);
+        if (buffer.limit() < minSampleSize) {
+            throw new NotEnoughSampleError();
+        }
+        throw new SampledNoticeError(buffer);
+    }
+
+    private static Buffer readSample(FileInput fileInput, int sampleSize)
+    {
+        if (!fileInput.nextFile()) {
+            // no input files
+            return Buffer.EMPTY;
+        }
+
+        Buffer sample = Buffer.allocate(sampleSize);
+        int offset = 0;
+
+        for (Buffer buffer : each(fileInput)) {
+            int size = Math.min(buffer.limit(), sample.capacity() - offset);
+            sample.setBytes(offset, buffer, 0, size);
+            offset += size;
+            buffer.release();
+            if (offset >= sampleSize) {
+                break;
+            }
+        }
+        sample.limit(offset);
+        return sample;
+    }
+}

data/embulk-core/src/main/java/org/embulk/exec/SetCurrentThreadName.java

@@ -0,0 +1,19 @@
+package org.embulk.exec;
+
+public class SetCurrentThreadName
+        implements AutoCloseable
+{
+    private final String original;
+
+    public SetCurrentThreadName(String name)
+    {
+        this.original = Thread.currentThread().getName();
+        Thread.currentThread().setName(name);
+    }
+
+    @Override
+    public void close()
+    {
+        Thread.currentThread().setName(original);
+    }
+}

data/embulk-core/src/main/java/org/embulk/exec/SystemConfigModule.java

@@ -0,0 +1,24 @@
+package org.embulk.exec;
+
+import com.google.inject.Module;
+import com.google.inject.Binder;
+import org.embulk.config.ConfigSource;
+
+public class SystemConfigModule
+        implements Module
+{
+    private final ConfigSource systemConfig;
+
+    public SystemConfigModule(ConfigSource systemConfig)
+    {
+        this.systemConfig = systemConfig;
+    }
+
+    @Override
+    public void configure(Binder binder)
+    {
+        binder.bind(ConfigSource.class)
+            .annotatedWith(ForSystemConfig.class)
+            .toInstance(systemConfig);
+    }
+}

data/embulk-core/src/main/java/org/embulk/exec/TempFileAllocator.java

@@ -0,0 +1,35 @@
+package org.embulk.exec;
+
+import java.io.File;
+import com.google.inject.Inject;
+import org.embulk.config.ConfigSource;
+import org.embulk.spi.TempFileSpace;
+
+// TODO change this class to interface
+// TODO don't use this class directly. Use spi.Exec.getTempFileSpace() instead.
+public class TempFileAllocator
+{
+    private final File[] dirs;
+
+    @Inject
+    public TempFileAllocator(@ForSystemConfig ConfigSource systemConfig)
+    {
+        // TODO get `temp_dirs` from system config
+        String s = System.getProperty("java.io.tmpdir");
+        if (s == null || s.isEmpty()) {
+            s = "/tmp";
+        }
+        this.dirs = new File[] {
+            new File(s, "embulk")
+        };
+    }
+
+    public TempFileSpace newSpace(String subdir)
+    {
+        // TODO support multiple directories
+        // UNIX/Linux cannot include '/' as file name.
+        // Windows cannot include ':' as file name.
+        subdir = subdir.replace('/', '-').replace(':', '-');
+        return new TempFileSpace(new File(dirs[0], subdir));
+    }
+}

data/embulk-core/src/main/java/org/embulk/guice/Bootstrap.java

@@ -0,0 +1,157 @@
+/*
+ * Copyright 2015 Sadayuki Furuhashi
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.embulk.guice;
+
+import com.google.common.base.Function;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.annotations.Beta;
+
+import com.google.inject.Binder;
+import com.google.inject.Guice;
+import com.google.inject.Module;
+import com.google.inject.Stage;
+import com.google.inject.Injector;
+import java.util.Arrays;
+import java.util.List;
+
+@Beta
+public class Bootstrap
+{
+    private List<Module> modules;
+
+    private boolean requireExplicitBindings = true;
+
+    private boolean started;
+
+    public Bootstrap(Module... modules)
+    {
+        this(Arrays.asList(modules));
+    }
+
+    public Bootstrap(Iterable<? extends Module> modules)
+    {
+        this.modules = ImmutableList.copyOf(modules);
+    }
+
+    public Bootstrap requireExplicitBindings(boolean requireExplicitBindings)
+    {
+        this.requireExplicitBindings = requireExplicitBindings;
+        return this;
+    }
+
+    public Bootstrap addModules(Module... additionalModules)
+    {
+        return addModules(Arrays.asList(additionalModules));
+    }
+
+    public Bootstrap addModules(Iterable<? extends Module> additionalModules)
+    {
+        modules = ImmutableList.copyOf(Iterables.concat(modules, additionalModules));
+        return this;
+    }
+
+    //public Bootstrap onPreDestroy()
+    //{
+    //}
+
+    //public Bootstrap onPreDestroyException()
+    //{
+    //}
+
+    //public Bootstrap onStop()
+    //{
+    //}
+
+    //public Bootstrap forEachModule(Consumer<? super Module> function)
+    //{
+    //    for (Module module : modules) {
+    //        function.accept(module);
+    //    }
+    //    return this;
+    //}
+
+    //public <T> Bootstrap forEachModule(Class<T> ifClass, Consumer<? super T> function)
+    //{
+    //    for (Module module : modules) {
+    //        if (ifClass.instance(module) {
+    //            function.accept(module);
+    //        }
+    //    }
+    //    return this;
+    //}
+
+    public Bootstrap overrideModules(Function<? super List<Module>, ? extends Iterable<? extends Module>> function)
+    {
+        modules = ImmutableList.copyOf(function.apply(modules));
+        return this;
+    }
+
+    public LifeCycleInjector initialize()
+    {
+        return build(true);
+    }
+
+    public CloseableInjector initializeCloseable()
+    {
+        return build(false);
+    }
+
+    private LifeCycleInjectorProxy build(boolean destroyOnShutdownHook)
+    {
+        Injector injector = start();
+        LifeCycleManager lifeCycleManager = injector.getInstance(LifeCycleManager.class);
+        if (destroyOnShutdownHook) {
+            lifeCycleManager.destroyOnShutdownHook();
+        }
+        return new LifeCycleInjectorProxy(injector, lifeCycleManager);
+    }
+
+    private Injector start()
+    {
+        if (started) {
+            throw new IllegalStateException("System already initialized");
+        }
+        started = true;
+
+        ImmutableList.Builder<Module> moduleList = ImmutableList.builder();
+
+        moduleList.addAll(modules);
+
+        moduleList.add(new Module()
+        {
+            @Override
+            public void configure(Binder binder)
+            {
+                binder.disableCircularProxies();
+                if (requireExplicitBindings) {
+                    binder.requireExplicitBindings();
+                }
+            }
+        });
+
+        moduleList.add(new LifeCycleModule());
+
+        Injector injector = Guice.createInjector(Stage.PRODUCTION, moduleList.build());
+
+        LifeCycleManager lifeCycleManager = injector.getInstance(LifeCycleManager.class);
+        if (lifeCycleManager.size() > 0) {
+            lifeCycleManager.start();
+        }
+
+        return injector;
+    }
+}