embulk 0.7.0-java

data/embulk-standards/src/main/java/org/embulk/standards/GzipFileDecoderPlugin.java

@@ -0,0 +1,55 @@
+package org.embulk.standards;
+
+import java.io.InputStream;
+import java.io.IOException;
+import java.util.zip.GZIPInputStream;
+import org.embulk.config.Task;
+import org.embulk.config.TaskSource;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.ConfigInject;
+import org.embulk.spi.DecoderPlugin;
+import org.embulk.spi.BufferAllocator;
+import org.embulk.spi.FileInput;
+import org.embulk.spi.util.FileInputInputStream;
+import org.embulk.spi.util.InputStreamFileInput;
+
+public class GzipFileDecoderPlugin
+        implements DecoderPlugin
+{
+    public interface PluginTask
+            extends Task
+    {
+        @ConfigInject
+        BufferAllocator getBufferAllocator();
+    }
+
+    @Override
+    public void transaction(ConfigSource config, DecoderPlugin.Control control)
+    {
+        PluginTask task = config.loadConfig(PluginTask.class);
+        control.run(task.dump());
+    }
+
+    @Override
+    public FileInput open(TaskSource taskSource, FileInput fileInput)
+    {
+        PluginTask task = taskSource.loadTask(PluginTask.class);
+        final FileInputInputStream files = new FileInputInputStream(fileInput);
+        return new InputStreamFileInput(
+                task.getBufferAllocator(),
+                new InputStreamFileInput.Provider() {
+                    public InputStream openNext() throws IOException
+                    {
+                        if (!files.nextFile()) {
+                            return null;
+                        }
+                        return new GZIPInputStream(files);
+                    }
+
+                    public void close() throws IOException
+                    {
+                        files.close();
+                    }
+                });
+    }
+}

data/embulk-standards/src/main/java/org/embulk/standards/GzipFileEncoderPlugin.java

@@ -0,0 +1,71 @@
+package org.embulk.standards;
+
+import java.io.OutputStream;
+import java.io.IOException;
+import java.util.zip.GZIPOutputStream;
+import javax.validation.constraints.Min;
+import javax.validation.constraints.Max;
+import org.embulk.config.Task;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigInject;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.TaskSource;
+import org.embulk.config.ConfigSource;
+import org.embulk.spi.EncoderPlugin;
+import org.embulk.spi.FileOutput;
+import org.embulk.spi.BufferAllocator;
+import org.embulk.spi.util.FileOutputOutputStream;
+import org.embulk.spi.util.OutputStreamFileOutput;
+
+public class GzipFileEncoderPlugin
+        implements EncoderPlugin
+{
+    public interface PluginTask
+            extends Task
+    {
+        @Config("level")
+        @ConfigDefault("6")
+        @Min(0)
+        @Max(9)
+        int getLevel();
+
+        @ConfigInject
+        BufferAllocator getBufferAllocator();
+    }
+
+    public void transaction(ConfigSource config, EncoderPlugin.Control control)
+    {
+        PluginTask task = config.loadConfig(PluginTask.class);
+        control.run(task.dump());
+    }
+
+    @Override
+    public FileOutput open(TaskSource taskSource, final FileOutput fileOutput)
+    {
+        final PluginTask task = taskSource.loadTask(PluginTask.class);
+
+        final FileOutputOutputStream output = new FileOutputOutputStream(fileOutput, task.getBufferAllocator(), FileOutputOutputStream.CloseMode.FLUSH);
+
+        return new OutputStreamFileOutput(new OutputStreamFileOutput.Provider() {
+            public OutputStream openNext() throws IOException
+            {
+                output.nextFile();
+                return new GZIPOutputStream(output) {
+                    {
+                        this.def.setLevel(task.getLevel());
+                    }
+                };
+            }
+
+            public void finish() throws IOException
+            {
+                fileOutput.finish();
+            }
+
+            public void close() throws IOException
+            {
+                fileOutput.close();
+            }
+        });
+    }
+}

data/embulk-standards/src/main/java/org/embulk/standards/LocalFileInputPlugin.java

@@ -0,0 +1,203 @@
+package org.embulk.standards;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.Files;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.FileVisitResult;
+import java.nio.file.attribute.BasicFileAttributes;
+import com.google.common.collect.ImmutableList;
+import com.google.common.base.Optional;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigInject;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.Task;
+import org.embulk.config.TaskSource;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.TaskReport;
+import org.embulk.spi.BufferAllocator;
+import org.embulk.spi.Exec;
+import org.embulk.spi.FileInputPlugin;
+import org.embulk.spi.TransactionalFileInput;
+import org.embulk.spi.util.InputStreamTransactionalFileInput;
+import org.slf4j.Logger;
+
+public class LocalFileInputPlugin
+        implements FileInputPlugin
+{
+    public interface PluginTask
+            extends Task
+    {
+        @Config("path_prefix")
+        String getPathPrefix();
+
+        @Config("last_path")
+        @ConfigDefault("null")
+        Optional<String> getLastPath();
+
+        List<String> getFiles();
+        void setFiles(List<String> files);
+
+        @ConfigInject
+        BufferAllocator getBufferAllocator();
+    }
+
+    private final Logger log = Exec.getLogger(getClass());
+
+    private final static Path CURRENT_DIR = Paths.get(".").normalize();
+
+    @Override
+    public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control)
+    {
+        PluginTask task = config.loadConfig(PluginTask.class);
+
+        // list files recursively
+        List<String> files = listFiles(task);
+        log.info("Loading files {}", files);
+        task.setFiles(files);
+
+        // number of processors is same with number of files
+        int taskCount = task.getFiles().size();
+        return resume(task.dump(), taskCount, control);
+    }
+
+    @Override
+    public ConfigDiff resume(TaskSource taskSource,
+            int taskCount,
+            FileInputPlugin.Control control)
+    {
+        PluginTask task = taskSource.loadTask(PluginTask.class);
+
+        control.run(taskSource, taskCount);
+
+        // build next config
+        ConfigDiff configDiff = Exec.newConfigDiff();
+
+        // last_path
+        if (task.getFiles().isEmpty()) {
+            // keep the last value
+            if (task.getLastPath().isPresent()) {
+                configDiff.set("last_path", task.getLastPath().get());
+            }
+        } else {
+            List<String> files = new ArrayList<String>(task.getFiles());
+            Collections.sort(files);
+            configDiff.set("last_path", files.get(files.size() - 1));
+        }
+
+        return configDiff;
+    }
+
+    @Override
+    public void cleanup(TaskSource taskSource,
+            int taskCount,
+            List<TaskReport> successTaskReports)
+    { }
+
+    public List<String> listFiles(PluginTask task)
+    {
+        Path pathPrefix = Paths.get(task.getPathPrefix()).normalize();
+        final Path directory;
+        final String fileNamePrefix;
+        if (Files.isDirectory(pathPrefix)) {
+            directory = pathPrefix;
+            fileNamePrefix = "";
+        } else {
+            fileNamePrefix = pathPrefix.getFileName().toString();
+            Path d = pathPrefix.getParent();
+            directory = (d == null ? CURRENT_DIR : d);
+        }
+
+        final ImmutableList.Builder<String> builder = ImmutableList.builder();
+        final String lastPath = task.getLastPath().orNull();
+        try {
+            log.info("Listing local files at directory '{}' filtering filename by prefix '{}'", directory.equals(CURRENT_DIR) ? "." : directory.toString(), fileNamePrefix);
+            Files.walkFileTree(directory, new SimpleFileVisitor<Path>() {
+                @Override
+                public FileVisitResult preVisitDirectory(Path path, BasicFileAttributes attrs)
+                {
+                    if (path.equals(directory)) {
+                        return FileVisitResult.CONTINUE;
+                    } else if (lastPath != null && path.toString().compareTo(lastPath) <= 0) {
+                        return FileVisitResult.SKIP_SUBTREE;
+                    } else {
+                        Path parent = path.getParent();
+                        if (parent == null) {
+                            parent = CURRENT_DIR;
+                        }
+                        if (parent.equals(directory)) {
+                            if (path.getFileName().toString().startsWith(fileNamePrefix)) {
+                                return FileVisitResult.CONTINUE;
+                            } else {
+                                return FileVisitResult.SKIP_SUBTREE;
+                            }
+                        } else {
+                            return FileVisitResult.CONTINUE;
+                        }
+                    }
+                }
+
+                @Override
+                public FileVisitResult visitFile(Path path, BasicFileAttributes attrs)
+                {
+                    if (lastPath != null && path.toString().compareTo(lastPath) <= 0) {
+                        return FileVisitResult.CONTINUE;
+                    } else {
+                        Path parent = path.getParent();
+                        if (parent == null) {
+                            parent = CURRENT_DIR;
+                        }
+                        if (parent.equals(directory)) {
+                            if (path.getFileName().toString().startsWith(fileNamePrefix)) {
+                                builder.add(path.toString());
+                                return FileVisitResult.CONTINUE;
+                            }
+                        } else {
+                            builder.add(path.toString());
+                        }
+                        return FileVisitResult.CONTINUE;
+                    }
+                }
+            });
+        } catch (IOException ex) {
+            throw new RuntimeException(String.format("Failed get a list of local files at '%s'", directory), ex);
+        }
+        return builder.build();
+    }
+
+    @Override
+    public TransactionalFileInput open(TaskSource taskSource, int taskIndex)
+    {
+        final PluginTask task = taskSource.loadTask(PluginTask.class);
+
+        final File file = new File(task.getFiles().get(taskIndex));
+
+        return new InputStreamTransactionalFileInput(
+                task.getBufferAllocator(),
+                new InputStreamTransactionalFileInput.Opener() {
+                    public InputStream open() throws IOException
+                    {
+                        return new FileInputStream(file);
+                    }
+                })
+        {
+            @Override
+            public void abort()
+            { }
+
+            @Override
+            public TaskReport commit()
+            {
+                return Exec.newTaskReport();
+            }
+        };
+    }
+}

data/embulk-standards/src/main/java/org/embulk/standards/LocalFileOutputPlugin.java

@@ -0,0 +1,148 @@
+package org.embulk.standards;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.IllegalFormatException;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.TaskReport;
+import org.embulk.config.Task;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.Buffer;
+import org.embulk.spi.FileOutputPlugin;
+import org.embulk.spi.TransactionalFileOutput;
+import org.embulk.spi.Exec;
+import org.slf4j.Logger;
+
+public class LocalFileOutputPlugin
+        implements FileOutputPlugin
+{
+    public interface PluginTask
+            extends Task
+    {
+        @Config("path_prefix")
+        String getPathPrefix();
+
+        @Config("file_ext")
+        String getFileNameExtension();
+
+        @Config("sequence_format")
+        @ConfigDefault("\"%03d.%02d.\"")
+        String getSequenceFormat();
+    }
+
+    private final Logger log = Exec.getLogger(getClass());
+
+    @Override
+    public ConfigDiff transaction(ConfigSource config, int taskCount,
+            FileOutputPlugin.Control control)
+    {
+        PluginTask task = config.loadConfig(PluginTask.class);
+
+        // validate sequence_format
+        try {
+            String dontCare = String.format(Locale.ENGLISH, task.getSequenceFormat(), 0, 0);
+        } catch (IllegalFormatException ex) {
+            throw new ConfigException("Invalid sequence_format: parameter for file output plugin", ex);
+        }
+
+        return resume(task.dump(), taskCount, control);
+    }
+
+    @Override
+    public ConfigDiff resume(TaskSource taskSource,
+            int taskCount,
+            FileOutputPlugin.Control control)
+    {
+        control.run(taskSource);
+        return Exec.newConfigDiff();
+    }
+
+    @Override
+    public void cleanup(TaskSource taskSource,
+            int taskCount,
+            List<TaskReport> successTaskReports)
+    { }
+
+    @Override
+    public TransactionalFileOutput open(TaskSource taskSource, final int taskIndex)
+    {
+        PluginTask task = taskSource.loadTask(PluginTask.class);
+
+        final String pathPrefix = task.getPathPrefix();
+        final String pathSuffix = task.getFileNameExtension();
+        final String sequenceFormat = task.getSequenceFormat();
+
+        return new TransactionalFileOutput() {
+            private final List<String> fileNames = new ArrayList<>();
+            private int fileIndex = 0;
+            private FileOutputStream output = null;
+
+            public void nextFile()
+            {
+                closeFile();
+                String path = pathPrefix + String.format(sequenceFormat, taskIndex, fileIndex) + pathSuffix;
+                log.info("Writing local file '{}'", path);
+                fileNames.add(path);
+                try {
+                    output = new FileOutputStream(new File(path));
+                } catch (FileNotFoundException ex) {
+                    throw new RuntimeException(ex);  // TODO exception class
+                }
+                fileIndex++;
+            }
+
+            private void closeFile()
+            {
+                if (output != null) {
+                    try {
+                        output.close();
+                    } catch (IOException ex) {
+                        throw new RuntimeException(ex);
+                    }
+                }
+            }
+
+            public void add(Buffer buffer)
+            {
+                try {
+                    output.write(buffer.array(), buffer.offset(), buffer.limit());
+                } catch (IOException ex) {
+                    throw new RuntimeException(ex);
+                } finally {
+                    buffer.release();
+                }
+            }
+
+            public void finish()
+            {
+                closeFile();
+            }
+
+            public void close()
+            {
+                closeFile();
+            }
+
+            public void abort()
+            { }
+
+            public TaskReport commit()
+            {
+                TaskReport report = Exec.newTaskReport();
+                // TODO better setting for Report
+                // report.set("file_names", fileNames);
+                // report.set("file_sizes", fileSizes);
+                return report;
+            }
+        };
+    }
+}