embulk 0.8.37-java → 0.10.26-java

data/embulk-standards/src/main/java/org/embulk/standards/GzipFileDecoderPlugin.java

@@ -1,55 +0,0 @@
-package org.embulk.standards;
-
-import java.io.InputStream;
-import java.io.IOException;
-import java.util.zip.GZIPInputStream;
-import org.embulk.config.Task;
-import org.embulk.config.TaskSource;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.ConfigInject;
-import org.embulk.spi.DecoderPlugin;
-import org.embulk.spi.BufferAllocator;
-import org.embulk.spi.FileInput;
-import org.embulk.spi.util.FileInputInputStream;
-import org.embulk.spi.util.InputStreamFileInput;
-
-public class GzipFileDecoderPlugin
-        implements DecoderPlugin
-{
-    public interface PluginTask
-            extends Task
-    {
-        @ConfigInject
-        BufferAllocator getBufferAllocator();
-    }
-
-    @Override
-    public void transaction(ConfigSource config, DecoderPlugin.Control control)
-    {
-        PluginTask task = config.loadConfig(PluginTask.class);
-        control.run(task.dump());
-    }
-
-    @Override
-    public FileInput open(TaskSource taskSource, FileInput fileInput)
-    {
-        PluginTask task = taskSource.loadTask(PluginTask.class);
-        final FileInputInputStream files = new FileInputInputStream(fileInput);
-        return new InputStreamFileInput(
-                task.getBufferAllocator(),
-                new InputStreamFileInput.Provider() {
-                    public InputStream openNext() throws IOException
-                    {
-                        if (!files.nextFile()) {
-                            return null;
-                        }
-                        return new GZIPInputStream(files, 8*1024);
-                    }
-
-                    public void close() throws IOException
-                    {
-                        files.close();
-                    }
-                });
-    }
-}

data/embulk-standards/src/main/java/org/embulk/standards/GzipFileEncoderPlugin.java

@@ -1,71 +0,0 @@
-package org.embulk.standards;
-
-import java.io.OutputStream;
-import java.io.IOException;
-import java.util.zip.GZIPOutputStream;
-import javax.validation.constraints.Min;
-import javax.validation.constraints.Max;
-import org.embulk.config.Task;
-import org.embulk.config.Config;
-import org.embulk.config.ConfigInject;
-import org.embulk.config.ConfigDefault;
-import org.embulk.config.TaskSource;
-import org.embulk.config.ConfigSource;
-import org.embulk.spi.EncoderPlugin;
-import org.embulk.spi.FileOutput;
-import org.embulk.spi.BufferAllocator;
-import org.embulk.spi.util.FileOutputOutputStream;
-import org.embulk.spi.util.OutputStreamFileOutput;
-
-public class GzipFileEncoderPlugin
-        implements EncoderPlugin
-{
-    public interface PluginTask
-            extends Task
-    {
-        @Config("level")
-        @ConfigDefault("6")
-        @Min(0)
-        @Max(9)
-        int getLevel();
-
-        @ConfigInject
-        BufferAllocator getBufferAllocator();
-    }
-
-    public void transaction(ConfigSource config, EncoderPlugin.Control control)
-    {
-        PluginTask task = config.loadConfig(PluginTask.class);
-        control.run(task.dump());
-    }
-
-    @Override
-    public FileOutput open(TaskSource taskSource, final FileOutput fileOutput)
-    {
-        final PluginTask task = taskSource.loadTask(PluginTask.class);
-
-        final FileOutputOutputStream output = new FileOutputOutputStream(fileOutput, task.getBufferAllocator(), FileOutputOutputStream.CloseMode.FLUSH);
-
-        return new OutputStreamFileOutput(new OutputStreamFileOutput.Provider() {
-            public OutputStream openNext() throws IOException
-            {
-                output.nextFile();
-                return new GZIPOutputStream(output) {
-                    {
-                        this.def.setLevel(task.getLevel());
-                    }
-                };
-            }
-
-            public void finish() throws IOException
-            {
-                fileOutput.finish();
-            }
-
-            public void close() throws IOException
-            {
-                fileOutput.close();
-            }
-        });
-    }
-}

data/embulk-standards/src/main/java/org/embulk/standards/JsonParserPlugin.java

@@ -1,235 +0,0 @@
-package org.embulk.standards;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Function;
-import com.google.common.collect.Lists;
-import com.google.common.io.CharSource;
-import com.google.common.io.CharStreams;
-import org.embulk.config.Config;
-import org.embulk.config.ConfigDefault;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.Task;
-import org.embulk.config.TaskSource;
-import org.embulk.spi.Column;
-import org.embulk.spi.DataException;
-import org.embulk.spi.Exec;
-import org.embulk.spi.FileInput;
-import org.embulk.spi.PageBuilder;
-import org.embulk.spi.PageOutput;
-import org.embulk.spi.ParserPlugin;
-import org.embulk.spi.Schema;
-import org.embulk.spi.json.JsonParseException;
-import org.embulk.spi.json.JsonParser;
-import org.embulk.spi.type.Types;
-import org.embulk.spi.util.FileInputInputStream;
-import org.jruby.embed.io.ReaderInputStream;
-import org.msgpack.core.Preconditions;
-import org.msgpack.value.Value;
-import org.slf4j.Logger;
-
-import javax.annotation.Nullable;
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.util.regex.Pattern;
-
-public class JsonParserPlugin
-        implements ParserPlugin
-{
-
-    public enum InvalidEscapeStringPolicy
-    {
-        PASSTHROUGH("PASSTHROUGH"),
-        SKIP("SKIP"),
-        UNESCAPE("UNESCAPE");
-
-        private final String string;
-
-        private InvalidEscapeStringPolicy(String string)
-        {
-            this.string = string;
-        }
-
-        public String getString()
-        {
-            return string;
-        }
-    }
-
-    public interface PluginTask
-            extends Task
-    {
-        @Config("stop_on_invalid_record")
-        @ConfigDefault("false")
-        boolean getStopOnInvalidRecord();
-
-        @Config("invalid_string_escapes")
-        @ConfigDefault("\"PASSTHROUGH\"")
-        InvalidEscapeStringPolicy getInvalidEscapeStringPolicy();
-    }
-
-    private final Logger log;
-
-    public JsonParserPlugin()
-    {
-        this.log = Exec.getLogger(JsonParserPlugin.class);
-    }
-
-    @Override
-    public void transaction(ConfigSource configSource, Control control)
-    {
-        PluginTask task = configSource.loadConfig(PluginTask.class);
-        control.run(task.dump(), newSchema());
-    }
-
-    @VisibleForTesting
-    Schema newSchema()
-    {
-        return Schema.builder().add("record", Types.JSON).build(); // generate a schema
-    }
-
-    @Override
-    public void run(TaskSource taskSource, Schema schema, FileInput input, PageOutput output)
-    {
-        PluginTask task = taskSource.loadTask(PluginTask.class);
-
-        final boolean stopOnInvalidRecord = task.getStopOnInvalidRecord();
-        final Column column = schema.getColumn(0); // record column
-
-        try (PageBuilder pageBuilder = newPageBuilder(schema, output);
-                FileInputInputStream in = new FileInputInputStream(input)) {
-            while (in.nextFile()) {
-                boolean evenOneJsonParsed = false;
-                try (JsonParser.Stream stream = newJsonStream(in, task)) {
-                    Value value;
-                    while ((value = stream.next()) != null) {
-                        try {
-                            if (!value.isMapValue()) {
-                                throw new JsonRecordValidateException(
-                                        String.format("A Json record must not represent map value but it's %s", value.getValueType().name()));
-                            }
-
-                            pageBuilder.setJson(column, value);
-                            pageBuilder.addRecord();
-                            evenOneJsonParsed = true;
-                        }
-                        catch (JsonRecordValidateException e) {
-                            if (stopOnInvalidRecord) {
-                                throw new DataException(String.format("Invalid record: %s", value.toJson()), e);
-                            }
-                            log.warn(String.format("Skipped record (%s): %s", e.getMessage(), value.toJson()));
-                        }
-                    }
-                }
-                catch (IOException | JsonParseException e) {
-                    if (Exec.isPreview() && evenOneJsonParsed) {
-			// JsonParseException occurs when it cannot parse the last part of sampling buffer. Because
-			// the last part is sometimes invalid as JSON data. Therefore JsonParseException can be
-			// ignore in preview if at least one JSON is already parsed.
-                        break;
-                    }
-                    throw new DataException(e);
-                }
-            }
-
-            pageBuilder.finish();
-        }
-    }
-
-    private PageBuilder newPageBuilder(Schema schema, PageOutput output)
-    {
-        return new PageBuilder(Exec.getBufferAllocator(), schema, output);
-    }
-
-    private JsonParser.Stream newJsonStream(FileInputInputStream in, PluginTask task)
-            throws IOException
-    {
-        InvalidEscapeStringPolicy policy = task.getInvalidEscapeStringPolicy();
-        switch (policy) {
-            case SKIP:
-            case UNESCAPE:
-                Iterable<CharSource> lines = Lists.transform(CharStreams.readLines(new BufferedReader(new InputStreamReader(in))),
-                        invalidEscapeStringFunction(policy));
-                return new JsonParser().open(new ReaderInputStream(CharSource.concat(lines).openStream()));
-            case PASSTHROUGH:
-            default:
-                return new JsonParser().open(in);
-        }
-    }
-
-    Function<String, CharSource> invalidEscapeStringFunction(final InvalidEscapeStringPolicy policy)
-    {
-        return new Function<String, CharSource>()
-        {
-            final Pattern digitsPattern = Pattern.compile("\\p{XDigit}+");
-
-            @Override
-            public CharSource apply(@Nullable String input)
-            {
-                Preconditions.checkNotNull(input);
-                if (policy == InvalidEscapeStringPolicy.PASSTHROUGH) {
-                    return CharSource.wrap(input);
-                }
-                StringBuilder builder = new StringBuilder();
-                char[] charArray = input.toCharArray();
-                for (int characterIndex = 0; characterIndex < charArray.length; characterIndex++) {
-                    char c = charArray[characterIndex];
-                    if (c == '\\') {
-                        if (charArray.length > characterIndex + 1) {
-                            char next = charArray[characterIndex + 1];
-                            switch (next) {
-                                case 'b':
-                                case 'f':
-                                case 'n':
-                                case 'r':
-                                case 't':
-                                case '"':
-                                case '\\':
-                                case '/':
-                                    builder.append(c);
-                                    break;
-                                case 'u': // hexstring such as \u0001
-                                    if (charArray.length > characterIndex + 5) {
-                                        char[] hexChars = {charArray[characterIndex + 2], charArray[characterIndex + 3], charArray[characterIndex + 4],
-                                                           charArray[characterIndex + 5]};
-                                        String hexString = new String(hexChars);
-                                        if (digitsPattern.matcher(hexString).matches()) {
-                                            builder.append(c);
-                                        } else {
-                                            if (policy == InvalidEscapeStringPolicy.SKIP) {
-                                                // remove \\u
-                                                characterIndex++;
-                                            }
-                                        }
-                                    }
-                                    break;
-                                default:
-                                    switch (policy) {
-                                        case SKIP:
-                                            characterIndex++;
-                                            break;
-                                        case UNESCAPE:
-                                            break;
-                                    }
-                                    break;
-                            }
-                        }
-                    }
-                    else {
-                        builder.append(c);
-                    }
-                }
-                return CharSource.wrap(builder.toString());
-            }
-        };
-    }
-
-    static class JsonRecordValidateException
-            extends DataException
-    {
-        JsonRecordValidateException(String message)
-        {
-            super(message);
-        }
-    }
-}

data/embulk-standards/src/main/java/org/embulk/standards/LocalFileInputPlugin.java

@@ -1,232 +0,0 @@
-package org.embulk.standards;
-
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.InputStream;
-import java.io.IOException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.Files;
-import java.nio.file.SimpleFileVisitor;
-import java.nio.file.FileVisitResult;
-import java.nio.file.attribute.BasicFileAttributes;
-import com.google.common.collect.ImmutableList;
-import com.google.common.base.Optional;
-import org.embulk.config.Config;
-import org.embulk.config.ConfigInject;
-import org.embulk.config.ConfigDefault;
-import org.embulk.config.Task;
-import org.embulk.config.TaskSource;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.ConfigDiff;
-import org.embulk.config.TaskReport;
-import org.embulk.spi.BufferAllocator;
-import org.embulk.spi.Exec;
-import org.embulk.spi.FileInputPlugin;
-import org.embulk.spi.TransactionalFileInput;
-import org.embulk.spi.util.InputStreamTransactionalFileInput;
-import org.slf4j.Logger;
-
-import java.nio.file.FileVisitOption;
-import java.util.EnumSet;
-import java.util.Set;
-
-public class LocalFileInputPlugin
-        implements FileInputPlugin
-{
-    public interface PluginTask
-            extends Task
-    {
-        @Config("path_prefix")
-        String getPathPrefix();
-
-        @Config("last_path")
-        @ConfigDefault("null")
-        Optional<String> getLastPath();
-
-        @Config("follow_symlinks")
-        @ConfigDefault("false")
-        boolean getFollowSymlinks();
-
-        List<String> getFiles();
-        void setFiles(List<String> files);
-
-        @ConfigInject
-        BufferAllocator getBufferAllocator();
-    }
-
-    private final Logger log = Exec.getLogger(getClass());
-
-    private final static Path CURRENT_DIR = Paths.get(".").normalize();
-
-    @Override
-    public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control)
-    {
-        PluginTask task = config.loadConfig(PluginTask.class);
-
-        // list files recursively
-        List<String> files = listFiles(task);
-        log.info("Loading files {}", files);
-        task.setFiles(files);
-
-        // number of processors is same with number of files
-        int taskCount = task.getFiles().size();
-        return resume(task.dump(), taskCount, control);
-    }
-
-    @Override
-    public ConfigDiff resume(TaskSource taskSource,
-            int taskCount,
-            FileInputPlugin.Control control)
-    {
-        PluginTask task = taskSource.loadTask(PluginTask.class);
-
-        control.run(taskSource, taskCount);
-
-        // build next config
-        ConfigDiff configDiff = Exec.newConfigDiff();
-
-        // last_path
-        if (task.getFiles().isEmpty()) {
-            // keep the last value
-            if (task.getLastPath().isPresent()) {
-                configDiff.set("last_path", task.getLastPath().get());
-            }
-        } else {
-            List<String> files = new ArrayList<String>(task.getFiles());
-            Collections.sort(files);
-            configDiff.set("last_path", files.get(files.size() - 1));
-        }
-
-        return configDiff;
-    }
-
-    @Override
-    public void cleanup(TaskSource taskSource,
-            int taskCount,
-            List<TaskReport> successTaskReports)
-    { }
-
-    public List<String> listFiles(PluginTask task)
-    {
-        Path pathPrefix = Paths.get(task.getPathPrefix()).normalize();
-        final Path directory;
-        final String fileNamePrefix;
-        if (Files.isDirectory(pathPrefix)) {
-            directory = pathPrefix;
-            fileNamePrefix = "";
-        } else {
-            fileNamePrefix = pathPrefix.getFileName().toString();
-            Path d = pathPrefix.getParent();
-            directory = (d == null ? CURRENT_DIR : d);
-        }
-
-        final ImmutableList.Builder<String> builder = ImmutableList.builder();
-        final String lastPath = task.getLastPath().orNull();
-        try {
-            log.info("Listing local files at directory '{}' filtering filename by prefix '{}'", directory.equals(CURRENT_DIR) ? "." : directory.toString(), fileNamePrefix);
-
-            int maxDepth = Integer.MAX_VALUE;
-            Set<FileVisitOption> opts;
-            if (task.getFollowSymlinks()) {
-                opts = EnumSet.of(FileVisitOption.FOLLOW_LINKS);
-            } else {
-                opts = EnumSet.noneOf(FileVisitOption.class);
-                log.info("\"follow_symlinks\" is set false. Note that symbolic links to directories are skipped.");
-            }
-
-            Files.walkFileTree(directory, opts, maxDepth, new SimpleFileVisitor<Path>() {
-                @Override
-                public FileVisitResult preVisitDirectory(Path path, BasicFileAttributes attrs)
-                {
-                    if (path.equals(directory)) {
-                        return FileVisitResult.CONTINUE;
-                    } else if (lastPath != null && path.toString().compareTo(lastPath) <= 0) {
-                        return FileVisitResult.SKIP_SUBTREE;
-                    } else {
-                        Path parent = path.getParent();
-                        if (parent == null) {
-                            parent = CURRENT_DIR;
-                        }
-                        if (parent.equals(directory)) {
-                            if (path.getFileName().toString().startsWith(fileNamePrefix)) {
-                                return FileVisitResult.CONTINUE;
-                            } else {
-                                return FileVisitResult.SKIP_SUBTREE;
-                            }
-                        } else {
-                            return FileVisitResult.CONTINUE;
-                        }
-                    }
-                }
-
-                @Override
-                public FileVisitResult visitFile(Path path, BasicFileAttributes attrs)
-                {
-                    try {
-                        // Avoid directories from listing.
-                        // Directories are normally unvisited with |FileVisitor#visitFile|, but symbolic links to
-                        // directories are visited like files unless |FOLLOW_LINKS| is set in |Files#walkFileTree|.
-                        // Symbolic links to directories are explicitly skipped here by checking with |Path#toReadlPath|.
-                        if (Files.isDirectory(path.toRealPath())) {
-                            return FileVisitResult.CONTINUE;
-                        }
-                    } catch (IOException ex){
-                        throw new RuntimeException("Can't resolve symbolic link", ex);
-                    }
-                    if (lastPath != null && path.toString().compareTo(lastPath) <= 0) {
-                        return FileVisitResult.CONTINUE;
-                    } else {
-                        Path parent = path.getParent();
-                        if (parent == null) {
-                            parent = CURRENT_DIR;
-                        }
-                        if (parent.equals(directory)) {
-                            if (path.getFileName().toString().startsWith(fileNamePrefix)) {
-                                builder.add(path.toString());
-                                return FileVisitResult.CONTINUE;
-                            }
-                        } else {
-                            builder.add(path.toString());
-                        }
-                        return FileVisitResult.CONTINUE;
-                    }
-                }
-            });
-        } catch (IOException ex) {
-            throw new RuntimeException(String.format("Failed get a list of local files at '%s'", directory), ex);
-        }
-        return builder.build();
-    }
-
-    @Override
-    public TransactionalFileInput open(TaskSource taskSource, int taskIndex)
-    {
-        final PluginTask task = taskSource.loadTask(PluginTask.class);
-
-        final File file = new File(task.getFiles().get(taskIndex));
-
-        return new InputStreamTransactionalFileInput(
-                task.getBufferAllocator(),
-                new InputStreamTransactionalFileInput.Opener() {
-                    public InputStream open() throws IOException
-                    {
-                        return new FileInputStream(file);
-                    }
-                })
-        {
-            @Override
-            public void abort()
-            { }
-
-            @Override
-            public TaskReport commit()
-            {
-                return Exec.newTaskReport();
-            }
-        };
-    }
-}