embulk-input-swift 0.1.0

data/gradlew.bat

@@ -0,0 +1,90 @@
+@if "%DEBUG%" == "" @echo off
+@rem ##########################################################################
+@rem
+@rem  Gradle startup script for Windows
+@rem
+@rem ##########################################################################
+
+@rem Set local scope for the variables with windows NT shell
+if "%OS%"=="Windows_NT" setlocal
+
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS=
+
+set DIRNAME=%~dp0
+if "%DIRNAME%" == "" set DIRNAME=.
+set APP_BASE_NAME=%~n0
+set APP_HOME=%DIRNAME%
+
+@rem Find java.exe
+if defined JAVA_HOME goto findJavaFromJavaHome
+
+set JAVA_EXE=java.exe
+%JAVA_EXE% -version >NUL 2>&1
+if "%ERRORLEVEL%" == "0" goto init
+
+echo.
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:findJavaFromJavaHome
+set JAVA_HOME=%JAVA_HOME:"=%
+set JAVA_EXE=%JAVA_HOME%/bin/java.exe
+
+if exist "%JAVA_EXE%" goto init
+
+echo.
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:init
+@rem Get command-line arguments, handling Windowz variants
+
+if not "%OS%" == "Windows_NT" goto win9xME_args
+if "%@eval[2+2]" == "4" goto 4NT_args
+
+:win9xME_args
+@rem Slurp the command line arguments.
+set CMD_LINE_ARGS=
+set _SKIP=2
+
+:win9xME_args_slurp
+if "x%~1" == "x" goto execute
+
+set CMD_LINE_ARGS=%*
+goto execute
+
+:4NT_args
+@rem Get arguments from the 4NT Shell from JP Software
+set CMD_LINE_ARGS=%$
+
+:execute
+@rem Setup the command line
+
+set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+
+@rem Execute Gradle
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
+
+:end
+@rem End local scope for the variables with windows NT shell
+if "%ERRORLEVEL%"=="0" goto mainEnd
+
+:fail
+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
+rem the _cmd.exe /c_ return code!
+if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
+exit /b 1
+
+:mainEnd
+if "%OS%"=="Windows_NT" endlocal
+
+:omega

data/lib/embulk/input/swift.rb

@@ -0,0 +1,3 @@
+Embulk::JavaPlugin.register_input(
+  "swift", "org.embulk.input.swift.SwiftFileInputPlugin",
+  File.expand_path('../../../../classpath', __FILE__))

data/src/main/java/org/embulk/input/swift/FileList.java

@@ -0,0 +1,300 @@
+package org.embulk.input.swift;
+
+import java.util.List;
+import java.util.AbstractList;
+import java.util.ArrayList;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+import java.util.regex.Pattern;
+import java.util.regex.Matcher;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.BufferedOutputStream;
+import java.io.BufferedInputStream;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigSource;
+import com.google.common.base.Throwables;
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableList;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonCreator;
+
+// this class was reused from embulk-input-s2(https://github.com/embulk/embulk-input-s3) plugin
+public class FileList {
+    public interface Task {
+        @Config("path_match_pattern")
+        @ConfigDefault("\".*\"")
+        String getPathMatchPattern();
+
+        @Config("total_file_count_limit")
+        @ConfigDefault("2147483647")
+        int getTotalFileCountLimit();
+
+        // TODO support more algorithms to combine tasks
+        @Config("min_task_size")
+        @ConfigDefault("0")
+        long getMinTaskSize();
+    }
+
+    public static class Entry {
+        private int index;
+        private long size;
+
+        @JsonCreator
+        public Entry(
+                @JsonProperty("index") int index,
+                @JsonProperty("size") long size) {
+            this.index = index;
+            this.size = size;
+        }
+
+        @JsonProperty("index")
+        public int getIndex() {
+            return index;
+        }
+
+        @JsonProperty("size")
+        public long getSize() {
+            return size;
+        }
+    }
+
+    public static class Builder {
+        private final ByteArrayOutputStream binary;
+        private final OutputStream stream;
+        private final List<Entry> entries = new ArrayList<>();
+        private String last = null;
+
+        private int limitCount = Integer.MAX_VALUE;
+        private long minTaskSize = 1;
+        private Pattern pathMatchPattern;
+
+        private final ByteBuffer castBuffer = ByteBuffer.allocate(4);
+
+        public Builder(Task task) {
+            this();
+            this.pathMatchPattern = Pattern.compile(task.getPathMatchPattern());
+            this.limitCount = task.getTotalFileCountLimit();
+            this.minTaskSize = task.getMinTaskSize();
+        }
+
+        public Builder(ConfigSource config) {
+            this();
+            this.pathMatchPattern = Pattern.compile(config.get(String.class, "path_match_pattern", ".*"));
+            this.limitCount = config.get(int.class, "total_file_count_limit", Integer.MAX_VALUE);
+            this.minTaskSize = config.get(long.class, "min_task_size", 0L);
+        }
+
+        public Builder() {
+            binary = new ByteArrayOutputStream();
+            try {
+                stream = new BufferedOutputStream(new GZIPOutputStream(binary));
+            } catch (IOException ex) {
+                throw Throwables.propagate(ex);
+            }
+        }
+
+        public Builder limitTotalFileCount(int limitCount) {
+            this.limitCount = limitCount;
+            return this;
+        }
+
+        public Builder minTaskSize(long bytes) {
+            this.minTaskSize = bytes;
+            return this;
+        }
+
+        public Builder pathMatchPattern(String pattern) {
+            this.pathMatchPattern = Pattern.compile(pattern);
+            return this;
+        }
+
+        public int size() {
+            return entries.size();
+        }
+
+        public boolean needsMore() {
+            return size() < limitCount;
+        }
+
+        // returns true if this file is used
+        public synchronized boolean add(String path, long size) {
+            // TODO throw IllegalStateException if stream is already closed
+
+            if (!needsMore()) {
+                return false;
+            }
+
+            if (!pathMatchPattern.matcher(path).find()) {
+                return false;
+            }
+
+            int index = entries.size();
+            entries.add(new Entry(index, size));
+
+            byte[] data = path.getBytes(StandardCharsets.UTF_8);
+            castBuffer.putInt(0, data.length);
+            try {
+                stream.write(castBuffer.array());
+                stream.write(data);
+            } catch (IOException ex) {
+                throw Throwables.propagate(ex);
+            }
+
+            last = path;
+            return true;
+        }
+
+        public FileList build() {
+            try {
+                stream.close();
+            } catch (IOException ex) {
+                throw Throwables.propagate(ex);
+            }
+            return new FileList(binary.toByteArray(), getSplits(entries), Optional.fromNullable(last));
+        }
+
+        private List<List<Entry>> getSplits(List<Entry> all) {
+            List<List<Entry>> tasks = new ArrayList<>();
+            long currentTaskSize = 0;
+            List<Entry> currentTask = new ArrayList<>();
+            for (Entry entry : all) {
+                currentTask.add(entry);
+                currentTaskSize += entry.getSize();  // TODO consider to multiply the size by cost_per_byte, and add cost_per_file
+                if (currentTaskSize >= minTaskSize) {
+                    tasks.add(currentTask);
+                    currentTask = new ArrayList<>();
+                    currentTaskSize = 0;
+                }
+            }
+            if (!currentTask.isEmpty()) {
+                tasks.add(currentTask);
+            }
+            return tasks;
+        }
+    }
+
+    private final byte[] data;
+    private final List<List<Entry>> tasks;
+    private final Optional<String> last;
+
+    @JsonCreator
+    @Deprecated
+    public FileList(
+            @JsonProperty("data") byte[] data,
+            @JsonProperty("tasks") List<List<Entry>> tasks,
+            @JsonProperty("last") Optional<String> last) {
+        this.data = data;
+        this.tasks = tasks;
+        this.last = last;
+    }
+
+    @JsonIgnore
+    public Optional<String> getLastPath(Optional<String> lastLastPath) {
+        if (last.isPresent()) {
+            return last;
+        }
+        return lastLastPath;
+    }
+
+    @JsonIgnore
+    public int getTaskCount() {
+        return tasks.size();
+    }
+
+    @JsonIgnore
+    public List<String> get(int i) {
+        return new EntryList(data, tasks.get(i));
+    }
+
+    @JsonProperty("data")
+    @Deprecated
+    public byte[] getData() {
+        return data;
+    }
+
+    @JsonProperty("tasks")
+    @Deprecated
+    public List<List<Entry>> getTasks() {
+        return tasks;
+    }
+
+    @JsonProperty("last")
+    @Deprecated
+    public Optional<String> getLast() {
+        return last;
+    }
+
+    private class EntryList
+            extends AbstractList<String> {
+        private final byte[] data;
+        private final List<Entry> entries;
+        private InputStream stream;
+        private int current;
+
+        private final ByteBuffer castBuffer = ByteBuffer.allocate(4);
+
+        public EntryList(byte[] data, List<Entry> entries) {
+            this.data = data;
+            this.entries = entries;
+            try {
+                this.stream = new BufferedInputStream(new GZIPInputStream(new ByteArrayInputStream(data)));
+            } catch (IOException ex) {
+                throw Throwables.propagate(ex);
+            }
+            this.current = 0;
+        }
+
+        @Override
+        public synchronized String get(int i) {
+            Entry e = entries.get(i);
+            if (e.getIndex() < current) {
+                // rewind to the head
+                try {
+                    stream.close();
+                    stream = new BufferedInputStream(new GZIPInputStream(new ByteArrayInputStream(data)));
+                } catch (IOException ex) {
+                    throw Throwables.propagate(ex);
+                }
+                current = 0;
+            }
+
+            while (current < e.getIndex()) {
+                readNext();
+            }
+            // now current == e.getIndex()
+            return readNextString();
+        }
+
+        @Override
+        public int size() {
+            return entries.size();
+        }
+
+        private byte[] readNext() {
+            try {
+                stream.read(castBuffer.array());
+                int n = castBuffer.getInt(0);
+                byte[] b = new byte[n];  // here should be able to use a pooled buffer because read data is ignored if readNextString doesn't call this method
+                stream.read(b);
+
+                current++;
+
+                return b;
+            } catch (IOException ex) {
+                throw Throwables.propagate(ex);
+            }
+        }
+
+        private String readNextString() {
+            return new String(readNext(), StandardCharsets.UTF_8);
+        }
+    }
+}

data/src/main/java/org/embulk/input/swift/SwiftFileInputPlugin.java

@@ -0,0 +1,307 @@
+package org.embulk.input.swift;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InterruptedIOException;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Optional;
+import com.google.common.base.Throwables;
+import org.embulk.config.*;
+import org.embulk.spi.Exec;
+import org.embulk.spi.FileInputPlugin;
+import org.embulk.spi.BufferAllocator;
+import org.embulk.spi.TransactionalFileInput;
+import org.embulk.spi.util.InputStreamFileInput;
+import org.embulk.spi.util.ResumableInputStream;
+import org.embulk.spi.util.RetryExecutor;
+import org.javaswift.joss.client.factory.AccountConfig;
+import org.javaswift.joss.client.factory.AccountFactory;
+import org.javaswift.joss.client.factory.AuthenticationMethod;
+import org.javaswift.joss.headers.object.range.ExcludeStartRange;
+import org.javaswift.joss.headers.object.range.LastPartRange;
+import org.javaswift.joss.headers.object.range.MidPartRange;
+import org.javaswift.joss.instructions.DownloadInstructions;
+import org.javaswift.joss.model.Account;
+import org.javaswift.joss.model.Container;
+import org.javaswift.joss.model.StoredObject;
+import org.slf4j.Logger;
+
+import static org.embulk.spi.util.RetryExecutor.retryExecutor;
+
+public class SwiftFileInputPlugin
+        implements FileInputPlugin {
+    public interface PluginTask
+            extends FileList.Task, Task {
+        @Config("username")
+        public String getUsername();
+
+        @Config("password")
+        public String getPassword();
+
+        @Config("auth_url")
+        public String getAuthUrl();
+
+        @Config("auth_type")
+        public String getAuthType();
+
+        @Config("tenant_id")
+        @ConfigDefault("null")
+        public Optional<String> getTenantId();
+
+        @Config("tenant_name")
+        @ConfigDefault("null")
+        public Optional<String> getTenantName();
+
+        @Config("container")
+        public String getContainer();
+
+        @Config("path_prefix")
+        public String getPathPrefix();
+
+        @Config("last_path")
+        @ConfigDefault("null")
+        public Optional<String> getLastPath();
+
+        @Config("incremental")
+        @ConfigDefault("true")
+        public boolean getIncremental();
+
+        public FileList getFiles();
+
+        public void setFiles(FileList files);
+
+        @ConfigInject
+        public BufferAllocator getBufferAllocator();
+    }
+
+    /**
+     * Logger
+     */
+    private static final Logger LOGGER = Exec.getLogger(SwiftFileInputPlugin.class);
+
+    private Account getAccount(PluginTask task) {
+        AccountConfig accountConfig = new AccountConfig();
+
+        String auth_type = task.getAuthType();
+        accountConfig.setAuthUrl(task.getAuthUrl());
+        accountConfig.setUsername(task.getUsername());
+        accountConfig.setPassword(task.getPassword());
+
+        Optional<String> tenant_id = task.getTenantId();
+        if (tenant_id.isPresent()) {
+            accountConfig.setTenantId(tenant_id.get());
+        }
+        Optional<String> tenant_name = task.getTenantName();
+        if (tenant_name.isPresent()) {
+            accountConfig.setTenantName(tenant_name.get());
+        }
+
+        if (auth_type.equals("keystone")) {
+            if (!tenant_id.isPresent() && !tenant_name.isPresent()) {
+                throw new ConfigException("if you choose keystone auth, you must specify to either tenant_id or tenant_name.");
+            }
+            accountConfig.setAuthenticationMethod(AuthenticationMethod.KEYSTONE);
+        } else if (auth_type.equals("tempauth")) {
+            accountConfig.setAuthenticationMethod(AuthenticationMethod.TEMPAUTH);
+        } else if (auth_type.equals("basic")) {
+            accountConfig.setAuthenticationMethod(AuthenticationMethod.BASIC);
+        } else {
+            throw new ConfigException("auth_type has to be either keystone, tempauth or basic.");
+        }
+
+        return new AccountFactory(accountConfig).createAccount();
+    }
+
+    /**
+     * retrieve target objects with specified prefix
+     * @param task PluginTsak
+     * @return List of Target Objects
+     */
+    private FileList listFiles(PluginTask task) {
+        FileList.Builder builder = new FileList.Builder(task);
+        Account account = this.getAccount(task);
+        Container container = account.getContainer(task.getContainer());
+
+        // if the container is not exist, cannot input.
+        if (container.exists() == false) {
+            throw new ConfigException("Container not found");
+        }
+
+        String marker = task.getLastPath().orNull();
+
+        do {
+            Collection<StoredObject> objectList = container.list(task.getPathPrefix(), marker, 1024);
+            if (objectList.size() > 0) {
+                for (StoredObject obj : objectList) {
+                    if (obj.getContentLength() > 0) {
+                        LOGGER.info("add {}", obj.getName());
+                        builder.add(obj.getName(), obj.getContentLength());
+                        if (!builder.needsMore()) {
+                            return builder.build();
+                        }
+                    }
+                    marker = obj.getName();
+                }
+            } else {
+                break;
+            }
+        } while (marker != null);
+
+        return builder.build();
+    }
+
+    @Override
+    public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control) {
+        PluginTask task = config.loadConfig(PluginTask.class);
+
+        //set input files
+        task.setFiles(this.listFiles(task));
+        int taskCount = task.getFiles().getTaskCount();
+
+        return resume(task.dump(), taskCount, control);
+    }
+
+    @Override
+    public ConfigDiff resume(TaskSource taskSource,
+                             int taskCount,
+                             FileInputPlugin.Control control) {
+        PluginTask task = taskSource.loadTask(PluginTask.class);
+
+        //validate
+        this.getAccount(task);
+
+        control.run(taskSource, taskCount);
+
+        ConfigDiff configDiff = Exec.newConfigDiff();
+
+        if (task.getIncremental()) {
+            configDiff.set("last_path", task.getFiles().getLastPath(task.getLastPath()));
+        }
+
+        return configDiff;
+    }
+
+    @Override
+    public void cleanup(TaskSource taskSource,
+                        int taskCount,
+                        List<TaskReport> successTaskReports) {
+    }
+
+    @Override
+    public TransactionalFileInput open(TaskSource taskSource, int taskIndex) {
+        final PluginTask task = taskSource.loadTask(PluginTask.class);
+
+        return new SwiftFileInput(task, taskIndex);
+    }
+
+    //private static InputStream openInputStream(PluginTask task, String path)
+    //{
+    //    return new MyInputStream(file);
+    //}
+
+
+    @VisibleForTesting
+    static class SwiftInputStreamReopener
+            implements ResumableInputStream.Reopener {
+        private final Logger LOGGER = Exec.getLogger(SwiftInputStreamReopener.class);
+
+        private final StoredObject obj;
+
+        public SwiftInputStreamReopener(StoredObject obj) {
+            this.obj = obj;
+        }
+
+        @Override
+        public InputStream reopen(final long offset, final Exception closedCause) throws IOException {
+            try {
+                return retryExecutor()
+                        .withRetryLimit(3)
+                        .withInitialRetryWait(500)
+                        .withMaxRetryWait(30 * 1000)
+                        .runInterruptible(new RetryExecutor.Retryable<InputStream>() {
+                            @Override
+                            public InputStream call() throws InterruptedIOException {
+                                LOGGER.warn(String.format("Swift read failed. Retrying GET request with %,d bytes offset", offset), closedCause);
+                                return obj.downloadObjectAsInputStream(new DownloadInstructions().setRange(new ExcludeStartRange((int) offset)));
+                            }
+
+                            @Override
+                            public boolean isRetryableException(Exception exception) {
+                                return true;  // TODO
+                            }
+
+                            @Override
+                            public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
+                                    throws RetryExecutor.RetryGiveupException {
+                                String message = String.format("Swift GET request failed. Retrying %d/%d after %d seconds. Message: %s",
+                                        retryCount, retryLimit, retryWait / 1000, exception.getMessage());
+                                if (retryCount % 3 == 0) {
+                                    LOGGER.warn(message, exception);
+                                } else {
+                                    LOGGER.warn(message);
+                                }
+                            }
+
+                            @Override
+                            public void onGiveup(Exception firstException, Exception lastException)
+                                    throws RetryExecutor.RetryGiveupException {
+                            }
+                        });
+            } catch (RetryExecutor.RetryGiveupException ex) {
+                Throwables.propagateIfInstanceOf(ex.getCause(), IOException.class);
+                throw Throwables.propagate(ex.getCause());
+            } catch (InterruptedException ex) {
+                throw new InterruptedIOException();
+            }
+        }
+    }
+
+    public class SwiftFileInput
+            extends InputStreamFileInput
+            implements TransactionalFileInput {
+        public SwiftFileInput(PluginTask task, int taskIndex) {
+            super(task.getBufferAllocator(), new SingleFileProvider(task, taskIndex));
+        }
+
+        public void abort() {
+        }
+
+        public TaskReport commit() {
+            return Exec.newTaskReport();
+        }
+
+        @Override
+        public void close() {
+        }
+    }
+
+    private class SingleFileProvider
+            implements InputStreamFileInput.Provider {
+        private Account account;
+        private final String containerName;
+        private final Iterator<String> iterator;
+
+        public SingleFileProvider(PluginTask task, int taskIndex) {
+            this.account = getAccount(task);
+            this.containerName = task.getContainer();
+            this.iterator = task.getFiles().get(taskIndex).iterator();
+        }
+
+        @Override
+        public InputStream openNext() throws IOException {
+            if (!iterator.hasNext()) {
+                return null;
+            }
+            StoredObject obj = this.account.getContainer(this.containerName).getObject(iterator.next());
+            return new ResumableInputStream(obj.downloadObjectAsInputStream(), new SwiftInputStreamReopener(obj));
+        }
+
+        @Override
+        public void close() {
+        }
+    }
+}