embulk-input-gcs 0.2.5 → 0.2.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 01e9c70bf2d1f4c9a25784ab64828913c056078d
-  data.tar.gz: 22d6b84e12d965045719b49026924f5247a6609a
+  metadata.gz: 04cb5b37d8fb8c70e1c9c4c306cf792ac1ad1ec9
+  data.tar.gz: 44ec9518fc188320a19ffbe2ce7db0b07cda30f3
 SHA512:
-  metadata.gz: d090628aec5d9512976ceff29215f99e04eb939e4e927106bca4bee47fe95cef7e636fc74b239cde7222cc71f5b259b43905b36ad90d3c515f8b97f8b6c9c220
-  data.tar.gz: 0ad764f0f9f85818b5924b3477bfe7d65806113ca56e689b45e919d3e8c0b156bfbb39cf60e9f349bb1005264655eaf0ad4446115f0254c554c5e425edac746a
+  metadata.gz: 3d2a59336002f07d48bcf8b5b22b3626e9c59df7aad93a4f91721667f6f7e1552bd1db72ebd1323e19ce1dc67f32da64b9ad88bc6d6785591904b8d87059249f
+  data.tar.gz: 1533eccae86b7355303ab91bc5ab08cf49c816438fef3541ad1318fc24f0eab948667132952746173a2dd3dabea41bc06cfc2493b8b4701c1908d2e07ff9b635

data/.travis.yml

@@ -1,3 +1,4 @@
+dist: precise
 language: java
 
 jdk:
@@ -21,3 +22,7 @@ env:
 script:
   - ./gradlew gem
   - ./gradlew --info check jacocoTestReport
+addons:
+  hosts:
+    - example.com
+  hostname: example.com

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.2.6 - 2018-03-05
+* [maintenance] Support "path_match_pattern" option [#32](https://github.com/embulk/embulk-input-gcs/pull/32)
+
 ## 0.2.5 - 2017-05-19
 * [maintenance] Fix InputStream handling to avoid plugin get less records than expected [#27](https://github.com/embulk/embulk-input-gcs/pull/27)
 

data/README.md

@@ -40,6 +40,7 @@ embulk run /path/to/config.yml
 - **bucket** Google Cloud Storage bucket name (string, required)
 - **path_prefix** prefix of target keys (string, either of "path_prefix" or "paths" is required)
 - **paths** list of target keys (array of string, either of "path_prefix" or "paths" is required)
+* **path_match_pattern**: regexp to match file paths. If a file path doesn't match with this pattern, the file will be skipped (regexp string, optional)
 - **incremental**: enables incremental loading(boolean, optional. default: true. If incremental loading is enabled, config diff for the next execution will include `last_path` parameter so that next execution skips files before the path. Otherwise, `last_path` will not be included.
 - **auth_method**  (string, optional, "private_key", "json_key" or "compute_engine". default value is "private_key")
 - **service_account_email** Google Cloud Storage service_account_email (string, required when auth_method is private_key)
@@ -89,6 +90,21 @@ in:
 out: {type: stdout}
 ```
 
+To skip files using regexp:
+
+```yaml
+in:
+  type: gcs
+  bucket: my-gcs-bucket
+  path_prefix: logs/csv-
+  # ...
+  path_match_pattern: \.csv$   # a file will be skipped if its path doesn't match with this pattern
+  ## some examples of regexp:
+  #path_match_pattern: /archive/         # match files in .../archive/... directory
+  #path_match_pattern: /data1/|/data2/   # match files in .../data1/... or .../data2/... directory
+  #path_match_pattern: .csv$|.csv.gz$    # match files whose suffix is .csv or .csv.gz
+```
+
 ## Authentication
 
 There are three methods supported to fetch access token for the service account.

data/build.gradle

@@ -17,7 +17,7 @@ configurations {
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
-version = "0.2.5"
+version = "0.2.6"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.8.2"

data/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
-#Wed Jan 13 12:41:02 JST 2016
+#Sun Jan 08 00:35:58 PST 2017
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-2.10-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-3.2.1-bin.zip

data/src/main/java/org/embulk/input/gcs/FileList.java

@@ -0,0 +1,335 @@
+package org.embulk.input.gcs;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Optional;
+import com.google.common.base.Throwables;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigSource;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.AbstractList;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Pattern;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+
+public class FileList
+{
+    public interface Task
+    {
+        @Config("path_match_pattern")
+        @ConfigDefault("\".*\"")
+        String getPathMatchPattern();
+
+        @Config("total_file_count_limit")
+        @ConfigDefault("2147483647")
+        int getTotalFileCountLimit();
+
+        // TODO support more algorithms to combine tasks
+        @Config("min_task_size")
+        @ConfigDefault("0")
+        long getMinTaskSize();
+    }
+
+    public static class Entry
+    {
+        private int index;
+        private long size;
+
+        @JsonCreator
+        public Entry(
+                @JsonProperty("index") int index,
+                @JsonProperty("size") long size)
+        {
+            this.index = index;
+            this.size = size;
+        }
+
+        @JsonProperty("index")
+        public int getIndex()
+        {
+            return index;
+        }
+
+        @JsonProperty("size")
+        public long getSize()
+        {
+            return size;
+        }
+    }
+
+    public static class Builder
+    {
+        private final ByteArrayOutputStream binary;
+        private final OutputStream stream;
+        private final List<Entry> entries = new ArrayList<>();
+        private String last = null;
+
+        private int limitCount = Integer.MAX_VALUE;
+        private long minTaskSize = 1;
+        private Pattern pathMatchPattern;
+
+        private final ByteBuffer castBuffer = ByteBuffer.allocate(4);
+
+        public Builder(Task task)
+        {
+            this();
+            this.pathMatchPattern = Pattern.compile(task.getPathMatchPattern());
+            this.limitCount = task.getTotalFileCountLimit();
+            this.minTaskSize = task.getMinTaskSize();
+        }
+
+        public Builder(ConfigSource config)
+        {
+            this();
+            this.pathMatchPattern = Pattern.compile(config.get(String.class, "path_match_pattern", ".*"));
+            this.limitCount = config.get(int.class, "total_file_count_limit", Integer.MAX_VALUE);
+            this.minTaskSize = config.get(long.class, "min_task_size", 0L);
+        }
+
+        public Builder()
+        {
+            binary = new ByteArrayOutputStream();
+            try {
+                stream = new BufferedOutputStream(new GZIPOutputStream(binary));
+            }
+            catch (IOException ex) {
+                throw Throwables.propagate(ex);
+            }
+        }
+
+        public Builder limitTotalFileCount(int limitCount)
+        {
+            this.limitCount = limitCount;
+            return this;
+        }
+
+        public Builder minTaskSize(long bytes)
+        {
+            this.minTaskSize = bytes;
+            return this;
+        }
+
+        public Builder pathMatchPattern(String pattern)
+        {
+            this.pathMatchPattern = Pattern.compile(pattern);
+            return this;
+        }
+
+        public int size()
+        {
+            return entries.size();
+        }
+
+        public boolean needsMore()
+        {
+            return size() < limitCount;
+        }
+
+        // returns true if this file is used
+        public synchronized boolean add(String path, long size)
+        {
+            // TODO throw IllegalStateException if stream is already closed
+
+            if (!needsMore()) {
+                return false;
+            }
+
+            if (!pathMatchPattern.matcher(path).find()) {
+                return false;
+            }
+
+            int index = entries.size();
+            entries.add(new Entry(index, size));
+
+            byte[] data = path.getBytes(StandardCharsets.UTF_8);
+            castBuffer.putInt(0, data.length);
+            try {
+                stream.write(castBuffer.array());
+                stream.write(data);
+            }
+            catch (IOException ex) {
+                throw Throwables.propagate(ex);
+            }
+
+            last = path;
+            return true;
+        }
+
+        public FileList build()
+        {
+            try {
+                stream.close();
+            }
+            catch (IOException ex) {
+                throw Throwables.propagate(ex);
+            }
+            return new FileList(binary.toByteArray(), getSplits(entries), Optional.fromNullable(last));
+        }
+
+        private List<List<Entry>> getSplits(List<Entry> all)
+        {
+            List<List<Entry>> tasks = new ArrayList<>();
+            long currentTaskSize = 0;
+            List<Entry> currentTask = new ArrayList<>();
+            for (Entry entry : all) {
+                currentTask.add(entry);
+                currentTaskSize += entry.getSize();  // TODO consider to multiply the size by cost_per_byte, and add cost_per_file
+                if (currentTaskSize >= minTaskSize) {
+                    tasks.add(currentTask);
+                    currentTask = new ArrayList<>();
+                    currentTaskSize = 0;
+                }
+            }
+            if (!currentTask.isEmpty()) {
+                tasks.add(currentTask);
+            }
+            return tasks;
+        }
+    }
+
+    private final byte[] data;
+    private final List<List<Entry>> tasks;
+    private final Optional<String> last;
+
+    @JsonCreator
+    @Deprecated
+    public FileList(
+            @JsonProperty("data") byte[] data,
+            @JsonProperty("tasks") List<List<Entry>> tasks,
+            @JsonProperty("last") Optional<String> last)
+    {
+        this.data = data;
+        this.tasks = tasks;
+        this.last = last;
+    }
+
+    @JsonIgnore
+    public Optional<String> getLastPath(Optional<String> lastLastPath)
+    {
+        if (last.isPresent()) {
+            return last;
+        }
+        return lastLastPath;
+    }
+
+    @JsonIgnore
+    public int getTaskCount()
+    {
+        return tasks.size();
+    }
+
+    @JsonIgnore
+    public List<String> get(int i)
+    {
+        return new EntryList(data, tasks.get(i));
+    }
+
+    @JsonProperty("data")
+    @Deprecated
+    public byte[] getData()
+    {
+        return data;
+    }
+
+    @JsonProperty("tasks")
+    @Deprecated
+    public List<List<Entry>> getTasks()
+    {
+        return tasks;
+    }
+
+    @JsonProperty("last")
+    @Deprecated
+    public Optional<String> getLast()
+    {
+        return last;
+    }
+
+    private class EntryList
+            extends AbstractList<String>
+    {
+        private final byte[] data;
+        private final List<Entry> entries;
+        private InputStream stream;
+        private int current;
+
+        private final ByteBuffer castBuffer = ByteBuffer.allocate(4);
+
+        public EntryList(byte[] data, List<Entry> entries)
+        {
+            this.data = data;
+            this.entries = entries;
+            try {
+                this.stream = new BufferedInputStream(new GZIPInputStream(new ByteArrayInputStream(data)));
+            }
+            catch (IOException ex) {
+                throw Throwables.propagate(ex);
+            }
+            this.current = 0;
+        }
+
+        @Override
+        public synchronized String get(int i)
+        {
+            Entry e = entries.get(i);
+            if (e.getIndex() < current) {
+                // rewind to the head
+                try {
+                    stream.close();
+                    stream = new BufferedInputStream(new GZIPInputStream(new ByteArrayInputStream(data)));
+                }
+                catch (IOException ex) {
+                    throw Throwables.propagate(ex);
+                }
+                current = 0;
+            }
+
+            while (current < e.getIndex()) {
+                readNext();
+            }
+            // now current == e.getIndex()
+            return readNextString();
+        }
+
+        @Override
+        public int size()
+        {
+            return entries.size();
+        }
+
+        private byte[] readNext()
+        {
+            try {
+                stream.read(castBuffer.array());
+                int n = castBuffer.getInt(0);
+                byte[] b = new byte[n];  // here should be able to use a pooled buffer because read data is ignored if readNextString doesn't call this method
+                stream.read(b);
+
+                current++;
+
+                return b;
+            }
+            catch (IOException ex) {
+                throw Throwables.propagate(ex);
+            }
+        }
+
+        private String readNextString()
+        {
+            return new String(readNext(), StandardCharsets.UTF_8);
+        }
+    }
+}

data/src/main/java/org/embulk/input/gcs/GcsFileInput.java

@@ -0,0 +1,195 @@
+package org.embulk.input.gcs;
+
+import com.google.api.client.http.HttpResponseException;
+import com.google.api.services.storage.Storage;
+import com.google.api.services.storage.model.Bucket;
+import com.google.api.services.storage.model.Objects;
+import com.google.api.services.storage.model.StorageObject;
+import com.google.common.base.Charsets;
+import com.google.common.base.Function;
+import com.google.common.base.Optional;
+import com.google.common.io.BaseEncoding;
+import org.embulk.config.ConfigException;
+import org.embulk.config.TaskReport;
+import org.embulk.spi.Exec;
+import org.embulk.spi.TransactionalFileInput;
+import org.embulk.spi.unit.LocalFile;
+import org.embulk.spi.util.InputStreamFileInput;
+import org.slf4j.Logger;
+
+import java.io.IOException;
+import java.math.BigInteger;
+import java.security.GeneralSecurityException;
+import java.util.List;
+
+public class GcsFileInput
+        extends InputStreamFileInput
+        implements TransactionalFileInput
+{
+    private static final Logger log = Exec.getLogger(org.embulk.input.gcs.GcsFileInput.class);
+
+    public GcsFileInput(PluginTask task, int taskIndex)
+    {
+        super(task.getBufferAllocator(), new SingleFileProvider(task, taskIndex));
+    }
+
+    public void abort()
+    {
+    }
+
+    public TaskReport commit()
+    {
+        return Exec.newTaskReport();
+    }
+
+    @Override
+    public void close()
+    {
+    }
+
+    public static GcsAuthentication newGcsAuth(PluginTask task)
+    {
+        try {
+            return new GcsAuthentication(
+                    task.getAuthMethod().getString(),
+                    task.getServiceAccountEmail(),
+                    task.getP12Keyfile().transform(localFileToPathString()),
+                    task.getJsonKeyfile().transform(localFileToPathString()),
+                    task.getApplicationName()
+            );
+        }
+        catch (GeneralSecurityException | IOException ex) {
+            throw new ConfigException(ex);
+        }
+    }
+
+    protected static Storage newGcsClient(final PluginTask task, final GcsAuthentication auth)
+    {
+        Storage client = null;
+        try {
+            client = auth.getGcsClient(task.getBucket(), task.getMaxConnectionRetry());
+        }
+        catch (IOException ex) {
+            throw new ConfigException(ex);
+        }
+
+        return client;
+    }
+
+    private static Function<LocalFile, String> localFileToPathString()
+    {
+        return new Function<LocalFile, String>()
+        {
+            public String apply(LocalFile file)
+            {
+                return file.getPath().toString();
+            }
+        };
+    }
+
+    public static FileList listFiles(PluginTask task, Storage client)
+    {
+        String bucket = task.getBucket();
+
+        FileList.Builder builder = new FileList.Builder(task);
+        listGcsFilesByPrefix(builder, client, bucket, task.getPathPrefix().get(), task.getLastPath());
+        return builder.build();
+    }
+
+    /**
+     * Lists GCS filenames filtered by prefix.
+     *
+     * The resulting list does not include the file that's size == 0.
+     */
+    public static void listGcsFilesByPrefix(FileList.Builder builder, Storage client, String bucket,
+                                             String prefix, Optional<String> lastPath)
+    {
+        String lastKey = lastPath.isPresent() ? base64Encode(lastPath.get()) : null;
+
+        // @see https://cloud.google.com/storage/docs/json_api/v1/objects#resource
+        if (log.isDebugEnabled()) {
+            try {
+                Storage.Buckets.Get getBucket = client.buckets().get(bucket);
+                getBucket.setProjection("full");
+                Bucket bk = getBucket.execute();
+
+                log.debug("bucket name: " + bucket);
+                log.debug("bucket location: " + bk.getLocation());
+                log.debug("bucket timeCreated: " + bk.getTimeCreated());
+                log.debug("bucket owner: " + bk.getOwner());
+            }
+            catch (IOException e) {
+                log.warn("Could not access to bucket:" + bucket);
+                log.warn(e.getMessage());
+            }
+        }
+
+        try {
+            // @see https://cloud.google.com/storage/docs/json_api/v1/objects/list
+            Storage.Objects.List listObjects = client.objects().list(bucket);
+            listObjects.setPrefix(prefix);
+            listObjects.setPageToken(lastKey);
+            do {
+                Objects objects = listObjects.execute();
+                List<StorageObject> items = objects.getItems();
+                if (items == null) {
+                    log.info(String.format("No file was found in bucket:%s prefix:%s", bucket, prefix));
+                    break;
+                }
+                for (StorageObject o : items) {
+                    if (o.getSize().compareTo(BigInteger.ZERO) > 0) {
+                        builder.add(o.getName(), o.getSize().longValue());
+                    }
+                    log.debug("filename: " + o.getName());
+                    log.debug("updated: " + o.getUpdated());
+                }
+                lastKey = objects.getNextPageToken();
+                listObjects.setPageToken(lastKey);
+            } while (lastKey != null);
+        }
+        catch (IOException e) {
+            if ((e instanceof HttpResponseException) && ((HttpResponseException) e).getStatusCode() == 400) {
+                throw new ConfigException(String.format("Files listing failed: bucket:%s, prefix:%s, last_path:%s", bucket, prefix, lastKey), e);
+            }
+
+            log.warn(String.format("Could not get file list from bucket:%s", bucket));
+            log.warn(e.getMessage());
+        }
+    }
+
+    // String nextToken = base64Encode(0x0a + 0x01~0x27 + filePath);
+    private static String base64Encode(String path)
+    {
+        byte[] encoding;
+        byte[] utf8 = path.getBytes(Charsets.UTF_8);
+        log.debug(String.format("path string: %s ,path length:%s \" + ", path, utf8.length));
+
+        encoding = new byte[utf8.length + 2];
+        encoding[0] = 0x0a;
+        encoding[1] = new Byte(String.valueOf(path.length()));
+        System.arraycopy(utf8, 0, encoding, 2, utf8.length);
+
+        String s = BaseEncoding.base64().encode(encoding);
+        log.debug(String.format("last_path(base64 encoded): %s", s));
+        return s;
+    }
+
+    public enum AuthMethod
+    {
+        private_key("private_key"),
+        compute_engine("compute_engine"),
+        json_key("json_key");
+
+        private final String string;
+
+        AuthMethod(String string)
+        {
+            this.string = string;
+        }
+
+        public String getString()
+        {
+            return string;
+        }
+    }
+}