embulk-input-s3 0.2.6 → 0.2.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ee2494b43bcc58768baa64772d07f73c521b87f4
-  data.tar.gz: e12a159f3ad9e6e3cf012bcdc2e779be3242580a
+  metadata.gz: 3fde7e23b02972cc6b615c1845ab820c3e960770
+  data.tar.gz: 7ab94c1a292321bbee5c4642be906c1404db35d2
 SHA512:
-  metadata.gz: 8cf67d08d7f4b11b0c4c35e2b15e2537e9bf984f3a9a77adc0ecd30222c958d493b28817c273045d8bcbfd99bde327ed79760e8fd5118f1dc8c466b93b6f74de
-  data.tar.gz: 36f4ee1d0bcddd46a5945865b9d2f2f41e997b562d15783b96f94b2389e3037acdc104ebe967108110550eeaed12ebf5a1a961c3826358f247c8b06f21fa1e0c
+  metadata.gz: ed3cfe60161e427fdffe19b0622feae7aa1e5138d2c3f10243125eb82349ee91efbda869ab21644fb7141125415c29172e36c77f20cf352128c85855222a8fc6
+  data.tar.gz: 7235b4b49d0f7df056098084c70d8dea60be33dbe0335b589eac8e0478a835514caa4a82ab1ee1b5a45370a2b0353985b2f6b66a01ca9ce123202417917fcbbf

data/src/main/java/org/embulk/input/s3/FileList.java

@@ -38,6 +38,11 @@ public class FileList
         @Config("total_file_count_limit")
         @ConfigDefault("2147483647")
         int getTotalFileCountLimit();
+
+        // TODO support more algorithms to combine tasks
+        @Config("min_task_size")
+        @ConfigDefault("0")
+        long getMinTaskSize();
     }
 
     public static class Entry
@@ -69,6 +74,7 @@ public class FileList
         private String last = null;
 
         private int limitCount = Integer.MAX_VALUE;
+        private long minTaskSize = 1;
         private Pattern pathMatchPattern;
 
         private final ByteBuffer castBuffer = ByteBuffer.allocate(4);
@@ -76,8 +82,9 @@ public class FileList
         public Builder(Task task)
         {
             this();
-            this.limitCount = task.getTotalFileCountLimit();
             this.pathMatchPattern = Pattern.compile(task.getPathMatchPattern());
+            this.limitCount = task.getTotalFileCountLimit();
+            this.minTaskSize = task.getMinTaskSize();
         }
 
         public Builder(ConfigSource config)
@@ -85,6 +92,7 @@ public class FileList
             this();
             this.pathMatchPattern = Pattern.compile(config.get(String.class, "path_match_pattern", ".*"));
             this.limitCount = config.get(int.class, "total_file_count_limit", Integer.MAX_VALUE);
+            this.minTaskSize = config.get(long.class, "min_task_size", 0L);
         }
 
         public Builder()
@@ -104,6 +112,12 @@ public class FileList
             return this;
         }
 
+        public Builder minTaskSize(long bytes)
+        {
+            this.minTaskSize = bytes;
+            return this;
+        }
+
         public Builder pathMatchPattern(String pattern)
         {
             this.pathMatchPattern = Pattern.compile(pattern);
@@ -163,10 +177,20 @@ public class FileList
 
         private List<List<Entry>> getSplits(List<Entry> all)
         {
-            // TODO combine multiple entries into one task using some configuration parameters
             List<List<Entry>> tasks = new ArrayList<>();
+            long currentTaskSize = 0;
+            List<Entry> currentTask = new ArrayList<>();
             for (Entry entry : all) {
-                tasks.add(ImmutableList.of(entry));
+                currentTask.add(entry);
+                currentTaskSize += entry.getSize();  // TODO consider to multiply the size by cost_per_byte, and add cost_per_file
+                if (currentTaskSize >= minTaskSize) {
+                    tasks.add(currentTask);
+                    currentTask = new ArrayList<>();
+                    currentTaskSize = 0;
+                }
+            }
+            if (!currentTask.isEmpty()) {
+                tasks.add(currentTask);
             }
             return tasks;
         }

data/src/test/java/org/embulk/input/s3/TestFileList.java

@@ -0,0 +1,87 @@
+package org.embulk.input.s3;
+
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.config.ConfigSource;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestFileList
+{
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    private ConfigSource config;
+
+    @Before
+    public void createConfigSource()
+    {
+        config = runtime.getExec().newConfigSource();
+    }
+
+    @Test
+    public void checkMinTaskSize()
+            throws Exception
+    {
+        { // not specify min_task_size
+            FileList fileList = newFileList(config.deepCopy(),
+                    "sample_00", 100L,
+                    "sample_01", 150L,
+                    "sample_02", 350L);
+
+            assertEquals(3, fileList.getTaskCount());
+            assertEquals("sample_00", fileList.get(0).get(0));
+            assertEquals("sample_01", fileList.get(1).get(0));
+            assertEquals("sample_02", fileList.get(2).get(0));
+        }
+
+        {
+            FileList fileList = newFileList(config.deepCopy().set("min_task_size", 100),
+                    "sample_00", 100L,
+                    "sample_01", 150L,
+                    "sample_02", 350L);
+
+            assertEquals(3, fileList.getTaskCount());
+            assertEquals("sample_00", fileList.get(0).get(0));
+            assertEquals("sample_01", fileList.get(1).get(0));
+            assertEquals("sample_02", fileList.get(2).get(0));
+        }
+
+        {
+            FileList fileList = newFileList(config.deepCopy().set("min_task_size", 200),
+                    "sample_00", 100L,
+                    "sample_01", 150L,
+                    "sample_02", 350L);
+
+            assertEquals(2, fileList.getTaskCount());
+            assertEquals("sample_00", fileList.get(0).get(0));
+            assertEquals("sample_01", fileList.get(0).get(1));
+            assertEquals("sample_02", fileList.get(1).get(0));
+        }
+
+        {
+            FileList fileList = newFileList(config.deepCopy().set("min_task_size", 700),
+                    "sample_00", 100L,
+                    "sample_01", 150L,
+                    "sample_02", 350L);
+
+            assertEquals(1, fileList.getTaskCount());
+            assertEquals("sample_00", fileList.get(0).get(0));
+            assertEquals("sample_01", fileList.get(0).get(1));
+            assertEquals("sample_02", fileList.get(0).get(2));
+        }
+    }
+
+    private static FileList newFileList(ConfigSource config, Object... nameAndSize)
+    {
+        FileList.Builder builder = new FileList.Builder(config);
+
+        for (int i = 0; i < nameAndSize.length; i += 2) {
+            builder.add((String) nameAndSize[i], (long) nameAndSize[i + 1]);
+        }
+
+        return builder.build();
+    }
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-s3
 version: !ruby/object:Gem::Version
-  version: 0.2.6
+  version: 0.2.7
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-12-05 00:00:00.000000000 Z
+date: 2015-12-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -53,6 +53,7 @@ files:
 - src/main/java/org/embulk/input/s3/FileList.java
 - src/main/java/org/embulk/input/s3/S3FileInputPlugin.java
 - src/test/java/org/embulk/input/s3/TestAwsCredentials.java
+- src/test/java/org/embulk/input/s3/TestFileList.java
 - src/test/java/org/embulk/input/s3/TestS3FileInputPlugin.java
 - src/test/java/org/embulk/input/s3/TestS3InputStreamReopener.java
 - src/test/resources/sample_01.csv
@@ -60,7 +61,7 @@ files:
 - classpath/aws-java-sdk-kms-1.10.33.jar
 - classpath/aws-java-sdk-s3-1.10.33.jar
 - classpath/commons-codec-1.6.jar
-- classpath/embulk-input-s3-0.2.6.jar
+- classpath/embulk-input-s3-0.2.7.jar
 - classpath/httpclient-4.3.6.jar
 - classpath/httpcore-4.3.3.jar
 - classpath/jcl-over-slf4j-1.7.12.jar