embulk-output-s3_per_record 0.1.2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: df24af87908ca267e9a6518308f0bfb319d14808
-  data.tar.gz: 1f9f6af566dff9c786c1f9531b7527d039eadcf2
+  metadata.gz: 59291fd824e6db4a6f56e4e3b107f1be43a71dfa
+  data.tar.gz: 0cb03ee9ae9d61130c53c5a0f5b80e5bd4ae34d7
 SHA512:
-  metadata.gz: 083299386ca77263932553aaab9bb0e735f73b24ebe502328568ab70e4d7d0ff5c9711792c7a4758a0a3daeb5c74f2b5567473cc9310b325f78f6d8fe25f333e
-  data.tar.gz: 38912a1f91df7c26d1db2a023babd24b7a696b8f132f8218add7b59890bd668fd172205fff76105fd851d7ed346a22314502176703861983a37a7aff73824605
+  metadata.gz: 7c446fa5ddc162c53763695e8b36d5ca6ec213d5e677ecf5df0fe31097e5aae445ae34176642dcdfbe51b05085d5fc82d771a1572588386c9587c22c9f3b2b8a
+  data.tar.gz: 7c7e563beeede3252e82ed481a9e4bba1785bf4e6f1abfb508d602041a8ad37df6db7a6f5d6b61ba78c042274d2a8b5abc13a3423f75edda8cdbbe4c05624220

data/.gitignore

@@ -10,3 +10,4 @@ build/
 /.metadata/
 .classpath
 .project
+*.iml

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.1.2"
+version = "0.2.0"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7

data/src/main/java/org/embulk/output/s3_per_record/S3PerRecordOutputPlugin.java

@@ -6,6 +6,7 @@ import java.io.InputStream;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.concurrent.atomic.AtomicLong;
 
 import javax.validation.constraints.NotNull;
 
@@ -32,6 +33,7 @@ import com.amazonaws.services.s3.transfer.TransferManager;
 import com.amazonaws.services.s3.transfer.Upload;
 import com.amazonaws.util.Base64;
 import com.google.common.base.Optional;
+import org.slf4j.Logger;
 
 
 public class S3PerRecordOutputPlugin
@@ -41,6 +43,11 @@ public class S3PerRecordOutputPlugin
     public static final String KEY_COLUMN_START_MARKER = "${";
     public static final String KEY_COLUMN_END_MARKER = "}";
 
+    private static final Logger logger = Exec.getLogger(S3PerRecordOutputPlugin.class);
+    private static volatile long nextLoggingRowCount = 1000;
+    private static AtomicLong processedRows = new AtomicLong(0);
+    private static long startTime = System.currentTimeMillis();
+
     public interface PluginTask
             extends Task
     {
@@ -129,6 +136,7 @@ public class S3PerRecordOutputPlugin
                 credentials = new DefaultAWSCredentialsProviderChain().getCredentials();
             }
             transferManager = new TransferManager(credentials);
+            logger.info("Start Upload to bucket \"{}\"", bucket);
         }
 
         private List<KeyPart> makeKeyPattern(final String key) {
@@ -175,6 +183,12 @@ public class S3PerRecordOutputPlugin
                 try (InputStream is = new ByteArrayInputStream(payloadBytes)) {
                     Upload upload = transferManager.upload(bucket, key, is, metadata);
                     upload.waitForUploadResult();
+                    long rows = processedRows.incrementAndGet();
+                    if (rows == nextLoggingRowCount) {
+                        double seconds = (System.currentTimeMillis() - startTime) / 1000.0;
+                        logger.info(String.format("> Uploaded %,d rows in %.2f seconds", rows, seconds));
+                        nextLoggingRowCount *= 2;
+                    }
                 } catch (InterruptedException | IOException e) {
                     throw new RuntimeException(e);
                 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-s3_per_record
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
 platform: ruby
 authors:
 - tomykaira
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-04-22 00:00:00.000000000 Z
+date: 2016-05-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -63,8 +63,8 @@ files:
 - classpath/commons-logging-1.1.3.jar
 - classpath/commons-codec-1.6.jar
 - classpath/aws-java-sdk-s3-1.10.71.jar
-- classpath/embulk-output-s3_per_record-0.1.2.jar
 - classpath/aws-java-sdk-core-1.10.71.jar
+- classpath/embulk-output-s3_per_record-0.2.0.jar
 - classpath/httpcore-4.3.3.jar
 - classpath/aws-java-sdk-kms-1.10.71.jar
 homepage: https://github.com/tomykaira/embulk-output-s3_per_record