embulk-output-larges3 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f84d3811ca05bdbd6a2ec5e15ed8f454538482c5
-  data.tar.gz: b905b1d625e3bc5e8ec48c249152d9d969db8748
+  metadata.gz: d645c8ac94cb9dcbdf6a4d95b78e860d1d74dbf8
+  data.tar.gz: debcdf0a002be4394b7a4d5a4dd124d5cc32f3ac
 SHA512:
-  metadata.gz: 2a4d47467b0eec06e010b19c19a65b84a5ade9312b9638756cc842435da0de6ac610398880c448176cabf92642e6e2ab7dcad6f15265132ddbdc229223214dad
-  data.tar.gz: 97f5c770a102e8827a3b196f24fb72df1267614a711d4484098595b2f809a055f46c21e3d7afa42879f801d2a03adf0d002cb5c0d43f3b0975ad6a317bd80473
+  metadata.gz: 54395b570321c91ddd267ed75925350afd09ec5ed6764d729bbc103ed49d9bd66d27f1ca1a964b74c6cc11589cc7c022e2a9281defa387c7777df3f97ebce2cb
+  data.tar.gz: d95a396cbc798a2c6537eeb72cb0f092ea7deb442154b2a7d054b726d107ad12f6d054dde28c17d67008613f1e5fbd40604b9f531c5b12ecb0ddf337cb4a239f

data/README.md

@@ -1,6 +1,9 @@
-# Embulk Output Large S3 output plugin for Embulk
+# Large S3 File Output Plugin for Embulk
+Embulk File Output Plugin: Handle and Upload really large files to AWS S3 using multipart upload.
+This plugin is an extension of classic s3 output plugin. [https://github.com/llibra/embulk-output-s3] 
+## Developers
 
-TODO: Write short description here and build.gradle file.
+* Angelos Alexopoulos <alexopoulos7@gmail.com>
 
 ## Overview
 
@@ -11,22 +14,52 @@ TODO: Write short description here and build.gradle file.
 
 ## Configuration
 
-- **option1**: description (integer, required)
-- **option2**: description (string, default: `"myvalue"`)
-- **option3**: description (string, default: `null`)
+- **path_prefix**: prefix of target keys (string, required)
+- **file_ext**: suffix of target keys (string, required)
+- **sequence_format**: format for sequence part of target keys (string, default: '.%03d.%02d')
+- **bucket**: S3 bucket name (string, required)
+- **endpoint**: S3 endpoint login user name (string, optional)
+- **access_key_id**: AWS access key id. This parameter is required when your agent is not running on EC2 instance with an IAM Role. (string, defualt: null)
+- **secret_access_key**: AWS secret key. This parameter is required when your agent is not running on EC2 instance with an IAM Role. (string, defualt: null)
+- **tmp_path**: temporary file directory. If null, it is associated with the default FileSystem. (string, default: null)
+- **tmp_path_prefix**: prefix of temporary files (string, default: 'embulk-output-s3-')
+- **canned_acl**: canned access control list for created objects ([enum](#cannedaccesscontrollist), default: null)
+- **proxy_host**: proxy host to use when accessing AWS S3 via proxy. (string, default: null )
+- **proxy_port**: proxy port to use when accessing AWS S3 via proxy. (string, default: null )
+- **part_size**: Size in Bytes of each part for multipart upload to S3, defaults to 50 MB (int, default: 52428800 )
+
+### CannedAccessControlList
+you can choose one of the below list.
+
+- AuthenticatedRead
+- AwsExecRead
+- BucketOwnerFullControl
+- BucketOwnerRead
+- LogDeliveryWrite
+- Private
+- PublicRead
+- PublicReadWrite
+
+cf. http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/model/CannedAccessControlList.html
 
 ## Example
 
 ```yaml
 out:
-  type: embulk_output_larges3
-  option1: example1
-  option2: example2
+  type: larges3
+  path_prefix: logs/out
+  file_ext: .csv
+  bucket: my-s3-bucket
+  endpoint: s3-us-west-1.amazonaws.com
+  access_key_id: ABCXYZ123ABCXYZ123
+  secret_access_key: AbCxYz123aBcXyZ123
+  formatter:
+    type: csv
 ```
 
 
 ## Build
 
 ```
-$ ./gradlew gem  # -t to watch change of files and rebuild continuously
-```
+$ ./gradlew gem
+```

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.1.2"
+version = "0.1.3"
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8

data/src/main/java/org/embulk/output/Larges3FileOutputPlugin.java

@@ -10,6 +10,7 @@ import java.util.IllegalFormatException;
 import java.util.List;
 import java.util.ArrayList;
 import java.util.Locale;
+import java.lang.Long;
 
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
@@ -89,6 +90,10 @@ public class Larges3FileOutputPlugin
         @Config("canned_acl")
         @ConfigDefault("null")
         Optional<CannedAccessControlList> getCannedAccessControlList();
+
+        @Config("part_size")
+        @ConfigDefault("52428800")
+        Optional<String> getPartSize();
     }
 
     public static class S3FileOutput
@@ -102,6 +107,7 @@ public class Larges3FileOutputPlugin
         private final String sequenceFormat;
         private final String fileNameExtension;
         private final String tempPathPrefix;
+        private final String partSize;
         private final Optional<CannedAccessControlList> cannedAccessControlListOptional;
 
         private int taskIndex;
@@ -153,6 +159,7 @@ public class Larges3FileOutputPlugin
             this.sequenceFormat = task.getSequenceFormat();
             this.fileNameExtension = task.getFileNameExtension();
             this.tempPathPrefix = task.getTempPathPrefix();
+            this.partSize = task.getPartSize().get();
             if (task.getTempPath().isPresent()) {
                 this.tempPath = task.getTempPath().get();
             }
@@ -201,13 +208,13 @@ public class Larges3FileOutputPlugin
 
             File file = new File(from.toString());
             long contentLength = file.length();
-            long partSize = 52428800; // Set part size to 50 MB.
+            long partSize = Long.parseLong(this.partSize); // Set part size to 50 MB.
 
             try {
                 // Step 2: Upload parts.
                 long filePosition = 0;
                 for (int i = 1; filePosition < contentLength; i++) {
-                    // Last part can be less than 5 MB. Adjust part size.
+                    // Last part can be less than 50 MB. Adjust part size.
                     partSize = Math.min(partSize, (contentLength - filePosition));
 
                     // Create request to upload a part.

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-larges3
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Angelos Alexopoulos
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-04-18 00:00:00.000000000 Z
+date: 2018-04-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -67,9 +67,9 @@ files:
 - classpath/httpclient-4.5.2.jar
 - classpath/commons-codec-1.9.jar
 - classpath/aws-java-sdk-s3-1.11.271.jar
+- classpath/embulk-output-larges3-0.1.3.jar
 - classpath/jmespath-java-1.11.271.jar
 - classpath/jackson-dataformat-cbor-2.6.7.jar
-- classpath/embulk-output-larges3-0.1.2.jar
 - classpath/jackson-databind-2.6.7.1.jar
 - classpath/httpcore-4.4.4.jar
 - classpath/aws-java-sdk-kms-1.11.271.jar