embulk-output-s3v2 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 38c87b938e84dcad47a5b47f65872b15d6a549877003bc1379bf9ca32c44eddb
-  data.tar.gz: 3505a9c76d0c385e25d8b1b11676143cca4b3c798d130a0e0c0724abd63dc1a1
+  metadata.gz: 5b2d6890502b2b8e6bf2769d5b9952ed18bfdc5f32b49ad863208fbb4acfb65c
+  data.tar.gz: 8775e499e5f2f94be1d1d20af588318ce333cd5e7223dc1b3bc8ca01e60decc8
 SHA512:
-  metadata.gz: bc999df40b640047ba878092a623b22ad25b8c975600a4615165446703b3827e823d24ed7df17b7bcc1aa64ef181847414ef72381a529b41642573a868e0c045
-  data.tar.gz: 3d3b4f6360528d541fff6d84963dd96b4d44360c0d9ab0f47f657ec0fee550858ff03faa64f4c0d0f2b30ee8910516a497a43b6a2ba38ae4bf67cd8272caa848
+  metadata.gz: 44bdad7736ff31c9e7037d2c0d567f3074eedd53c83865372730963f46a43f126cf7ef9a68a701245c4ead500d8f7071c9ddac7399f98b396ae0af4a803d5665
+  data.tar.gz: 1bdf96573eb326f63daa531f9dc2b6b54f455b0f3bf10454bb375b3ffebd1547e13322ab9cb99c590a733c02dd68de24e8fe9268de382be559a10115f801c05a

data/.github/workflows/codeql-analysis.yml

@@ -0,0 +1,71 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ master ]
+  schedule:
+    - cron: '36 5 * * 2'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'java' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+        # Learn more:
+        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

data/README.md

@@ -13,7 +13,7 @@ Files stores on Amazon S3.
 
 ## Configuration
 - **region**: AWS region name. (string, required)
-- **enable_profile**: If true, AWS credentials profile will be used when authenticating AWS. (boolean, default: `false`)
+- **enable_profile**: If true, AWS credentials profile will be used when authenticating AWS. If false, IamRole will be used. (boolean, default: `false`)
   - Supported in v0.2.0 or later
 - **profile**: AWS credentials profile name. If `enable_profile` is false, this parameter will be ignored. (string, default: `default`)
   - Supported in v0.2.0 or later
@@ -27,7 +27,7 @@ Files stores on Amazon S3.
   - Maximum size: `2GB`
   - Enable semantics
     - Same as that of `multipart_threshold`
-- **multipart_threshold**: If `enable_multi_part_upload` is false, this parameter will be ignored. (string, default: `8MB`)
+- **multipart_threshold**: The size threshold the plugin uses for multipart transfers of individual divided bulk-data. If `enable_multi_part_upload` is false, this parameter will be ignored. (string, default: `8MB`)
   - Enable semantics
     - `KB`
     - `MB`
@@ -38,6 +38,7 @@ Files stores on Amazon S3.
 - **temp_path**: Directory for temp file output. (string, default: `/tmp`)
 - **temp_file_prefix**: Prefix of temp file name. (string, default: `embulk-output-s3v2`)
 ### Example
+#### Basic sample with IAMRole authentication
 ```yaml
 out:
   type: s3v2
@@ -50,6 +51,40 @@ out:
     type: csv
     delimeter: ","
 ```
+#### Basic sample with Credentials-Profile authentication
+```yaml
+out:
+  type: s3v2
+  region: ap-northeast-1
+  bucket: s3-bucket-name
+  object_key_prefix: embulk/embulk-output-s3v2
+  temp_path: /tmp
+  enable_profile: true
+  profile: default
+  extension: .csv
+  formatter:
+    type: csv
+    delimeter: ","
+```
+#### Multipart Upload Sample with gzip encode
+```yaml
+out:
+  type: s3v2
+  region: ap-northeast-1
+  bucket: s3-bucket-name
+  object_key_prefix: embulk/embulk-output-s3v2
+  temp_path: /tmp
+  enable_multi_part_upload: true
+  multipart_chunksize: 10MB
+  max_concurrent_requests: 20
+  extension: csv.gz
+  formatter:
+    type: csv
+    delimeter: ","
+  encoders:
+  - type: gzip
+    level: 1
+```
 
 ## Usage
 ### Build

data/build.gradle

@@ -6,7 +6,7 @@ plugins {
 import com.github.jrubygradle.JRubyExec
 
 group 'com.github.ttksm'
-version '0.2.0'
+version '0.3.0'
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8
@@ -28,6 +28,8 @@ dependencies {
     testCompile 'org.mockito:mockito-core:3.+'
     testCompile 'org.mockito:mockito-junit-jupiter:3.+'
     testCompile 'org.embulk:embulk-core:0.9.23:tests'
+    compile 'software.amazon.awssdk:bom:2.17.282'
+    compile 'software.amazon.awssdk:sso:2.17.282'
 }
 
 test {

data/src/main/java/org/embulk/output/s3v2/s3/S3ClientManager.java

@@ -1,13 +1,11 @@
 package org.embulk.output.s3v2.s3;
 
-import java.io.BufferedOutputStream;
-import java.io.FileOutputStream;
+import java.io.BufferedInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.FileInputStream;
 import java.io.IOException;
 import java.nio.ByteBuffer;
-import java.nio.channels.FileChannel;
-import java.nio.file.Files;
 import java.nio.file.Path;
-import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
@@ -91,26 +89,23 @@ public class S3ClientManager
         CreateMultipartUploadResponse response = s3.createMultipartUpload(createMultipartUploadRequest);
         String uploadId = response.uploadId();
 
-        try (FileChannel fc = FileChannel.open(sourceFile)) {
-            String multipartChunksize = status.getMultipartChunksize();
-            ByteBuffer buffer = ByteBuffer.allocate(ChunksizeComputation.getChunksizeBytes(multipartChunksize));
-
+        try (BufferedInputStream bufferStream = new BufferedInputStream(
+                new FileInputStream(sourceFile.toFile()))) {
             ExecutorService es = Executors.newFixedThreadPool(status.getMaxConcurrentRequests());
             List<CompletableFuture<String>> futureList = new ArrayList<>();
+
+            int multipartChunksize = ChunksizeComputation.getChunksizeBytes(status.getMultipartChunksize());
+            byte[] data = new byte[multipartChunksize];
             int i = 1;
             while (true) {
-                buffer.clear();
-                if (fc.read(buffer) == -1) {
+                int n = bufferStream.read(data);
+                if (n == -1) {
                     break;
                 }
-                buffer.flip();
 
-                // Create tmpFile per size of multipart_chunksize
-                String tmpFile = sourceFile + "_" + i;
-                try (BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(tmpFile))) {
-                    bos.write(buffer.array(), buffer.arrayOffset(), buffer.limit());
-                    bos.flush();
-                }
+                ByteArrayOutputStream out = new ByteArrayOutputStream();
+                out.write(data, 0, n);
+                byte[] chunk = out.toByteArray();
 
                 UploadPartRequest uploadPartRequest = UploadPartRequest.builder()
                         .bucket(bucket).key(objectKey).uploadId(uploadId)
@@ -119,7 +114,7 @@ public class S3ClientManager
 
                 // Async upload to S3
                 CompletableFuture<String> future = CompletableFuture.supplyAsync(() -> {
-                    return s3.uploadPart(uploadPartRequest, RequestBody.fromFile(Paths.get(tmpFile))).eTag();
+                    return s3.uploadPart(uploadPartRequest, RequestBody.fromBytes(chunk)).eTag();
                 }, es);
                 futureList.add(future);
 
@@ -131,8 +126,6 @@ public class S3ClientManager
             for (int j = 1; j <= futureList.size(); j++) {
                 CompletedPart part = CompletedPart.builder().partNumber(j).eTag(futureList.get(j - 1).get()).build();
                 partList.add(part);
-                // Remove tmpFile
-                Files.delete(Paths.get(sourceFile + "_" + j));
             }
 
             CompletedMultipartUpload completedMultipartUpload = CompletedMultipartUpload.builder()

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-s3v2
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Toshihiro Takushima
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-01-11 00:00:00.000000000 Z
+date: 2022-09-29 00:00:00.000000000 Z
 dependencies: []
 description: Stores files on Amazon S3 using aws-sdk-java-v2.
 email:
@@ -18,50 +18,50 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".github/workflows/build.yml"
+- ".github/workflows/codeql-analysis.yml"
 - ".github/workflows/release.yml"
 - ".gitignore"
 - LICENSE
 - README.md
 - build.gradle
-- classpath/annotations-2.15.61.jar
-- classpath/apache-client-2.15.61.jar
-- classpath/arns-2.15.61.jar
-- classpath/auth-2.15.61.jar
-- classpath/aws-core-2.15.61.jar
-- classpath/aws-query-protocol-2.15.61.jar
-- classpath/aws-xml-protocol-2.15.61.jar
+- classpath/annotations-2.17.283.jar
+- classpath/apache-client-2.17.283.jar
+- classpath/arns-2.17.283.jar
+- classpath/auth-2.17.283.jar
+- classpath/aws-core-2.17.283.jar
+- classpath/aws-json-protocol-2.17.282.jar
+- classpath/aws-query-protocol-2.17.283.jar
+- classpath/aws-xml-protocol-2.17.283.jar
 - classpath/commons-codec-1.11.jar
 - classpath/commons-logging-1.2.jar
-- classpath/embulk-output-s3v2-0.2.0.jar
+- classpath/embulk-output-s3v2-0.3.0.jar
 - classpath/eventstream-1.0.1.jar
-- classpath/http-client-spi-2.15.61.jar
+- classpath/http-client-spi-2.17.283.jar
 - classpath/httpclient-4.5.13.jar
 - classpath/httpcore-4.4.13.jar
-- classpath/jackson-annotations-2.10.5.jar
-- classpath/jackson-core-2.10.5.jar
-- classpath/jackson-databind-2.10.5.1.jar
-- classpath/metrics-spi-2.15.61.jar
-- classpath/netty-buffer-4.1.53.Final.jar
-- classpath/netty-codec-4.1.53.Final.jar
-- classpath/netty-codec-http-4.1.53.Final.jar
-- classpath/netty-codec-http2-4.1.53.Final.jar
-- classpath/netty-common-4.1.53.Final.jar
-- classpath/netty-handler-4.1.53.Final.jar
-- classpath/netty-nio-client-2.15.61.jar
-- classpath/netty-reactive-streams-2.0.4.jar
-- classpath/netty-reactive-streams-http-2.0.4.jar
-- classpath/netty-resolver-4.1.53.Final.jar
-- classpath/netty-transport-4.1.53.Final.jar
-- classpath/netty-transport-native-epoll-4.1.53.Final-linux-x86_64.jar
-- classpath/netty-transport-native-unix-common-4.1.53.Final.jar
-- classpath/profiles-2.15.61.jar
-- classpath/protocol-core-2.15.61.jar
+- classpath/json-utils-2.17.283.jar
+- classpath/metrics-spi-2.17.283.jar
+- classpath/netty-buffer-4.1.77.Final.jar
+- classpath/netty-codec-4.1.77.Final.jar
+- classpath/netty-codec-http-4.1.77.Final.jar
+- classpath/netty-codec-http2-4.1.77.Final.jar
+- classpath/netty-common-4.1.77.Final.jar
+- classpath/netty-handler-4.1.77.Final.jar
+- classpath/netty-nio-client-2.17.283.jar
+- classpath/netty-resolver-4.1.77.Final.jar
+- classpath/netty-transport-4.1.77.Final.jar
+- classpath/netty-transport-classes-epoll-4.1.77.Final.jar
+- classpath/netty-transport-native-unix-common-4.1.77.Final.jar
+- classpath/profiles-2.17.283.jar
+- classpath/protocol-core-2.17.283.jar
 - classpath/reactive-streams-1.0.3.jar
-- classpath/regions-2.15.61.jar
-- classpath/s3-2.15.61.jar
-- classpath/sdk-core-2.15.61.jar
-- classpath/slf4j-api-1.7.28.jar
-- classpath/utils-2.15.61.jar
+- classpath/regions-2.17.283.jar
+- classpath/s3-2.17.283.jar
+- classpath/sdk-core-2.17.283.jar
+- classpath/slf4j-api-1.7.30.jar
+- classpath/sso-2.17.282.jar
+- classpath/third-party-jackson-core-2.17.283.jar
+- classpath/utils-2.17.283.jar
 - config/checkstyle/checkstyle.xml
 - config/checkstyle/default.xml
 - gradle/wrapper/gradle-wrapper.jar