embulk-output-dynamodb 0.1.3 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 65c351383c9859353bc121a5f054b63de4876773
-  data.tar.gz: e171f2369c6582ef90c8fb7d4a6acc0a0300061b
+  metadata.gz: 82cba589f5f74209f19cff43e295c2c6fb48fb20
+  data.tar.gz: b0645e814a4937376ac7f9a10b06f29cf4b5873f
 SHA512:
-  metadata.gz: 862214066e48cb94be626d68b9a014dcdccf1a4a74f1808e4ff87b43beef2115f55b0d5547aa8d3460404bf85856ab9ed64488190bf1dbdde71ce7f44c07dd47
-  data.tar.gz: 8466fe13984980117b6a7bba30efbafca953ad8011758fcfd4b6a0eb6589efc45389051ed2de2e7c2eeb11342a5abf2e5be9b9c82a188d28f28652c5b7b5e5c6
+  metadata.gz: 25b60a3454c49cb41e10a633ff55b7dc80637d35766a69d56994a29541e06283a47b310e99240ef02b30f7b376c0a7cefface78ed20b29b52601dc05e796d575
+  data.tar.gz: af0c6c9e88343be88c1c010e48d197f3f7c1f41c269754ebc2615e0611bced90694149ea2ae67810d99c7606c115ee3517c9669868f4b28c4a8f446997f5b3dc

data/CHANGELOG.md

@@ -1 +1,27 @@
-## 0.1.3 - 2016-07-11
+## 0.1.5 - 2022-11-09
+
+[new feature] Support both ECS task role and EC2 instance profile in auth_method 'instance' [#12](https://github.com/sakama/embulk-output-dynamodb/pull/12) @grass-fed thanks!
+
+## 0.1.4 - 2017-06-21
+
+[new feature] HashKey/RangeKey table support at upsert_with_expression mode [#9](https://github.com/sakama/embulk-output-dynamodb/pull/9) @takasick thanks!
+
+## 0.1.3 - 2016-07-11
+
+[maintenance] Fix JSON column handling(Array type) [#7](https://github.com/sakama/embulk-output-dynamodb/pull/7) @joker1007 thanks!
+[maintenance] Add validation for auto_create_table option [#8](https://github.com/sakama/embulk-output-dynamodb/pull/8)
+
+## 0.1.2 - 2016-05-24
+
+[maintenance] Small fix
+
+## 0.1.1 - 2016-05-16
+
+[maintenance] Check if table exists before creating table [#5](https://github.com/sakama/embulk-output-dynamodb/pull/5)
+
+## 0.1.0 - 2016-05-13
+
+Thanks for contribution!
+
+@joker1007 [#1](https://github.com/sakama/embulk-output-dynamodb/pull/1) [#2](https://github.com/sakama/embulk-output-dynamodb/pull/2)
+@satoshihirose [#3](https://github.com/sakama/embulk-output-dynamodb/pull/3)

data/README.md

@@ -115,7 +115,7 @@
 
 - **instance**
     
-    Use EC2 instance profile
+    Use ECS task role or EC2 instance profile.
     
     ```yaml
     auth_method: instance
@@ -212,4 +212,4 @@ $ cd /path/to/dynamodb
 $ wget http://dynamodb-local.s3-website-us-west-2.amazonaws.com/dynamodb_local_latest.tar.gz
 $ tar zxvf dynamodb_local_latest.tar.gz
 $ java -Djava.library.path=./DynamoDBLocal_lib -jar DynamoDBLocal.jar -sharedDb
-```
+```

data/build.gradle

@@ -14,7 +14,7 @@ configurations {
     provided
 }
 
-version = "0.1.3"
+version = "0.1.5"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
@@ -22,7 +22,7 @@ targetCompatibility = 1.7
 dependencies {
     compile  "org.embulk:embulk-core:0.8.8"
     provided "org.embulk:embulk-core:0.8.8"
-    compile "com.amazonaws:aws-java-sdk-dynamodb:1.10.50"
+    compile "com.amazonaws:aws-java-sdk-dynamodb:1.11.16"
 
     testCompile "junit:junit:4.+"
     testCompile "org.embulk:embulk-core:0.8.8:tests"

data/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
-#Wed Jan 13 12:41:02 JST 2016
+#Sun Jan 08 00:35:58 PST 2017
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-2.10-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-3.2.1-bin.zip

data/gradlew.bat

@@ -1,90 +1,90 @@
-@if "%DEBUG%" == "" @echo off
-@rem ##########################################################################
-@rem
-@rem  Gradle startup script for Windows
-@rem
-@rem ##########################################################################
-
-@rem Set local scope for the variables with windows NT shell
-if "%OS%"=="Windows_NT" setlocal
-
-@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-set DEFAULT_JVM_OPTS=
-
-set DIRNAME=%~dp0
-if "%DIRNAME%" == "" set DIRNAME=.
-set APP_BASE_NAME=%~n0
-set APP_HOME=%DIRNAME%
-
-@rem Find java.exe
-if defined JAVA_HOME goto findJavaFromJavaHome
-
-set JAVA_EXE=java.exe
-%JAVA_EXE% -version >NUL 2>&1
-if "%ERRORLEVEL%" == "0" goto init
-
-echo.
-echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
-
-goto fail
-
-:findJavaFromJavaHome
-set JAVA_HOME=%JAVA_HOME:"=%
-set JAVA_EXE=%JAVA_HOME%/bin/java.exe
-
-if exist "%JAVA_EXE%" goto init
-
-echo.
-echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
-
-goto fail
-
-:init
-@rem Get command-line arguments, handling Windowz variants
-
-if not "%OS%" == "Windows_NT" goto win9xME_args
-if "%@eval[2+2]" == "4" goto 4NT_args
-
-:win9xME_args
-@rem Slurp the command line arguments.
-set CMD_LINE_ARGS=
-set _SKIP=2
-
-:win9xME_args_slurp
-if "x%~1" == "x" goto execute
-
-set CMD_LINE_ARGS=%*
-goto execute
-
-:4NT_args
-@rem Get arguments from the 4NT Shell from JP Software
-set CMD_LINE_ARGS=%$
-
-:execute
-@rem Setup the command line
-
-set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
-
-@rem Execute Gradle
-"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
-
-:end
-@rem End local scope for the variables with windows NT shell
-if "%ERRORLEVEL%"=="0" goto mainEnd
-
-:fail
-rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
-rem the _cmd.exe /c_ return code!
-if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
-exit /b 1
-
-:mainEnd
-if "%OS%"=="Windows_NT" endlocal
-
-:omega
+@if "%DEBUG%" == "" @echo off
+@rem ##########################################################################
+@rem
+@rem  Gradle startup script for Windows
+@rem
+@rem ##########################################################################
+
+@rem Set local scope for the variables with windows NT shell
+if "%OS%"=="Windows_NT" setlocal
+
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS=
+
+set DIRNAME=%~dp0
+if "%DIRNAME%" == "" set DIRNAME=.
+set APP_BASE_NAME=%~n0
+set APP_HOME=%DIRNAME%
+
+@rem Find java.exe
+if defined JAVA_HOME goto findJavaFromJavaHome
+
+set JAVA_EXE=java.exe
+%JAVA_EXE% -version >NUL 2>&1
+if "%ERRORLEVEL%" == "0" goto init
+
+echo.
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:findJavaFromJavaHome
+set JAVA_HOME=%JAVA_HOME:"=%
+set JAVA_EXE=%JAVA_HOME%/bin/java.exe
+
+if exist "%JAVA_EXE%" goto init
+
+echo.
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:init
+@rem Get command-line arguments, handling Windowz variants
+
+if not "%OS%" == "Windows_NT" goto win9xME_args
+if "%@eval[2+2]" == "4" goto 4NT_args
+
+:win9xME_args
+@rem Slurp the command line arguments.
+set CMD_LINE_ARGS=
+set _SKIP=2
+
+:win9xME_args_slurp
+if "x%~1" == "x" goto execute
+
+set CMD_LINE_ARGS=%*
+goto execute
+
+:4NT_args
+@rem Get arguments from the 4NT Shell from JP Software
+set CMD_LINE_ARGS=%$
+
+:execute
+@rem Setup the command line
+
+set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+
+@rem Execute Gradle
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
+
+:end
+@rem End local scope for the variables with windows NT shell
+if "%ERRORLEVEL%"=="0" goto mainEnd
+
+:fail
+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
+rem the _cmd.exe /c_ return code!
+if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
+exit /b 1
+
+:mainEnd
+if "%OS%"=="Windows_NT" endlocal
+
+:omega

data/src/main/java/org/embulk/output/dynamodb/AwsCredentials.java

@@ -7,8 +7,8 @@ import com.amazonaws.auth.AWSSessionCredentialsProvider;
 import com.amazonaws.auth.AnonymousAWSCredentials;
 import com.amazonaws.auth.BasicAWSCredentials;
 import com.amazonaws.auth.BasicSessionCredentials;
+import com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper;
 import com.amazonaws.auth.EnvironmentVariableCredentialsProvider;
-import com.amazonaws.auth.InstanceProfileCredentialsProvider;
 import com.amazonaws.auth.SystemPropertiesCredentialsProvider;
 import com.amazonaws.auth.profile.ProfileCredentialsProvider;
 import com.amazonaws.auth.profile.ProfilesConfigFile;
@@ -85,7 +85,7 @@ public abstract class AwsCredentials
                 reject(task.getSessionToken(), "session_token");
                 reject(task.getProfileFile(), "profile_file");
                 reject(task.getProfileName(), "profile_name");
-                return new InstanceProfileCredentialsProvider();
+                return new EC2ContainerCredentialsProviderWrapper();
 
             case "profile":
             {

data/src/main/java/org/embulk/output/dynamodb/DynamodbOutputPlugin.java

@@ -199,7 +199,7 @@ public class DynamodbOutputPlugin
         private final String table;
         private final Mode mode;
         private final Optional<String> updateExpression;
-        private final String primaryKey;
+        private final List primaryKeyElements;
         private final int maxPutItems;
 
         public DynamodbPageOutput(PluginTask task, DynamoDB dynamoDB)
@@ -210,7 +210,7 @@ public class DynamodbOutputPlugin
             this.table = task.getTable();
             this.mode = task.getMode();
             this.updateExpression = task.getUpdateExpression();
-            this.primaryKey = (mode.equals(Mode.UPSERT_WITH_EXPRESSION)) ? dynamodbUtils.getPrimaryKeyName(dynamoDB, table) : null;
+            this.primaryKeyElements = (mode.equals(Mode.UPSERT_WITH_EXPRESSION)) ? dynamodbUtils.getPrimaryKey(dynamoDB, table) : null;
             this.maxPutItems = task.getMaxPutItems();
         }
 
@@ -400,7 +400,7 @@ public class DynamodbOutputPlugin
         public void updateItem(Item item)
         {
             try {
-                dynamodbUtils.updateItem(dynamoDB, table, item, primaryKey, updateExpression);
+                dynamodbUtils.updateItem(dynamoDB, table, item, primaryKeyElements, updateExpression);
                 totalWroteItemSize++;
                 if (totalWroteItemSize % 1000 == 0) {
                     log.info(String.format("Updated %s items", totalWroteItemSize));

data/src/main/java/org/embulk/output/dynamodb/DynamodbUtils.java

@@ -143,17 +143,34 @@ public class DynamodbUtils
         }
     }
 
-    protected void updateItem(DynamoDB dynamoDB, String tableName, Item item, String primaryKey, Optional<String> expression)
+    protected void updateItem(DynamoDB dynamoDB, String tableName, Item item, List primaryKeyelements, Optional<String> expression)
     {
-        Object primaryKeyValue = null;
+        String hashKeyname = null;
+        String rangeKeyname = null;
+        Object hashKeyValue = null;
+        Object rangeKeyValue = null;
         Map<String, String> attributeNames = new HashMap<>();
         Map<String, Object> attributeValues = new HashMap<>();
 
+        Iterator it = primaryKeyelements.iterator();
+        while (it.hasNext()) {
+            KeySchemaElement element = (KeySchemaElement) it.next();
+            if (element.getKeyType().equals(KeyType.HASH.toString())) {
+                hashKeyname = element.getAttributeName();
+            }
+            else if (element.getKeyType().equals(KeyType.RANGE.toString())) {
+                rangeKeyname = element.getAttributeName();
+            }
+        }
+
         Map<String, Object> itemMap = item.asMap();
         for (Map.Entry<String, Object> e : itemMap.entrySet()) {
             String keyName = e.getKey();
-            if (keyName.equals(primaryKey)) {
-                primaryKeyValue = e.getValue();
+            if (keyName.equals(hashKeyname)) {
+                hashKeyValue = e.getValue();
+            }
+            else if (keyName.equals(rangeKeyname)) {
+                rangeKeyValue = e.getValue();
             }
             else {
                 if (expression.get().indexOf(keyName) > 0) {
@@ -164,9 +181,17 @@ public class DynamodbUtils
         }
         log.debug("attribute names: " + attributeNames.toString());
         log.debug("attribute values: " + attributeValues.toString());
-        log.debug(String.format("primary key %s:%s", primaryKey, primaryKeyValue));
+
         Table table = dynamoDB.getTable(tableName);
-        table.updateItem(primaryKey, primaryKeyValue, expression.get(), attributeNames, attributeValues);
+
+        log.debug(String.format("hash key %s:%s", hashKeyname, hashKeyValue));
+        if (rangeKeyValue == null) {
+            table.updateItem(hashKeyname, hashKeyValue, expression.get(), attributeNames, attributeValues);
+        }
+        else {
+            log.debug(String.format("range key %s:%s", rangeKeyname, rangeKeyValue));
+            table.updateItem(hashKeyname, hashKeyValue, rangeKeyname, rangeKeyValue, expression.get(), attributeNames, attributeValues);
+        }
     }
 
     protected String getPrimaryKeyName(DynamoDB dynamoDB, String tableName)
@@ -178,11 +203,21 @@ public class DynamodbUtils
         String primaryKey = null;
         while (schema.hasNext()) {
             KeySchemaElement element = schema.next();
-            primaryKey = element.getAttributeName();
+            if (element.getKeyType().equals(KeyType.HASH.toString())) {
+                primaryKey = element.getAttributeName();
+            }
         }
         return primaryKey;
     }
 
+    protected List getPrimaryKey(DynamoDB dynamoDB, String tableName)
+    {
+        Table table = dynamoDB.getTable(tableName);
+        TableDescription description = table.describe();
+        List<KeySchemaElement>  keyelements = description.getKeySchema();
+        return keyelements;
+    }
+
     protected void createTable(DynamoDB dynamoDB, DynamodbOutputPlugin.PluginTask task)
             throws InterruptedException
     {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-dynamodb
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.5
 platform: ruby
 authors:
 - Satoshi Akama
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-07-11 00:00:00.000000000 Z
+date: 2022-11-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -65,15 +65,19 @@ files:
 - src/test/java/org/embulk/output/dynamodb/TestDynamodbOutputPlugin.java
 - src/test/java/org/embulk/output/dynamodb/TestDynamodbUtils.java
 - src/test/resources/sample_01.csv
-- classpath/aws-java-sdk-core-1.10.50.jar
-- classpath/aws-java-sdk-dynamodb-1.10.50.jar
-- classpath/aws-java-sdk-kms-1.10.50.jar
-- classpath/aws-java-sdk-s3-1.10.50.jar
-- classpath/commons-codec-1.6.jar
-- classpath/commons-logging-1.1.3.jar
-- classpath/embulk-output-dynamodb-0.1.3.jar
-- classpath/httpclient-4.3.6.jar
-- classpath/httpcore-4.3.3.jar
+- classpath/commons-logging-1.2.jar
+- classpath/aws-java-sdk-s3-1.11.16.jar
+- classpath/embulk-output-dynamodb-0.1.5.jar
+- classpath/aws-java-sdk-dynamodb-1.11.16.jar
+- classpath/jackson-databind-2.6.6.jar
+- classpath/jackson-core-2.6.6.jar
+- classpath/httpclient-4.5.2.jar
+- classpath/commons-codec-1.9.jar
+- classpath/aws-java-sdk-core-1.11.16.jar
+- classpath/aws-java-sdk-kms-1.11.16.jar
+- classpath/jackson-dataformat-cbor-2.6.6.jar
+- classpath/httpcore-4.4.4.jar
+- classpath/jackson-annotations-2.6.0.jar
 homepage: https://github.com/sakama/embulk-output-dynamodb
 licenses:
 - Apache-2.0