embulk-input-dynamodb 0.3.0 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (11) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +7 -1
  3. data/build.gradle +2 -2
  4. data/src/main/scala/org/embulk/input/dynamodb/operation/DynamodbQueryOperation.scala +1 -1
  5. data/src/main/scala/org/embulk/input/dynamodb/operation/DynamodbScanOperation.scala +1 -1
  6. data/src/test/scala/org/embulk/input/dynamodb/AwsCredentialsTest.scala +62 -59
  7. data/src/test/scala/org/embulk/input/dynamodb/DynamodbOperationTest.scala +131 -0
  8. data/src/test/scala/org/embulk/input/dynamodb/{DynamodbQueryOperationTest.scala → DynamodbQueryOperationBackwardCompatibilityTest.scala} +41 -48
  9. data/src/test/scala/org/embulk/input/dynamodb/{DynamodbScanOperationTest.scala → DynamodbScanOperationBackwardCompatibilityTest.scala} +39 -45
  10. data/src/test/scala/org/embulk/input/dynamodb/testutil/EmbulkTestBase.scala +31 -17
  11. metadata +6 -5
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 85976c690e023ebd1405fa2990e5dc6995511f08
4
- data.tar.gz: e7aa6e0eed7b15581be57fbbb69028ed6a7d1621
3
+ metadata.gz: e7ae13caaaaee6d19eb328725cf064e1771042a9
4
+ data.tar.gz: f2b96cdf15c20762e1c3bc9e41d0666286a6db59
5
5
  SHA512:
6
- metadata.gz: 8ad60d0ea024cab88469c6afec78c37eb6341035b33effdbe2b30ba823e3239f4d6c07d347da8e095f3d1f43cf61f77f3df654629c2a073f7a4daebb8550dce9
7
- data.tar.gz: 67718ce3f0d8fd97ce3a2c33bbdfb77e1891b9b50ad59815c2282ec1ea56a56e8b7c0e8725d44f0c9fa88f5e50c9542aeb80e62108662ed1c23c85d59cb56aba
6
+ metadata.gz: 32bccbd2fe8aa2bc08b0130ffcc73c77e0ed9a5907a90a4bb5e65d23de2e4811433916f4a5b8b7c0e748917367715347aabe24b441297341755bd549503df1db
7
+ data.tar.gz: 1247ab5ed561520e3f4ac2164602b85b909a2e8c112e3ac9a3b357935787aa295a90bdf7bab9cc916e3fbc98df529df36709414172a43a4714ff69001bd9796e
data/CHANGELOG.md CHANGED
@@ -1,3 +1,9 @@
1
+ 0.3.1 (2020-03-29)
2
+ ==================
3
+
4
+ - [BugFix] [#23](https://github.com/lulichn/embulk-input-dynamodb/pull/23) Throw scala.MatchError when using "limit" option. (Fix [#21](https://github.com/lulichn/embulk-input-dynamodb/issues/21))
5
+ - [Enhancement] [#23](https://github.com/lulichn/embulk-input-dynamodb/pull/23) Use embulk-core tests library instead of embulk-test.
6
+
1
7
  0.3.0 (2020-03-09)
2
8
  ==================
3
9
 
@@ -46,4 +52,4 @@
46
52
  - [Enhancement] [#19](https://github.com/lulichn/embulk-input-dynamodb/pull/19) You can specify the `AttributeValue` type (like `"S"`, `"N"`, `"SS"` and so on) used when converting AttributeValue to Embulk type.
47
53
  - [BugFix] [#19](https://github.com/lulichn/embulk-input-dynamodb/pull/19) Avoid `NullPointerException` when Type `N` AttributeValue has `null` in the deprecated operation.
48
54
  - [Enhancement] [#19](https://github.com/lulichn/embulk-input-dynamodb/pull/19) Examples work without real Dynamodb.
49
- - [Enhancement] [#19](https://github.com/lulichn/embulk-input-dynamodb/pull/19) Add more examples.
55
+ - [Enhancement] [#19](https://github.com/lulichn/embulk-input-dynamodb/pull/19) Add more examples.
data/build.gradle CHANGED
@@ -14,7 +14,7 @@ configurations {
14
14
  provided
15
15
  }
16
16
 
17
- version = "0.3.0"
17
+ version = "0.3.1"
18
18
 
19
19
  sourceCompatibility = 1.8
20
20
  targetCompatibility = 1.8
@@ -31,10 +31,10 @@ dependencies {
31
31
  compile "dev.zio:zio-macros-core_2.13:0.6.2"
32
32
 
33
33
  testCompile "junit:junit:4.+"
34
+ testCompile "org.embulk:embulk-core:0.9.23:tests"
34
35
  testCompile "org.embulk:embulk-standards:0.9.23"
35
36
  testCompile "org.embulk:embulk-deps-buffer:0.9.23"
36
37
  testCompile "org.embulk:embulk-deps-config:0.9.23"
37
- testCompile "org.embulk:embulk-test:0.9.23"
38
38
  }
39
39
 
40
40
  compileScala {
data/src/main/scala/org/embulk/input/dynamodb/operation/DynamodbQueryOperation.scala CHANGED
@@ -51,7 +51,7 @@ case class DynamodbQueryOperation(task: DynamodbQueryOperation.Task)
51
51
  loadableRecords match {
52
52
  case Some(v) if (result.getCount > v) =>
53
53
  f(result.getItems.asScala.take(v.toInt).map(_.asScala.toMap).toSeq)
54
- case None =>
54
+ case _ =>
55
55
  f(result.getItems.asScala.map(_.asScala.toMap).toSeq)
56
56
  Option(result.getLastEvaluatedKey).foreach { lastEvaluatedKey =>
57
57
  runInternal(
data/src/main/scala/org/embulk/input/dynamodb/operation/DynamodbScanOperation.scala CHANGED
@@ -71,7 +71,7 @@ case class DynamodbScanOperation(task: DynamodbScanOperation.Task)
71
71
  loadableRecords match {
72
72
  case Some(v) if (result.getCount > v) =>
73
73
  f(result.getItems.asScala.take(v.toInt).map(_.asScala.toMap).toSeq)
74
- case None =>
74
+ case _ =>
75
75
  f(result.getItems.asScala.map(_.asScala.toMap).toSeq)
76
76
  Option(result.getLastEvaluatedKey).foreach { lastEvaluatedKey =>
77
77
  runInternal(
data/src/test/scala/org/embulk/input/dynamodb/AwsCredentialsTest.scala CHANGED
@@ -6,7 +6,7 @@ import org.embulk.input.dynamodb.aws.AwsCredentials
6
6
  import org.embulk.input.dynamodb.testutil.EmbulkTestBase
7
7
  import org.hamcrest.CoreMatchers._
8
8
  import org.hamcrest.MatcherAssert.assertThat
9
- import org.junit.{Assume, Test}
9
+ import org.junit.{Assert, Test}
10
10
 
11
11
  class AwsCredentialsTest extends EmbulkTestBase {
12
12
 
@@ -44,32 +44,31 @@ class AwsCredentialsTest extends EmbulkTestBase {
44
44
  }
45
45
 
46
46
  def defaultInConfig: ConfigSource = {
47
- embulk.configLoader().fromYamlString(s"""
48
- |type: dynamodb
49
- |region: us-east-1
50
- |table: hoge
51
- |operation: scan
52
- |columns:
53
- | - {name: key1, type: string}
54
- | - {name: key2, type: long}
55
- | - {name: value1, type: string}
56
- |""".stripMargin)
47
+ loadConfigSourceFromYamlString(s"""
48
+ |type: dynamodb
49
+ |region: us-east-1
50
+ |table: hoge
51
+ |operation: scan
52
+ |columns:
53
+ | - {name: key1, type: string}
54
+ | - {name: key2, type: long}
55
+ | - {name: value1, type: string}
56
+ |""".stripMargin)
57
57
  }
58
58
 
59
59
  @deprecated(since = "0.3.0")
60
60
  @Test
61
- def notSetAuthMethod_SetCredentials_deprecated(): Unit = {
62
- Assume.assumeTrue(runAwsCredentialsTest)
63
- val inConfig: ConfigSource = defaultInConfig
64
- .set("access_key", EMBULK_DYNAMODB_TEST_ACCESS_KEY)
65
- .set("secret_key", EMBULK_DYNAMODB_TEST_SECRET_KEY)
61
+ def notSetAuthMethod_SetCredentials_deprecated(): Unit =
62
+ if (runAwsCredentialsTest) {
63
+ val inConfig: ConfigSource = defaultInConfig
64
+ .set("access_key", EMBULK_DYNAMODB_TEST_ACCESS_KEY)
65
+ .set("secret_key", EMBULK_DYNAMODB_TEST_SECRET_KEY)
66
66
 
67
- doTest(inConfig)
68
- }
67
+ doTest(inConfig)
68
+ }
69
69
 
70
70
  @Test
71
- def notSetAuthMethod_SetCredentials(): Unit = {
72
- Assume.assumeTrue(runAwsCredentialsTest)
71
+ def notSetAuthMethod_SetCredentials(): Unit = if (runAwsCredentialsTest) {
73
72
  val inConfig: ConfigSource = defaultInConfig
74
73
  .set("access_key_id", EMBULK_DYNAMODB_TEST_ACCESS_KEY)
75
74
  .set("secret_access_key", EMBULK_DYNAMODB_TEST_SECRET_KEY)
@@ -79,8 +78,7 @@ class AwsCredentialsTest extends EmbulkTestBase {
79
78
 
80
79
  @deprecated(since = "0.3.0")
81
80
  @Test
82
- def setAuthMethod_Basic_deprecated(): Unit = {
83
- Assume.assumeTrue(runAwsCredentialsTest)
81
+ def setAuthMethod_Basic_deprecated(): Unit = if (runAwsCredentialsTest) {
84
82
  val inConfig: ConfigSource = defaultInConfig
85
83
  .set("auth_method", "basic")
86
84
  .set("access_key", EMBULK_DYNAMODB_TEST_ACCESS_KEY)
@@ -90,8 +88,7 @@ class AwsCredentialsTest extends EmbulkTestBase {
90
88
  }
91
89
 
92
90
  @Test
93
- def setAuthMethod_Basic(): Unit = {
94
- Assume.assumeTrue(runAwsCredentialsTest)
91
+ def setAuthMethod_Basic(): Unit = if (runAwsCredentialsTest) {
95
92
  val inConfig: ConfigSource = defaultInConfig
96
93
  .set("auth_method", "basic")
97
94
  .set("access_key_id", EMBULK_DYNAMODB_TEST_ACCESS_KEY)
@@ -101,42 +98,46 @@ class AwsCredentialsTest extends EmbulkTestBase {
101
98
  }
102
99
 
103
100
  @deprecated(since = "0.3.0")
104
- @Test(expected = classOf[ConfigException])
105
- def throwIfSetAccessKeyAndAccessKeyId(): Unit = {
106
- Assume.assumeTrue(runAwsCredentialsTest)
101
+ @Test
102
+ def throwIfSetAccessKeyAndAccessKeyId(): Unit = if (runAwsCredentialsTest) {
107
103
  val inConfig: ConfigSource = defaultInConfig
108
104
  .set("auth_method", "basic")
109
105
  .set("access_key", EMBULK_DYNAMODB_TEST_ACCESS_KEY)
110
106
  .set("access_key_id", EMBULK_DYNAMODB_TEST_ACCESS_KEY)
111
107
  .set("secret_key", EMBULK_DYNAMODB_TEST_SECRET_KEY)
112
108
 
113
- doTest(inConfig)
109
+ Assert.assertThrows(classOf[ConfigException], () => {
110
+ doTest(inConfig)
111
+ })
114
112
  }
115
113
 
116
114
  @deprecated(since = "0.3.0")
117
- @Test(expected = classOf[ConfigException])
118
- def throwIfSetSecretKeyAndSecretAccessKeyId(): Unit = {
119
- Assume.assumeTrue(runAwsCredentialsTest)
120
- val inConfig: ConfigSource = defaultInConfig
121
- .set("auth_method", "basic")
122
- .set("access_key", EMBULK_DYNAMODB_TEST_ACCESS_KEY)
123
- .set("secret_key", EMBULK_DYNAMODB_TEST_SECRET_KEY)
124
- .set("secret_access_key", EMBULK_DYNAMODB_TEST_SECRET_KEY)
125
-
126
- doTest(inConfig)
127
- }
115
+ @Test
116
+ def throwIfSetSecretKeyAndSecretAccessKeyId(): Unit =
117
+ if (runAwsCredentialsTest) {
118
+ val inConfig: ConfigSource = defaultInConfig
119
+ .set("auth_method", "basic")
120
+ .set("access_key", EMBULK_DYNAMODB_TEST_ACCESS_KEY)
121
+ .set("secret_key", EMBULK_DYNAMODB_TEST_SECRET_KEY)
122
+ .set("secret_access_key", EMBULK_DYNAMODB_TEST_SECRET_KEY)
123
+
124
+ Assert.assertThrows(classOf[ConfigException], () => {
125
+ doTest(inConfig)
126
+ })
127
+ }
128
128
 
129
- @Test(expected = classOf[ConfigException])
129
+ @Test
130
130
  def setAuthMethod_Basic_NotSet(): Unit = {
131
131
  val inConfig: ConfigSource = defaultInConfig
132
132
  .set("auth_method", "basic")
133
133
 
134
- doTest(inConfig)
134
+ Assert.assertThrows(classOf[ConfigException], () => {
135
+ doTest(inConfig)
136
+ })
135
137
  }
136
138
 
137
139
  @Test
138
- def setAuthMethod_Env(): Unit = {
139
- Assume.assumeTrue(runAwsCredentialsTest)
140
+ def setAuthMethod_Env(): Unit = if (runAwsCredentialsTest) {
140
141
  // NOTE: Requires to set the env vars like 'AWS_ACCESS_KEY_ID' and so on when testing.
141
142
  val inConfig: ConfigSource = defaultInConfig
142
143
  .set("auth_method", "env")
@@ -145,8 +146,7 @@ class AwsCredentialsTest extends EmbulkTestBase {
145
146
  }
146
147
 
147
148
  @Test
148
- def setAuthMethod_Profile(): Unit = {
149
- Assume.assumeTrue(runAwsCredentialsTest)
149
+ def setAuthMethod_Profile(): Unit = if (runAwsCredentialsTest) {
150
150
  // NOTE: Requires to set credentials to '~/.aws' when testing.
151
151
  val inConfig: ConfigSource = defaultInConfig
152
152
  .set("auth_method", "profile")
@@ -155,18 +155,19 @@ class AwsCredentialsTest extends EmbulkTestBase {
155
155
  doTest(inConfig)
156
156
  }
157
157
 
158
- @Test(expected = classOf[IllegalArgumentException])
158
+ @Test
159
159
  def setAuthMethod_Profile_NotExistProfileName(): Unit = {
160
160
  val inConfig: ConfigSource = defaultInConfig
161
161
  .set("auth_method", "profile")
162
162
  .set("profile_name", "DO_NOT_EXIST")
163
163
 
164
- doTest(inConfig)
164
+ Assert.assertThrows(classOf[IllegalArgumentException], () => {
165
+ doTest(inConfig)
166
+ })
165
167
  }
166
168
 
167
169
  @Test
168
- def setAuthMethod_assume_role(): Unit = {
169
- Assume.assumeTrue(runAwsCredentialsTest)
170
+ def setAuthMethod_assume_role(): Unit = if (runAwsCredentialsTest) {
170
171
  val inConfig: ConfigSource = defaultInConfig
171
172
  .set("auth_method", "assume_role")
172
173
  .set("role_arn", EMBULK_DYNAMODB_TEST_ASSUME_ROLE_ROLE_ARN)
@@ -175,14 +176,16 @@ class AwsCredentialsTest extends EmbulkTestBase {
175
176
  doTest(inConfig)
176
177
  }
177
178
 
178
- @Test(expected = classOf[AWSSecurityTokenServiceException])
179
- def setAuthMethod_assume_role_NotExistRoleArn(): Unit = {
180
- Assume.assumeTrue(runAwsCredentialsTest)
181
- val inConfig: ConfigSource = defaultInConfig
182
- .set("auth_method", "assume_role")
183
- .set("role_arn", "DO_NOT_EXIST")
184
- .set("role_session_name", "dummy")
185
-
186
- doTest(inConfig)
187
- }
179
+ @Test
180
+ def setAuthMethod_assume_role_NotExistRoleArn(): Unit =
181
+ if (runAwsCredentialsTest) {
182
+ val inConfig: ConfigSource = defaultInConfig
183
+ .set("auth_method", "assume_role")
184
+ .set("role_arn", "DO_NOT_EXIST")
185
+ .set("role_session_name", "dummy")
186
+
187
+ Assert.assertThrows(classOf[AWSSecurityTokenServiceException], () => {
188
+ doTest(inConfig)
189
+ })
190
+ }
188
191
  }
data/src/test/scala/org/embulk/input/dynamodb/DynamodbOperationTest.scala ADDED
@@ -0,0 +1,131 @@
1
+ package org.embulk.input.dynamodb
2
+
3
+ import com.amazonaws.services.dynamodbv2.model.{
4
+ AttributeDefinition,
5
+ AttributeValue,
6
+ BillingMode,
7
+ CreateTableRequest,
8
+ KeySchemaElement,
9
+ KeyType,
10
+ PutItemRequest,
11
+ ScalarAttributeType
12
+ }
13
+ import org.embulk.config.ConfigSource
14
+ import org.embulk.input.dynamodb.testutil.EmbulkTestBase
15
+ import org.junit.Test
16
+
17
+ import scala.jdk.CollectionConverters._
18
+
19
+ class DynamodbOperationTest extends EmbulkTestBase {
20
+
21
+ @Test
22
+ def limitTest(): Unit = {
23
+ val tableName = "limit_test"
24
+ cleanupTable(tableName)
25
+ withDynamodb { dynamodb =>
26
+ dynamodb.createTable(
27
+ new CreateTableRequest()
28
+ .withTableName(tableName)
29
+ .withAttributeDefinitions(
30
+ new AttributeDefinition()
31
+ .withAttributeName("pk")
32
+ .withAttributeType(ScalarAttributeType.S)
33
+ )
34
+ .withKeySchema(
35
+ new KeySchemaElement()
36
+ .withAttributeName("pk")
37
+ .withKeyType(KeyType.HASH)
38
+ )
39
+ .withBillingMode(BillingMode.PAY_PER_REQUEST)
40
+ )
41
+ dynamodb.putItem(
42
+ new PutItemRequest()
43
+ .withTableName(tableName)
44
+ .withItem(
45
+ Map
46
+ .newBuilder[String, AttributeValue]
47
+ .addOne("pk", new AttributeValue().withS("a"))
48
+ .result()
49
+ .asJava
50
+ )
51
+ )
52
+ dynamodb.putItem(
53
+ new PutItemRequest()
54
+ .withTableName(tableName)
55
+ .withItem(
56
+ Map
57
+ .newBuilder[String, AttributeValue]
58
+ .addOne("pk", new AttributeValue().withS("a"))
59
+ .result()
60
+ .asJava
61
+ )
62
+ )
63
+ dynamodb.putItem(
64
+ new PutItemRequest()
65
+ .withTableName(tableName)
66
+ .withItem(
67
+ Map
68
+ .newBuilder[String, AttributeValue]
69
+ .addOne("pk", new AttributeValue().withS("a"))
70
+ .result()
71
+ .asJava
72
+ )
73
+ )
74
+ dynamodb.putItem(
75
+ new PutItemRequest()
76
+ .withTableName(tableName)
77
+ .withItem(
78
+ Map
79
+ .newBuilder[String, AttributeValue]
80
+ .addOne("pk", new AttributeValue().withS("b"))
81
+ .result()
82
+ .asJava
83
+ )
84
+ )
85
+ dynamodb.putItem(
86
+ new PutItemRequest()
87
+ .withTableName(tableName)
88
+ .withItem(
89
+ Map
90
+ .newBuilder[String, AttributeValue]
91
+ .addOne("pk", new AttributeValue().withS("b"))
92
+ .result()
93
+ .asJava
94
+ )
95
+ )
96
+ }
97
+
98
+ val inScanConfig: ConfigSource = loadConfigSourceFromYamlString(s"""
99
+ |type: dynamodb
100
+ |table: $tableName
101
+ |endpoint: http://$dynamoDBHost:$dynamoDBPort/
102
+ |auth_method: basic
103
+ |access_key_id: dummy
104
+ |secret_access_key: dummy
105
+ |scan:
106
+ | limit: 1
107
+ |""".stripMargin)
108
+ runInput(inScanConfig, { result =>
109
+ assert(result.size.equals(1))
110
+ })
111
+
112
+ val inQueryConfig: ConfigSource = loadConfigSourceFromYamlString(s"""
113
+ |type: dynamodb
114
+ |table: $tableName
115
+ |endpoint: http://$dynamoDBHost:$dynamoDBPort/
116
+ |auth_method: basic
117
+ |access_key_id: dummy
118
+ |secret_access_key: dummy
119
+ |query:
120
+ | key_condition_expression: "#x = :v"
121
+ | expression_attribute_names:
122
+ | "#x": pk
123
+ | expression_attribute_values:
124
+ | ":v": {S: a}
125
+ | limit: 1
126
+ |""".stripMargin)
127
+ runInput(inQueryConfig, { result =>
128
+ assert(result.size.equals(1))
129
+ })
130
+ }
131
+ }
data/src/test/scala/org/embulk/input/dynamodb/{DynamodbQueryOperationTest.scala → DynamodbQueryOperationBackwardCompatibilityTest.scala} RENAMED
@@ -12,7 +12,6 @@ import com.amazonaws.services.dynamodbv2.model.{
12
12
  }
13
13
  import org.embulk.config.ConfigSource
14
14
  import org.embulk.input.dynamodb.testutil.EmbulkTestBase
15
- import org.embulk.spi.util.Pages
16
15
  import org.hamcrest.CoreMatchers._
17
16
  import org.hamcrest.MatcherAssert.assertThat
18
17
  import org.junit.Test
@@ -20,7 +19,7 @@ import org.msgpack.value.Value
20
19
 
21
20
  import scala.jdk.CollectionConverters._
22
21
 
23
- class DynamodbQueryOperationTest extends EmbulkTestBase {
22
+ class DynamodbQueryOperationBackwardCompatibilityTest extends EmbulkTestBase {
24
23
 
25
24
  private def testBackwardCompatibility(embulkInConfig: ConfigSource): Unit = {
26
25
  cleanupTable("EMBULK_DYNAMODB_TEST_TABLE")
@@ -86,58 +85,52 @@ class DynamodbQueryOperationTest extends EmbulkTestBase {
86
85
  .asJava
87
86
  )
88
87
  )
88
+ }
89
89
 
90
- val path = embulk.createTempFile("csv")
91
- val result = embulk
92
- .inputBuilder()
93
- .in(embulkInConfig)
94
- .outputPath(path)
95
- .preview()
96
-
97
- val pages = result.getPages
98
- val head = Pages.toObjects(result.getSchema, pages.get(0)).get(0)
99
-
100
- assertThat(head(0).toString, is("key-1"))
101
- assertThat(head(1).asInstanceOf[Long], is(0L))
102
- assertThat(head(2).asInstanceOf[Double], is(42.195))
103
- assertThat(head(3).asInstanceOf[Boolean], is(true))
90
+ runInput(
91
+ embulkInConfig, { result: Seq[Seq[AnyRef]] =>
92
+ val head = result.head
93
+ assertThat(head(0).toString, is("key-1"))
94
+ assertThat(head(1).asInstanceOf[Long], is(0L))
95
+ assertThat(head(2).asInstanceOf[Double], is(42.195))
96
+ assertThat(head(3).asInstanceOf[Boolean], is(true))
104
97
 
105
- val arrayValue = head(4).asInstanceOf[Value].asArrayValue()
106
- assertThat(arrayValue.size(), is(2))
107
- assertThat(arrayValue.get(0).asStringValue().toString, is("list-value"))
108
- assertThat(arrayValue.get(1).asIntegerValue().asLong(), is(123L))
98
+ val arrayValue = head(4).asInstanceOf[Value].asArrayValue()
99
+ assertThat(arrayValue.size(), is(2))
100
+ assertThat(arrayValue.get(0).asStringValue().toString, is("list-value"))
101
+ assertThat(arrayValue.get(1).asIntegerValue().asLong(), is(123L))
109
102
 
110
- val mapValue = head(5).asInstanceOf[Value].asMapValue()
111
- assert(mapValue.keySet().asScala.map(_.toString).contains("map-key-1"))
112
- assertThat(
113
- mapValue
114
- .entrySet()
115
- .asScala
116
- .filter(_.getKey.toString.equals("map-key-1"))
117
- .head
118
- .getValue
119
- .toString,
120
- is("map-value-1")
121
- )
122
- assert(mapValue.keySet().asScala.map(_.toString).contains("map-key-2"))
123
- assertThat(
124
- mapValue
125
- .entrySet()
126
- .asScala
127
- .filter(_.getKey.toString.equals("map-key-2"))
128
- .head
129
- .getValue
130
- .asIntegerValue()
131
- .asLong(),
132
- is(456L)
133
- )
134
-
135
- }
103
+ val mapValue = head(5).asInstanceOf[Value].asMapValue()
104
+ assert(mapValue.keySet().asScala.map(_.toString).contains("map-key-1"))
105
+ assertThat(
106
+ mapValue
107
+ .entrySet()
108
+ .asScala
109
+ .filter(_.getKey.toString.equals("map-key-1"))
110
+ .head
111
+ .getValue
112
+ .toString,
113
+ is("map-value-1")
114
+ )
115
+ assert(mapValue.keySet().asScala.map(_.toString).contains("map-key-2"))
116
+ assertThat(
117
+ mapValue
118
+ .entrySet()
119
+ .asScala
120
+ .filter(_.getKey.toString.equals("map-key-2"))
121
+ .head
122
+ .getValue
123
+ .asIntegerValue()
124
+ .asLong(),
125
+ is(456L)
126
+ )
127
+ }
128
+ )
136
129
  }
137
130
 
138
131
  @Test
139
132
  def deprecatedQueryOperationTest(): Unit = {
140
- val inConfig: ConfigSource = embulk.configLoader().fromYamlString(s"""
133
+ val inConfig: ConfigSource = loadConfigSourceFromYamlString(s"""
141
134
  |type: dynamodb
142
135
  |end_point: http://${dynamoDBHost}:${dynamoDBPort}/
143
136
  |table: EMBULK_DYNAMODB_TEST_TABLE
@@ -161,7 +154,7 @@ class DynamodbQueryOperationTest extends EmbulkTestBase {
161
154
 
162
155
  @Test
163
156
  def keepTheSameBehaviourAsDeprecatedQueryOperationTest(): Unit = {
164
- val inConfig: ConfigSource = embulk.configLoader().fromYamlString(s"""
157
+ val inConfig: ConfigSource = loadConfigSourceFromYamlString(s"""
165
158
  |type: dynamodb
166
159
  |endpoint: http://${dynamoDBHost}:${dynamoDBPort}/
167
160
  |table: EMBULK_DYNAMODB_TEST_TABLE
data/src/test/scala/org/embulk/input/dynamodb/{DynamodbScanOperationTest.scala → DynamodbScanOperationBackwardCompatibilityTest.scala} RENAMED
@@ -12,7 +12,6 @@ import com.amazonaws.services.dynamodbv2.model.{
12
12
  }
13
13
  import org.embulk.config.ConfigSource
14
14
  import org.embulk.input.dynamodb.testutil.EmbulkTestBase
15
- import org.embulk.spi.util.Pages
16
15
  import org.hamcrest.CoreMatchers._
17
16
  import org.hamcrest.MatcherAssert.assertThat
18
17
  import org.junit.Test
@@ -20,7 +19,7 @@ import org.msgpack.value.Value
20
19
 
21
20
  import scala.jdk.CollectionConverters._
22
21
 
23
- class DynamodbScanOperationTest extends EmbulkTestBase {
22
+ class DynamodbScanOperationBackwardCompatibilityTest extends EmbulkTestBase {
24
23
 
25
24
  private def testBackwardCompatibility(embulkInConfig: ConfigSource): Unit = {
26
25
  cleanupTable("EMBULK_DYNAMODB_TEST_TABLE")
@@ -88,56 +87,51 @@ class DynamodbScanOperationTest extends EmbulkTestBase {
88
87
  )
89
88
  }
90
89
 
91
- val path = embulk.createTempFile("csv")
92
- val result = embulk
93
- .inputBuilder()
94
- .in(embulkInConfig)
95
- .outputPath(path)
96
- .preview()
90
+ runInput(
91
+ embulkInConfig, { result: Seq[Seq[AnyRef]] =>
92
+ val head = result.head
93
+ assertThat(head(0).toString, is("key-1"))
94
+ assertThat(head(1).asInstanceOf[Long], is(0L))
95
+ assertThat(head(2).asInstanceOf[Double], is(42.195))
96
+ assertThat(head(3).asInstanceOf[Boolean], is(true))
97
97
 
98
- val pages = result.getPages
99
- val head = Pages.toObjects(result.getSchema, pages.get(0)).get(0)
98
+ val arrayValue = head(4).asInstanceOf[Value].asArrayValue()
99
+ assertThat(arrayValue.size(), is(2))
100
+ assertThat(arrayValue.get(0).asStringValue().toString, is("list-value"))
101
+ assertThat(arrayValue.get(1).asIntegerValue().asLong(), is(123L))
100
102
 
101
- assertThat(head(0).toString, is("key-1"))
102
- assertThat(head(1).asInstanceOf[Long], is(0L))
103
- assertThat(head(2).asInstanceOf[Double], is(42.195))
104
- assertThat(head(3).asInstanceOf[Boolean], is(true))
105
-
106
- val arrayValue = head(4).asInstanceOf[Value].asArrayValue()
107
- assertThat(arrayValue.size(), is(2))
108
- assertThat(arrayValue.get(0).asStringValue().toString, is("list-value"))
109
- assertThat(arrayValue.get(1).asIntegerValue().asLong(), is(123L))
110
-
111
- val mapValue = head(5).asInstanceOf[Value].asMapValue()
112
- assert(mapValue.keySet().asScala.map(_.toString).contains("map-key-1"))
113
- assertThat(
114
- mapValue
115
- .entrySet()
116
- .asScala
117
- .filter(_.getKey.toString.equals("map-key-1"))
118
- .head
119
- .getValue
120
- .toString,
121
- is("map-value-1")
122
- )
123
- assert(mapValue.keySet().asScala.map(_.toString).contains("map-key-2"))
124
- assertThat(
125
- mapValue
126
- .entrySet()
127
- .asScala
128
- .filter(_.getKey.toString.equals("map-key-2"))
129
- .head
130
- .getValue
131
- .asIntegerValue()
132
- .asLong(),
133
- is(456L)
103
+ val mapValue = head(5).asInstanceOf[Value].asMapValue()
104
+ assert(mapValue.keySet().asScala.map(_.toString).contains("map-key-1"))
105
+ assertThat(
106
+ mapValue
107
+ .entrySet()
108
+ .asScala
109
+ .filter(_.getKey.toString.equals("map-key-1"))
110
+ .head
111
+ .getValue
112
+ .toString,
113
+ is("map-value-1")
114
+ )
115
+ assert(mapValue.keySet().asScala.map(_.toString).contains("map-key-2"))
116
+ assertThat(
117
+ mapValue
118
+ .entrySet()
119
+ .asScala
120
+ .filter(_.getKey.toString.equals("map-key-2"))
121
+ .head
122
+ .getValue
123
+ .asIntegerValue()
124
+ .asLong(),
125
+ is(456L)
126
+ )
127
+ }
134
128
  )
135
129
  }
136
130
 
137
131
  @Test
138
132
  def deprecatedScanOperationTest(): Unit = {
139
133
 
140
- val inConfig: ConfigSource = embulk.configLoader().fromYamlString(s"""
134
+ val inConfig: ConfigSource = loadConfigSourceFromYamlString(s"""
141
135
  |type: dynamodb
142
136
  |end_point: http://${dynamoDBHost}:${dynamoDBPort}/
143
137
  |table: EMBULK_DYNAMODB_TEST_TABLE
@@ -159,7 +153,7 @@ class DynamodbScanOperationTest extends EmbulkTestBase {
159
153
 
160
154
  @Test
161
155
  def keepTheSameBehaviourAsDeprecatedScanOperationTest(): Unit = {
162
- val inConfig: ConfigSource = embulk.configLoader().fromYamlString(s"""
156
+ val inConfig: ConfigSource = loadConfigSourceFromYamlString(s"""
163
157
  |type: dynamodb
164
158
  |endpoint: http://${dynamoDBHost}:${dynamoDBPort}/
165
159
  |table: EMBULK_DYNAMODB_TEST_TABLE
data/src/test/scala/org/embulk/input/dynamodb/testutil/EmbulkTestBase.scala CHANGED
@@ -6,11 +6,15 @@ import com.amazonaws.services.dynamodbv2.{
6
6
  AmazonDynamoDB,
7
7
  AmazonDynamoDBClientBuilder
8
8
  }
9
+ import org.embulk.config.{ConfigLoader, ConfigSource, TaskReport, TaskSource}
9
10
  import org.embulk.input.dynamodb.DynamodbInputPlugin
10
- import org.embulk.spi.InputPlugin
11
- import org.embulk.test.TestingEmbulk
12
- import org.junit.{After, Rule}
11
+ import org.embulk.spi.Schema
12
+ import org.embulk.EmbulkTestRuntime
13
+ import org.embulk.spi.TestPageBuilderReader.MockPageOutput
14
+ import org.embulk.spi.util.Pages
15
+ import org.junit.Rule
13
16
 
17
+ import scala.jdk.CollectionConverters._
14
18
  import scala.util.{Failure, Success, Try}
15
19
  import scala.util.chaining._
16
20
 
@@ -49,20 +53,7 @@ trait EmbulkTestBase {
49
53
  }
50
54
 
51
55
  @Rule
52
- def embulk: TestingEmbulk =
53
- TestingEmbulk
54
- .builder()
55
- .registerPlugin(
56
- classOf[InputPlugin],
57
- "dynamodb",
58
- classOf[DynamodbInputPlugin]
59
- )
60
- .build()
61
-
62
- @After
63
- def destroyEmbulk(): Unit = {
64
- embulk.destroy()
65
- }
56
+ def runtime: EmbulkTestRuntime = new EmbulkTestRuntime()
66
57
 
67
58
  def getEnvironmentVariableOrShowErrorMessage(name: String): String = {
68
59
  try {
@@ -82,4 +73,27 @@ trait EmbulkTestBase {
82
73
  )
83
74
  }
84
75
  }
76
+
77
+ def runInput(inConfig: ConfigSource, test: Seq[Seq[AnyRef]] => Unit): Unit = {
78
+ runtime.getInstance(classOf[DynamodbInputPlugin]).tap { plugin =>
79
+ plugin.transaction(
80
+ inConfig,
81
+ (taskSource: TaskSource, schema: Schema, taskCount: Int) => {
82
+ val output: MockPageOutput = new MockPageOutput()
83
+ val reports: Seq[TaskReport] = 0.until(taskCount).map { taskIndex =>
84
+ plugin.run(taskSource, schema, taskIndex, output)
85
+ }
86
+ output.finish()
87
+
88
+ test(Pages.toObjects(schema, output.pages).asScala.toSeq.map(_.toSeq))
89
+
90
+ reports.asJava
91
+ }
92
+ )
93
+ }
94
+ }
95
+
96
+ def loadConfigSourceFromYamlString(yaml: String): ConfigSource = {
97
+ new ConfigLoader(runtime.getModelManager).fromYamlString(yaml)
98
+ }
85
99
  }
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-input-dynamodb
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.0
4
+ version: 0.3.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Daisuke Higashi
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2020-03-08 00:00:00.000000000 Z
12
+ date: 2020-03-28 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  requirement: !ruby/object:Gem::Requirement
@@ -62,7 +62,7 @@ files:
62
62
  - classpath/aws-java-sdk-sts-1.11.711.jar
63
63
  - classpath/commons-codec-1.11.jar
64
64
  - classpath/commons-logging-1.2.jar
65
- - classpath/embulk-input-dynamodb-0.3.0.jar
65
+ - classpath/embulk-input-dynamodb-0.3.1.jar
66
66
  - classpath/httpclient-4.5.9.jar
67
67
  - classpath/httpcore-4.4.11.jar
68
68
  - classpath/ion-java-1.0.2.jar
@@ -118,8 +118,9 @@ files:
118
118
  - src/test/resources/json/test.template
119
119
  - src/test/scala/org/embulk/input/dynamodb/AttributeValueHelperTest.scala
120
120
  - src/test/scala/org/embulk/input/dynamodb/AwsCredentialsTest.scala
121
- - src/test/scala/org/embulk/input/dynamodb/DynamodbQueryOperationTest.scala
122
- - src/test/scala/org/embulk/input/dynamodb/DynamodbScanOperationTest.scala
121
+ - src/test/scala/org/embulk/input/dynamodb/DynamodbOperationTest.scala
122
+ - src/test/scala/org/embulk/input/dynamodb/DynamodbQueryOperationBackwardCompatibilityTest.scala
123
+ - src/test/scala/org/embulk/input/dynamodb/DynamodbScanOperationBackwardCompatibilityTest.scala
123
124
  - src/test/scala/org/embulk/input/dynamodb/testutil/EmbulkTestBase.scala
124
125
  homepage: https://github.com/lulichn/embulk-input-dynamodb
125
126
  licenses: