embulk-input-dynamodb 0.3.0 → 0.3.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (11) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +7 -1
  3. data/build.gradle +2 -2
  4. data/src/main/scala/org/embulk/input/dynamodb/operation/DynamodbQueryOperation.scala +1 -1
  5. data/src/main/scala/org/embulk/input/dynamodb/operation/DynamodbScanOperation.scala +1 -1
  6. data/src/test/scala/org/embulk/input/dynamodb/AwsCredentialsTest.scala +62 -59
  7. data/src/test/scala/org/embulk/input/dynamodb/DynamodbOperationTest.scala +131 -0
  8. data/src/test/scala/org/embulk/input/dynamodb/{DynamodbQueryOperationTest.scala → DynamodbQueryOperationBackwardCompatibilityTest.scala} +41 -48
  9. data/src/test/scala/org/embulk/input/dynamodb/{DynamodbScanOperationTest.scala → DynamodbScanOperationBackwardCompatibilityTest.scala} +39 -45
  10. data/src/test/scala/org/embulk/input/dynamodb/testutil/EmbulkTestBase.scala +31 -17
  11. metadata +6 -5
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 85976c690e023ebd1405fa2990e5dc6995511f08
4
- data.tar.gz: e7aa6e0eed7b15581be57fbbb69028ed6a7d1621
3
+ metadata.gz: e7ae13caaaaee6d19eb328725cf064e1771042a9
4
+ data.tar.gz: f2b96cdf15c20762e1c3bc9e41d0666286a6db59
5
5
  SHA512:
6
- metadata.gz: 8ad60d0ea024cab88469c6afec78c37eb6341035b33effdbe2b30ba823e3239f4d6c07d347da8e095f3d1f43cf61f77f3df654629c2a073f7a4daebb8550dce9
7
- data.tar.gz: 67718ce3f0d8fd97ce3a2c33bbdfb77e1891b9b50ad59815c2282ec1ea56a56e8b7c0e8725d44f0c9fa88f5e50c9542aeb80e62108662ed1c23c85d59cb56aba
6
+ metadata.gz: 32bccbd2fe8aa2bc08b0130ffcc73c77e0ed9a5907a90a4bb5e65d23de2e4811433916f4a5b8b7c0e748917367715347aabe24b441297341755bd549503df1db
7
+ data.tar.gz: 1247ab5ed561520e3f4ac2164602b85b909a2e8c112e3ac9a3b357935787aa295a90bdf7bab9cc916e3fbc98df529df36709414172a43a4714ff69001bd9796e
data/CHANGELOG.md CHANGED
@@ -1,3 +1,9 @@
1
+ 0.3.1 (2020-03-29)
2
+ ==================
3
+
4
+ - [BugFix] [#23](https://github.com/lulichn/embulk-input-dynamodb/pull/23) Throw scala.MatchError when using "limit" option. (Fix [#21](https://github.com/lulichn/embulk-input-dynamodb/issues/21))
5
+ - [Enhancement] [#23](https://github.com/lulichn/embulk-input-dynamodb/pull/23) Use embulk-core tests library instead of embulk-test.
6
+
1
7
  0.3.0 (2020-03-09)
2
8
  ==================
3
9
 
@@ -46,4 +52,4 @@
46
52
  - [Enhancement] [#19](https://github.com/lulichn/embulk-input-dynamodb/pull/19) You can specify the `AttributeValue` type (like `"S"`, `"N"`, `"SS"` and so on) used when converting AttributeValue to Embulk type.
47
53
  - [BugFix] [#19](https://github.com/lulichn/embulk-input-dynamodb/pull/19) Avoid `NullPointerException` when Type `N` AttributeValue has `null` in the deprecated operation.
48
54
  - [Enhancement] [#19](https://github.com/lulichn/embulk-input-dynamodb/pull/19) Examples work without real Dynamodb.
49
- - [Enhancement] [#19](https://github.com/lulichn/embulk-input-dynamodb/pull/19) Add more examples.
55
+ - [Enhancement] [#19](https://github.com/lulichn/embulk-input-dynamodb/pull/19) Add more examples.
data/build.gradle CHANGED
@@ -14,7 +14,7 @@ configurations {
14
14
  provided
15
15
  }
16
16
 
17
- version = "0.3.0"
17
+ version = "0.3.1"
18
18
 
19
19
  sourceCompatibility = 1.8
20
20
  targetCompatibility = 1.8
@@ -31,10 +31,10 @@ dependencies {
31
31
  compile "dev.zio:zio-macros-core_2.13:0.6.2"
32
32
 
33
33
  testCompile "junit:junit:4.+"
34
+ testCompile "org.embulk:embulk-core:0.9.23:tests"
34
35
  testCompile "org.embulk:embulk-standards:0.9.23"
35
36
  testCompile "org.embulk:embulk-deps-buffer:0.9.23"
36
37
  testCompile "org.embulk:embulk-deps-config:0.9.23"
37
- testCompile "org.embulk:embulk-test:0.9.23"
38
38
  }
39
39
 
40
40
  compileScala {
data/src/main/scala/org/embulk/input/dynamodb/operation/DynamodbQueryOperation.scala CHANGED
@@ -51,7 +51,7 @@ case class DynamodbQueryOperation(task: DynamodbQueryOperation.Task)
51
51
  loadableRecords match {
52
52
  case Some(v) if (result.getCount > v) =>
53
53
  f(result.getItems.asScala.take(v.toInt).map(_.asScala.toMap).toSeq)
54
- case None =>
54
+ case _ =>
55
55
  f(result.getItems.asScala.map(_.asScala.toMap).toSeq)
56
56
  Option(result.getLastEvaluatedKey).foreach { lastEvaluatedKey =>
57
57
  runInternal(
data/src/main/scala/org/embulk/input/dynamodb/operation/DynamodbScanOperation.scala CHANGED
@@ -71,7 +71,7 @@ case class DynamodbScanOperation(task: DynamodbScanOperation.Task)
71
71
  loadableRecords match {
72
72
  case Some(v) if (result.getCount > v) =>
73
73
  f(result.getItems.asScala.take(v.toInt).map(_.asScala.toMap).toSeq)
74
- case None =>
74
+ case _ =>
75
75
  f(result.getItems.asScala.map(_.asScala.toMap).toSeq)
76
76
  Option(result.getLastEvaluatedKey).foreach { lastEvaluatedKey =>
77
77
  runInternal(
data/src/test/scala/org/embulk/input/dynamodb/AwsCredentialsTest.scala CHANGED
@@ -6,7 +6,7 @@ import org.embulk.input.dynamodb.aws.AwsCredentials
6
6
  import org.embulk.input.dynamodb.testutil.EmbulkTestBase
7
7
  import org.hamcrest.CoreMatchers._
8
8
  import org.hamcrest.MatcherAssert.assertThat
9
- import org.junit.{Assume, Test}
9
+ import org.junit.{Assert, Test}
10
10
 
11
11
  class AwsCredentialsTest extends EmbulkTestBase {
12
12
 
@@ -44,32 +44,31 @@ class AwsCredentialsTest extends EmbulkTestBase {
44
44
  }
45
45
 
46
46
  def defaultInConfig: ConfigSource = {
47
- embulk.configLoader().fromYamlString(s"""
48
- |type: dynamodb
49
- |region: us-east-1
50
- |table: hoge
51
- |operation: scan
52
- |columns:
53
- | - {name: key1, type: string}
54
- | - {name: key2, type: long}
55
- | - {name: value1, type: string}
56
- |""".stripMargin)
47
+ loadConfigSourceFromYamlString(s"""
48
+ |type: dynamodb
49
+ |region: us-east-1
50
+ |table: hoge
51
+ |operation: scan
52
+ |columns:
53
+ | - {name: key1, type: string}
54
+ | - {name: key2, type: long}
55
+ | - {name: value1, type: string}
56
+ |""".stripMargin)
57
57
  }
58
58
 
59
59
  @deprecated(since = "0.3.0")
60
60
  @Test
61
- def notSetAuthMethod_SetCredentials_deprecated(): Unit = {
62
- Assume.assumeTrue(runAwsCredentialsTest)
63
- val inConfig: ConfigSource = defaultInConfig
64
- .set("access_key", EMBULK_DYNAMODB_TEST_ACCESS_KEY)
65
- .set("secret_key", EMBULK_DYNAMODB_TEST_SECRET_KEY)
61
+ def notSetAuthMethod_SetCredentials_deprecated(): Unit =
62
+ if (runAwsCredentialsTest) {
63
+ val inConfig: ConfigSource = defaultInConfig
64
+ .set("access_key", EMBULK_DYNAMODB_TEST_ACCESS_KEY)
65
+ .set("secret_key", EMBULK_DYNAMODB_TEST_SECRET_KEY)
66
66
 
67
- doTest(inConfig)
68
- }
67
+ doTest(inConfig)
68
+ }
69
69
 
70
70
  @Test
71
- def notSetAuthMethod_SetCredentials(): Unit = {
72
- Assume.assumeTrue(runAwsCredentialsTest)
71
+ def notSetAuthMethod_SetCredentials(): Unit = if (runAwsCredentialsTest) {
73
72
  val inConfig: ConfigSource = defaultInConfig
74
73
  .set("access_key_id", EMBULK_DYNAMODB_TEST_ACCESS_KEY)
75
74
  .set("secret_access_key", EMBULK_DYNAMODB_TEST_SECRET_KEY)
@@ -79,8 +78,7 @@ class AwsCredentialsTest extends EmbulkTestBase {
79
78
 
80
79
  @deprecated(since = "0.3.0")
81
80
  @Test
82
- def setAuthMethod_Basic_deprecated(): Unit = {
83
- Assume.assumeTrue(runAwsCredentialsTest)
81
+ def setAuthMethod_Basic_deprecated(): Unit = if (runAwsCredentialsTest) {
84
82
  val inConfig: ConfigSource = defaultInConfig
85
83
  .set("auth_method", "basic")
86
84
  .set("access_key", EMBULK_DYNAMODB_TEST_ACCESS_KEY)
@@ -90,8 +88,7 @@ class AwsCredentialsTest extends EmbulkTestBase {
90
88
  }
91
89
 
92
90
  @Test
93
- def setAuthMethod_Basic(): Unit = {
94
- Assume.assumeTrue(runAwsCredentialsTest)
91
+ def setAuthMethod_Basic(): Unit = if (runAwsCredentialsTest) {
95
92
  val inConfig: ConfigSource = defaultInConfig
96
93
  .set("auth_method", "basic")
97
94
  .set("access_key_id", EMBULK_DYNAMODB_TEST_ACCESS_KEY)
@@ -101,42 +98,46 @@ class AwsCredentialsTest extends EmbulkTestBase {
101
98
  }
102
99
 
103
100
  @deprecated(since = "0.3.0")
104
- @Test(expected = classOf[ConfigException])
105
- def throwIfSetAccessKeyAndAccessKeyId(): Unit = {
106
- Assume.assumeTrue(runAwsCredentialsTest)
101
+ @Test
102
+ def throwIfSetAccessKeyAndAccessKeyId(): Unit = if (runAwsCredentialsTest) {
107
103
  val inConfig: ConfigSource = defaultInConfig
108
104
  .set("auth_method", "basic")
109
105
  .set("access_key", EMBULK_DYNAMODB_TEST_ACCESS_KEY)
110
106
  .set("access_key_id", EMBULK_DYNAMODB_TEST_ACCESS_KEY)
111
107
  .set("secret_key", EMBULK_DYNAMODB_TEST_SECRET_KEY)
112
108
 
113
- doTest(inConfig)
109
+ Assert.assertThrows(classOf[ConfigException], () => {
110
+ doTest(inConfig)
111
+ })
114
112
  }
115
113
 
116
114
  @deprecated(since = "0.3.0")
117
- @Test(expected = classOf[ConfigException])
118
- def throwIfSetSecretKeyAndSecretAccessKeyId(): Unit = {
119
- Assume.assumeTrue(runAwsCredentialsTest)
120
- val inConfig: ConfigSource = defaultInConfig
121
- .set("auth_method", "basic")
122
- .set("access_key", EMBULK_DYNAMODB_TEST_ACCESS_KEY)
123
- .set("secret_key", EMBULK_DYNAMODB_TEST_SECRET_KEY)
124
- .set("secret_access_key", EMBULK_DYNAMODB_TEST_SECRET_KEY)
125
-
126
- doTest(inConfig)
127
- }
115
+ @Test
116
+ def throwIfSetSecretKeyAndSecretAccessKeyId(): Unit =
117
+ if (runAwsCredentialsTest) {
118
+ val inConfig: ConfigSource = defaultInConfig
119
+ .set("auth_method", "basic")
120
+ .set("access_key", EMBULK_DYNAMODB_TEST_ACCESS_KEY)
121
+ .set("secret_key", EMBULK_DYNAMODB_TEST_SECRET_KEY)
122
+ .set("secret_access_key", EMBULK_DYNAMODB_TEST_SECRET_KEY)
123
+
124
+ Assert.assertThrows(classOf[ConfigException], () => {
125
+ doTest(inConfig)
126
+ })
127
+ }
128
128
 
129
- @Test(expected = classOf[ConfigException])
129
+ @Test
130
130
  def setAuthMethod_Basic_NotSet(): Unit = {
131
131
  val inConfig: ConfigSource = defaultInConfig
132
132
  .set("auth_method", "basic")
133
133
 
134
- doTest(inConfig)
134
+ Assert.assertThrows(classOf[ConfigException], () => {
135
+ doTest(inConfig)
136
+ })
135
137
  }
136
138
 
137
139
  @Test
138
- def setAuthMethod_Env(): Unit = {
139
- Assume.assumeTrue(runAwsCredentialsTest)
140
+ def setAuthMethod_Env(): Unit = if (runAwsCredentialsTest) {
140
141
  // NOTE: Requires to set the env vars like 'AWS_ACCESS_KEY_ID' and so on when testing.
141
142
  val inConfig: ConfigSource = defaultInConfig
142
143
  .set("auth_method", "env")
@@ -145,8 +146,7 @@ class AwsCredentialsTest extends EmbulkTestBase {
145
146
  }
146
147
 
147
148
  @Test
148
- def setAuthMethod_Profile(): Unit = {
149
- Assume.assumeTrue(runAwsCredentialsTest)
149
+ def setAuthMethod_Profile(): Unit = if (runAwsCredentialsTest) {
150
150
  // NOTE: Requires to set credentials to '~/.aws' when testing.
151
151
  val inConfig: ConfigSource = defaultInConfig
152
152
  .set("auth_method", "profile")
@@ -155,18 +155,19 @@ class AwsCredentialsTest extends EmbulkTestBase {
155
155
  doTest(inConfig)
156
156
  }
157
157
 
158
- @Test(expected = classOf[IllegalArgumentException])
158
+ @Test
159
159
  def setAuthMethod_Profile_NotExistProfileName(): Unit = {
160
160
  val inConfig: ConfigSource = defaultInConfig
161
161
  .set("auth_method", "profile")
162
162
  .set("profile_name", "DO_NOT_EXIST")
163
163
 
164
- doTest(inConfig)
164
+ Assert.assertThrows(classOf[IllegalArgumentException], () => {
165
+ doTest(inConfig)
166
+ })
165
167
  }
166
168
 
167
169
  @Test
168
- def setAuthMethod_assume_role(): Unit = {
169
- Assume.assumeTrue(runAwsCredentialsTest)
170
+ def setAuthMethod_assume_role(): Unit = if (runAwsCredentialsTest) {
170
171
  val inConfig: ConfigSource = defaultInConfig
171
172
  .set("auth_method", "assume_role")
172
173
  .set("role_arn", EMBULK_DYNAMODB_TEST_ASSUME_ROLE_ROLE_ARN)
@@ -175,14 +176,16 @@ class AwsCredentialsTest extends EmbulkTestBase {
175
176
  doTest(inConfig)
176
177
  }
177
178
 
178
- @Test(expected = classOf[AWSSecurityTokenServiceException])
179
- def setAuthMethod_assume_role_NotExistRoleArn(): Unit = {
180
- Assume.assumeTrue(runAwsCredentialsTest)
181
- val inConfig: ConfigSource = defaultInConfig
182
- .set("auth_method", "assume_role")
183
- .set("role_arn", "DO_NOT_EXIST")
184
- .set("role_session_name", "dummy")
185
-
186
- doTest(inConfig)
187
- }
179
+ @Test
180
+ def setAuthMethod_assume_role_NotExistRoleArn(): Unit =
181
+ if (runAwsCredentialsTest) {
182
+ val inConfig: ConfigSource = defaultInConfig
183
+ .set("auth_method", "assume_role")
184
+ .set("role_arn", "DO_NOT_EXIST")
185
+ .set("role_session_name", "dummy")
186
+
187
+ Assert.assertThrows(classOf[AWSSecurityTokenServiceException], () => {
188
+ doTest(inConfig)
189
+ })
190
+ }
188
191
  }
data/src/test/scala/org/embulk/input/dynamodb/DynamodbOperationTest.scala ADDED
@@ -0,0 +1,131 @@
1
+ package org.embulk.input.dynamodb
2
+
3
+ import com.amazonaws.services.dynamodbv2.model.{
4
+ AttributeDefinition,
5
+ AttributeValue,
6
+ BillingMode,
7
+ CreateTableRequest,
8
+ KeySchemaElement,
9
+ KeyType,
10
+ PutItemRequest,
11
+ ScalarAttributeType
12
+ }
13
+ import org.embulk.config.ConfigSource
14
+ import org.embulk.input.dynamodb.testutil.EmbulkTestBase
15
+ import org.junit.Test
16
+
17
+ import scala.jdk.CollectionConverters._
18
+
19
+ class DynamodbOperationTest extends EmbulkTestBase {
20
+
21
+ @Test
22
+ def limitTest(): Unit = {
23
+ val tableName = "limit_test"
24
+ cleanupTable(tableName)
25
+ withDynamodb { dynamodb =>
26
+ dynamodb.createTable(
27
+ new CreateTableRequest()
28
+ .withTableName(tableName)
29
+ .withAttributeDefinitions(
30
+ new AttributeDefinition()
31
+ .withAttributeName("pk")
32
+ .withAttributeType(ScalarAttributeType.S)
33
+ )
34
+ .withKeySchema(
35
+ new KeySchemaElement()
36
+ .withAttributeName("pk")
37
+ .withKeyType(KeyType.HASH)
38
+ )
39
+ .withBillingMode(BillingMode.PAY_PER_REQUEST)
40
+ )
41
+ dynamodb.putItem(
42
+ new PutItemRequest()
43
+ .withTableName(tableName)
44
+ .withItem(
45
+ Map
46
+ .newBuilder[String, AttributeValue]
47
+ .addOne("pk", new AttributeValue().withS("a"))
48
+ .result()
49
+ .asJava
50
+ )
51
+ )
52
+ dynamodb.putItem(
53
+ new PutItemRequest()
54
+ .withTableName(tableName)
55
+ .withItem(
56
+ Map
57
+ .newBuilder[String, AttributeValue]
58
+ .addOne("pk", new AttributeValue().withS("a"))
59
+ .result()
60
+ .asJava
61
+ )
62
+ )
63
+ dynamodb.putItem(
64
+ new PutItemRequest()
65
+ .withTableName(tableName)
66
+ .withItem(
67
+ Map
68
+ .newBuilder[String, AttributeValue]
69
+ .addOne("pk", new AttributeValue().withS("a"))
70
+ .result()
71
+ .asJava
72
+ )
73
+ )
74
+ dynamodb.putItem(
75
+ new PutItemRequest()
76
+ .withTableName(tableName)
77
+ .withItem(
78
+ Map
79
+ .newBuilder[String, AttributeValue]
80
+ .addOne("pk", new AttributeValue().withS("b"))
81
+ .result()
82
+ .asJava
83
+ )
84
+ )
85
+ dynamodb.putItem(
86
+ new PutItemRequest()
87
+ .withTableName(tableName)
88
+ .withItem(
89
+ Map
90
+ .newBuilder[String, AttributeValue]
91
+ .addOne("pk", new AttributeValue().withS("b"))
92
+ .result()
93
+ .asJava
94
+ )
95
+ )
96
+ }
97
+
98
+ val inScanConfig: ConfigSource = loadConfigSourceFromYamlString(s"""
99
+ |type: dynamodb
100
+ |table: $tableName
101
+ |endpoint: http://$dynamoDBHost:$dynamoDBPort/
102
+ |auth_method: basic
103
+ |access_key_id: dummy
104
+ |secret_access_key: dummy
105
+ |scan:
106
+ | limit: 1
107
+ |""".stripMargin)
108
+ runInput(inScanConfig, { result =>
109
+ assert(result.size.equals(1))
110
+ })
111
+
112
+ val inQueryConfig: ConfigSource = loadConfigSourceFromYamlString(s"""
113
+ |type: dynamodb
114
+ |table: $tableName
115
+ |endpoint: http://$dynamoDBHost:$dynamoDBPort/
116
+ |auth_method: basic
117
+ |access_key_id: dummy
118
+ |secret_access_key: dummy
119
+ |query:
120
+ | key_condition_expression: "#x = :v"
121
+ | expression_attribute_names:
122
+ | "#x": pk
123
+ | expression_attribute_values:
124
+ | ":v": {S: a}
125
+ | limit: 1
126
+ |""".stripMargin)
127
+ runInput(inQueryConfig, { result =>
128
+ assert(result.size.equals(1))
129
+ })
130
+ }
131
+ }
data/src/test/scala/org/embulk/input/dynamodb/{DynamodbQueryOperationTest.scala → DynamodbQueryOperationBackwardCompatibilityTest.scala} RENAMED
@@ -12,7 +12,6 @@ import com.amazonaws.services.dynamodbv2.model.{
12
12
  }
13
13
  import org.embulk.config.ConfigSource
14
14
  import org.embulk.input.dynamodb.testutil.EmbulkTestBase
15
- import org.embulk.spi.util.Pages
16
15
  import org.hamcrest.CoreMatchers._
17
16
  import org.hamcrest.MatcherAssert.assertThat
18
17
  import org.junit.Test
@@ -20,7 +19,7 @@ import org.msgpack.value.Value
20
19
 
21
20
  import scala.jdk.CollectionConverters._
22
21
 
23
- class DynamodbQueryOperationTest extends EmbulkTestBase {
22
+ class DynamodbQueryOperationBackwardCompatibilityTest extends EmbulkTestBase {
24
23
 
25
24
  private def testBackwardCompatibility(embulkInConfig: ConfigSource): Unit = {
26
25
  cleanupTable("EMBULK_DYNAMODB_TEST_TABLE")
@@ -86,58 +85,52 @@ class DynamodbQueryOperationTest extends EmbulkTestBase {
86
85
  .asJava
87
86
  )
88
87
  )
88
+ }
89
89
 
90
- val path = embulk.createTempFile("csv")
91
- val result = embulk
92
- .inputBuilder()
93
- .in(embulkInConfig)
94
- .outputPath(path)
95
- .preview()
96
-
97
- val pages = result.getPages
98
- val head = Pages.toObjects(result.getSchema, pages.get(0)).get(0)
99
-
100
- assertThat(head(0).toString, is("key-1"))
101
- assertThat(head(1).asInstanceOf[Long], is(0L))
102
- assertThat(head(2).asInstanceOf[Double], is(42.195))
103
- assertThat(head(3).asInstanceOf[Boolean], is(true))
90
+ runInput(
91
+ embulkInConfig, { result: Seq[Seq[AnyRef]] =>
92
+ val head = result.head
93
+ assertThat(head(0).toString, is("key-1"))
94
+ assertThat(head(1).asInstanceOf[Long], is(0L))
95
+ assertThat(head(2).asInstanceOf[Double], is(42.195))
96
+ assertThat(head(3).asInstanceOf[Boolean], is(true))
104
97
 
105
- val arrayValue = head(4).asInstanceOf[Value].asArrayValue()
106
- assertThat(arrayValue.size(), is(2))
107
- assertThat(arrayValue.get(0).asStringValue().toString, is("list-value"))
108
- assertThat(arrayValue.get(1).asIntegerValue().asLong(), is(123L))
98
+ val arrayValue = head(4).asInstanceOf[Value].asArrayValue()
99
+ assertThat(arrayValue.size(), is(2))
100
+ assertThat(arrayValue.get(0).asStringValue().toString, is("list-value"))
101
+ assertThat(arrayValue.get(1).asIntegerValue().asLong(), is(123L))
109
102
 
110
- val mapValue = head(5).asInstanceOf[Value].asMapValue()
111
- assert(mapValue.keySet().asScala.map(_.toString).contains("map-key-1"))
112
- assertThat(
113
- mapValue
114
- .entrySet()
115
- .asScala
116
- .filter(_.getKey.toString.equals("map-key-1"))
117
- .head
118
- .getValue
119
- .toString,
120
- is("map-value-1")
121
- )
122
- assert(mapValue.keySet().asScala.map(_.toString).contains("map-key-2"))
123
- assertThat(
124
- mapValue
125
- .entrySet()
126
- .asScala
127
- .filter(_.getKey.toString.equals("map-key-2"))
128
- .head
129
- .getValue
130
- .asIntegerValue()
131
- .asLong(),
132
- is(456L)
133
- )
134
-
135
- }
103
+ val mapValue = head(5).asInstanceOf[Value].asMapValue()
104
+ assert(mapValue.keySet().asScala.map(_.toString).contains("map-key-1"))
105
+ assertThat(
106
+ mapValue
107
+ .entrySet()
108
+ .asScala
109
+ .filter(_.getKey.toString.equals("map-key-1"))
110
+ .head
111
+ .getValue
112
+ .toString,
113
+ is("map-value-1")
114
+ )
115
+ assert(mapValue.keySet().asScala.map(_.toString).contains("map-key-2"))
116
+ assertThat(
117
+ mapValue
118
+ .entrySet()
119
+ .asScala
120
+ .filter(_.getKey.toString.equals("map-key-2"))
121
+ .head
122
+ .getValue
123
+ .asIntegerValue()
124
+ .asLong(),
125
+ is(456L)
126
+ )
127
+ }
128
+ )
136
129
  }
137
130
 
138
131
  @Test
139
132
  def deprecatedQueryOperationTest(): Unit = {
140
- val inConfig: ConfigSource = embulk.configLoader().fromYamlString(s"""
133
+ val inConfig: ConfigSource = loadConfigSourceFromYamlString(s"""
141
134
  |type: dynamodb
142
135
  |end_point: http://${dynamoDBHost}:${dynamoDBPort}/
143
136
  |table: EMBULK_DYNAMODB_TEST_TABLE
@@ -161,7 +154,7 @@ class DynamodbQueryOperationTest extends EmbulkTestBase {
161
154
 
162
155
  @Test
163
156
  def keepTheSameBehaviourAsDeprecatedQueryOperationTest(): Unit = {
164
- val inConfig: ConfigSource = embulk.configLoader().fromYamlString(s"""
157
+ val inConfig: ConfigSource = loadConfigSourceFromYamlString(s"""
165
158
  |type: dynamodb
166
159
  |endpoint: http://${dynamoDBHost}:${dynamoDBPort}/
167
160
  |table: EMBULK_DYNAMODB_TEST_TABLE
data/src/test/scala/org/embulk/input/dynamodb/{DynamodbScanOperationTest.scala → DynamodbScanOperationBackwardCompatibilityTest.scala} RENAMED
@@ -12,7 +12,6 @@ import com.amazonaws.services.dynamodbv2.model.{
12
12
  }
13
13
  import org.embulk.config.ConfigSource
14
14
  import org.embulk.input.dynamodb.testutil.EmbulkTestBase
15
- import org.embulk.spi.util.Pages
16
15
  import org.hamcrest.CoreMatchers._
17
16
  import org.hamcrest.MatcherAssert.assertThat
18
17
  import org.junit.Test
@@ -20,7 +19,7 @@ import org.msgpack.value.Value
20
19
 
21
20
  import scala.jdk.CollectionConverters._
22
21
 
23
- class DynamodbScanOperationTest extends EmbulkTestBase {
22
+ class DynamodbScanOperationBackwardCompatibilityTest extends EmbulkTestBase {
24
23
 
25
24
  private def testBackwardCompatibility(embulkInConfig: ConfigSource): Unit = {
26
25
  cleanupTable("EMBULK_DYNAMODB_TEST_TABLE")
@@ -88,56 +87,51 @@ class DynamodbScanOperationTest extends EmbulkTestBase {
88
87
  )
89
88
  }
90
89
 
91
- val path = embulk.createTempFile("csv")
92
- val result = embulk
93
- .inputBuilder()
94
- .in(embulkInConfig)
95
- .outputPath(path)
96
- .preview()
90
+ runInput(
91
+ embulkInConfig, { result: Seq[Seq[AnyRef]] =>
92
+ val head = result.head
93
+ assertThat(head(0).toString, is("key-1"))
94
+ assertThat(head(1).asInstanceOf[Long], is(0L))
95
+ assertThat(head(2).asInstanceOf[Double], is(42.195))
96
+ assertThat(head(3).asInstanceOf[Boolean], is(true))
97
97
 
98
- val pages = result.getPages
99
- val head = Pages.toObjects(result.getSchema, pages.get(0)).get(0)
98
+ val arrayValue = head(4).asInstanceOf[Value].asArrayValue()
99
+ assertThat(arrayValue.size(), is(2))
100
+ assertThat(arrayValue.get(0).asStringValue().toString, is("list-value"))
101
+ assertThat(arrayValue.get(1).asIntegerValue().asLong(), is(123L))
100
102
 
101
- assertThat(head(0).toString, is("key-1"))
102
- assertThat(head(1).asInstanceOf[Long], is(0L))
103
- assertThat(head(2).asInstanceOf[Double], is(42.195))
104
- assertThat(head(3).asInstanceOf[Boolean], is(true))
105
-
106
- val arrayValue = head(4).asInstanceOf[Value].asArrayValue()
107
- assertThat(arrayValue.size(), is(2))
108
- assertThat(arrayValue.get(0).asStringValue().toString, is("list-value"))
109
- assertThat(arrayValue.get(1).asIntegerValue().asLong(), is(123L))
110
-
111
- val mapValue = head(5).asInstanceOf[Value].asMapValue()
112
- assert(mapValue.keySet().asScala.map(_.toString).contains("map-key-1"))
113
- assertThat(
114
- mapValue
115
- .entrySet()
116
- .asScala
117
- .filter(_.getKey.toString.equals("map-key-1"))
118
- .head
119
- .getValue
120
- .toString,
121
- is("map-value-1")
122
- )
123
- assert(mapValue.keySet().asScala.map(_.toString).contains("map-key-2"))
124
- assertThat(
125
- mapValue
126
- .entrySet()
127
- .asScala
128
- .filter(_.getKey.toString.equals("map-key-2"))
129
- .head
130
- .getValue
131
- .asIntegerValue()
132
- .asLong(),
133
- is(456L)
103
+ val mapValue = head(5).asInstanceOf[Value].asMapValue()
104
+ assert(mapValue.keySet().asScala.map(_.toString).contains("map-key-1"))
105
+ assertThat(
106
+ mapValue
107
+ .entrySet()
108
+ .asScala
109
+ .filter(_.getKey.toString.equals("map-key-1"))
110
+ .head
111
+ .getValue
112
+ .toString,
113
+ is("map-value-1")
114
+ )
115
+ assert(mapValue.keySet().asScala.map(_.toString).contains("map-key-2"))
116
+ assertThat(
117
+ mapValue
118
+ .entrySet()
119
+ .asScala
120
+ .filter(_.getKey.toString.equals("map-key-2"))
121
+ .head
122
+ .getValue
123
+ .asIntegerValue()
124
+ .asLong(),
125
+ is(456L)
126
+ )
127
+ }
134
128
  )
135
129
  }
136
130
 
137
131
  @Test
138
132
  def deprecatedScanOperationTest(): Unit = {
139
133
 
140
- val inConfig: ConfigSource = embulk.configLoader().fromYamlString(s"""
134
+ val inConfig: ConfigSource = loadConfigSourceFromYamlString(s"""
141
135
  |type: dynamodb
142
136
  |end_point: http://${dynamoDBHost}:${dynamoDBPort}/
143
137
  |table: EMBULK_DYNAMODB_TEST_TABLE
@@ -159,7 +153,7 @@ class DynamodbScanOperationTest extends EmbulkTestBase {
159
153
 
160
154
  @Test
161
155
  def keepTheSameBehaviourAsDeprecatedScanOperationTest(): Unit = {
162
- val inConfig: ConfigSource = embulk.configLoader().fromYamlString(s"""
156
+ val inConfig: ConfigSource = loadConfigSourceFromYamlString(s"""
163
157
  |type: dynamodb
164
158
  |endpoint: http://${dynamoDBHost}:${dynamoDBPort}/
165
159
  |table: EMBULK_DYNAMODB_TEST_TABLE
data/src/test/scala/org/embulk/input/dynamodb/testutil/EmbulkTestBase.scala CHANGED
@@ -6,11 +6,15 @@ import com.amazonaws.services.dynamodbv2.{
6
6
  AmazonDynamoDB,
7
7
  AmazonDynamoDBClientBuilder
8
8
  }
9
+ import org.embulk.config.{ConfigLoader, ConfigSource, TaskReport, TaskSource}
9
10
  import org.embulk.input.dynamodb.DynamodbInputPlugin
10
- import org.embulk.spi.InputPlugin
11
- import org.embulk.test.TestingEmbulk
12
- import org.junit.{After, Rule}
11
+ import org.embulk.spi.Schema
12
+ import org.embulk.EmbulkTestRuntime
13
+ import org.embulk.spi.TestPageBuilderReader.MockPageOutput
14
+ import org.embulk.spi.util.Pages
15
+ import org.junit.Rule
13
16
 
17
+ import scala.jdk.CollectionConverters._
14
18
  import scala.util.{Failure, Success, Try}
15
19
  import scala.util.chaining._
16
20
 
@@ -49,20 +53,7 @@ trait EmbulkTestBase {
49
53
  }
50
54
 
51
55
  @Rule
52
- def embulk: TestingEmbulk =
53
- TestingEmbulk
54
- .builder()
55
- .registerPlugin(
56
- classOf[InputPlugin],
57
- "dynamodb",
58
- classOf[DynamodbInputPlugin]
59
- )
60
- .build()
61
-
62
- @After
63
- def destroyEmbulk(): Unit = {
64
- embulk.destroy()
65
- }
56
+ def runtime: EmbulkTestRuntime = new EmbulkTestRuntime()
66
57
 
67
58
  def getEnvironmentVariableOrShowErrorMessage(name: String): String = {
68
59
  try {
@@ -82,4 +73,27 @@ trait EmbulkTestBase {
82
73
  )
83
74
  }
84
75
  }
76
+
77
+ def runInput(inConfig: ConfigSource, test: Seq[Seq[AnyRef]] => Unit): Unit = {
78
+ runtime.getInstance(classOf[DynamodbInputPlugin]).tap { plugin =>
79
+ plugin.transaction(
80
+ inConfig,
81
+ (taskSource: TaskSource, schema: Schema, taskCount: Int) => {
82
+ val output: MockPageOutput = new MockPageOutput()
83
+ val reports: Seq[TaskReport] = 0.until(taskCount).map { taskIndex =>
84
+ plugin.run(taskSource, schema, taskIndex, output)
85
+ }
86
+ output.finish()
87
+
88
+ test(Pages.toObjects(schema, output.pages).asScala.toSeq.map(_.toSeq))
89
+
90
+ reports.asJava
91
+ }
92
+ )
93
+ }
94
+ }
95
+
96
+ def loadConfigSourceFromYamlString(yaml: String): ConfigSource = {
97
+ new ConfigLoader(runtime.getModelManager).fromYamlString(yaml)
98
+ }
85
99
  }
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-input-dynamodb
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.0
4
+ version: 0.3.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Daisuke Higashi
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2020-03-08 00:00:00.000000000 Z
12
+ date: 2020-03-28 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  requirement: !ruby/object:Gem::Requirement
@@ -62,7 +62,7 @@ files:
62
62
  - classpath/aws-java-sdk-sts-1.11.711.jar
63
63
  - classpath/commons-codec-1.11.jar
64
64
  - classpath/commons-logging-1.2.jar
65
- - classpath/embulk-input-dynamodb-0.3.0.jar
65
+ - classpath/embulk-input-dynamodb-0.3.1.jar
66
66
  - classpath/httpclient-4.5.9.jar
67
67
  - classpath/httpcore-4.4.11.jar
68
68
  - classpath/ion-java-1.0.2.jar
@@ -118,8 +118,9 @@ files:
118
118
  - src/test/resources/json/test.template
119
119
  - src/test/scala/org/embulk/input/dynamodb/AttributeValueHelperTest.scala
120
120
  - src/test/scala/org/embulk/input/dynamodb/AwsCredentialsTest.scala
121
- - src/test/scala/org/embulk/input/dynamodb/DynamodbQueryOperationTest.scala
122
- - src/test/scala/org/embulk/input/dynamodb/DynamodbScanOperationTest.scala
121
+ - src/test/scala/org/embulk/input/dynamodb/DynamodbOperationTest.scala
122
+ - src/test/scala/org/embulk/input/dynamodb/DynamodbQueryOperationBackwardCompatibilityTest.scala
123
+ - src/test/scala/org/embulk/input/dynamodb/DynamodbScanOperationBackwardCompatibilityTest.scala
123
124
  - src/test/scala/org/embulk/input/dynamodb/testutil/EmbulkTestBase.scala
124
125
  homepage: https://github.com/lulichn/embulk-input-dynamodb
125
126
  licenses: