embulk-output-dynamodb 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +23 -0
- data/CHANGELOG.md +1 -1
- data/README.md +1 -0
- data/build.gradle +1 -1
- data/src/main/java/org/embulk/output/dynamodb/DynamodbOutputPlugin.java +53 -1
- data/src/main/java/org/embulk/output/dynamodb/DynamodbUtils.java +3 -0
- data/src/test/java/org/embulk/output/dynamodb/TestDynamodbOutputPlugin.java +67 -29
- data/src/test/resources/sample_01.csv +5 -5
- metadata +4 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 65c351383c9859353bc121a5f054b63de4876773
|
4
|
+
data.tar.gz: e171f2369c6582ef90c8fb7d4a6acc0a0300061b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 862214066e48cb94be626d68b9a014dcdccf1a4a74f1808e4ff87b43beef2115f55b0d5547aa8d3460404bf85856ab9ed64488190bf1dbdde71ce7f44c07dd47
|
7
|
+
data.tar.gz: 8466fe13984980117b6a7bba30efbafca953ad8011758fcfd4b6a0eb6589efc45389051ed2de2e7c2eeb11342a5abf2e5be9b9c82a188d28f28652c5b7b5e5c6
|
data/.travis.yml
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
language: java
|
2
|
+
|
3
|
+
jdk:
|
4
|
+
- oraclejdk8
|
5
|
+
- oraclejdk7
|
6
|
+
- openjdk7
|
7
|
+
|
8
|
+
install:
|
9
|
+
- mkdir ./dynamodb
|
10
|
+
- cd ./dynamodb
|
11
|
+
- wget http://dynamodb-local.s3-website-us-west-2.amazonaws.com/dynamodb_local_latest.tar.gz
|
12
|
+
- tar zxvf dynamodb_local_latest.tar.gz
|
13
|
+
- java -Djava.library.path=./DynamoDBLocal_lib -jar DynamoDBLocal.jar -sharedDb &
|
14
|
+
- cd ../
|
15
|
+
|
16
|
+
before_script:
|
17
|
+
- echo "Wait DynamoDB wakeup"
|
18
|
+
- sleep 10
|
19
|
+
- date
|
20
|
+
|
21
|
+
script:
|
22
|
+
- ./gradlew gem
|
23
|
+
- ./gradlew --info check jacocoTestReport
|
data/CHANGELOG.md
CHANGED
@@ -1 +1 @@
|
|
1
|
-
## 0.1.
|
1
|
+
## 0.1.3 - 2016-07-11
|
data/README.md
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
# AWS DynamoDB output plugin for Embulk
|
2
|
+
[](https://travis-ci.org/sakama/embulk-output-dynamodb)
|
2
3
|
|
3
4
|
[Embulk](http://www.embulk.org/) output plugin to load/insert data into [AWS DynamoDB](http://aws.amazon.com/dynamodb/)
|
4
5
|
|
data/build.gradle
CHANGED
@@ -20,16 +20,24 @@ import org.embulk.config.TaskReport;
|
|
20
20
|
import org.embulk.config.TaskSource;
|
21
21
|
import org.embulk.spi.Column;
|
22
22
|
import org.embulk.spi.ColumnVisitor;
|
23
|
+
import org.embulk.spi.DataException;
|
23
24
|
import org.embulk.spi.Exec;
|
24
25
|
import org.embulk.spi.OutputPlugin;
|
25
26
|
import org.embulk.spi.Page;
|
26
27
|
import org.embulk.spi.PageReader;
|
27
28
|
import org.embulk.spi.Schema;
|
28
29
|
import org.embulk.spi.TransactionalPageOutput;
|
30
|
+
import org.msgpack.value.BooleanValue;
|
31
|
+
import org.msgpack.value.FloatValue;
|
32
|
+
import org.msgpack.value.IntegerValue;
|
33
|
+
import org.msgpack.value.Value;
|
29
34
|
import org.slf4j.Logger;
|
30
35
|
|
36
|
+
import java.util.ArrayList;
|
37
|
+
import java.util.LinkedHashMap;
|
31
38
|
import java.util.List;
|
32
39
|
import java.util.Locale;
|
40
|
+
import java.util.Map;
|
33
41
|
|
34
42
|
public class DynamodbOutputPlugin
|
35
43
|
implements OutputPlugin
|
@@ -285,7 +293,17 @@ public class DynamodbOutputPlugin
|
|
285
293
|
addNullValue(column.getName());
|
286
294
|
}
|
287
295
|
else {
|
288
|
-
|
296
|
+
Value jsonValue = pageReader.getJson(column);
|
297
|
+
if (jsonValue.isArrayValue()) {
|
298
|
+
List<Object> list = new ArrayList<Object>();
|
299
|
+
for (Value v : jsonValue.asArrayValue()) {
|
300
|
+
list.add(getRawFromValue(v));
|
301
|
+
}
|
302
|
+
item.withList(column.getName(), list);
|
303
|
+
}
|
304
|
+
else {
|
305
|
+
item.withJSON(column.getName(), jsonValue.toJson());
|
306
|
+
}
|
289
307
|
}
|
290
308
|
}
|
291
309
|
|
@@ -293,6 +311,40 @@ public class DynamodbOutputPlugin
|
|
293
311
|
{
|
294
312
|
item.withNull(name);
|
295
313
|
}
|
314
|
+
|
315
|
+
private Object getRawFromValue(Value value)
|
316
|
+
{
|
317
|
+
if (value.isBooleanValue()) {
|
318
|
+
return ((BooleanValue) value).getBoolean();
|
319
|
+
}
|
320
|
+
else if (value.isStringValue()) {
|
321
|
+
return value.toString();
|
322
|
+
}
|
323
|
+
else if (value.isIntegerValue()) {
|
324
|
+
return ((IntegerValue) value).asLong();
|
325
|
+
}
|
326
|
+
else if (value.isFloatValue()) {
|
327
|
+
return ((FloatValue) value).toDouble();
|
328
|
+
}
|
329
|
+
else if (value.isArrayValue()) {
|
330
|
+
List<Object> list = new ArrayList<>();
|
331
|
+
for (Value v : value.asArrayValue()) {
|
332
|
+
list.add(getRawFromValue(v));
|
333
|
+
}
|
334
|
+
return list;
|
335
|
+
}
|
336
|
+
else if (value.isMapValue()) {
|
337
|
+
Map<String, Object> map = new LinkedHashMap<>();
|
338
|
+
for (Map.Entry<Value, Value> entry : value.asMapValue().entrySet()) {
|
339
|
+
map.put(entry.getKey().toString(), getRawFromValue(entry.getValue()));
|
340
|
+
}
|
341
|
+
return map;
|
342
|
+
}
|
343
|
+
else if (value.isNilValue()) {
|
344
|
+
return null;
|
345
|
+
}
|
346
|
+
throw new DataException("Record has invalid json column value");
|
347
|
+
}
|
296
348
|
});
|
297
349
|
|
298
350
|
if (mode.equals(Mode.UPSERT)) {
|
@@ -111,6 +111,9 @@ public class DynamodbUtils
|
|
111
111
|
if (!task.getPrimaryKey().isPresent() || !task.getPrimaryKeyType().isPresent()) {
|
112
112
|
throw new ConfigException("If auto_create_table is true, both primary_key and primary_key_type is necessary");
|
113
113
|
}
|
114
|
+
if (!task.getReadCapacityUnits().isPresent() || !task.getWriteCapacityUnits().isPresent()) {
|
115
|
+
throw new ConfigException("If auto_create_table is true, 'read_capacity_units' and 'write_capacity_units' is required.");
|
116
|
+
}
|
114
117
|
}
|
115
118
|
}
|
116
119
|
|
@@ -1,6 +1,10 @@
|
|
1
1
|
package org.embulk.output.dynamodb;
|
2
2
|
|
3
3
|
import com.amazonaws.services.dynamodbv2.document.DynamoDB;
|
4
|
+
import com.amazonaws.services.dynamodbv2.document.Item;
|
5
|
+
import com.amazonaws.services.dynamodbv2.document.ItemCollection;
|
6
|
+
import com.amazonaws.services.dynamodbv2.document.ScanOutcome;
|
7
|
+
import com.amazonaws.services.dynamodbv2.document.Table;
|
4
8
|
import com.google.common.collect.ImmutableList;
|
5
9
|
import com.google.common.collect.ImmutableMap;
|
6
10
|
import com.google.common.collect.Lists;
|
@@ -22,9 +26,15 @@ import org.junit.Before;
|
|
22
26
|
import org.junit.BeforeClass;
|
23
27
|
import org.junit.Rule;
|
24
28
|
import org.junit.Test;
|
29
|
+
import org.msgpack.value.Value;
|
30
|
+
import org.msgpack.value.ValueFactory;
|
25
31
|
|
32
|
+
import java.math.BigDecimal;
|
33
|
+
import java.util.ArrayList;
|
26
34
|
import java.util.Arrays;
|
35
|
+
import java.util.LinkedHashMap;
|
27
36
|
import java.util.List;
|
37
|
+
import java.util.Map;
|
28
38
|
|
29
39
|
import static org.junit.Assert.assertEquals;
|
30
40
|
|
@@ -134,39 +144,65 @@ public class TestDynamodbOutputPlugin
|
|
134
144
|
});
|
135
145
|
TransactionalPageOutput output = plugin.open(task.dump(), schema, 0);
|
136
146
|
|
137
|
-
|
147
|
+
Map<Value, Value> map = new LinkedHashMap<>();
|
148
|
+
map.put(ValueFactory.newString("foo"), ValueFactory.newString("bar"));
|
149
|
+
map.put(ValueFactory.newString("key1"), ValueFactory.newString("val1"));
|
150
|
+
List<Page> pages = PageTestUtils.buildPage(runtime.getBufferAllocator(), schema,
|
151
|
+
1L,
|
152
|
+
32864L,
|
153
|
+
1L,
|
154
|
+
1L,
|
155
|
+
true,
|
156
|
+
123.45,
|
157
|
+
"embulk",
|
158
|
+
ValueFactory.newArray(ValueFactory.newInteger(1), ValueFactory.newInteger(2), ValueFactory.newInteger(3), ValueFactory.newArray(ValueFactory.newString("inner"))),
|
159
|
+
ValueFactory.newMap(map)
|
160
|
+
);
|
138
161
|
assertEquals(1, pages.size());
|
139
162
|
for (Page page : pages) {
|
140
163
|
output.add(page);
|
141
164
|
}
|
142
165
|
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
166
|
+
output.finish();
|
167
|
+
output.commit();
|
168
|
+
|
169
|
+
DynamodbUtils dynamoDbUtils = new DynamodbUtils();
|
170
|
+
DynamoDB dynamoDB = null;
|
171
|
+
try {
|
172
|
+
dynamoDB = dynamoDbUtils.createDynamoDB(task);
|
173
|
+
|
174
|
+
Table table = dynamoDB.getTable(task.getTable());
|
175
|
+
ItemCollection<ScanOutcome> items = table.scan();
|
176
|
+
|
177
|
+
for (Item item1 : items) {
|
178
|
+
assertEquals(1L, item1.getLong("id"));
|
179
|
+
assertEquals(32864L, item1.getLong("account"));
|
180
|
+
assertEquals("1970-01-01 00:00:01 UTC", item1.getString("time"));
|
181
|
+
assertEquals("1970-01-01 00:00:01 UTC", item1.getString("purchase"));
|
182
|
+
assertEquals(true, item1.getBoolean("flg"));
|
183
|
+
assertEquals(new BigDecimal("123.45"), item1.get("score"));
|
184
|
+
assertEquals("embulk", item1.getString("comment"));
|
185
|
+
|
186
|
+
List<Object> list = new ArrayList<>();
|
187
|
+
List<Object> inner = new ArrayList<>();
|
188
|
+
inner.add("inner");
|
189
|
+
list.add(new BigDecimal(1));
|
190
|
+
list.add(new BigDecimal(2));
|
191
|
+
list.add(new BigDecimal(3));
|
192
|
+
list.add(inner);
|
193
|
+
assertEquals(list, item1.getList("list"));
|
194
|
+
|
195
|
+
Map<String, Object> expectedMap = new LinkedHashMap<>();
|
196
|
+
expectedMap.put("foo", "bar");
|
197
|
+
expectedMap.put("key1", "val1");
|
198
|
+
assertEquals(expectedMap, item1.getMap("map"));
|
199
|
+
}
|
200
|
+
}
|
201
|
+
finally {
|
202
|
+
if (dynamoDB != null) {
|
203
|
+
dynamoDB.shutdown();
|
204
|
+
}
|
205
|
+
}
|
170
206
|
}
|
171
207
|
|
172
208
|
@Test
|
@@ -192,7 +228,7 @@ public class TestDynamodbOutputPlugin
|
|
192
228
|
.set("region", "us-west-1")
|
193
229
|
.set("table", "dummy")
|
194
230
|
.set("primary_key", "id")
|
195
|
-
.set("primary_key_type", "
|
231
|
+
.set("primary_key_type", "number")
|
196
232
|
.set("read_capacity_units", capacityUnitConfig())
|
197
233
|
.set("write_capacity_units", capacityUnitConfig())
|
198
234
|
.set("auth_method", "basic")
|
@@ -244,6 +280,8 @@ public class TestDynamodbOutputPlugin
|
|
244
280
|
builder.add(ImmutableMap.of("name", "flg", "type", "boolean"));
|
245
281
|
builder.add(ImmutableMap.of("name", "score", "type", "double"));
|
246
282
|
builder.add(ImmutableMap.of("name", "comment", "type", "string"));
|
283
|
+
builder.add(ImmutableMap.of("name", "list", "type", "json"));
|
284
|
+
builder.add(ImmutableMap.of("name", "map", "type", "json"));
|
247
285
|
return builder.build();
|
248
286
|
}
|
249
287
|
}
|
@@ -1,5 +1,5 @@
|
|
1
|
-
id,account,time,purchase,flg,score,comment
|
2
|
-
1,32864,2015-01-27 19:23:49,20150127,1,123.45,embulk
|
3
|
-
2,14824,2015-01-27 19:01:23,20150127,0,234,56,embulk
|
4
|
-
3,27559,2015-01-28 02:20:02,20150128,1,678.90,embulk
|
5
|
-
4,11270,2015-01-29 11:54:36,20150129,0,100.00,embulk
|
1
|
+
id,account,time,purchase,flg,score,comment,list
|
2
|
+
1,32864,2015-01-27 19:23:49,20150127,1,123.45,embulk,[1,2,3],{"foo": "bar", "key1": "val1"}
|
3
|
+
2,14824,2015-01-27 19:01:23,20150127,0,234,56,embulk,[1,2,3],{"foo": "bar", "key1": "val1"}
|
4
|
+
3,27559,2015-01-28 02:20:02,20150128,1,678.90,embulk,[1,2,3],{"foo": "bar", "key1": "val1"}
|
5
|
+
4,11270,2015-01-29 11:54:36,20150129,0,100.00,embulk,[1,2,3],{"foo": "bar", "key1": "val1"}
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-dynamodb
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Satoshi Akama
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-07-11 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -46,6 +46,7 @@ extensions: []
|
|
46
46
|
extra_rdoc_files: []
|
47
47
|
files:
|
48
48
|
- .gitignore
|
49
|
+
- .travis.yml
|
49
50
|
- CHANGELOG.md
|
50
51
|
- README.md
|
51
52
|
- build.gradle
|
@@ -70,7 +71,7 @@ files:
|
|
70
71
|
- classpath/aws-java-sdk-s3-1.10.50.jar
|
71
72
|
- classpath/commons-codec-1.6.jar
|
72
73
|
- classpath/commons-logging-1.1.3.jar
|
73
|
-
- classpath/embulk-output-dynamodb-0.1.
|
74
|
+
- classpath/embulk-output-dynamodb-0.1.3.jar
|
74
75
|
- classpath/httpclient-4.3.6.jar
|
75
76
|
- classpath/httpcore-4.3.3.jar
|
76
77
|
homepage: https://github.com/sakama/embulk-output-dynamodb
|