embulk-output-dynamodb 0.1.2 → 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: f96ba155f4fbef98fcbaf6160538d6602b0fec15
4
- data.tar.gz: 32bbb567f67c2c74d1ced286d83a36244b96bcd0
3
+ metadata.gz: 65c351383c9859353bc121a5f054b63de4876773
4
+ data.tar.gz: e171f2369c6582ef90c8fb7d4a6acc0a0300061b
5
5
  SHA512:
6
- metadata.gz: 1b16cacfd4f8c4832ba3bee7eb4c83d422fe0d924e02bcaf4e47c70b1af840cba27113ea6ca3a9d53341508ad06883fe6ac8753934f298244fd702466f0d9e15
7
- data.tar.gz: 1b58e87bb2daf8763a8770d36f2f6e9a98a9a7c2a4126511ee44aa28f7bf05c2525024b4c1f085c4fd7b68830c9bbb3b0210392d3716e00b7621755e3b0b6587
6
+ metadata.gz: 862214066e48cb94be626d68b9a014dcdccf1a4a74f1808e4ff87b43beef2115f55b0d5547aa8d3460404bf85856ab9ed64488190bf1dbdde71ce7f44c07dd47
7
+ data.tar.gz: 8466fe13984980117b6a7bba30efbafca953ad8011758fcfd4b6a0eb6589efc45389051ed2de2e7c2eeb11342a5abf2e5be9b9c82a188d28f28652c5b7b5e5c6
data/.travis.yml ADDED
@@ -0,0 +1,23 @@
1
+ language: java
2
+
3
+ jdk:
4
+ - oraclejdk8
5
+ - oraclejdk7
6
+ - openjdk7
7
+
8
+ install:
9
+ - mkdir ./dynamodb
10
+ - cd ./dynamodb
11
+ - wget http://dynamodb-local.s3-website-us-west-2.amazonaws.com/dynamodb_local_latest.tar.gz
12
+ - tar zxvf dynamodb_local_latest.tar.gz
13
+ - java -Djava.library.path=./DynamoDBLocal_lib -jar DynamoDBLocal.jar -sharedDb &
14
+ - cd ../
15
+
16
+ before_script:
17
+ - echo "Wait DynamoDB wakeup"
18
+ - sleep 10
19
+ - date
20
+
21
+ script:
22
+ - ./gradlew gem
23
+ - ./gradlew --info check jacocoTestReport
data/CHANGELOG.md CHANGED
@@ -1 +1 @@
1
- ## 0.1.2 - 2016-05-24
1
+ ## 0.1.3 - 2016-07-11
data/README.md CHANGED
@@ -1,4 +1,5 @@
1
1
  # AWS DynamoDB output plugin for Embulk
2
+ [![Build Status](https://travis-ci.org/sakama/embulk-output-dynamodb.svg?branch=master)](https://travis-ci.org/sakama/embulk-output-dynamodb)
2
3
 
3
4
  [Embulk](http://www.embulk.org/) output plugin to load/insert data into [AWS DynamoDB](http://aws.amazon.com/dynamodb/)
4
5
 
data/build.gradle CHANGED
@@ -14,7 +14,7 @@ configurations {
14
14
  provided
15
15
  }
16
16
 
17
- version = "0.1.2"
17
+ version = "0.1.3"
18
18
 
19
19
  sourceCompatibility = 1.7
20
20
  targetCompatibility = 1.7
@@ -20,16 +20,24 @@ import org.embulk.config.TaskReport;
20
20
  import org.embulk.config.TaskSource;
21
21
  import org.embulk.spi.Column;
22
22
  import org.embulk.spi.ColumnVisitor;
23
+ import org.embulk.spi.DataException;
23
24
  import org.embulk.spi.Exec;
24
25
  import org.embulk.spi.OutputPlugin;
25
26
  import org.embulk.spi.Page;
26
27
  import org.embulk.spi.PageReader;
27
28
  import org.embulk.spi.Schema;
28
29
  import org.embulk.spi.TransactionalPageOutput;
30
+ import org.msgpack.value.BooleanValue;
31
+ import org.msgpack.value.FloatValue;
32
+ import org.msgpack.value.IntegerValue;
33
+ import org.msgpack.value.Value;
29
34
  import org.slf4j.Logger;
30
35
 
36
+ import java.util.ArrayList;
37
+ import java.util.LinkedHashMap;
31
38
  import java.util.List;
32
39
  import java.util.Locale;
40
+ import java.util.Map;
33
41
 
34
42
  public class DynamodbOutputPlugin
35
43
  implements OutputPlugin
@@ -285,7 +293,17 @@ public class DynamodbOutputPlugin
285
293
  addNullValue(column.getName());
286
294
  }
287
295
  else {
288
- item.withJSON(column.getName(), pageReader.getJson(column).toString());
296
+ Value jsonValue = pageReader.getJson(column);
297
+ if (jsonValue.isArrayValue()) {
298
+ List<Object> list = new ArrayList<Object>();
299
+ for (Value v : jsonValue.asArrayValue()) {
300
+ list.add(getRawFromValue(v));
301
+ }
302
+ item.withList(column.getName(), list);
303
+ }
304
+ else {
305
+ item.withJSON(column.getName(), jsonValue.toJson());
306
+ }
289
307
  }
290
308
  }
291
309
 
@@ -293,6 +311,40 @@ public class DynamodbOutputPlugin
293
311
  {
294
312
  item.withNull(name);
295
313
  }
314
+
315
+ private Object getRawFromValue(Value value)
316
+ {
317
+ if (value.isBooleanValue()) {
318
+ return ((BooleanValue) value).getBoolean();
319
+ }
320
+ else if (value.isStringValue()) {
321
+ return value.toString();
322
+ }
323
+ else if (value.isIntegerValue()) {
324
+ return ((IntegerValue) value).asLong();
325
+ }
326
+ else if (value.isFloatValue()) {
327
+ return ((FloatValue) value).toDouble();
328
+ }
329
+ else if (value.isArrayValue()) {
330
+ List<Object> list = new ArrayList<>();
331
+ for (Value v : value.asArrayValue()) {
332
+ list.add(getRawFromValue(v));
333
+ }
334
+ return list;
335
+ }
336
+ else if (value.isMapValue()) {
337
+ Map<String, Object> map = new LinkedHashMap<>();
338
+ for (Map.Entry<Value, Value> entry : value.asMapValue().entrySet()) {
339
+ map.put(entry.getKey().toString(), getRawFromValue(entry.getValue()));
340
+ }
341
+ return map;
342
+ }
343
+ else if (value.isNilValue()) {
344
+ return null;
345
+ }
346
+ throw new DataException("Record has invalid json column value");
347
+ }
296
348
  });
297
349
 
298
350
  if (mode.equals(Mode.UPSERT)) {
@@ -111,6 +111,9 @@ public class DynamodbUtils
111
111
  if (!task.getPrimaryKey().isPresent() || !task.getPrimaryKeyType().isPresent()) {
112
112
  throw new ConfigException("If auto_create_table is true, both primary_key and primary_key_type is necessary");
113
113
  }
114
+ if (!task.getReadCapacityUnits().isPresent() || !task.getWriteCapacityUnits().isPresent()) {
115
+ throw new ConfigException("If auto_create_table is true, 'read_capacity_units' and 'write_capacity_units' is required.");
116
+ }
114
117
  }
115
118
  }
116
119
 
@@ -1,6 +1,10 @@
1
1
  package org.embulk.output.dynamodb;
2
2
 
3
3
  import com.amazonaws.services.dynamodbv2.document.DynamoDB;
4
+ import com.amazonaws.services.dynamodbv2.document.Item;
5
+ import com.amazonaws.services.dynamodbv2.document.ItemCollection;
6
+ import com.amazonaws.services.dynamodbv2.document.ScanOutcome;
7
+ import com.amazonaws.services.dynamodbv2.document.Table;
4
8
  import com.google.common.collect.ImmutableList;
5
9
  import com.google.common.collect.ImmutableMap;
6
10
  import com.google.common.collect.Lists;
@@ -22,9 +26,15 @@ import org.junit.Before;
22
26
  import org.junit.BeforeClass;
23
27
  import org.junit.Rule;
24
28
  import org.junit.Test;
29
+ import org.msgpack.value.Value;
30
+ import org.msgpack.value.ValueFactory;
25
31
 
32
+ import java.math.BigDecimal;
33
+ import java.util.ArrayList;
26
34
  import java.util.Arrays;
35
+ import java.util.LinkedHashMap;
27
36
  import java.util.List;
37
+ import java.util.Map;
28
38
 
29
39
  import static org.junit.Assert.assertEquals;
30
40
 
@@ -134,39 +144,65 @@ public class TestDynamodbOutputPlugin
134
144
  });
135
145
  TransactionalPageOutput output = plugin.open(task.dump(), schema, 0);
136
146
 
137
- List<Page> pages = PageTestUtils.buildPage(runtime.getBufferAllocator(), schema, 1L, 32864L, "2015-01-27T19:23:49", "2015-01-27T00:00:00", true, 123.45, "embulk");
147
+ Map<Value, Value> map = new LinkedHashMap<>();
148
+ map.put(ValueFactory.newString("foo"), ValueFactory.newString("bar"));
149
+ map.put(ValueFactory.newString("key1"), ValueFactory.newString("val1"));
150
+ List<Page> pages = PageTestUtils.buildPage(runtime.getBufferAllocator(), schema,
151
+ 1L,
152
+ 32864L,
153
+ 1L,
154
+ 1L,
155
+ true,
156
+ 123.45,
157
+ "embulk",
158
+ ValueFactory.newArray(ValueFactory.newInteger(1), ValueFactory.newInteger(2), ValueFactory.newInteger(3), ValueFactory.newArray(ValueFactory.newString("inner"))),
159
+ ValueFactory.newMap(map)
160
+ );
138
161
  assertEquals(1, pages.size());
139
162
  for (Page page : pages) {
140
163
  output.add(page);
141
164
  }
142
165
 
143
- // output.finish();
144
- // output.commit();
145
- //
146
- // DynamodbUtils dynamoDbUtils = new DynamodbUtils();
147
- // DynamoDB dynamoDB = null;
148
- // try {
149
- // dynamoDB = dynamoDbUtils.createDynamoDB(task);
150
- //
151
- // Table table = dynamoDB.getTable(task.getTable());
152
- // ItemCollection<ScanOutcome> items = table.scan();
153
- //
154
- // while (items.iterator().hasNext()) {
155
- // Map<String, Object> item = items.iterator().next().asMap();
156
- // assertEquals(1, item.get("id"));
157
- // assertEquals(32864, item.get("account"));
158
- // assertEquals("2015-01-27T19:23:49", item.get("time"));
159
- // assertEquals("2015-01-27T00:00:00", item.get("purchase"));
160
- // assertEquals(true, item.get("flg"));
161
- // assertEquals(123.45, item.get("score"));
162
- // assertEquals("embulk", item.get("comment"));
163
- // }
164
- // }
165
- // finally {
166
- // if (dynamoDB != null) {
167
- // dynamoDB.shutdown();
168
- // }
169
- // }
166
+ output.finish();
167
+ output.commit();
168
+
169
+ DynamodbUtils dynamoDbUtils = new DynamodbUtils();
170
+ DynamoDB dynamoDB = null;
171
+ try {
172
+ dynamoDB = dynamoDbUtils.createDynamoDB(task);
173
+
174
+ Table table = dynamoDB.getTable(task.getTable());
175
+ ItemCollection<ScanOutcome> items = table.scan();
176
+
177
+ for (Item item1 : items) {
178
+ assertEquals(1L, item1.getLong("id"));
179
+ assertEquals(32864L, item1.getLong("account"));
180
+ assertEquals("1970-01-01 00:00:01 UTC", item1.getString("time"));
181
+ assertEquals("1970-01-01 00:00:01 UTC", item1.getString("purchase"));
182
+ assertEquals(true, item1.getBoolean("flg"));
183
+ assertEquals(new BigDecimal("123.45"), item1.get("score"));
184
+ assertEquals("embulk", item1.getString("comment"));
185
+
186
+ List<Object> list = new ArrayList<>();
187
+ List<Object> inner = new ArrayList<>();
188
+ inner.add("inner");
189
+ list.add(new BigDecimal(1));
190
+ list.add(new BigDecimal(2));
191
+ list.add(new BigDecimal(3));
192
+ list.add(inner);
193
+ assertEquals(list, item1.getList("list"));
194
+
195
+ Map<String, Object> expectedMap = new LinkedHashMap<>();
196
+ expectedMap.put("foo", "bar");
197
+ expectedMap.put("key1", "val1");
198
+ assertEquals(expectedMap, item1.getMap("map"));
199
+ }
200
+ }
201
+ finally {
202
+ if (dynamoDB != null) {
203
+ dynamoDB.shutdown();
204
+ }
205
+ }
170
206
  }
171
207
 
172
208
  @Test
@@ -192,7 +228,7 @@ public class TestDynamodbOutputPlugin
192
228
  .set("region", "us-west-1")
193
229
  .set("table", "dummy")
194
230
  .set("primary_key", "id")
195
- .set("primary_key_type", "string")
231
+ .set("primary_key_type", "number")
196
232
  .set("read_capacity_units", capacityUnitConfig())
197
233
  .set("write_capacity_units", capacityUnitConfig())
198
234
  .set("auth_method", "basic")
@@ -244,6 +280,8 @@ public class TestDynamodbOutputPlugin
244
280
  builder.add(ImmutableMap.of("name", "flg", "type", "boolean"));
245
281
  builder.add(ImmutableMap.of("name", "score", "type", "double"));
246
282
  builder.add(ImmutableMap.of("name", "comment", "type", "string"));
283
+ builder.add(ImmutableMap.of("name", "list", "type", "json"));
284
+ builder.add(ImmutableMap.of("name", "map", "type", "json"));
247
285
  return builder.build();
248
286
  }
249
287
  }
@@ -1,5 +1,5 @@
1
- id,account,time,purchase,flg,score,comment
2
- 1,32864,2015-01-27 19:23:49,20150127,1,123.45,embulk
3
- 2,14824,2015-01-27 19:01:23,20150127,0,234,56,embulk
4
- 3,27559,2015-01-28 02:20:02,20150128,1,678.90,embulk
5
- 4,11270,2015-01-29 11:54:36,20150129,0,100.00,embulk
1
+ id,account,time,purchase,flg,score,comment,list
2
+ 1,32864,2015-01-27 19:23:49,20150127,1,123.45,embulk,[1,2,3],{"foo": "bar", "key1": "val1"}
3
+ 2,14824,2015-01-27 19:01:23,20150127,0,234,56,embulk,[1,2,3],{"foo": "bar", "key1": "val1"}
4
+ 3,27559,2015-01-28 02:20:02,20150128,1,678.90,embulk,[1,2,3],{"foo": "bar", "key1": "val1"}
5
+ 4,11270,2015-01-29 11:54:36,20150129,0,100.00,embulk,[1,2,3],{"foo": "bar", "key1": "val1"}
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-output-dynamodb
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.2
4
+ version: 0.1.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Satoshi Akama
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-05-24 00:00:00.000000000 Z
11
+ date: 2016-07-11 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -46,6 +46,7 @@ extensions: []
46
46
  extra_rdoc_files: []
47
47
  files:
48
48
  - .gitignore
49
+ - .travis.yml
49
50
  - CHANGELOG.md
50
51
  - README.md
51
52
  - build.gradle
@@ -70,7 +71,7 @@ files:
70
71
  - classpath/aws-java-sdk-s3-1.10.50.jar
71
72
  - classpath/commons-codec-1.6.jar
72
73
  - classpath/commons-logging-1.1.3.jar
73
- - classpath/embulk-output-dynamodb-0.1.2.jar
74
+ - classpath/embulk-output-dynamodb-0.1.3.jar
74
75
  - classpath/httpclient-4.3.6.jar
75
76
  - classpath/httpcore-4.3.3.jar
76
77
  homepage: https://github.com/sakama/embulk-output-dynamodb