embulk-output-dynamodb 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f96ba155f4fbef98fcbaf6160538d6602b0fec15
-  data.tar.gz: 32bbb567f67c2c74d1ced286d83a36244b96bcd0
+  metadata.gz: 65c351383c9859353bc121a5f054b63de4876773
+  data.tar.gz: e171f2369c6582ef90c8fb7d4a6acc0a0300061b
 SHA512:
-  metadata.gz: 1b16cacfd4f8c4832ba3bee7eb4c83d422fe0d924e02bcaf4e47c70b1af840cba27113ea6ca3a9d53341508ad06883fe6ac8753934f298244fd702466f0d9e15
-  data.tar.gz: 1b58e87bb2daf8763a8770d36f2f6e9a98a9a7c2a4126511ee44aa28f7bf05c2525024b4c1f085c4fd7b68830c9bbb3b0210392d3716e00b7621755e3b0b6587
+  metadata.gz: 862214066e48cb94be626d68b9a014dcdccf1a4a74f1808e4ff87b43beef2115f55b0d5547aa8d3460404bf85856ab9ed64488190bf1dbdde71ce7f44c07dd47
+  data.tar.gz: 8466fe13984980117b6a7bba30efbafca953ad8011758fcfd4b6a0eb6589efc45389051ed2de2e7c2eeb11342a5abf2e5be9b9c82a188d28f28652c5b7b5e5c6

data/.travis.yml

@@ -0,0 +1,23 @@
+language: java
+
+jdk:
+  - oraclejdk8
+  - oraclejdk7
+  - openjdk7
+
+install:
+  - mkdir ./dynamodb
+  - cd ./dynamodb
+  - wget http://dynamodb-local.s3-website-us-west-2.amazonaws.com/dynamodb_local_latest.tar.gz
+  - tar zxvf dynamodb_local_latest.tar.gz
+  - java -Djava.library.path=./DynamoDBLocal_lib -jar DynamoDBLocal.jar -sharedDb &
+  - cd ../
+
+before_script:
+  - echo "Wait DynamoDB wakeup"
+  - sleep 10
+  - date
+
+script:
+  - ./gradlew gem
+  - ./gradlew --info check jacocoTestReport

data/CHANGELOG.md

@@ -1 +1 @@
-## 0.1.2 - 2016-05-24
+## 0.1.3 - 2016-07-11

data/README.md

@@ -1,4 +1,5 @@
 # AWS DynamoDB output plugin for Embulk
+[![Build Status](https://travis-ci.org/sakama/embulk-output-dynamodb.svg?branch=master)](https://travis-ci.org/sakama/embulk-output-dynamodb)
 
 [Embulk](http://www.embulk.org/) output plugin to load/insert data into [AWS DynamoDB](http://aws.amazon.com/dynamodb/)
 

data/build.gradle

@@ -14,7 +14,7 @@ configurations {
     provided
 }
 
-version = "0.1.2"
+version = "0.1.3"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7

data/src/main/java/org/embulk/output/dynamodb/DynamodbOutputPlugin.java

@@ -20,16 +20,24 @@ import org.embulk.config.TaskReport;
 import org.embulk.config.TaskSource;
 import org.embulk.spi.Column;
 import org.embulk.spi.ColumnVisitor;
+import org.embulk.spi.DataException;
 import org.embulk.spi.Exec;
 import org.embulk.spi.OutputPlugin;
 import org.embulk.spi.Page;
 import org.embulk.spi.PageReader;
 import org.embulk.spi.Schema;
 import org.embulk.spi.TransactionalPageOutput;
+import org.msgpack.value.BooleanValue;
+import org.msgpack.value.FloatValue;
+import org.msgpack.value.IntegerValue;
+import org.msgpack.value.Value;
 import org.slf4j.Logger;
 
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
 
 public class DynamodbOutputPlugin
         implements OutputPlugin
@@ -285,7 +293,17 @@ public class DynamodbOutputPlugin
                                 addNullValue(column.getName());
                             }
                             else {
-                                item.withJSON(column.getName(), pageReader.getJson(column).toString());
+                                Value jsonValue = pageReader.getJson(column);
+                                if (jsonValue.isArrayValue()) {
+                                    List<Object> list = new ArrayList<Object>();
+                                    for (Value v : jsonValue.asArrayValue()) {
+                                        list.add(getRawFromValue(v));
+                                    }
+                                    item.withList(column.getName(), list);
+                                }
+                                else {
+                                    item.withJSON(column.getName(), jsonValue.toJson());
+                                }
                             }
                         }
 
@@ -293,6 +311,40 @@ public class DynamodbOutputPlugin
                         {
                             item.withNull(name);
                         }
+
+                        private Object getRawFromValue(Value value)
+                        {
+                            if (value.isBooleanValue()) {
+                                return ((BooleanValue) value).getBoolean();
+                            }
+                            else if (value.isStringValue()) {
+                                return value.toString();
+                            }
+                            else if (value.isIntegerValue()) {
+                                return ((IntegerValue) value).asLong();
+                            }
+                            else if (value.isFloatValue()) {
+                                return ((FloatValue) value).toDouble();
+                            }
+                            else if (value.isArrayValue()) {
+                                List<Object> list = new ArrayList<>();
+                                for (Value v : value.asArrayValue()) {
+                                    list.add(getRawFromValue(v));
+                                }
+                                return list;
+                            }
+                            else if (value.isMapValue()) {
+                                Map<String, Object> map = new LinkedHashMap<>();
+                                for (Map.Entry<Value, Value> entry : value.asMapValue().entrySet()) {
+                                    map.put(entry.getKey().toString(), getRawFromValue(entry.getValue()));
+                                }
+                                return map;
+                            }
+                            else if (value.isNilValue()) {
+                                return null;
+                            }
+                            throw new DataException("Record has invalid json column value");
+                        }
                     });
 
                     if (mode.equals(Mode.UPSERT)) {

data/src/main/java/org/embulk/output/dynamodb/DynamodbUtils.java

@@ -111,6 +111,9 @@ public class DynamodbUtils
             if (!task.getPrimaryKey().isPresent() || !task.getPrimaryKeyType().isPresent()) {
                 throw new ConfigException("If auto_create_table is true, both primary_key and primary_key_type is necessary");
             }
+            if (!task.getReadCapacityUnits().isPresent() || !task.getWriteCapacityUnits().isPresent()) {
+                throw new ConfigException("If auto_create_table is true, 'read_capacity_units' and 'write_capacity_units' is required.");
+            }
         }
     }
 

data/src/test/java/org/embulk/output/dynamodb/TestDynamodbOutputPlugin.java

@@ -1,6 +1,10 @@
 package org.embulk.output.dynamodb;
 
 import com.amazonaws.services.dynamodbv2.document.DynamoDB;
+import com.amazonaws.services.dynamodbv2.document.Item;
+import com.amazonaws.services.dynamodbv2.document.ItemCollection;
+import com.amazonaws.services.dynamodbv2.document.ScanOutcome;
+import com.amazonaws.services.dynamodbv2.document.Table;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
@@ -22,9 +26,15 @@ import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Rule;
 import org.junit.Test;
+import org.msgpack.value.Value;
+import org.msgpack.value.ValueFactory;
 
+import java.math.BigDecimal;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Map;
 
 import static org.junit.Assert.assertEquals;
 
@@ -134,39 +144,65 @@ public class TestDynamodbOutputPlugin
         });
         TransactionalPageOutput output = plugin.open(task.dump(), schema, 0);
 
-        List<Page> pages = PageTestUtils.buildPage(runtime.getBufferAllocator(), schema, 1L, 32864L, "2015-01-27T19:23:49", "2015-01-27T00:00:00",  true, 123.45, "embulk");
+        Map<Value, Value> map = new LinkedHashMap<>();
+        map.put(ValueFactory.newString("foo"), ValueFactory.newString("bar"));
+        map.put(ValueFactory.newString("key1"), ValueFactory.newString("val1"));
+        List<Page> pages = PageTestUtils.buildPage(runtime.getBufferAllocator(), schema,
+                1L,
+                32864L,
+                1L,
+                1L,
+                true,
+                123.45,
+                "embulk",
+                ValueFactory.newArray(ValueFactory.newInteger(1), ValueFactory.newInteger(2), ValueFactory.newInteger(3), ValueFactory.newArray(ValueFactory.newString("inner"))),
+                ValueFactory.newMap(map)
+        );
         assertEquals(1, pages.size());
         for (Page page : pages) {
             output.add(page);
         }
 
-//        output.finish();
-//        output.commit();
-//
-//        DynamodbUtils dynamoDbUtils = new DynamodbUtils();
-//        DynamoDB dynamoDB = null;
-//        try {
-//            dynamoDB = dynamoDbUtils.createDynamoDB(task);
-//
-//            Table table = dynamoDB.getTable(task.getTable());
-//            ItemCollection<ScanOutcome> items = table.scan();
-//
-//            while (items.iterator().hasNext()) {
-//                Map<String, Object> item = items.iterator().next().asMap();
-//                assertEquals(1, item.get("id"));
-//                assertEquals(32864, item.get("account"));
-//                assertEquals("2015-01-27T19:23:49", item.get("time"));
-//                assertEquals("2015-01-27T00:00:00", item.get("purchase"));
-//                assertEquals(true, item.get("flg"));
-//                assertEquals(123.45, item.get("score"));
-//                assertEquals("embulk", item.get("comment"));
-//            }
-//        }
-//        finally {
-//            if (dynamoDB != null) {
-//                dynamoDB.shutdown();
-//            }
-//        }
+        output.finish();
+        output.commit();
+
+        DynamodbUtils dynamoDbUtils = new DynamodbUtils();
+        DynamoDB dynamoDB = null;
+        try {
+            dynamoDB = dynamoDbUtils.createDynamoDB(task);
+
+            Table table = dynamoDB.getTable(task.getTable());
+            ItemCollection<ScanOutcome> items = table.scan();
+
+            for (Item item1 : items) {
+                assertEquals(1L, item1.getLong("id"));
+                assertEquals(32864L, item1.getLong("account"));
+                assertEquals("1970-01-01 00:00:01 UTC", item1.getString("time"));
+                assertEquals("1970-01-01 00:00:01 UTC", item1.getString("purchase"));
+                assertEquals(true, item1.getBoolean("flg"));
+                assertEquals(new BigDecimal("123.45"), item1.get("score"));
+                assertEquals("embulk", item1.getString("comment"));
+
+                List<Object> list = new ArrayList<>();
+                List<Object> inner = new ArrayList<>();
+                inner.add("inner");
+                list.add(new BigDecimal(1));
+                list.add(new BigDecimal(2));
+                list.add(new BigDecimal(3));
+                list.add(inner);
+                assertEquals(list, item1.getList("list"));
+
+                Map<String, Object> expectedMap = new LinkedHashMap<>();
+                expectedMap.put("foo", "bar");
+                expectedMap.put("key1", "val1");
+                assertEquals(expectedMap, item1.getMap("map"));
+            }
+        }
+        finally {
+            if (dynamoDB != null) {
+                dynamoDB.shutdown();
+            }
+        }
     }
 
     @Test
@@ -192,7 +228,7 @@ public class TestDynamodbOutputPlugin
                 .set("region", "us-west-1")
                 .set("table", "dummy")
                 .set("primary_key", "id")
-                .set("primary_key_type", "string")
+                .set("primary_key_type", "number")
                 .set("read_capacity_units", capacityUnitConfig())
                 .set("write_capacity_units", capacityUnitConfig())
                 .set("auth_method", "basic")
@@ -244,6 +280,8 @@ public class TestDynamodbOutputPlugin
         builder.add(ImmutableMap.of("name", "flg", "type", "boolean"));
         builder.add(ImmutableMap.of("name", "score", "type", "double"));
         builder.add(ImmutableMap.of("name", "comment", "type", "string"));
+        builder.add(ImmutableMap.of("name", "list", "type", "json"));
+        builder.add(ImmutableMap.of("name", "map", "type", "json"));
         return builder.build();
     }
 }

data/src/test/resources/sample_01.csv

@@ -1,5 +1,5 @@
-id,account,time,purchase,flg,score,comment
-1,32864,2015-01-27 19:23:49,20150127,1,123.45,embulk
-2,14824,2015-01-27 19:01:23,20150127,0,234,56,embulk
-3,27559,2015-01-28 02:20:02,20150128,1,678.90,embulk
-4,11270,2015-01-29 11:54:36,20150129,0,100.00,embulk
+id,account,time,purchase,flg,score,comment,list
+1,32864,2015-01-27 19:23:49,20150127,1,123.45,embulk,[1,2,3],{"foo": "bar", "key1": "val1"}
+2,14824,2015-01-27 19:01:23,20150127,0,234,56,embulk,[1,2,3],{"foo": "bar", "key1": "val1"}
+3,27559,2015-01-28 02:20:02,20150128,1,678.90,embulk,[1,2,3],{"foo": "bar", "key1": "val1"}
+4,11270,2015-01-29 11:54:36,20150129,0,100.00,embulk,[1,2,3],{"foo": "bar", "key1": "val1"}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-dynamodb
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Satoshi Akama
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-05-24 00:00:00.000000000 Z
+date: 2016-07-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -46,6 +46,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .travis.yml
 - CHANGELOG.md
 - README.md
 - build.gradle
@@ -70,7 +71,7 @@ files:
 - classpath/aws-java-sdk-s3-1.10.50.jar
 - classpath/commons-codec-1.6.jar
 - classpath/commons-logging-1.1.3.jar
-- classpath/embulk-output-dynamodb-0.1.2.jar
+- classpath/embulk-output-dynamodb-0.1.3.jar
 - classpath/httpclient-4.3.6.jar
 - classpath/httpcore-4.3.3.jar
 homepage: https://github.com/sakama/embulk-output-dynamodb