embulk-input-dynamodb 0.2.0 → 0.3.0

data/src/test/resources/yaml/authMethodProfile.yml

@@ -1,20 +0,0 @@
-in:
-  type: dynamodb
-  region: ENV_VAR
-  operation: scan
-  table: ENV_VAR
-  auth_method: profile
-  profile_name: ENV_VAR
-  columns:
-    - {name: pri-key,  type: string}
-    - {name: sort-key, type: long}
-    - {name: value, type: string}
-
-out:
-  type: file
-  path_prefix: result
-  file_ext: tsv
-  formatter:
-    type: csv
-    delimiter: "\t"
-    header_line: false

data/src/test/resources/yaml/dynamodb-local-query.yml

@@ -1,25 +0,0 @@
-in:
-  type: dynamodb
-  end_point: http://localhost:8000/
-  table: ENV_VAR
-  auth_method: basic
-  access_key: dummy
-  secret_key: dummy
-  filters:
-    - {name: pri-key, type: string, condition: EQ, value: key-1}
-  columns:
-    - {name: pri-key,     type: string}
-    - {name: sort-key,    type: long}
-    - {name: doubleValue, type: double}
-    - {name: boolValue,   type: boolean}
-    - {name: listValue,   type: json}
-    - {name: mapValue,    type: json}
-
-out:
-  type: file
-  path_prefix: dynamodb-local-result
-  file_ext: tsv
-  formatter:
-    type: csv
-    delimiter: "\t"
-    header_line: false

data/src/test/resources/yaml/dynamodb-local-scan.yml

@@ -1,23 +0,0 @@
-in:
-  type: dynamodb
-  end_point: http://localhost:8000/
-  table: ENV_VAR
-  auth_method: basic
-  access_key: dummy
-  secret_key: dummy
-  columns:
-    - {name: pri-key,     type: string}
-    - {name: sort-key,    type: long}
-    - {name: doubleValue, type: double}
-    - {name: boolValue,   type: boolean}
-    - {name: listValue,   type: json}
-    - {name: mapValue,    type: json}
-
-out:
-  type: file
-  path_prefix: dynamodb-local-result
-  file_ext: tsv
-  formatter:
-    type: csv
-    delimiter: "\t"
-    header_line: false

data/src/test/resources/yaml/notSetAuthMethod.yml

@@ -1,20 +0,0 @@
-in:
-  type: dynamodb
-  region: ENV_VAR
-  operation: scan
-  table: ENV_VAR
-  access_key: ENV_VAR
-  secret_key: ENV_VAR
-  columns:
-    - {name: pri-key,  type: string}
-    - {name: sort-key, type: long}
-    - {name: value, type: string}
-
-out:
-  type: file
-  path_prefix: result
-  file_ext: tsv
-  formatter:
-    type: csv
-    delimiter: "\t"
-    header_line: false

data/src/test/scala/org/embulk/input/dynamodb/ope/QueryOperationTest.scala

@@ -1,83 +0,0 @@
-package org.embulk.input.dynamodb.ope
-
-import java.io.File
-import java.nio.charset.Charset
-import java.nio.file.{FileSystems, Files}
-
-import com.fasterxml.jackson.databind.ObjectMapper
-import com.google.inject.{Binder, Module}
-import org.embulk.EmbulkEmbed
-import org.embulk.config.ConfigSource
-import org.embulk.input.dynamodb.DynamodbInputPlugin
-import org.embulk.plugin.InjectedPluginSource
-import org.embulk.spi.InputPlugin
-import org.hamcrest.CoreMatchers._
-import org.junit.Assert._
-import org.junit.{Before, Test}
-
-class QueryOperationTest {
-  private var embulk: EmbulkEmbed = null
-
-  private var EMBULK_DYNAMODB_TEST_TABLE: String = null
-  private var mapper: ObjectMapper = null
-
-  @Before
-  def createResources() {
-    // Get Environments
-    EMBULK_DYNAMODB_TEST_TABLE = System.getenv("EMBULK_DYNAMODB_TEST_TABLE")
-
-    val bootstrap = new EmbulkEmbed.Bootstrap()
-    bootstrap.addModules(new Module {
-      def configure(binder: Binder): Unit = {
-        InjectedPluginSource.registerPluginTo(binder,
-          classOf[InputPlugin],
-          "dynamodb",
-          classOf[DynamodbInputPlugin])
-      }
-    })
-
-    embulk = bootstrap.initializeCloseable()
-
-    mapper = new ObjectMapper()
-  }
-
-
-  def doTest(config: ConfigSource) {
-    embulk.run(config)
-
-    val fs = FileSystems.getDefault
-    val lines = Files.readAllLines(fs.getPath("dynamodb-local-result000.00.tsv"), Charset.forName("UTF-8"))
-    assertEquals(lines.size, 1)
-
-    val head = lines.get(0)
-    val values = head.split("\t")
-
-    assertThat(values(0), is("key-1"))
-    assertThat(values(1), is("0"))
-    assertThat(values(2), is("42.195"))
-    assertThat(values(3), is("true"))
-
-    val listValue = mapper.readValue(values(4).replaceAll("\"(?!\")", ""), classOf[java.util.List[Object]])
-    assertThat(listValue.size(), is(2))
-    assertThat(listValue.get(0).asInstanceOf[String], is("list-value"))
-    assertThat(listValue.get(1).asInstanceOf[Int], is(123))
-
-    val mapValue = mapper.readValue(values(5).replaceAll("\"(?!\")", ""), classOf[java.util.Map[String, Object]])
-    assert(mapValue.containsKey("map-key-1"))
-    assertThat(mapValue.get("map-key-1").asInstanceOf[String], is("map-value-1"))
-    assert(mapValue.containsKey("map-key-2"))
-    assertThat(mapValue.get("map-key-2").asInstanceOf[Int], is(456))
-  }
-
-  @Test
-  def queryTest() {
-    val config = embulk.newConfigLoader().fromYamlFile(
-      new File("src/test/resources/yaml/dynamodb-local-query.yml"))
-
-    config.getNested("in")
-      .set("operation", "query")
-      .set("table", EMBULK_DYNAMODB_TEST_TABLE)
-
-    doTest(config)
-  }
-}

data/src/test/scala/org/embulk/input/dynamodb/ope/ScanOperationTest.scala

@@ -1,83 +0,0 @@
-package org.embulk.input.dynamodb.ope
-
-import java.io.File
-import java.nio.charset.Charset
-import java.nio.file.{FileSystems, Files}
-
-import com.fasterxml.jackson.databind.ObjectMapper
-import com.google.inject.{Binder, Module}
-import org.embulk.EmbulkEmbed
-import org.embulk.config.ConfigSource
-import org.embulk.input.dynamodb.DynamodbInputPlugin
-import org.embulk.plugin.InjectedPluginSource
-import org.embulk.spi.InputPlugin
-import org.hamcrest.CoreMatchers._
-import org.junit.Assert._
-import org.junit.{Before, Test}
-
-class ScanOperationTest {
-  private var embulk: EmbulkEmbed = null
-
-  private var EMBULK_DYNAMODB_TEST_TABLE: String = null
-  private var mapper: ObjectMapper = null
-
-  @Before
-  def createResources() {
-    // Get Environments
-    EMBULK_DYNAMODB_TEST_TABLE = System.getenv("EMBULK_DYNAMODB_TEST_TABLE")
-
-    val bootstrap = new EmbulkEmbed.Bootstrap()
-    bootstrap.addModules(new Module {
-      def configure(binder: Binder): Unit = {
-        InjectedPluginSource.registerPluginTo(binder,
-          classOf[InputPlugin],
-          "dynamodb",
-          classOf[DynamodbInputPlugin])
-      }
-    })
-
-    embulk = bootstrap.initializeCloseable()
-
-    mapper = new ObjectMapper()
-  }
-
-
-  def doTest(config: ConfigSource) {
-    embulk.run(config)
-
-    val fs = FileSystems.getDefault
-    val lines = Files.readAllLines(fs.getPath("dynamodb-local-result000.00.tsv"), Charset.forName("UTF-8"))
-    assertEquals(lines.size, 1)
-
-    val head = lines.get(0)
-    val values = head.split("\t")
-
-    assertThat(values(0), is("key-1"))
-    assertThat(values(1), is("0"))
-    assertThat(values(2), is("42.195"))
-    assertThat(values(3), is("true"))
-
-    val listValue = mapper.readValue(values(4).replaceAll("\"(?!\")", ""), classOf[java.util.List[Object]])
-    assertThat(listValue.size(), is(2))
-    assertThat(listValue.get(0).asInstanceOf[String], is("list-value"))
-    assertThat(listValue.get(1).asInstanceOf[Int], is(123))
-
-    val mapValue = mapper.readValue(values(5).replaceAll("\"(?!\")", ""), classOf[java.util.Map[String, Object]])
-    assert(mapValue.containsKey("map-key-1"))
-    assertThat(mapValue.get("map-key-1").asInstanceOf[String], is("map-value-1"))
-    assert(mapValue.containsKey("map-key-2"))
-    assertThat(mapValue.get("map-key-2").asInstanceOf[Int], is(456))
-  }
-
-  @Test
-  def scanTest() {
-    val config = embulk.newConfigLoader().fromYamlFile(
-      new File("src/test/resources/yaml/dynamodb-local-scan.yml"))
-
-    config.getNested("in")
-      .set("operation", "scan")
-      .set("table", EMBULK_DYNAMODB_TEST_TABLE)
-
-    doTest(config)
-  }
-}

data/test/create_table.sh

@@ -1,16 +0,0 @@
-#!/bin/sh
-
-aws dynamodb create-table \
-	--table-name='EMBULK_DYNAMODB_TEST_TABLE' \
-	--attribute-definitions='[
-		{"AttributeName":"pri-key","AttributeType":"S"},
-		{"AttributeName":"sort-key","AttributeType":"N"}
-	]' \
-	--key-schema='[
-		{"AttributeName":"pri-key","KeyType":"HASH"},
-		{"AttributeName":"sort-key","KeyType":"RANGE"}
-	]' \
-	--provisioned-throughput='{"ReadCapacityUnits":5, "WriteCapacityUnits":5}' \
-	--endpoint-url http://localhost:8000 \
-	--region us-east-1
-

data/test/put_items.sh

@@ -1,25 +0,0 @@
-#!/bin/sh
-
-aws dynamodb put-item \
-	--table-name='EMBULK_DYNAMODB_TEST_TABLE' \
-	--item='{
-		"pri-key" : { "S" : "key-1" },
-		"sort-key" : { "N" : "0" },
-		"doubleValue" : { "N" : "42.195" },
-		"boolValue" : { "BOOL" : true },
-		"listValue" : { "L":
-			[
-				{ "S" : "list-value"},
-				{ "N" : "123"}
-			]
-		},
-		"mapValue" : { "M":
-			{
-				"map-key-1" : { "S" : "map-value-1" },
-				"map-key-2" : { "N" : "456" }
-			}
-		}
-	}' \
-	--endpoint-url http://localhost:8000 \
-	--region us-east-1
-