embulk-output-cassandra 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e128e8bd86c8b5573a718700e449160a47e9fa8d
-  data.tar.gz: 5bf5acfb7cc3d84360a5f7889f60f5575608000f
+  metadata.gz: ab4a3fa7b0b477c2c6ebed457fa48f248ce7a6ae
+  data.tar.gz: a7355ea7423d856a3533e2b3d4dbbb50d05c60be
 SHA512:
-  metadata.gz: 623b2b2b23945c8ebd2519b41e580636581b4ac4fcd305c1f60d84ccb73f7bad89ae0abfbfa20d35cb5598b507b161ead4ae0b8c9235650de538611a8a763d4c
-  data.tar.gz: 7c9a6cfd7e008a4621b866b516057bb0d6277a3c875a6a8f95eeb735b977ae7b21e8325cf40d1f5e53c7da76004e45d54ec942e47b8958d24b7441a47e54aa53
+  metadata.gz: 0a8b7043a53d758515a06274ffd00955013d8533d447e512a01228f17e63e378e9d70a5f149f54e9451b52a74718951261e3e6a6493889f8d6213b06db480ea3
+  data.tar.gz: f51f4f392c31bb8928e97b14ef4059a523578883517ea390e3beac76ffd84dc1f30b464cac999d05e478fc6ae4ad102739a26590b219dd5437901c00debc1181

data/README.md

@@ -11,10 +11,53 @@ Apache Cassandra output plugin for Embulk.
 
 ## Caution
 In current, version of netty components conflicts to one that is used by embulk-core.
+
 This probrem is very severe.
+
 I tested this plugin on embulk-0.9.7.
 But future embulk version may break this plugin.
 
+## Support Data types
+
+| CQL Type                    | Embulk Type                                    | Descritpion                                                             |
+| --------                    | -----------                                    | --------------                                                          |
+| ascii                       | string, boolean, long, double, timestamp, json | use `toString` or `toJson`                                              |
+| bigint                      | string, boolean(as 0 or 1), long, double       |                                                                         |
+| blob                        | unsupported                                    |                                                                         |
+| boolean                     | boolean, long, double                          | 0 == false, 1 == true                                                   |
+| counter                     | unsupported                                    |                                                                         |
+| date                        | string, timestamp                              | timestamp use `toEpochMilli`                                            |
+| decimal                     | string, boolean(as 0 or 1), long, double       |                                                                         |
+| double                      | string, boolean(as 0 or 1), long, double       |                                                                         |
+| float                       | string, boolean(as 0 or 1), long, double       |                                                                         |
+| inet                        | string                                         |                                                                         |
+| int                         | string, boolean(as 0 or 1), long, double       | overflowed value is reset to 0                                          |
+| list                        | json                                           |                                                                         |
+| map (support only text key) | json                                           |                                                                         |
+| set                         | json                                           |                                                                         |
+| smallint                    | string, boolean(as 0 or 1), long, double       | overflowed value is reset to 0                                          |
+| text                        | string, boolean, long, double, timestamp, json | use `toString` or `toJson`                                              |
+| time                        | string, long, double, timestamp                | long and double as nano seconds of day,<br>timestamp use `toEpochMilli` |
+| timestamp                   | long, double, timestamp                        | long and double as epoch second                                         |
+| timeuuid                    | null                                           |
+| uuid                        | null                                           |
+| varchar                     | string, boolean, long, double, timestamp, json | use `toString` or `toJson`                                              |
+| varint                      | string, boolean(as 0 or 1), long, double       |                                                                         |
+| UDT                         | unsupported                                    |                                                                         |
+
+## Insert Behavior
+If embulk record does not have a column, it is treated as `unset`.
+If same key record already exists, the column is not touched.
+
+### Counter table
+This plugin supports counter table.
+
+But counter table supports only increment/decrement update.
+
+Because of it, This plugin uses input value as increment value;
+
+For example, If input data = {id: 1, count: 5}, Executed Statement is `UPDATE tablename SET count = count + 5 WHERE id = 1`
+
 ## Configuration
 
 - **hosts**: list of seed hosts (list<string>, required)

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.1.1"
+version = "0.2.0"
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8

data/docker-compose.yml

@@ -0,0 +1,6 @@
+version: "3"
+services:
+  cassandra:
+    image: cassandra:latest
+    ports:
+      - '9042:9042'

data/embulk-output-cassandra.iml

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<module external.linked.project.id="embulk-output-cassandra" external.linked.project.path="$MODULE_DIR$" external.root.project.path="$MODULE_DIR$" external.system.id="GRADLE" external.system.module.group="" external.system.module.version="0.1.0" type="JAVA_MODULE" version="4">
+<module external.linked.project.id="embulk-output-cassandra" external.linked.project.path="$MODULE_DIR$" external.root.project.path="$MODULE_DIR$" external.system.id="GRADLE" external.system.module.group="" external.system.module.version="0.1.1" type="JAVA_MODULE" version="4">
   <component name="NewModuleRootManager" inherit-compiler-output="true">
     <exclude-output />
     <content url="file://$MODULE_DIR$">

data/example.yml

@@ -0,0 +1,23 @@
+in:
+  type: file
+  path_prefix: ./src/test/resources/org/embulk/output/cassandra/test1.csv
+  parser:
+    type: csv
+    skip_header_lines: 1
+    columns:
+      - {name: id, type: string}
+      - {name: int_item, type: long}
+      - {name: int32_item, type: long}
+      - {name: smallint_item, type: long}
+      - {name: tinyint_item, type: long}
+      - {name: boolean_item, type: boolean}
+      - {name: varchar_item, type: string}
+      - {name: timestamp_item, type: timestamp}
+
+out:
+  type: cassandra
+  hosts:
+    - 127.0.0.1
+  keyspace: embulk_test
+  table: test_basic
+  idempotent: true

data/src/main/java/org/embulk/output/cassandra/CassandraOutputPlugin.java

@@ -3,13 +3,16 @@ package org.embulk.output.cassandra;
 import com.datastax.driver.core.BoundStatement;
 import com.datastax.driver.core.Cluster;
 import com.datastax.driver.core.ColumnMetadata;
+import com.datastax.driver.core.DataType;
 import com.datastax.driver.core.KeyspaceMetadata;
 import com.datastax.driver.core.PreparedStatement;
 import com.datastax.driver.core.Session;
 import com.datastax.driver.core.SocketOptions;
 import com.datastax.driver.core.TableMetadata;
+import com.datastax.driver.core.querybuilder.BuiltStatement;
 import com.datastax.driver.core.querybuilder.Insert;
 import com.datastax.driver.core.querybuilder.QueryBuilder;
+import com.datastax.driver.core.querybuilder.Update;
 import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
@@ -86,6 +89,10 @@ public class CassandraOutputPlugin
         @Config("request_timeout")
         @ConfigDefault("12000")
         public int getRequestTimeout();
+
+        @Config("counter_columnName")
+        @ConfigDefault("null")
+        public Optional<String> getCounterColumnName();
     }
 
     private final Logger logger = Exec.getLogger(CassandraOutputPlugin.class);
@@ -135,18 +142,42 @@ public class CassandraOutputPlugin
             throw new RuntimeException("table `" + task.getTable() + "` is not found");
         }
 
-        Insert insert = QueryBuilder.insertInto(task.getKeyspace(), task.getTable());
-        if (task.getIfNotExists()) {
-            insert.ifNotExists();
+        List<ColumnMetadata> columns = tableMetadata.getColumns();
+        boolean isCounterTable = columns.stream().anyMatch(col -> col.getType().getName() == DataType.Name.COUNTER);
+
+        BuiltStatement query;
+        if (isCounterTable) {
+            Update update = QueryBuilder.update(task.getKeyspace(), task.getTable());
+            if (task.getTtl().isPresent()) {
+                update.using(QueryBuilder.ttl(task.getTtl().get()));
+            }
+            for (ColumnMetadata column : tableMetadata.getColumns()) {
+                if (column.getType().getName() == DataType.Name.COUNTER) {
+                    update.with(QueryBuilder.incr(column.getName(), QueryBuilder.bindMarker(column.getName())));
+                }
+                else{
+                    update.where(QueryBuilder.eq(column.getName(), QueryBuilder.bindMarker(column.getName())));
+                }
+            }
+            query = update;
         }
-        if (task.getTtl().isPresent()) {
-            insert.using(QueryBuilder.ttl(task.getTtl().get()));
+        else {
+            Insert insert = QueryBuilder.insertInto(task.getKeyspace(), task.getTable());
+            if (task.getIfNotExists()) {
+                insert.ifNotExists();
+            }
+            if (task.getTtl().isPresent()) {
+                insert.using(QueryBuilder.ttl(task.getTtl().get()));
+            }
+            for (ColumnMetadata column : tableMetadata.getColumns()) {
+                insert.value(column.getName(), QueryBuilder.bindMarker(column.getName()));
+            }
+            query = insert;
         }
 
         ImmutableMap.Builder<String, CassandraColumnSetter> columnSettersBuilder = ImmutableMap.builder();
         ImmutableList.Builder<String> uuidColumnsBuilder = ImmutableList.builder();
         for (ColumnMetadata column : tableMetadata.getColumns()) {
-            insert.value(column.getName(), QueryBuilder.bindMarker(column.getName()));
             columnSettersBuilder.put(column.getName(), CassandraColumnSetterFactory.createColumnSetter(column, cluster));
             switch (column.getType().getName()) {
                 case UUID:
@@ -159,9 +190,9 @@ public class CassandraOutputPlugin
         List<ColumnSetterVisitor> columnVisitors = Lists.transform(schema.getColumns(), (column) ->
                 new ColumnSetterVisitor(pageReader, columnSetters.get(column.getName())));
 
-        logger.info("Insert Query: {}", insert.getQueryString());
+        logger.info("Insert Query: {}", query.getQueryString());
 
-        PreparedStatement prepared = session.prepare(insert);
+        PreparedStatement prepared = session.prepare(query);
         if (task.getIdempotent()) {
             prepared.setIdempotent(task.getIdempotent());
         }

data/src/main/java/org/embulk/output/cassandra/setter/CassandraColumnSetterFactory.java

@@ -19,6 +19,7 @@ public class CassandraColumnSetterFactory
             case BOOLEAN:
                 return new BooleanColumnSetter(cassandraColumn);
             case BIGINT:
+            case COUNTER:
                 return new BigintColumnSetter(cassandraColumn);
             case INT:
                 return new IntColumnSetter(cassandraColumn);

data/src/test/java/org/embulk/output/cassandra/TestCassandraOutputPlugin.java

@@ -3,6 +3,7 @@ package org.embulk.output.cassandra;
 import com.datastax.driver.core.Cluster;
 import com.datastax.driver.core.DataType;
 import com.datastax.driver.core.LocalDate;
+import com.datastax.driver.core.ResultSet;
 import com.datastax.driver.core.Row;
 import com.datastax.driver.core.Session;
 import com.datastax.driver.core.TupleType;
@@ -26,6 +27,7 @@ import java.nio.file.Path;
 import java.time.ZoneId;
 import java.time.ZonedDateTime;
 import java.util.Arrays;
+import java.util.Comparator;
 import java.util.Date;
 import java.util.List;
 import java.util.Map;
@@ -83,13 +85,16 @@ public class TestCassandraOutputPlugin
         String createTableBasic = EmbulkTests.readResource(RESOURCE_PATH + "create_table_test_basic.cql");
         String createTableUuid = EmbulkTests.readResource(RESOURCE_PATH + "create_table_test_uuid.cql");
         String createTableComplex = EmbulkTests.readResource(RESOURCE_PATH + "create_table_test_complex.cql");
+        String createTableCounter = EmbulkTests.readResource(RESOURCE_PATH + "create_table_test_counter.cql");
         session.execute(createKeyspace);
         session.execute(createTableBasic);
         session.execute(createTableUuid);
         session.execute(createTableComplex);
+        session.execute(createTableCounter);
         session.execute("TRUNCATE embulk_test.test_basic");
         session.execute("TRUNCATE embulk_test.test_uuid");
         session.execute("TRUNCATE embulk_test.test_complex");
+        session.execute("TRUNCATE embulk_test.test_counter");
     }
 
     @After
@@ -139,6 +144,43 @@ public class TestCassandraOutputPlugin
         assertEquals(createDate(2018, 7, 1, 10, 0, 2, 0), row3.getTimestamp("timestamp_item"));
     }
 
+    @Test
+    public void testCounter() throws IOException
+    {
+        Path input = getInputPath("test1.csv");
+        ConfigSource config = loadYamlResource("test_counter.yaml");
+        config.set("hosts", getCassandraHostAsList());
+
+        assertEquals(0, session.execute("SELECT * FROM embulk_test.test_counter").all().size());
+
+        TestingEmbulk.RunResult result = embulk.runOutput(config, input);
+        assertEquals(3, result.getOutputTaskReports().get(0).get(Long.class, "inserted_record_count").longValue());
+
+        List<Row> rows = session.execute("SELECT * FROM embulk_test.test_counter").all();
+        rows.sort(Comparator.comparing(row -> row.getString("id")));
+        Row row1 = rows.get(0);
+        Row row2 = rows.get(1);
+        Row row3 = rows.get(2);
+        assertEquals("A001", row1.getString("id"));
+        assertEquals(9, row1.getLong("int_item"));
+        assertEquals(1, row1.getInt("int32_item"));
+        assertEquals(2, row1.getShort("smallint_item"));
+        assertTrue(row1.getBool("boolean_item"));
+        assertEquals(createDate(2018, 7, 1, 10, 0, 0, 0), row1.getTimestamp("timestamp_item"));
+        assertEquals("A002", row2.getString("id"));
+        assertEquals(0, row2.getLong("int_item"));
+        assertEquals(0, row2.getInt("int32_item"));
+        assertEquals(4, row2.getShort("smallint_item"));
+        assertTrue(row2.getBool("boolean_item"));
+        assertEquals(createDate(2018, 7, 1, 10, 0, 1, 0), row2.getTimestamp("timestamp_item"));
+        assertEquals("A003", row3.getString("id"));
+        assertEquals(9, row3.getLong("int_item"));
+        assertEquals(0, row3.getInt("int32_item"));
+        assertEquals(8, row3.getShort("smallint_item"));
+        assertFalse(row3.getBool("boolean_item"));
+        assertEquals(createDate(2018, 7, 1, 10, 0, 2, 0), row3.getTimestamp("timestamp_item"));
+    }
+
     @Test
     public void testBasicWithTtl() throws IOException
     {

data/src/test/resources/org/embulk/output/cassandra/create_table_test_counter.cql

@@ -0,0 +1,11 @@
+CREATE TABLE IF NOT EXISTS embulk_test.test_counter (
+  id text,
+  int_item counter,
+  int32_item int,
+  smallint_item smallint,
+  tinyint_item tinyint,
+  boolean_item boolean,
+  double_item double,
+  timestamp_item timestamp,
+  primary key (id, int32_item, smallint_item, tinyint_item, boolean_item, double_item, timestamp_item)
+) WITH compaction = { 'class' : 'SizeTieredCompactionStrategy' };

data/src/test/resources/org/embulk/output/cassandra/test1.csv

@@ -1,4 +1,4 @@
-id:string,int_item:long,int32_item:long,smallint_item:long,tinyint_item:long,boolean_item:boolean,varchar_item:string,timestamp_item:timestamp
-A001,9,4294967297,2,0,true,a,2018-07-01 10:00:00.000 UTC
-A002,0,0,4,0,true,b,2018-07-01 10:00:01.000 UTC
-A003,9,0,8,1,false,c,2018-07-01 10:00:02.000 UTC
+id:string,int_item:long,int32_item:long,smallint_item:long,tinyint_item:long,boolean_item:boolean,varchar_item:string,timestamp_item:timestamp,double_item:double
+A001,9,4294967297,2,0,true,a,2018-07-01 10:00:00.000 UTC,1.1
+A002,0,0,4,0,true,b,2018-07-01 10:00:01.000 UTC,1.2
+A003,9,0,8,1,false,c,2018-07-01 10:00:02.000 UTC,1.3

data/src/test/resources/org/embulk/output/cassandra/test_counter.yaml

@@ -0,0 +1,3 @@
+type: cassandra
+keyspace: embulk_test
+table: "test_counter"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-cassandra
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - joker1007
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-07-02 00:00:00.000000000 Z
+date: 2018-08-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -55,7 +55,7 @@ files:
 - classpath/asm-tree-5.0.3.jar
 - classpath/asm-util-5.0.3.jar
 - classpath/cassandra-driver-core-3.5.0.jar
-- classpath/embulk-output-cassandra-0.1.1.jar
+- classpath/embulk-output-cassandra-0.2.0.jar
 - classpath/guava-19.0.jar
 - classpath/jffi-1.2.16-native.jar
 - classpath/jffi-1.2.16.jar
@@ -70,7 +70,9 @@ files:
 - classpath/slf4j-api-1.7.25.jar
 - config/checkstyle/checkstyle.xml
 - config/checkstyle/default.xml
+- docker-compose.yml
 - embulk-output-cassandra.iml
+- example.yml
 - gradle/wrapper/gradle-wrapper.jar
 - gradle/wrapper/gradle-wrapper.properties
 - gradlew
@@ -105,12 +107,14 @@ files:
 - src/test/resources/org/embulk/output/cassandra/create_keyspace.cql
 - src/test/resources/org/embulk/output/cassandra/create_table_test_basic.cql
 - src/test/resources/org/embulk/output/cassandra/create_table_test_complex.cql
+- src/test/resources/org/embulk/output/cassandra/create_table_test_counter.cql
 - src/test/resources/org/embulk/output/cassandra/create_table_test_uuid.cql
 - src/test/resources/org/embulk/output/cassandra/test1.csv
 - src/test/resources/org/embulk/output/cassandra/test2.csv
 - src/test/resources/org/embulk/output/cassandra/test3.csv
 - src/test/resources/org/embulk/output/cassandra/test_basic.yaml
 - src/test/resources/org/embulk/output/cassandra/test_complex.yaml
+- src/test/resources/org/embulk/output/cassandra/test_counter.yaml
 - src/test/resources/org/embulk/output/cassandra/test_uuid.yaml
 homepage: https://github.com/joker1007/embulk-output-cassandra
 licenses: