embulk-output-cassandra 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ab4a3fa7b0b477c2c6ebed457fa48f248ce7a6ae
-  data.tar.gz: a7355ea7423d856a3533e2b3d4dbbb50d05c60be
+  metadata.gz: f3a5a33db3984bac71c043a99cf1306c12aed2fa
+  data.tar.gz: 6351267b4e17e6d75617fa4ddda29f4c5b2551e5
 SHA512:
-  metadata.gz: 0a8b7043a53d758515a06274ffd00955013d8533d447e512a01228f17e63e378e9d70a5f149f54e9451b52a74718951261e3e6a6493889f8d6213b06db480ea3
-  data.tar.gz: f51f4f392c31bb8928e97b14ef4059a523578883517ea390e3beac76ffd84dc1f30b464cac999d05e478fc6ae4ad102739a26590b219dd5437901c00debc1181
+  metadata.gz: 45f9a322c118f1b4786d730f1db68eefbdccfb1f63fcc9c14967319c9a441bb6b003f0f06a075228c042fdd217f82064f215fb807812cd9e6152c2e76a557a02
+  data.tar.gz: 6a29cfbd03d3ecb2f070563af73265b711c093c0b88ffd68b57d27f98b541964f8002f9f9d59aa3dcba8c33afb8ea680fab6ce77560c44653bd390a87b48cef6

data/README.md

@@ -67,7 +67,9 @@ For example, If input data = {id: 1, count: 5}, Executed Statement is `UPDATE ta
 - **cluster_name**: cluster name (string, default: `null`)
 - **keyspace**: target keyspace name (string, required)
 - **table**: target table name (string, required)
+- **mode**: insert or update (string, default: `"insert"`)
 - **if_not_exists**: Add "IF NOT EXISTS" to INSERT query (boolean, default: `false`)
+- **if_exists**: Add "IF EXISTS" to UPDATE query (boolean, default: `false`)
 - **ttl**: Add "TTL" to INSERT query (integer, default: `null`)
 - **idempotent**: Treat INSERT query as idempotent (boolean, default: `false`)
 - **connect_timeout**: Set connect timeout millisecond (integer, default: `5000`)

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.2.0"
+version = "0.3.0"
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8

data/embulk-output-cassandra.iml

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<module external.linked.project.id="embulk-output-cassandra" external.linked.project.path="$MODULE_DIR$" external.root.project.path="$MODULE_DIR$" external.system.id="GRADLE" external.system.module.group="" external.system.module.version="0.1.1" type="JAVA_MODULE" version="4">
+<module external.linked.project.id="embulk-output-cassandra" external.linked.project.path="$MODULE_DIR$" external.root.project.path="$MODULE_DIR$" external.system.id="GRADLE" external.system.module.group="" external.system.module.version="0.2.0" type="JAVA_MODULE" version="4">
   <component name="NewModuleRootManager" inherit-compiler-output="true">
     <exclude-output />
     <content url="file://$MODULE_DIR$">

data/src/main/java/org/embulk/output/cassandra/CassandraOutputPlugin.java

@@ -13,6 +13,8 @@ import com.datastax.driver.core.querybuilder.BuiltStatement;
 import com.datastax.driver.core.querybuilder.Insert;
 import com.datastax.driver.core.querybuilder.QueryBuilder;
 import com.datastax.driver.core.querybuilder.Update;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonValue;
 import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
@@ -20,6 +22,7 @@ import com.google.common.collect.Lists;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigException;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.Task;
 import org.embulk.config.TaskReport;
@@ -27,6 +30,7 @@ import org.embulk.config.TaskSource;
 import org.embulk.output.cassandra.setter.CassandraColumnSetter;
 import org.embulk.output.cassandra.setter.CassandraColumnSetterFactory;
 import org.embulk.output.cassandra.setter.ColumnSetterVisitor;
+import org.embulk.spi.Column;
 import org.embulk.spi.Exec;
 import org.embulk.spi.OutputPlugin;
 import org.embulk.spi.Page;
@@ -37,7 +41,9 @@ import org.slf4j.Logger;
 
 import java.net.InetSocketAddress;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 public class CassandraOutputPlugin
         implements OutputPlugin
@@ -46,7 +52,7 @@ public class CassandraOutputPlugin
             extends Task
     {
         @Config("hosts")
-        public List<String> gethosts();
+        public List<String> getHosts();
 
         @Config("port")
         @ConfigDefault("9042")
@@ -70,10 +76,18 @@ public class CassandraOutputPlugin
         @Config("table")
         public String getTable();
 
+        @Config("mode")
+        @ConfigDefault("\"insert\"")
+        public Mode getMode();
+
         @Config("if_not_exists")
         @ConfigDefault("false")
         public Boolean getIfNotExists();
 
+        @Config("if_exists")
+        @ConfigDefault("false")
+        public Boolean getIfExists();
+
         @Config("ttl")
         @ConfigDefault("null")
         public Optional<Integer> getTtl();
@@ -89,10 +103,6 @@ public class CassandraOutputPlugin
         @Config("request_timeout")
         @ConfigDefault("12000")
         public int getRequestTimeout();
-
-        @Config("counter_columnName")
-        @ConfigDefault("null")
-        public Optional<String> getCounterColumnName();
     }
 
     private final Logger logger = Exec.getLogger(CassandraOutputPlugin.class);
@@ -155,12 +165,36 @@ public class CassandraOutputPlugin
                 if (column.getType().getName() == DataType.Name.COUNTER) {
                     update.with(QueryBuilder.incr(column.getName(), QueryBuilder.bindMarker(column.getName())));
                 }
-                else{
+                else {
                     update.where(QueryBuilder.eq(column.getName(), QueryBuilder.bindMarker(column.getName())));
                 }
             }
             query = update;
         }
+        else if (task.getMode() == Mode.UPDATE) {
+            Update update = QueryBuilder.update(task.getKeyspace(), task.getTable());
+            List<String> primaryKeys = tableMetadata.getPrimaryKey().stream().map(ColumnMetadata::getName).collect(Collectors.toList());
+            List<String> columnNames = tableMetadata.getColumns().stream().map(ColumnMetadata::getName).collect(Collectors.toList());
+
+            if (task.getIfExists()) {
+                update.where().ifExists();
+            }
+            if (task.getTtl().isPresent()) {
+                update.using(QueryBuilder.ttl(task.getTtl().get()));
+            }
+            for (String pkey : primaryKeys) {
+                update.where(QueryBuilder.eq(pkey, QueryBuilder.bindMarker(pkey)));
+            }
+            for (Column col : schema.getColumns()) {
+                if (primaryKeys.contains(col.getName())) {
+                    continue;
+                }
+                if (columnNames.contains(col.getName())) {
+                    update.with(QueryBuilder.set(col.getName(), QueryBuilder.bindMarker(col.getName())));
+                }
+            }
+            query = update;
+        }
         else {
             Insert insert = QueryBuilder.insertInto(task.getKeyspace(), task.getTable());
             if (task.getIfNotExists()) {
@@ -190,7 +224,7 @@ public class CassandraOutputPlugin
         List<ColumnSetterVisitor> columnVisitors = Lists.transform(schema.getColumns(), (column) ->
                 new ColumnSetterVisitor(pageReader, columnSetters.get(column.getName())));
 
-        logger.info("Insert Query: {}", query.getQueryString());
+        logger.info("Query: {}", query.getQueryString());
 
         PreparedStatement prepared = session.prepare(query);
         if (task.getIdempotent()) {
@@ -203,7 +237,7 @@ public class CassandraOutputPlugin
     private Cluster getCluster(PluginTask task)
     {
         Cluster.Builder builder = Cluster.builder();
-        for (String host : task.gethosts()) {
+        for (String host : task.getHosts()) {
             builder.addContactPointsWithPorts(new InetSocketAddress(host, task.getPort()));
         }
 
@@ -311,4 +345,29 @@ public class CassandraOutputPlugin
             return report;
         }
     }
+
+    public enum Mode {
+        INSERT,
+        UPDATE;
+
+        @JsonValue
+        @Override
+        public String toString()
+        {
+            return name().toLowerCase(Locale.ENGLISH);
+        }
+
+        @JsonCreator
+        public static Mode fromString(String value)
+        {
+            switch(value) {
+                case "insert":
+                    return INSERT;
+                case "update":
+                    return UPDATE;
+                default:
+                    throw new ConfigException(String.format("Unknown mode '%s'", value));
+            }
+        }
+    }
 }

data/src/test/java/org/embulk/output/cassandra/TestCassandraOutputPlugin.java

@@ -3,7 +3,6 @@ package org.embulk.output.cassandra;
 import com.datastax.driver.core.Cluster;
 import com.datastax.driver.core.DataType;
 import com.datastax.driver.core.LocalDate;
-import com.datastax.driver.core.ResultSet;
 import com.datastax.driver.core.Row;
 import com.datastax.driver.core.Session;
 import com.datastax.driver.core.TupleType;
@@ -33,7 +32,12 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
 
 public class TestCassandraOutputPlugin
 {
@@ -249,6 +253,56 @@ public class TestCassandraOutputPlugin
         assertTrue(UUIDs.unixTimestamp(row1.getUUID("timeuuid_item")) < ZonedDateTime.now().toInstant().toEpochMilli());
     }
 
+    @Test
+    public void testBasicWithUpdateMode() throws IOException
+    {
+        Path input = getInputPath("test1.csv");
+        ConfigSource config = loadYamlResource("test_basic.yaml");
+        config.set("hosts", getCassandraHostAsList());
+        config.set("mode", "update");
+
+        assertEquals(0, session.execute("SELECT * FROM embulk_test.test_basic").all().size());
+
+        embulk.runOutput(config, input);
+
+        Row row1 = session.execute("SELECT * FROM embulk_test.test_basic WHERE id = 'A001'").one();
+        Row row2 = session.execute("SELECT * FROM embulk_test.test_basic WHERE id = 'A002'").one();
+        Row row3 = session.execute("SELECT * FROM embulk_test.test_basic WHERE id = 'A003'").one();
+        assertEquals("A001", row1.getString("id"));
+        assertEquals(9, row1.getLong("int_item"));
+        assertEquals(createDate(2018, 7, 1, 10, 0, 0, 0), row1.getTimestamp("timestamp_item"));
+        assertEquals("A002", row2.getString("id"));
+        assertEquals(0, row2.getLong("int_item"));
+        assertEquals(createDate(2018, 7, 1, 10, 0, 1, 0), row2.getTimestamp("timestamp_item"));
+        assertEquals("A003", row3.getString("id"));
+        assertEquals(9, row3.getLong("int_item"));
+        assertEquals(createDate(2018, 7, 1, 10, 0, 2, 0), row3.getTimestamp("timestamp_item"));
+    }
+
+    @Test
+    public void testBasicWithIfExists() throws IOException
+    {
+        Path input = getInputPath("test1.csv");
+        ConfigSource config = loadYamlResource("test_basic.yaml");
+        config.set("hosts", getCassandraHostAsList());
+        config.set("mode", "update");
+        config.set("if_exists", true);
+
+        session.execute("INSERT INTO embulk_test.test_basic (id) VALUES ('A001')");
+        assertEquals(1, session.execute("SELECT * FROM embulk_test.test_basic").all().size());
+
+        embulk.runOutput(config, input);
+
+        Row row1 = session.execute("SELECT * FROM embulk_test.test_basic WHERE id = 'A001'").one();
+        Row row2 = session.execute("SELECT * FROM embulk_test.test_basic WHERE id = 'A002'").one();
+        Row row3 = session.execute("SELECT * FROM embulk_test.test_basic WHERE id = 'A003'").one();
+        assertEquals("A001", row1.getString("id"));
+        assertEquals(9, row1.getLong("int_item"));
+        assertEquals(createDate(2018, 7, 1, 10, 0, 0, 0), row1.getTimestamp("timestamp_item"));
+        assertNull(row2);
+        assertNull(row3);
+    }
+
     @Test
     public void testComplex() throws IOException
     {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-cassandra
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - joker1007
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-08-12 00:00:00.000000000 Z
+date: 2018-10-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -55,7 +55,7 @@ files:
 - classpath/asm-tree-5.0.3.jar
 - classpath/asm-util-5.0.3.jar
 - classpath/cassandra-driver-core-3.5.0.jar
-- classpath/embulk-output-cassandra-0.2.0.jar
+- classpath/embulk-output-cassandra-0.3.0.jar
 - classpath/guava-19.0.jar
 - classpath/jffi-1.2.16-native.jar
 - classpath/jffi-1.2.16.jar