embulk-output-cassandra 0.1.0

data/src/main/java/org/embulk/output/cassandra/setter/VarintColumnSetter.java

@@ -0,0 +1,43 @@
+package org.embulk.output.cassandra.setter;
+
+import com.datastax.driver.core.BoundStatement;
+import com.datastax.driver.core.ColumnMetadata;
+
+import java.math.BigInteger;
+
+public class VarintColumnSetter extends CassandraColumnSetter
+{
+    public VarintColumnSetter(ColumnMetadata cassandraColumn)
+    {
+        super(cassandraColumn);
+    }
+
+    @Override
+    public void setBooleanValue(Boolean value, BoundStatement statement)
+    {
+        if (value) {
+            statement.setVarint(cassandraColumn.getName(), BigInteger.ONE);
+        }
+        else {
+            statement.setVarint(cassandraColumn.getName(), BigInteger.ZERO);
+        }
+    }
+
+    @Override
+    public void setLongValue(Long value, BoundStatement statement)
+    {
+        statement.setVarint(cassandraColumn.getName(), BigInteger.valueOf(value));
+    }
+
+    @Override
+    public void setDoubleValue(Double value, BoundStatement statement)
+    {
+        statement.setVarint(cassandraColumn.getName(), BigInteger.valueOf(value.longValue()));
+    }
+
+    @Override
+    public void setStringValue(String value, BoundStatement statement)
+    {
+        statement.setVarint(cassandraColumn.getName(), new BigInteger(value));
+    }
+}

data/src/test/java/org/embulk/output/cassandra/TestCassandraOutputPlugin.java

@@ -0,0 +1,250 @@
+package org.embulk.output.cassandra;
+
+import com.datastax.driver.core.Cluster;
+import com.datastax.driver.core.DataType;
+import com.datastax.driver.core.LocalDate;
+import com.datastax.driver.core.Row;
+import com.datastax.driver.core.Session;
+import com.datastax.driver.core.TupleType;
+import com.datastax.driver.core.TupleValue;
+import com.datastax.driver.core.utils.UUIDs;
+import com.google.common.io.Resources;
+import org.embulk.config.ConfigSource;
+import org.embulk.spi.OutputPlugin;
+import org.embulk.test.EmbulkTests;
+import org.embulk.test.TestingEmbulk;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.net.InetAddress;
+import java.nio.file.Path;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.junit.Assert.*;
+
+public class TestCassandraOutputPlugin
+{
+    private static final String RESOURCE_PATH = "org/embulk/output/cassandra/";
+
+    @Rule
+    public TestingEmbulk embulk = TestingEmbulk.builder()
+            .registerPlugin(OutputPlugin.class, "cassandra", CassandraOutputPlugin.class)
+            .build();
+
+    private Cluster cluster;
+    private Session session;
+
+    private static String getCassandraHost()
+    {
+        String host = System.getenv("CASSANDRA_HOST");
+        if (host == null) {
+            host = "localhost";
+        }
+        return host;
+    }
+
+    private static List<String> getCassandraHostAsList()
+    {
+        return Arrays.asList(getCassandraHost());
+    }
+
+    private static String getCassandraPort()
+    {
+        String port = System.getenv("CASSANDRA_PORT");
+        if (port == null) {
+            port = "9042";
+        }
+        return port;
+    }
+
+    private static Cluster getCluster()
+    {
+        return Cluster.builder().addContactPoint(getCassandraHost())
+                .withPort(Integer.parseInt(getCassandraPort())).build();
+    }
+
+    @Before
+    public void setup()
+    {
+        cluster = getCluster();
+        session = cluster.connect();
+        String createKeyspace = EmbulkTests.readResource(RESOURCE_PATH + "create_keyspace.cql");
+        String createTableBasic = EmbulkTests.readResource(RESOURCE_PATH + "create_table_test_basic.cql");
+        String createTableUuid = EmbulkTests.readResource(RESOURCE_PATH + "create_table_test_uuid.cql");
+        String createTableComplex = EmbulkTests.readResource(RESOURCE_PATH + "create_table_test_complex.cql");
+        session.execute(createKeyspace);
+        session.execute(createTableBasic);
+        session.execute(createTableUuid);
+        session.execute(createTableComplex);
+        session.execute("TRUNCATE embulk_test.test_basic");
+        session.execute("TRUNCATE embulk_test.test_uuid");
+        session.execute("TRUNCATE embulk_test.test_complex");
+    }
+
+    @After
+    public void teardown()
+    {
+        session.close();
+        cluster.close();
+    }
+
+    private ConfigSource loadYamlResource(String filename)
+    {
+        return embulk.loadYamlResource(RESOURCE_PATH + filename);
+    }
+
+    @Test
+    public void testBasic() throws IOException
+    {
+        Path input = getInputPath("test1.csv");
+        ConfigSource config = loadYamlResource("test_basic.yaml");
+        config.set("hosts", getCassandraHostAsList());
+
+        assertEquals(0, session.execute("SELECT * FROM embulk_test.test_basic").all().size());
+
+        TestingEmbulk.RunResult result = embulk.runOutput(config, input);
+        assertEquals(3, result.getOutputTaskReports().get(0).get(Long.class, "inserted_record_count").longValue());
+
+        Row row1 = session.execute("SELECT * FROM embulk_test.test_basic WHERE id = 'A001'").one();
+        Row row2 = session.execute("SELECT * FROM embulk_test.test_basic WHERE id = 'A002'").one();
+        Row row3 = session.execute("SELECT * FROM embulk_test.test_basic WHERE id = 'A003'").one();
+        assertEquals("A001", row1.getString("id"));
+        assertEquals(9, row1.getLong("int_item"));
+        assertEquals(1, row1.getInt("int32_item"));
+        assertEquals(2, row1.getShort("smallint_item"));
+        assertTrue(row1.getBool("boolean_item"));
+        assertEquals(createDate(2018, 7, 1, 10, 0, 0, 0), row1.getTimestamp("timestamp_item"));
+        assertEquals("A002", row2.getString("id"));
+        assertEquals(0, row2.getLong("int_item"));
+        assertEquals(0, row2.getInt("int32_item"));
+        assertEquals(4, row2.getShort("smallint_item"));
+        assertTrue(row2.getBool("boolean_item"));
+        assertEquals(createDate(2018, 7, 1, 10, 0, 1, 0), row2.getTimestamp("timestamp_item"));
+        assertEquals("A003", row3.getString("id"));
+        assertEquals(9, row3.getLong("int_item"));
+        assertEquals(0, row3.getInt("int32_item"));
+        assertEquals(8, row3.getShort("smallint_item"));
+        assertFalse(row3.getBool("boolean_item"));
+        assertEquals(createDate(2018, 7, 1, 10, 0, 2, 0), row3.getTimestamp("timestamp_item"));
+    }
+
+    @Test
+    public void testBasicWithTtl() throws IOException
+    {
+        Path input = getInputPath("test1.csv");
+        ConfigSource config = loadYamlResource("test_basic.yaml");
+        config.set("hosts", getCassandraHostAsList());
+        config.set("ttl", 30);
+
+        assertEquals(0, session.execute("SELECT * FROM embulk_test.test_basic").all().size());
+
+        embulk.runOutput(config, input);
+
+        Row row1 = session.execute("SELECT * FROM embulk_test.test_basic WHERE id = 'A001'").one();
+        Row row2 = session.execute("SELECT * FROM embulk_test.test_basic WHERE id = 'A002'").one();
+        Row row3 = session.execute("SELECT * FROM embulk_test.test_basic WHERE id = 'A003'").one();
+        assertEquals("A001", row1.getString("id"));
+        assertEquals(9, row1.getLong("int_item"));
+        assertEquals(createDate(2018, 7, 1, 10, 0, 0, 0), row1.getTimestamp("timestamp_item"));
+        assertEquals("A002", row2.getString("id"));
+        assertEquals(0, row2.getLong("int_item"));
+        assertEquals(createDate(2018, 7, 1, 10, 0, 1, 0), row2.getTimestamp("timestamp_item"));
+        assertEquals("A003", row3.getString("id"));
+        assertEquals(9, row3.getLong("int_item"));
+        assertEquals(createDate(2018, 7, 1, 10, 0, 2, 0), row3.getTimestamp("timestamp_item"));
+    }
+
+    @Test
+    public void testBasicWithIfNotExists() throws IOException
+    {
+        Path input = getInputPath("test1.csv");
+        ConfigSource config = loadYamlResource("test_basic.yaml");
+        config.set("hosts", getCassandraHostAsList());
+        config.set("if_not_exists", true);
+
+        assertEquals(0, session.execute("SELECT * FROM embulk_test.test_basic").all().size());
+
+        embulk.runOutput(config, input);
+
+        Row row1 = session.execute("SELECT * FROM embulk_test.test_basic WHERE id = 'A001'").one();
+        Row row2 = session.execute("SELECT * FROM embulk_test.test_basic WHERE id = 'A002'").one();
+        Row row3 = session.execute("SELECT * FROM embulk_test.test_basic WHERE id = 'A003'").one();
+        assertEquals("A001", row1.getString("id"));
+        assertEquals(9, row1.getLong("int_item"));
+        assertEquals(createDate(2018, 7, 1, 10, 0, 0, 0), row1.getTimestamp("timestamp_item"));
+        assertEquals("A002", row2.getString("id"));
+        assertEquals(0, row2.getLong("int_item"));
+        assertEquals(createDate(2018, 7, 1, 10, 0, 1, 0), row2.getTimestamp("timestamp_item"));
+        assertEquals("A003", row3.getString("id"));
+        assertEquals(9, row3.getLong("int_item"));
+        assertEquals(createDate(2018, 7, 1, 10, 0, 2, 0), row3.getTimestamp("timestamp_item"));
+    }
+
+    @Test
+    public void testUuid() throws IOException
+    {
+        Path input = getInputPath("test2.csv");
+        ConfigSource config = loadYamlResource("test_uuid.yaml");
+        config.set("hosts", getCassandraHostAsList());
+        embulk.runOutput(config, input);
+
+        Row row1 = session.execute("SELECT * FROM embulk_test.test_uuid").one();
+        assertNotNull(row1.getUUID("id"));
+        assertEquals(9, row1.getLong("int_item"));
+        assertEquals(createDate(2018, 7, 1, 10, 0, 0, 0), row1.getTimestamp("timestamp_item"));
+        assertNotNull(row1.getUUID("timeuuid_item"));
+        assertTrue(UUIDs.unixTimestamp(row1.getUUID("timeuuid_item")) < ZonedDateTime.now().toInstant().toEpochMilli());
+    }
+
+    @Test
+    public void testComplex() throws IOException
+    {
+        Path input = getInputPath("test3.csv");
+        ConfigSource config = loadYamlResource("test_complex.yaml");
+        config.set("hosts", getCassandraHostAsList());
+        embulk.runOutput(config, input);
+
+        Row row1 = session.execute("SELECT * FROM embulk_test.test_complex").one();
+        assertNotNull(row1.getUUID("id"));
+        assertEquals(BigDecimal.valueOf(10), row1.getDecimal("decimal_item"));
+        assertEquals(LocalDate.fromYearMonthDay(2018, 7, 1), row1.getDate("date_item"));
+        assertEquals(3600L * 10 * 1000 * 1000 * 1000, row1.getTime("time_item"));
+
+        List<String> list = row1.getList("list_item", String.class);
+        assertArrayEquals(new String[] {"foo", "bar"}, list.toArray());
+
+        Map<String, Long> map = row1.getMap("map_item", String.class, Long.class);
+        assertEquals(1L, map.get("key1").longValue());
+
+        Set<Long> set = row1.getSet("set_item", Long.class);
+        assertArrayEquals(new Long[] {1L, 2L, 3L}, set.toArray());
+
+        InetAddress inet = row1.getInet("inet_item");
+        assertEquals(InetAddress.getByName("127.0.0.1"), inet);
+
+        TupleValue tuple = row1.getTupleValue("tuple_item");
+        TupleType tupleType = cluster.getMetadata().newTupleType(DataType.text(), DataType.cdouble());
+        assertEquals(tupleType.newValue("foo", 1.1), tuple);
+    }
+
+    private Path getInputPath(String filename)
+    {
+        return new File(Resources.getResource(RESOURCE_PATH + filename).getFile()).toPath();
+    }
+
+    private Date createDate(int year, int month, int day, int hour, int min, int sec, int nsec)
+    {
+        return Date.from(ZonedDateTime.of(year, month, day, hour, min, sec, nsec, ZoneId.of("UTC")).toInstant());
+    }
+}

data/src/test/resources/org/embulk/output/cassandra/create_keyspace.cql

@@ -0,0 +1,5 @@
+CREATE KEYSPACE IF NOT EXISTS embulk_test
+WITH REPLICATION = {
+  'class' : 'SimpleStrategy',
+  'replication_factor' : 1
+};

data/src/test/resources/org/embulk/output/cassandra/create_table_test_basic.cql

@@ -0,0 +1,11 @@
+CREATE TABLE IF NOT EXISTS embulk_test.test_basic (
+  id text,
+  int_item bigint,
+  int32_item int,
+  smallint_item smallint,
+  tinyint_item tinyint,
+  boolean_item boolean,
+  double_item double,
+  timestamp_item timestamp,
+  primary key (id)
+) WITH compaction = { 'class' : 'SizeTieredCompactionStrategy' };

data/src/test/resources/org/embulk/output/cassandra/create_table_test_complex.cql

@@ -0,0 +1,12 @@
+CREATE TABLE IF NOT EXISTS embulk_test.test_complex (
+  id uuid,
+  decimal_item decimal,
+  date_item date,
+  time_item time,
+  list_item list<text>,
+  map_item map<text, bigint>,
+  set_item set<bigint>,
+  inet_item inet,
+  tuple_item tuple<text, double>,
+  primary key (id)
+) WITH compaction = { 'class' : 'SizeTieredCompactionStrategy' };

data/src/test/resources/org/embulk/output/cassandra/create_table_test_uuid.cql

@@ -0,0 +1,8 @@
+CREATE TABLE IF NOT EXISTS embulk_test.test_uuid (
+  id uuid,
+  int_item bigint,
+  double_item double,
+  timestamp_item timestamp,
+  timeuuid_item timeuuid,
+  primary key (id)
+) WITH compaction = { 'class' : 'SizeTieredCompactionStrategy' };

data/src/test/resources/org/embulk/output/cassandra/test1.csv

@@ -0,0 +1,4 @@
+id:string,int_item:long,int32_item:long,smallint_item:long,tinyint_item:long,boolean_item:boolean,varchar_item:string,timestamp_item:timestamp
+A001,9,4294967297,2,0,true,a,2018-07-01 10:00:00.000 UTC
+A002,0,0,4,0,true,b,2018-07-01 10:00:01.000 UTC
+A003,9,0,8,1,false,c,2018-07-01 10:00:02.000 UTC

data/src/test/resources/org/embulk/output/cassandra/test2.csv

@@ -0,0 +1,2 @@
+int_item:long,varchar_item:string,timestamp_item:timestamp
+9,a,2018-07-01 10:00:00.000 UTC

data/src/test/resources/org/embulk/output/cassandra/test3.csv

@@ -0,0 +1,2 @@
+decimal_item:long,date_item:string,time_item:string,list_item:json,map_item:json,set_item:json,inet_item:string,tuple_item:json
+10,2018-07-01,10:00:00,"[""foo"", ""bar""]","{""key1"": 1, ""key2"": 2}","[1,2,3,3]",127.0.0.1,"[""foo"", 1.1]"

data/src/test/resources/org/embulk/output/cassandra/test_basic.yaml

@@ -0,0 +1,3 @@
+type: cassandra
+keyspace: embulk_test
+table: "test_basic"

data/src/test/resources/org/embulk/output/cassandra/test_complex.yaml

@@ -0,0 +1,3 @@
+type: cassandra
+keyspace: embulk_test
+table: "test_complex"

data/src/test/resources/org/embulk/output/cassandra/test_uuid.yaml

@@ -0,0 +1,3 @@
+type: cassandra
+keyspace: embulk_test
+table: "test_uuid"

metadata

@@ -0,0 +1,141 @@
+--- !ruby/object:Gem::Specification
+name: embulk-output-cassandra
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- joker1007
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2018-07-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: bundler
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  name: rake
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: Dumps records to Apache Cassandra.
+email:
+- kakyoin.hierophant@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- LICENSE.txt
+- README.md
+- build.gradle
+- config/checkstyle/checkstyle.xml
+- config/checkstyle/default.xml
+- embulk-output-cassandra.iml
+- gradle/wrapper/gradle-wrapper.jar
+- gradle/wrapper/gradle-wrapper.properties
+- gradlew
+- gradlew.bat
+- lib/embulk/output/cassandra.rb
+- src/main/java/org/embulk/output/cassandra/CassandraOutputPlugin.java
+- src/main/java/org/embulk/output/cassandra/setter/BigintColumnSetter.java
+- src/main/java/org/embulk/output/cassandra/setter/BooleanColumnSetter.java
+- src/main/java/org/embulk/output/cassandra/setter/CassandraColumnSetter.java
+- src/main/java/org/embulk/output/cassandra/setter/CassandraColumnSetterFactory.java
+- src/main/java/org/embulk/output/cassandra/setter/ColumnSetterVisitor.java
+- src/main/java/org/embulk/output/cassandra/setter/DateColumnSetter.java
+- src/main/java/org/embulk/output/cassandra/setter/DecimalColumnSetter.java
+- src/main/java/org/embulk/output/cassandra/setter/DoubleColumnSetter.java
+- src/main/java/org/embulk/output/cassandra/setter/FloatColumnSetter.java
+- src/main/java/org/embulk/output/cassandra/setter/InetColumnSetter.java
+- src/main/java/org/embulk/output/cassandra/setter/IntColumnSetter.java
+- src/main/java/org/embulk/output/cassandra/setter/ListColumnSetter.java
+- src/main/java/org/embulk/output/cassandra/setter/MapColumnSetter.java
+- src/main/java/org/embulk/output/cassandra/setter/SetColumnSetter.java
+- src/main/java/org/embulk/output/cassandra/setter/SmallintColumnSetter.java
+- src/main/java/org/embulk/output/cassandra/setter/TextColumnSetter.java
+- src/main/java/org/embulk/output/cassandra/setter/TimeColumnSetter.java
+- src/main/java/org/embulk/output/cassandra/setter/TimestampColumnSetter.java
+- src/main/java/org/embulk/output/cassandra/setter/TimeuuidColumnSetter.java
+- src/main/java/org/embulk/output/cassandra/setter/TinyintColumnSetter.java
+- src/main/java/org/embulk/output/cassandra/setter/TupleColumnSetter.java
+- src/main/java/org/embulk/output/cassandra/setter/UuidColumnSetter.java
+- src/main/java/org/embulk/output/cassandra/setter/ValueConverter.java
+- src/main/java/org/embulk/output/cassandra/setter/VarintColumnSetter.java
+- src/test/java/org/embulk/output/cassandra/TestCassandraOutputPlugin.java
+- src/test/resources/org/embulk/output/cassandra/create_keyspace.cql
+- src/test/resources/org/embulk/output/cassandra/create_table_test_basic.cql
+- src/test/resources/org/embulk/output/cassandra/create_table_test_complex.cql
+- src/test/resources/org/embulk/output/cassandra/create_table_test_uuid.cql
+- src/test/resources/org/embulk/output/cassandra/test1.csv
+- src/test/resources/org/embulk/output/cassandra/test2.csv
+- src/test/resources/org/embulk/output/cassandra/test3.csv
+- src/test/resources/org/embulk/output/cassandra/test_basic.yaml
+- src/test/resources/org/embulk/output/cassandra/test_complex.yaml
+- src/test/resources/org/embulk/output/cassandra/test_uuid.yaml
+- classpath/asm-util-5.0.3.jar
+- classpath/asm-analysis-5.0.3.jar
+- classpath/asm-tree-5.0.3.jar
+- classpath/guava-19.0.jar
+- classpath/jnr-constants-0.9.9.jar
+- classpath/jnr-posix-3.0.44.jar
+- classpath/cassandra-driver-core-3.5.0.jar
+- classpath/jnr-ffi-2.1.7.jar
+- classpath/netty-codec-4.0.56.Final.jar
+- classpath/asm-commons-5.0.3.jar
+- classpath/netty-buffer-4.0.56.Final.jar
+- classpath/metrics-core-3.2.2.jar
+- classpath/netty-common-4.0.56.Final.jar
+- classpath/jnr-x86asm-1.0.2.jar
+- classpath/asm-5.0.3.jar
+- classpath/netty-transport-4.0.56.Final.jar
+- classpath/embulk-output-cassandra-0.1.0.jar
+- classpath/jffi-1.2.16.jar
+- classpath/jffi-1.2.16-native.jar
+- classpath/slf4j-api-1.7.25.jar
+- classpath/netty-handler-4.0.56.Final.jar
+homepage: https://github.com/joker1007/embulk-output-cassandra
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.1.9
+signing_key:
+specification_version: 4
+summary: Cassandra output plugin for Embulk
+test_files: []