embulk-output-snowflake 0.1.1 → 0.2.0

data/src/main/java/org/embulk/output/snowflake/SnowflakeOutputConnector.java

@@ -1,34 +1,32 @@
 package org.embulk.output.snowflake;
 
 import com.google.common.base.Optional;
+import net.snowflake.client.jdbc.SnowflakeSQLException;
+import net.snowflake.client.jdbc.SnowflakeType;
 import org.embulk.output.jdbc.AbstractJdbcOutputConnector;
+import org.embulk.output.jdbc.JdbcColumn;
 import org.embulk.output.jdbc.JdbcOutputConnection;
 import org.embulk.output.jdbc.TransactionIsolation;
 
 import java.sql.Connection;
+import java.sql.Driver;
 import java.sql.DriverManager;
 import java.sql.SQLException;
+import java.util.Enumeration;
 import java.util.Properties;
 
-public class SnowflakeOutputConnector
-        extends AbstractJdbcOutputConnector
+public class SnowflakeOutputConnector extends AbstractJdbcOutputConnector
 {
     private final String url;
     private final Properties properties;
-    private final String schemaName;
 
-    public SnowflakeOutputConnector(String url, Properties properties, String schemaName,
-                                    Optional<TransactionIsolation> transactionIsolation)
+    public SnowflakeOutputConnector(String url, Properties properties,
+                                     Optional<TransactionIsolation> transactionIsolation)
     {
         super(transactionIsolation);
-        try {
-            Class.forName("com.snowflake.client.jdbc.SnowflakeDriver");
-        } catch (Exception ex) {
-            throw new RuntimeException(ex);
-        }
+
         this.url = url;
         this.properties = properties;
-        this.schemaName = schemaName;
     }
 
     @Override
@@ -36,7 +34,7 @@ public class SnowflakeOutputConnector
     {
         Connection c = DriverManager.getConnection(url, properties);
         try {
-            SnowflakeOutputConnection con = new SnowflakeOutputConnection(c, schemaName);
+            SnowflakeOutputConnection con = new SnowflakeOutputConnection(c);
             c = null;
             return con;
         } finally {
@@ -45,4 +43,5 @@ public class SnowflakeOutputConnector
             }
         }
     }
-}
+
+}

data/src/main/java/org/embulk/output/snowflake/SnowflakeUtils.java

@@ -0,0 +1,17 @@
+package org.embulk.output.snowflake;
+
+import java.util.Random;
+
+public class SnowflakeUtils {
+    public static final String SOURCES =
+            "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz1234567890";
+
+    public static String randomString(int length){
+        Random rand = new Random();
+        char[] text = new char[length];
+        for (int i = 0; i < length; i++) {
+            text[i] = SOURCES.charAt(rand.nextInt(SOURCES.length()));
+        }
+        return new String(text);
+    }
+}

data/src/main/java/org/embulk/output/snowflake/StageIdentifier.java

@@ -0,0 +1,61 @@
+package org.embulk.output.snowflake;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Optional;
+
+public class StageIdentifier {
+    private String database;
+    private String schemaName;
+    private String stageName;
+    private Optional<String> destPrefix;
+
+    public StageIdentifier(String database, String schemaName, String stageName, Optional<String> destPrefix) {
+        this.database = database;
+        this.schemaName = schemaName;
+        this.stageName = stageName;
+        this.destPrefix = destPrefix;
+    }
+
+    public StageIdentifier() {
+    }
+
+    @JsonProperty
+    public String getDatabase() {
+        return database;
+    }
+
+    @JsonProperty
+    public void setDatabase(String database) {
+        this.database = database;
+    }
+
+    @JsonProperty
+    public String getSchemaName() {
+        return schemaName;
+    }
+
+    @JsonProperty
+    public void setSchemaName(String schemaName) {
+        this.schemaName = schemaName;
+    }
+
+    @JsonProperty
+    public String getStageName() {
+        return stageName;
+    }
+
+    @JsonProperty
+    public void setStageName(String stageName) {
+        this.stageName = stageName;
+    }
+
+    @JsonProperty
+    public Optional<String> getDestPrefix() {
+        return destPrefix;
+    }
+
+    @JsonProperty
+    public void setDestPrefix(Optional<String> destPrefix) {
+        this.destPrefix = destPrefix;
+    }
+}

data/src/main/java/org/embulk/output/snowflake/StageIdentifierHolder.java

@@ -0,0 +1,20 @@
+package org.embulk.output.snowflake;
+
+import com.google.common.base.Optional;
+import org.embulk.output.SnowflakeOutputPlugin;
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+public class StageIdentifierHolder {
+    private static final String pattern = "yyyyMMdd";
+    private static final SimpleDateFormat simpleDateFormat = new SimpleDateFormat(pattern);
+    private static final String date = simpleDateFormat.format(new Date());
+    private static final String salt = SnowflakeUtils.randomString(6);
+    private static final String snowflakeStageName = "embulk_snowflake_" + date + salt;
+    private static final String snowflakeDestPrefix = date + "_" + salt;
+
+    public static StageIdentifier getStageIdentifier(SnowflakeOutputPlugin.SnowflakePluginTask t){
+        return new StageIdentifier(t.getDatabase(), t.getSchema(), snowflakeStageName, Optional.of(snowflakeDestPrefix));
+    }
+}

data/src/test/java/org/embulk/output/snowflake/TestSnowflakeOutputPlugin.java

@@ -0,0 +1,5 @@
+package org.embulk.output.snowflake;
+
+public class TestSnowflakeOutputPlugin
+{
+}

metadata

@@ -1,33 +1,75 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-snowflake
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
-- Piotr Wilkowski
+- giwa
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-05-29 00:00:00.000000000 Z
-dependencies: []
-description: Inserts or updates records to a table.
+date: 2020-11-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: bundler
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+  name: rake
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '12.0'
+description: Dumps records to Snowflake.
 email:
-- piwil@wp.pl
+- ugw.gi.world@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".gitignore"
+- LICENSE.txt
 - README.md
 - build.gradle
-- classpath/embulk-output-jdbc-0.1.1.jar
-- classpath/embulk-output-snowflake-0.1.1.jar
+- classpath/embulk-output-jdbc-0.8.7.jar
+- classpath/embulk-output-snowflake-0.2.0.jar
+- config/checkstyle/checkstyle.xml
+- config/checkstyle/default.xml
+- default_jdbc_driver/snowflake-jdbc-3.12.8.jar
+- gradle/wrapper/gradle-wrapper.jar
+- gradle/wrapper/gradle-wrapper.properties
+- gradlew
+- gradlew.bat
 - lib/embulk/output/snowflake.rb
 - src/main/java/org/embulk/output/SnowflakeOutputPlugin.java
+- src/main/java/org/embulk/output/snowflake/SnowflakeCopyBatchInsert.java
 - src/main/java/org/embulk/output/snowflake/SnowflakeOutputConnection.java
 - src/main/java/org/embulk/output/snowflake/SnowflakeOutputConnector.java
-homepage: ''
+- src/main/java/org/embulk/output/snowflake/SnowflakeUtils.java
+- src/main/java/org/embulk/output/snowflake/StageIdentifier.java
+- src/main/java/org/embulk/output/snowflake/StageIdentifierHolder.java
+- src/test/java/org/embulk/output/snowflake/TestSnowflakeOutputPlugin.java
+homepage: https://github.com/trocco-io/embulk-output-snowflake
 licenses:
-- Apache 2.0
+- MIT
 metadata: {}
 post_install_message:
 rdoc_options: []
@@ -45,7 +87,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.8
+rubygems_version: 2.6.13
 signing_key:
 specification_version: 4
 summary: Snowflake output plugin for Embulk