embulk-output-embulk_output_domo 0.3.2 → 0.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3978b59b3985b4520d50f23256d3708bda278d3c
-  data.tar.gz: 2324808afdd8b2ba25934dbbffd95eecfabd8cb0
+  metadata.gz: 369ef52a51a9004c3d21d03946c6034b24d359d4
+  data.tar.gz: 9d046241a16d2156fca7cff577ca72560ca6d86b
 SHA512:
-  metadata.gz: cef3cb441fb4d03a5e290747d61e08bc38fa31943cd165c70d0908d809aad1b3c023e2a8cd876f0ed2073a598eb461664bf749b8b1b7011850dfd5168f736fd8
-  data.tar.gz: 640715724ddb4e0ad5e9abc1dd9fe490dc803ad7ab4c9b16a6454ece9099d6a8d3f28fbce4bdc6e29a763fc61178f833653acc9aea7df130b0ffc3b35d28a221
+  metadata.gz: 9e463ffcac24d5cebf86265d8a37905d3b2e9802df8b90d61d60234528e938fc578f9720e09dde4869a86d0fb9a52a847d500b03e06e8ea096b0700bf9c7507c
+  data.tar.gz: 51229748f9c9994efc3ea2413e9e8a96464559ec466dabbc259b6a52d318c5555ce44ac0bc8cfbaccea3ac6c138f90566befacfeb24f9820addf71c9cf333781

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.3.2"
+version = "0.3.3"
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8

data/src/main/java/org/embulk/output/embulk_output_domo/EmbulkOutputDomoOutputPlugin.java

@@ -160,50 +160,6 @@ public class EmbulkOutputDomoOutputPlugin
         Newline getNewlineInField();
     }
 
-    public com.domo.sdk.datasets.model.Schema getDomoSchema(Schema schema){
-        /**
-         * We need to return domo Schema
-         * e.g. new com.domo.sdk.datasets.model.Schema(Lists.newArrayList(new Column(STRING, "Friend"), new Column(STRING, "Attending")))
-         */
-        ArrayList<com.domo.sdk.datasets.model.Column> domoSchema =  new ArrayList<com.domo.sdk.datasets.model.Column>();
-        for (int i = 0; i < schema.size(); i++) {
-            Column column = schema.getColumn(i);
-            Type type = column.getType();
-            System.out.println("{\n" +
-                    "      \"type\" : \""+type.getName().toUpperCase()+"\",\n" +
-                    "      \"name\" : \""+column.getName() +"\"\n" +
-                    "    },");
-            switch (type.getName()) {
-                case "long":
-                    domoSchema.add(new com.domo.sdk.datasets.model.Column(ColumnType.LONG, column.getName()));
-                    break;
-                case "double":
-                    //	     System.out.println("Double Column" + column.getName());
-                    domoSchema.add(new com.domo.sdk.datasets.model.Column(ColumnType.DOUBLE, column.getName()));
-                    break;
-                case "boolean":
-                    domoSchema.add(new com.domo.sdk.datasets.model.Column( ColumnType.LONG, column.getName()));
-                    break;
-                case "string":
-                    domoSchema.add(new com.domo.sdk.datasets.model.Column(ColumnType.STRING, column.getName()));
-                    break;
-                case "timestamp":
-                    domoSchema.add(new com.domo.sdk.datasets.model.Column( ColumnType.DATETIME, column.getName()));
-                    break;
-                default:
-                    logger.info("Unsupported type " + type.getName());
-                    break;
-            }
-        }
-        if(domoSchema != null && domoSchema.size()>0){
-            return new com.domo.sdk.datasets.model.Schema(domoSchema);
-        }
-        else{
-            logger.error("Cannot create domo schema");
-            throw new RuntimeException("Cannot create domo Schema");
-        }
-    }
-
     @Override
     public ConfigDiff transaction(ConfigSource config,
                                   Schema schema, int taskCount,
@@ -219,7 +175,7 @@ public class EmbulkOutputDomoOutputPlugin
 
         try {
             if (client == null) {
-                //getDomoSchema(schema);
+                com.domo.sdk.datasets.model.Schema ds_schema = getDomoSchema(schema);
                 com.domo.sdk.request.Config domoConfig = com.domo.sdk.request.Config.with()
                         .clientId(clientId)
                         .clientSecret(clientSecret)
@@ -231,18 +187,36 @@ public class EmbulkOutputDomoOutputPlugin
 
                 client = DomoClient.create(domoConfig);
                 streamClient = client.streamClient();
-
+                logger.info("Lets search for dataSource.name = "+task.getStreamName());
                 List<Stream> searchedSds = streamClient.search("dataSource.name:" + task.getStreamName());
-                sds = searchedSds.get(0);
-                logger.info("Stream "+ sds);
-                execution = streamClient.createExecution(sds.getId());
-                logger.info("Created Execution: " + execution);
-                timestampFormatters = Timestamps.newTimestampColumnFormatters(task, schema, task.getColumnOptions());
-                totalBatches = task.getBatchSize();
-                File directory = new File(TEMP_DIR);
-                if(!directory.exists()) {
-                    directory.mkdirs();
+                logger.info("Found search sds size = "+searchedSds.size());
+                if (searchedSds.size() >0) {
+                    sds = searchedSds.get(0);
+                }
+                else{
+                    logger.info("Lets create new Stream");
+                    CreateDataSetRequest ds = new CreateDataSetRequest();
+                    ds.setName(task.getStreamName());
+                    ds.setDescription(task.getStreamName());
+                    ds.setSchema(ds_schema);
+
+                    logger.info("We have to create sds");
+                    StreamRequest sdsRequest = new StreamRequest();
+
+                    sdsRequest.setDataSet(ds);
+                    sdsRequest.setUpdateMethod(UpdateMethod.REPLACE);
+
+                    sds = streamClient.create(sdsRequest);
                 }
+                    logger.info("Stream "+ sds);
+                    execution = streamClient.createExecution(sds.getId());
+                    logger.info("Created Execution: " + execution);
+                    timestampFormatters = Timestamps.newTimestampColumnFormatters(task, schema, task.getColumnOptions());
+                    totalBatches = task.getBatchSize();
+                    File directory = new File(TEMP_DIR);
+                    if(!directory.exists()) {
+                        directory.mkdirs();
+                    }
             }
         }
         catch(Exception ex){
@@ -254,6 +228,7 @@ public class EmbulkOutputDomoOutputPlugin
         control.run(task.dump());
         return Exec.newConfigDiff();
     }
+
     @Override
     public ConfigDiff resume(TaskSource taskSource,
                              Schema schema, int taskCount,
@@ -261,6 +236,7 @@ public class EmbulkOutputDomoOutputPlugin
     {
         throw new UnsupportedOperationException("embulk_output_domo output plugin does not support resuming");
     }
+
     @Override
     public void cleanup(TaskSource taskSource,
                         Schema schema, int taskCount,
@@ -424,6 +400,7 @@ public class EmbulkOutputDomoOutputPlugin
                         @Override
                         public void booleanColumn(Column column) {
                             addDelimiter(column);
+                            //logger.info(pageReader.getBoolean(column));
                             if (!pageReader.isNull(column)) {
                                 addValue(Boolean.toString(pageReader.getBoolean(column)));
                             } else {
@@ -484,7 +461,7 @@ public class EmbulkOutputDomoOutputPlugin
     /**
      *
      * @param v String value
-     * @param delimiter csv delimeter
+     * @param delimiter csv delimiter
      * @param policy enum QuotePolicy
      * @param quote Quote Character
      * @param escape Escape Character
@@ -700,6 +677,48 @@ public class EmbulkOutputDomoOutputPlugin
         bw.write(fileContent);
         bw.close();
     }
-
+    public com.domo.sdk.datasets.model.Schema getDomoSchema(Schema schema){
+        /**
+         * We need to return domo Schema
+         * e.g. new com.domo.sdk.datasets.model.Schema(Lists.newArrayList(new Column(STRING, "Friend"), new Column(STRING, "Attending")))
+         */
+        ArrayList<com.domo.sdk.datasets.model.Column> domoSchema =  new ArrayList<com.domo.sdk.datasets.model.Column>();
+        for (int i = 0; i < schema.size(); i++) {
+            Column column = schema.getColumn(i);
+            Type type = column.getType();
+            //            System.out.println("{\n" +
+            //                    "      \"type\" : \""+type.getName().toUpperCase()+"\",\n" +
+            //                    "      \"name\" : \""+column.getName() +"\"\n" +
+            //                    "    },");
+            switch (type.getName()) {
+                case "long":
+                    domoSchema.add(new com.domo.sdk.datasets.model.Column(ColumnType.LONG, column.getName()));
+                    break;
+                case "double":
+                    //	     System.out.println("Double Column" + column.getName());
+                    domoSchema.add(new com.domo.sdk.datasets.model.Column(ColumnType.DOUBLE, column.getName()));
+                    break;
+                case "boolean":
+                    domoSchema.add(new com.domo.sdk.datasets.model.Column( ColumnType.LONG, column.getName()));
+                    break;
+                case "string":
+                    domoSchema.add(new com.domo.sdk.datasets.model.Column(ColumnType.STRING, column.getName()));
+                    break;
+                case "timestamp":
+                    domoSchema.add(new com.domo.sdk.datasets.model.Column( ColumnType.DATETIME, column.getName()));
+                    break;
+                default:
+                    logger.info("Unsupported type " + type.getName());
+                    break;
+            }
+        }
+        if(domoSchema.size() > 0){
+            return new com.domo.sdk.datasets.model.Schema(domoSchema);
+        }
+        else{
+            logger.error("Cannot create domo schema");
+            throw new RuntimeException("Cannot create domo Schema");
+        }
+    }
 
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-embulk_output_domo
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.3.3
 platform: ruby
 authors:
 - Angelos Alexopoulos
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-10-31 00:00:00.000000000 Z
+date: 2019-09-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -67,7 +67,7 @@ files:
 - classpath/commons-io-2.6.jar
 - classpath/hamcrest-core-1.3.jar
 - classpath/okio-1.12.0.jar
-- classpath/embulk-output-embulk_output_domo-0.3.2.jar
+- classpath/embulk-output-embulk_output_domo-0.3.3.jar
 homepage: https://github.com/alexopoulos7/embulk-output-embulk_output_domo
 licenses:
 - MIT