embulk-output-embulk_output_domo 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a7b5bd4dca8bef7c4b4046f0d2e7d25d754e92cc
-  data.tar.gz: bfe92f0a853717a4f011ac0cffc5699b9af4c358
+  metadata.gz: 36ad85a24723e4f76d4c7acb15571978f78ecde9
+  data.tar.gz: b37e2fe3c7766a2c4faeb8d363e758933ada969c
 SHA512:
-  metadata.gz: dfaa81184ec2638d736fe17e6136e090daa46e45812116109a61ce3afd09adfc39b11ad7eba3c2e25dd09903fa9e67a02094abee411642f94e68e386c0fec52a
-  data.tar.gz: 7f5539af3ea82dd8810fd8b59c0d09a3d46f047fb9555282dbb549155e183f369726f693e0ed5d654e7b407a9bc1dc47d2bd31161ed228518d8762e94640d09c
+  metadata.gz: bf28f55db720cde8fc932521b0fae8f32d8be9d7154da7255ccdba63d1ab2266ad1600856b063e2473c5cf7874dad4437bffe08334631f76dffe40274e6b6c64
+  data.tar.gz: 86e84fa082c55f484676c5ca34d2bf3622d0e3530c256706fa6eeeada9b147af1e7d3c731406fe5db9761bcdc95d95e9b56d5023d9500d267fd52d17beaa915b

data/README.md

@@ -11,17 +11,31 @@ TODO: Write short description here and build.gradle file.
 
 ## Configuration
 
-- **option1**: description (integer, required)
-- **option2**: description (string, default: `"myvalue"`)
-- **option3**: description (string, default: `null`)
+- **clientId**: description (string, required)
+- **clientSecret**: description (string, required)
+- **apiHost**: description (string, default: `"api.domo.com"`)
+- **useHttps**: description (boolean, default: `true`)
+- **streamName**: description (string, required)
+- **column_options**: description (object, default: `Check embulk column options`)
+- **batchSize**: description (integer, default: `1000000`)
+- **quote**: description (string, default: `"\""`)
+- **quote_policy**: description (string, default: `MINIMAL`)
+- **escape**: description (string, default: `null`)
+- **newline_in_field**: description (string, default: `LF`)
+
 
 ## Example
 
 ```yaml
 out:
   type: embulk_output_domo
-  option1: example1
-  option2: example2
+  clientId: 209410f4-052a-4a25-8191-3475aecab631
+  clientSecret: 00fac62163e9ccea36de4614473f17e18dbf0c9860e7616007a70949c33dc2d6
+  apiHost: api.domo.com
+  useHttps: true
+  streamName: Daily Metrics Test--Angelos
+  updateMethod: APPEND
+  batchSize: 500
 ```
 
 

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.1.0"
+version = "0.2.0"
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8

data/src/main/java/org/embulk/output/embulk_output_domo/EmbulkOutputDomoOutputPlugin.java

@@ -33,6 +33,8 @@ import com.domo.sdk.streams.model.Execution;
 import com.domo.sdk.streams.model.Stream;
 import com.domo.sdk.streams.model.StreamRequest;
 import com.domo.sdk.streams.model.UpdateMethod;
+import com.domo.sdk.datasets.model.DataSet;
+import com.domo.sdk.datasets.DataSetClient;
 import com.domo.sdk.DomoClient;
 import com.domo.sdk.streams.StreamClient;
 
@@ -43,6 +45,7 @@ import static com.domo.sdk.request.Scope.DATA;
 import java.io.IOException;
 import java.util.List;
 import java.util.Map;
+import java.util.HashMap;
 import java.util.ArrayList;
 import java.util.Date;
 import java.text.SimpleDateFormat;
@@ -52,7 +55,17 @@ public class EmbulkOutputDomoOutputPlugin
         implements OutputPlugin
 {
     private static DomoClient client = null;
+    private static Execution execution = null;
+    private static StreamClient sdsClient = null;
+    private static Stream sds = null;
+    private static TimestampFormatter[] timestampFormatters = null;
+    private int partNum = 1;
+    private int totalRecords = 1;
     protected static Logger logger;
+    private static ArrayList<StringBuilder> allRecords = new ArrayList<StringBuilder>();
+    private static ArrayList<String> recordsParts = new ArrayList<String>();
+    private int currentPartCounter = 1;
+    private static int totalBatches = 1;
 
     public enum QuotePolicy
     {
@@ -137,6 +150,10 @@ public class EmbulkOutputDomoOutputPlugin
         for (int i = 0; i < schema.size(); i++) {
             Column column = schema.getColumn(i);
             Type type = column.getType();
+            logger.info("{\n" +
+                    "      \"type\" : \""+type.getName().toUpperCase()+"\",\n" +
+                    "      \"name\" : \""+column.getName() +"\"\n" +
+                    "    },");
             switch (type.getName()) {
                 case "long":
                     domoSchema.add(new com.domo.sdk.datasets.model.Column(ColumnType.LONG, column.getName()));
@@ -154,7 +171,7 @@ public class EmbulkOutputDomoOutputPlugin
                     domoSchema.add(new com.domo.sdk.datasets.model.Column( ColumnType.DATETIME, column.getName()));
                     break;
                 default:
-                    System.out.println("Unsupported type " + type.getName());
+                    logger.info("Unsupported type " + type.getName());
                     break;
             }
         }
@@ -162,12 +179,12 @@ public class EmbulkOutputDomoOutputPlugin
             return new com.domo.sdk.datasets.model.Schema(domoSchema);
         }
         else{
-            logger.error("");
+            logger.error("Cannot create domo schema");
             throw new RuntimeException("Cannot create domo Schema");
         }
-
     }
 
+
     @Override
     public ConfigDiff transaction(ConfigSource config,
                                   Schema schema, int taskCount,
@@ -182,9 +199,9 @@ public class EmbulkOutputDomoOutputPlugin
         final boolean useHttps = task.getUseHttps();
         String updateMethod = task.getUpdateMethod();
 
-
         try {
             if (client == null) {
+                //getDomoSchema(schema);
                 com.domo.sdk.request.Config domoConfig = com.domo.sdk.request.Config.with()
                         .clientId(clientId)
                         .clientSecret(clientSecret)
@@ -195,6 +212,16 @@ public class EmbulkOutputDomoOutputPlugin
                         .build();
 
                 client = DomoClient.create(domoConfig);
+                sdsClient = client.streamClient();
+
+                List<Stream> searchedSds = sdsClient.search("dataSource.name:" + task.getStreamName());
+                sds = searchedSds.get(0);
+                logger.info("Stream "+ sds);
+                execution = sdsClient.createExecution(sds.getId());
+                logger.info("Created Execution: " + execution);
+                timestampFormatters = Timestamps.newTimestampColumnFormatters(task, schema, task.getColumnOptions());
+                totalBatches = task.getBatchSize();
+
             }
         }
         catch(Exception ex){
@@ -220,6 +247,68 @@ public class EmbulkOutputDomoOutputPlugin
             Schema schema, int taskCount,
             List<TaskReport> successTaskReports)
     {
+        List<List<StringBuilder>> batchLists = batches(allRecords, totalBatches);
+        int i=1;
+        for(List<StringBuilder> l : batchLists){
+            logger.info("Uploading..."+i +" with records = "+l.size() );
+            sdsClient.uploadDataPart(sds.getId(), execution.getId(), i, stringifyList(l));
+
+            i++;
+        }
+        logger.info("Finished Uploading");
+        //sdsClient.uploadDataPart(sds.getId(), execution.getId(), 1, stringify(allRecords));
+        //Commit Execution
+        Execution committedExecution = sdsClient.commitExecution(sds.getId(), execution.getId());
+        logger.info("Committed Execution: " + committedExecution);
+    }
+
+    private String stringifyList(List<StringBuilder> records){
+        StringBuilder sb = new StringBuilder();
+        for (StringBuilder s : records)
+        {
+            if(s!=null) {
+                sb.append(s);
+                sb.append("\n");
+            }
+            else{
+                logger.info("NULL Found!");
+            }
+        }
+        return sb.toString();
+    }
+
+    private String stringify(ArrayList<StringBuilder> records) {
+        StringBuilder sb = new StringBuilder();
+        for (StringBuilder s : records)
+        {
+            sb.append(s);
+            sb.append("\n");
+        }
+        return sb.toString();
+    }
+
+
+    public static <T> List<List<T>> batches(List<T> input, int chunkSize) {
+
+        int inputSize = input.size();
+        int chunkCount = (int) Math.ceil(inputSize / (double) chunkSize);
+        logger.info("chunkCount = "+chunkCount);
+
+        Map<Integer, List<T>> map = new HashMap<>(chunkCount);
+        List<List<T>> chunks = new ArrayList<>(chunkCount);
+
+        for (int i = 0; i < inputSize; i++) {
+
+            map.computeIfAbsent(i / chunkSize, (ignore) -> {
+
+                List<T> chunk = new ArrayList<>();
+                chunks.add(chunk);
+                return chunk;
+
+            }).add(input.get(i));
+        }
+
+        return chunks;
     }
 
     @Override
@@ -238,17 +327,18 @@ public class EmbulkOutputDomoOutputPlugin
         private final PageReader pageReader;
         private DomoClient client;
         private PluginTask task;
-        private TimestampFormatter[] timestampFormatters;
+
         private Schema schema;
 
+
         public DomoPageOutput(final PageReader pageReader,
                                     DomoClient client, PluginTask task, Schema schema)
         {
+            //logger.info("NEW PAGE CONSTRUCTOR!!");
             this.pageReader = pageReader;
             this.client = client;
             this.task = task;
             this.schema = schema;
-            this.timestampFormatters = Timestamps.newTimestampColumnFormatters(task, schema, task.getColumnOptions());
         }
 
         @Override
@@ -256,6 +346,7 @@ public class EmbulkOutputDomoOutputPlugin
         {
             try {
                 pageReader.setPage(page);
+                //logger.info("NEW PAGE ADD!! " + page);
                 final char delimiter = ',';
                 final String delimiterString = ",";
                 final String nullString = "";
@@ -263,34 +354,8 @@ public class EmbulkOutputDomoOutputPlugin
                 final char quote = this.task.getQuoteChar() != '\0' ? this.task.getQuoteChar() : '"';
                 final char escape = this.task.getEscapeChar().or(quotePolicy == QuotePolicy.NONE ? '\\' : quote);
                 final String newlineInField = this.task.getNewlineInField().getString();
+                //ArrayList<StringBuilder> records = new ArrayList<StringBuilder>();
 
-                ArrayList<StringBuilder> records = new ArrayList<StringBuilder>();
-                StreamClient sdsClient = client.streamClient();
-
-                //Build DataSet to populate the create stream request
-                CreateDataSetRequest ds = new CreateDataSetRequest();
-                ds.setName(task.getStreamName());
-                ds.setDescription("Embulk Data Set");
-                // ds.setSchema(new com.domo.sdk.datasets.model.Schema(Lists.newArrayList(new Column(STRING, "Friend"), new Column(STRING, "Attending"))));
-                ds.setSchema(getDomoSchema(this.schema));
-
-                //Create Stream
-                StreamRequest sdsRequest = new StreamRequest();
-                sdsRequest.setDataSet(ds);
-                if (this.task.getUpdateMethod() == "REPLACE"){
-                    sdsRequest.setUpdateMethod(UpdateMethod.REPLACE);
-                }
-                else{
-                    sdsRequest.setUpdateMethod(UpdateMethod.APPEND);
-                }
-                Stream sds = sdsClient.create(sdsRequest);
-                System.out.println("Created:" + sds);
-
-                //Create Execution
-                Execution execution = sdsClient.createExecution(sds.getId());
-                System.out.println("Created Execution: " + execution);
-                int partNum = 1;
-                TimestampFormatter[] tFormatters = this.timestampFormatters;
                 while (pageReader.nextRecord()) {
                     StringBuilder lineBuilder = new StringBuilder();
                     pageReader.getSchema().visitColumns(new ColumnVisitor() {
@@ -324,7 +389,7 @@ public class EmbulkOutputDomoOutputPlugin
                             addDelimiter(column);
                             if (!pageReader.isNull(column)) {
                                 Timestamp value = pageReader.getTimestamp(column);
-                                addValue(tFormatters[column.getIndex()].format(value));
+                                addValue(timestampFormatters[column.getIndex()].format(value));
                             } else {
                                 addNullString();
                             }
@@ -369,19 +434,15 @@ public class EmbulkOutputDomoOutputPlugin
                         }
 
                     });
-                    records.add(lineBuilder);
+                    //records.add(lineBuilder);
+                    allRecords.add(lineBuilder);
+                    //logger.info("Records size = "+ records.size());
+                    totalRecords++;
+                }
+                logger.info(" records = " + totalRecords);
 
-                    if (records.size() >= this.task.getBatchSize()) {
-                        this.executeDomoUpload(partNum, this.stringify(records),  sdsClient, sds, execution);
-                        partNum++;
 
-                        logger.info("Number of processed records: {}", records.size());
-                    }
-                }
 
-                //Delete Stream
-                // sdsClient.delete(sds.getId());
-                System.out.println("Deleting Dataset: " + sds);
             }
             catch (Exception ex) {
                 throw new RuntimeException(ex);
@@ -391,6 +452,7 @@ public class EmbulkOutputDomoOutputPlugin
         @Override
         public void finish()
         {
+
         }
 
         @Override
@@ -414,20 +476,14 @@ public class EmbulkOutputDomoOutputPlugin
         for (StringBuilder s : records)
         {
             sb.append(s);
-            logger.info("Appending {}"+s.toString());
+            //logger.info("Appending {}"+s.toString());
             sb.append("\n");
         }
         return sb.toString();
-
     }
 
     private void executeDomoUpload(int partNum, String csvInput, StreamClient sdsClient, Stream sds, Execution execution) {
         sdsClient.uploadDataPart(sds.getId(), execution.getId(), partNum, csvInput);
-
-        //Commit Execution
-        Execution committedExecution = sdsClient.commitExecution(sds.getId(), execution.getId());
-        System.out.println("Committed Execution: " + committedExecution);
-
     }
 
     private String setEscapeAndQuoteValue(String v, char delimiter, QuotePolicy policy, char quote, char escape, String newline, String nullString)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-embulk_output_domo
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Angelos Alexopoulos
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-10-16 00:00:00.000000000 Z
+date: 2018-10-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -64,9 +64,9 @@ files:
 - classpath/junit-4.12.jar
 - classpath/okhttp-3.7.0.jar
 - classpath/logging-interceptor-3.7.0.jar
+- classpath/embulk-output-embulk_output_domo-0.2.0.jar
 - classpath/hamcrest-core-1.3.jar
 - classpath/okio-1.12.0.jar
-- classpath/embulk-output-embulk_output_domo-0.1.0.jar
 homepage: https://github.com/angelosalexopoulos/embulk-output-embulk_output_domo
 licenses:
 - MIT