embulk-output-embulk_output_domo 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 36ad85a24723e4f76d4c7acb15571978f78ecde9
-  data.tar.gz: b37e2fe3c7766a2c4faeb8d363e758933ada969c
+  metadata.gz: 3fb7cddec754cbf91746ecdf64cb1f1e60a83629
+  data.tar.gz: de3b04d810197478663fc110bff6b271738de0d6
 SHA512:
-  metadata.gz: bf28f55db720cde8fc932521b0fae8f32d8be9d7154da7255ccdba63d1ab2266ad1600856b063e2473c5cf7874dad4437bffe08334631f76dffe40274e6b6c64
-  data.tar.gz: 86e84fa082c55f484676c5ca34d2bf3622d0e3530c256706fa6eeeada9b147af1e7d3c731406fe5db9761bcdc95d95e9b56d5023d9500d267fd52d17beaa915b
+  metadata.gz: c172cf5cc559b1a1824f339b385a11a50300678f5d196879b2f22a58c8e1c3d835d01e8cdcdf83190c594f4096e9faa4b3bcee3a45267eb743e997d48b668849
+  data.tar.gz: 42dc9b0078667bf455be49bf3210befd62132e65e94a19a779e42a558894e2d19d848963e2f0150af07cb47cf4c8b2fb2d3b4e33fadc31de288e72871d52b427

data/README.md

@@ -1,14 +1,20 @@
 # Embulk Output Domo output plugin for Embulk
-
-TODO: Write short description here and build.gradle file.
+This embulk output plugin sends data to a Domo Dataset using domo Stream API.
+Using the [domo jdk](https://github.com/domoinc/domo-java-sdk) we can move data from embulk input to domo Stream API.
 
 ## Overview
 
 * **Plugin type**: output
-* **Load all or nothing**: no
+* **Load all or nothing**: yes
 * **Resume supported**: no
 * **Cleanup supported**: yes
 
+
+## Install
+```
+embulk gem install embulk-output-embulk_output_domo 
+```
+
 ## Configuration
 
 - **clientId**: description (string, required)
@@ -29,12 +35,11 @@ TODO: Write short description here and build.gradle file.
 ```yaml
 out:
   type: embulk_output_domo
-  clientId: 209410f4-052a-4a25-8191-3475aecab631
-  clientSecret: 00fac62163e9ccea36de4614473f17e18dbf0c9860e7616007a70949c33dc2d6
+  clientId: 209410f4
+  clientSecret: 00
   apiHost: api.domo.com
   useHttps: true
-  streamName: Daily Metrics Test--Angelos
-  updateMethod: APPEND
+  streamName: Daily Metrics Test
   batchSize: 500
 ```
 

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.2.0"
+version = "0.2.1"
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8

data/gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,6 @@
+#Thu Oct 18 13:21:42 EEST 2018
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-4.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.1-all.zip

data/src/main/java/org/embulk/output/embulk_output_domo/EmbulkOutputDomoOutputPlugin.java

@@ -66,6 +66,7 @@ public class EmbulkOutputDomoOutputPlugin
     private static ArrayList<String> recordsParts = new ArrayList<String>();
     private int currentPartCounter = 1;
     private static int totalBatches = 1;
+    private static int pageReaderCount = 0;
 
     public enum QuotePolicy
     {
@@ -86,7 +87,6 @@ public class EmbulkOutputDomoOutputPlugin
         }
     }
 
-
     public interface TimestampColumnOption
             extends Task, TimestampFormatter.TimestampColumnOption
     {
@@ -184,7 +184,6 @@ public class EmbulkOutputDomoOutputPlugin
         }
     }
 
-
     @Override
     public ConfigDiff transaction(ConfigSource config,
                                   Schema schema, int taskCount,
@@ -262,55 +261,6 @@ public class EmbulkOutputDomoOutputPlugin
         logger.info("Committed Execution: " + committedExecution);
     }
 
-    private String stringifyList(List<StringBuilder> records){
-        StringBuilder sb = new StringBuilder();
-        for (StringBuilder s : records)
-        {
-            if(s!=null) {
-                sb.append(s);
-                sb.append("\n");
-            }
-            else{
-                logger.info("NULL Found!");
-            }
-        }
-        return sb.toString();
-    }
-
-    private String stringify(ArrayList<StringBuilder> records) {
-        StringBuilder sb = new StringBuilder();
-        for (StringBuilder s : records)
-        {
-            sb.append(s);
-            sb.append("\n");
-        }
-        return sb.toString();
-    }
-
-
-    public static <T> List<List<T>> batches(List<T> input, int chunkSize) {
-
-        int inputSize = input.size();
-        int chunkCount = (int) Math.ceil(inputSize / (double) chunkSize);
-        logger.info("chunkCount = "+chunkCount);
-
-        Map<Integer, List<T>> map = new HashMap<>(chunkCount);
-        List<List<T>> chunks = new ArrayList<>(chunkCount);
-
-        for (int i = 0; i < inputSize; i++) {
-
-            map.computeIfAbsent(i / chunkSize, (ignore) -> {
-
-                List<T> chunk = new ArrayList<>();
-                chunks.add(chunk);
-                return chunk;
-
-            }).add(input.get(i));
-        }
-
-        return chunks;
-    }
-
     @Override
     public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int taskIndex)
     {
@@ -329,7 +279,7 @@ public class EmbulkOutputDomoOutputPlugin
         private PluginTask task;
 
         private Schema schema;
-
+        ArrayList<StringBuilder> recordsPage = new ArrayList<StringBuilder>();
 
         public DomoPageOutput(final PageReader pageReader,
                                     DomoClient client, PluginTask task, Schema schema)
@@ -346,7 +296,9 @@ public class EmbulkOutputDomoOutputPlugin
         {
             try {
                 pageReader.setPage(page);
-                //logger.info("NEW PAGE ADD!! " + page);
+                pageReaderCount+=pageReader.getRecordCount(page);
+                logger.info("Page Reader Count = "+pageReader.getRecordCount(page) +" and page reader total count = " + pageReaderCount);
+
                 final char delimiter = ',';
                 final String delimiterString = ",";
                 final String nullString = "";
@@ -354,8 +306,9 @@ public class EmbulkOutputDomoOutputPlugin
                 final char quote = this.task.getQuoteChar() != '\0' ? this.task.getQuoteChar() : '"';
                 final char escape = this.task.getEscapeChar().or(quotePolicy == QuotePolicy.NONE ? '\\' : quote);
                 final String newlineInField = this.task.getNewlineInField().getString();
-                //ArrayList<StringBuilder> records = new ArrayList<StringBuilder>();
 
+                //int countRecordsBefore = totalRecords;
+                //int i = 1;
                 while (pageReader.nextRecord()) {
                     StringBuilder lineBuilder = new StringBuilder();
                     pageReader.getSchema().visitColumns(new ColumnVisitor() {
@@ -434,15 +387,13 @@ public class EmbulkOutputDomoOutputPlugin
                         }
 
                     });
-                    //records.add(lineBuilder);
-                    allRecords.add(lineBuilder);
-                    //logger.info("Records size = "+ records.size());
+            //        allRecords.add(lineBuilder);
+                    recordsPage.add(lineBuilder);
                     totalRecords++;
+                //    i++;
                 }
-                logger.info(" records = " + totalRecords);
-
-
 
+              //  logger.info("Added " + (totalRecords-countRecordsBefore) +" records = " + totalRecords +" all records = "+allRecords.size() +" MISSING = "+(pageReader.getRecordCount(page) - i));
             }
             catch (Exception ex) {
                 throw new RuntimeException(ex);
@@ -452,12 +403,15 @@ public class EmbulkOutputDomoOutputPlugin
         @Override
         public void finish()
         {
-
+            logger.info("FINISH");
         }
 
         @Override
         public void close()
         {
+            logger.info("CLOSE");
+            logger.info("Records page = "+recordsPage.size());
+            allRecords.addAll(recordsPage);
         }
 
         @Override
@@ -468,22 +422,9 @@ public class EmbulkOutputDomoOutputPlugin
         @Override
         public TaskReport commit()
         {
-            return Exec.newTaskReport();
+            logger.info("COMMIT"); return Exec.newTaskReport();
         }
 
-    private String stringify(ArrayList<StringBuilder> records) {
-        StringBuilder sb = new StringBuilder();
-        for (StringBuilder s : records)
-        {
-            sb.append(s);
-            //logger.info("Appending {}"+s.toString());
-            sb.append("\n");
-        }
-        return sb.toString();
-    }
-
-    private void executeDomoUpload(int partNum, String csvInput, StreamClient sdsClient, Stream sds, Execution execution) {
-        sdsClient.uploadDataPart(sds.getId(), execution.getId(), partNum, csvInput);
     }
 
     private String setEscapeAndQuoteValue(String v, char delimiter, QuotePolicy policy, char quote, char escape, String newline, String nullString)
@@ -538,5 +479,51 @@ public class EmbulkOutputDomoOutputPlugin
 
         return String.valueOf(quote) + v + quote;
     }
+
+    private String stringifyList(List<StringBuilder> records){
+        StringBuilder sb = new StringBuilder();
+        for (StringBuilder s : records)
+        {
+            if(s!=null) {
+                sb.append(s);
+                sb.append("\n");
+            }
+            else{
+                logger.info("NULL Found!");
+            }
+        }
+        return sb.toString();
+    }
+
+    private String stringify(ArrayList<StringBuilder> records) {
+        StringBuilder sb = new StringBuilder();
+        for (StringBuilder s : records)
+        {
+            sb.append(s);
+            sb.append("\n");
+        }
+        return sb.toString();
+    }
+    public static <T> List<List<T>> batches(List<T> input, int chunkSize) {
+
+        int inputSize = input.size();
+        int chunkCount = (int) Math.ceil(inputSize / (double) chunkSize);
+        logger.info("chunkCount = "+chunkCount + " and input size = "+inputSize);
+
+        Map<Integer, List<T>> map = new HashMap<>(chunkCount);
+        List<List<T>> chunks = new ArrayList<>(chunkCount);
+
+        for (int i = 0; i < inputSize; i++) {
+
+            map.computeIfAbsent(i / chunkSize, (ignore) -> {
+
+                List<T> chunk = new ArrayList<>();
+                chunks.add(chunk);
+                return chunk;
+
+            }).add(input.get(i));
+        }
+
+        return chunks;
     }
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-embulk_output_domo
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Angelos Alexopoulos
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-10-17 00:00:00.000000000 Z
+date: 2018-10-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -64,7 +64,7 @@ files:
 - classpath/junit-4.12.jar
 - classpath/okhttp-3.7.0.jar
 - classpath/logging-interceptor-3.7.0.jar
-- classpath/embulk-output-embulk_output_domo-0.2.0.jar
+- classpath/embulk-output-embulk_output_domo-0.2.1.jar
 - classpath/hamcrest-core-1.3.jar
 - classpath/okio-1.12.0.jar
 homepage: https://github.com/angelosalexopoulos/embulk-output-embulk_output_domo