embulk-output-embulk_output_domo 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a0f0e72f5092ee17a2a9f653daf209c54afd686e
-  data.tar.gz: 9d4343c4a5f4972d6018a2f5e5e1c49b9dddfd3a
+  metadata.gz: a43474cd86b39442338ad57734bcc0c70fe123ae
+  data.tar.gz: 05c2e28fecf1292959564790a9c9a141e06292f9
 SHA512:
-  metadata.gz: a69775fdbc1c12340e899ccbf18d6f9c8fa53340fbf7af47ffc0b279183c3ff25d4cfc762529081ccae6cc52e73ef72955e35d052b26f4349a260e3793a883e8
-  data.tar.gz: 298b879ef0206ca9522330b13ae1e4b1678c0aad1d2141185598986d76e1f3be42a0883bc2d8ff16d4b386385803bfeb39233b409e52d40b902bcaa57523818c
+  metadata.gz: 2e41586ca7938d74a76776a589488cb493f873bf452f62fbef6b9012b7092b9e58890a810768e4ff9e2953df0a55db1a1a414dce2a3d33c09f2f18628e04f1f7
+  data.tar.gz: a41debd9b92b3febe60154e9a044e3b420315ad6c8f1cc8d846393ddff3199811192188f0550a63f0e34edbc7fea32d13ece05b987f99ca7cf0862d4af181bd2

data/README.md

@@ -23,7 +23,7 @@ embulk gem install embulk-output-embulk_output_domo
 - **useHttps**: description (boolean, default: `true`)
 - **streamName**: description (string, required)
 - **column_options**: description (object, default: `Check embulk column options`)
-- **batchSize**: description (integer, default: `1000000`)
+- **batchSize**: description (integer, default: `1000`)
 - **quote**: description (string, default: `"\""`)
 - **quote_policy**: description (string, default: `MINIMAL`)
 - **escape**: description (string, default: `null`)

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.3.0"
+version = "0.3.1"
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8

data/src/main/java/org/embulk/output/embulk_output_domo/EmbulkOutputDomoOutputPlugin.java

@@ -87,7 +87,7 @@ public class EmbulkOutputDomoOutputPlugin
     private static ArrayList<StringBuilder> allRecords = new ArrayList<StringBuilder>();
     private static ArrayList<String> recordsParts = new ArrayList<String>();
     private int currentPartCounter = 1;
-    private static int totalBatches = 1;
+    public static int totalBatches = 1;
     private static int pageReaderCount = 0;
     private static String TEMP_DIR = "/tmp/csv/" +RandomStringUtils.randomAlphabetic(10)+"/";
     public static int totalRecordsCounter = 0;
@@ -131,10 +131,6 @@ public class EmbulkOutputDomoOutputPlugin
         @ConfigDefault("true")
         public boolean getUseHttps();
 
-        @Config("updateMethod")
-        @ConfigDefault("REPLACE")
-        public String getUpdateMethod();
-
         @Config("streamName")
         public String getStreamName();
 
@@ -142,8 +138,9 @@ public class EmbulkOutputDomoOutputPlugin
         @ConfigDefault("{}")
         Map<String, TimestampColumnOption> getColumnOptions();
 
+        // Not used we get data in batches from input plugin, so we can configure there
         @Config("batchSize")
-        @ConfigDefault("1000000")
+        @ConfigDefault("1000")
         public int getBatchSize();
 
         @Config("quote")
@@ -162,48 +159,7 @@ public class EmbulkOutputDomoOutputPlugin
         @ConfigDefault("\"LF\"")
         Newline getNewlineInField();
     }
-    public com.domo.sdk.datasets.model.Schema getDomoSchema(Schema schema){
-        /**
-         * We need to return domo Schema
-         * e.g. new com.domo.sdk.datasets.model.Schema(Lists.newArrayList(new Column(STRING, "Friend"), new Column(STRING, "Attending")))
-         */
-        ArrayList<com.domo.sdk.datasets.model.Column> domoSchema =  new ArrayList<com.domo.sdk.datasets.model.Column>();
-        for (int i = 0; i < schema.size(); i++) {
-            Column column = schema.getColumn(i);
-            Type type = column.getType();
-            System.out.println("{\n" +
-                    "      \"type\" : \""+type.getName().toUpperCase()+"\",\n" +
-                    "      \"name\" : \""+column.getName() +"\"\n" +
-                    "    },");
-            switch (type.getName()) {
-                case "long":
-                    domoSchema.add(new com.domo.sdk.datasets.model.Column(ColumnType.LONG, column.getName()));
-                    break;
-                case "double":
-                    domoSchema.add(new com.domo.sdk.datasets.model.Column(ColumnType.DOUBLE, column.getName()));
-                    break;
-                case "boolean":
-                    domoSchema.add(new com.domo.sdk.datasets.model.Column( ColumnType.LONG, column.getName()));
-                    break;
-                case "string":
-                    domoSchema.add(new com.domo.sdk.datasets.model.Column(ColumnType.STRING, column.getName()));
-                    break;
-                case "timestamp":
-                    domoSchema.add(new com.domo.sdk.datasets.model.Column( ColumnType.DATETIME, column.getName()));
-                    break;
-                default:
-                    logger.info("Unsupported type " + type.getName());
-                    break;
-            }
-        }
-        if(domoSchema != null && domoSchema.size()>0){
-            return new com.domo.sdk.datasets.model.Schema(domoSchema);
-        }
-        else{
-            logger.error("Cannot create domo schema");
-            throw new RuntimeException("Cannot create domo Schema");
-        }
-    }
+
 
     @Override
     public ConfigDiff transaction(ConfigSource config,
@@ -217,7 +173,6 @@ public class EmbulkOutputDomoOutputPlugin
         final String clientSecret = task.getClientSecret();
         final String apiHost = task.getApiHost();
         final boolean useHttps = task.getUseHttps();
-        String updateMethod = task.getUpdateMethod();
 
         try {
             if (client == null) {
@@ -243,7 +198,6 @@ public class EmbulkOutputDomoOutputPlugin
                 totalBatches = task.getBatchSize();
                 File directory = new File(TEMP_DIR);
                 if(!directory.exists()) {
-
                     directory.mkdirs();
                 }
             }
@@ -345,17 +299,7 @@ public class EmbulkOutputDomoOutputPlugin
             this.client = client;
             this.task = task;
             this.schema = schema;
-            this.partPageNum = partNum++;
 
-            try {
-                File directory = new File(TEMP_DIR);
-                if (!directory.exists()) {
-                    directory.mkdir();
-                }
-            }
-            catch(Exception ex){
-                System.out.println(ex.getMessage());
-            }
             this.partPageNum = partNum++;
             this.quotePolicy = this.task.getQuotePolicy();
             this.quote = this.task.getQuoteChar() != '\0' ? this.task.getQuoteChar() : '"';
@@ -366,6 +310,10 @@ public class EmbulkOutputDomoOutputPlugin
             this.nullString = "";
         }
 
+        /**
+         * Main Transactional Page that loops
+         * @param page
+         */
         @Override
         public void add(Page page)
         {
@@ -487,8 +435,20 @@ public class EmbulkOutputDomoOutputPlugin
         {
             return Exec.newTaskReport();
         }
-
     }
+    /************************ H E L P E R   M E T H O D S *****************************/
+
+    /**
+     *
+     * @param v String value
+     * @param delimiter csv delimeter
+     * @param policy enum QuotePolicy
+     * @param quote Quote Character
+     * @param escape Escape Character
+     * @param newline NewLine Character
+     * @param nullString Null string
+     * @return String
+     */
     private String setEscapeAndQuoteValue(String v, char delimiter, QuotePolicy policy, char quote, char escape, String newline, String nullString)
     {
         StringBuilder escapedValue = new StringBuilder();
@@ -535,25 +495,41 @@ public class EmbulkOutputDomoOutputPlugin
             return escapedValue.toString();
         }
     }
+
+    /**
+     * Quote a string
+     * @param v
+     * @param quote
+     * @return String
+     */
     private String setQuoteValue(String v, char quote)
     {
-
         return String.valueOf(quote) + v + quote;
     }
-    private String stringifyList(List<StringBuilder> records){
-        StringBuilder sb = new StringBuilder();
-        for (StringBuilder s : records)
-        {
-            if(s!=null) {
-                sb.append(s);
-                sb.append("\n");
-            }
-            else{
-                logger.info("NULL Found!");
+
+    /**
+     * Return a list of all CSV files inside a folder
+     * @param searchFolder String
+     * @return an Arraylist of File Objects
+     */
+    public static ArrayList<File> loadCSVFiles (String searchFolder) {
+        File folder = new File(searchFolder);
+        File[] listOfFiles = folder.listFiles();
+        ArrayList<File> csvFiles = new ArrayList<File>();
+
+        for (File file : listOfFiles) {
+            if (file.isFile() && file.getName().indexOf(".csv")>0) {
+                csvFiles.add(file);
             }
         }
-        return sb.toString();
+        return csvFiles;
     }
+
+    /**
+     * Stringify an ArrayList of StringBuilder to a  String
+     * @param records ArrayList of <StringBuilder>
+     * @return String
+     */
     private String stringify(ArrayList<StringBuilder> records) {
         StringBuilder sb = new StringBuilder();
         for (StringBuilder s : records)
@@ -563,6 +539,14 @@ public class EmbulkOutputDomoOutputPlugin
         }
         return sb.toString();
     }
+
+    /**
+     * Not used currently. It slices a List of a Templated input to chunkSize
+     * @param input List<T>
+     * @param chunkSize Int
+     * @param <T>
+     * @return
+     */
     public static <T> List<List<T>> batches(List<T> input, int chunkSize) {
 
         int inputSize = input.size();
@@ -584,15 +568,15 @@ public class EmbulkOutputDomoOutputPlugin
 
         return chunks;
     }
-    public static String readFileAsString(String fileName)throws Exception
-    {
-        String data = "";
-        data = new String(Files.readAllBytes(Paths.get(fileName)));
-        return data;
-    }
+
+    /**
+     * Create a List of Zip files. Each zip file will contain a batch of csv files
+     * @param sourceFiles A List of Source <Files>
+     * @param path string
+     * @return List<File>
+     */
     public static List<File> toGzipFilesUTF8( List<File> sourceFiles, String path){
         List<File> files = new ArrayList<>();
-        int batchMaxCount = 1000;
         int currentCount = 0;
         int remaining = sourceFiles.size();
         ArrayList<File> batchFiles = new ArrayList<File>();
@@ -600,8 +584,8 @@ public class EmbulkOutputDomoOutputPlugin
         for (File sourceFile : sourceFiles) {
             currentCount++;
             batchFiles.add(sourceFile);
-            if(currentCount>=batchMaxCount || currentCount>=remaining){
-                remaining=remaining-batchMaxCount;
+            if(currentCount>=totalBatches || currentCount>=remaining){
+                remaining=remaining-totalBatches;
                 String zipFileName = sourceFile.getName().replace(".csv", ".zip");
                 files.add(toGzipFileUTF8(batchFiles, path + zipFileName));
                 // System.out.println("Add file "+sourceFile.getName()+"to zip file name = "+zipFileName+". Current count = "+currentCount +" Total records counter = "+totalRecordsCounter);
@@ -613,6 +597,13 @@ public class EmbulkOutputDomoOutputPlugin
         }
         return files;
     }
+
+    /**
+     * Read csv Files as UTF-8, convert to String
+     * @param csvFiles
+     * @param zipFilePath
+     * @return a Zip File
+     */
     public static File toGzipFileUTF8(ArrayList<File> csvFiles, String zipFilePath){
         File outputFile = new File(zipFilePath);
         try {
@@ -639,6 +630,13 @@ public class EmbulkOutputDomoOutputPlugin
 
         return outputFile;
     }
+
+    /**
+     * Writes a CSV File
+     * @param fileContent
+     * @param fileName
+     * @throws IOException
+     */
     public static void WriteToFile(String fileContent, String fileName) throws IOException {
         //logger.info("writing csv file to "+fileName);
 
@@ -660,18 +658,51 @@ public class EmbulkOutputDomoOutputPlugin
         bw.close();
     }
 
-    public static ArrayList<File> loadCSVFiles (String searchFolder) {
-        File folder = new File(searchFolder);
-        File[] listOfFiles = folder.listFiles();
-        ArrayList<File> csvFiles = new ArrayList<File>();
-
-            for (File file : listOfFiles) {
-                if (file.isFile() && file.getName().indexOf(".csv")>0) {
-                    csvFiles.add(file);
-                }
+    /**
+     * Get Domo Schema
+     * @param schema
+     * @return
+     */
+    public com.domo.sdk.datasets.model.Schema getDomoSchema(Schema schema){
+        /**
+         * We need to return domo Schema
+         * e.g. new com.domo.sdk.datasets.model.Schema(Lists.newArrayList(new Column(STRING, "Friend"), new Column(STRING, "Attending")))
+         */
+        ArrayList<com.domo.sdk.datasets.model.Column> domoSchema =  new ArrayList<com.domo.sdk.datasets.model.Column>();
+        for (int i = 0; i < schema.size(); i++) {
+            Column column = schema.getColumn(i);
+            Type type = column.getType();
+            System.out.println("{\n" +
+                    "      \"type\" : \""+type.getName().toUpperCase()+"\",\n" +
+                    "      \"name\" : \""+column.getName() +"\"\n" +
+                    "    },");
+            switch (type.getName()) {
+                case "long":
+                    domoSchema.add(new com.domo.sdk.datasets.model.Column(ColumnType.LONG, column.getName()));
+                    break;
+                case "double":
+                    domoSchema.add(new com.domo.sdk.datasets.model.Column(ColumnType.DOUBLE, column.getName()));
+                    break;
+                case "boolean":
+                    domoSchema.add(new com.domo.sdk.datasets.model.Column( ColumnType.LONG, column.getName()));
+                    break;
+                case "string":
+                    domoSchema.add(new com.domo.sdk.datasets.model.Column(ColumnType.STRING, column.getName()));
+                    break;
+                case "timestamp":
+                    domoSchema.add(new com.domo.sdk.datasets.model.Column( ColumnType.DATETIME, column.getName()));
+                    break;
+                default:
+                    logger.info("Unsupported type " + type.getName());
+                    break;
             }
-
-        //System.out.println("All csv files are "+ csvFiles);
-        return csvFiles;
+        }
+        if(domoSchema != null && domoSchema.size()>0){
+            return new com.domo.sdk.datasets.model.Schema(domoSchema);
+        }
+        else{
+            logger.error("Cannot create domo schema");
+            throw new RuntimeException("Cannot create domo Schema");
+        }
     }
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-embulk_output_domo
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Angelos Alexopoulos
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-10-22 00:00:00.000000000 Z
+date: 2018-10-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -67,7 +67,7 @@ files:
 - classpath/commons-io-2.6.jar
 - classpath/hamcrest-core-1.3.jar
 - classpath/okio-1.12.0.jar
-- classpath/embulk-output-embulk_output_domo-0.3.0.jar
+- classpath/embulk-output-embulk_output_domo-0.3.1.jar
 homepage: https://github.com/alexopoulos7/embulk-output-embulk_output_domo
 licenses:
 - MIT