embulk-output-embulk_output_domo 0.2.5 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 06ef088bc54d8412fbfc5fe0d250e64679c2191d
-  data.tar.gz: 54c0305194812b9c1bf056e9d61cc760f516a486
+  metadata.gz: a0f0e72f5092ee17a2a9f653daf209c54afd686e
+  data.tar.gz: 9d4343c4a5f4972d6018a2f5e5e1c49b9dddfd3a
 SHA512:
-  metadata.gz: 826c5d74751cdc1951a588cdaa0a1b98a3eb296a8720bd263c36f58ab463deed9e9108acb9382bdb4275fce4a7c12579a17e21c8f2309a9986f68b179cbbd451
-  data.tar.gz: 5804e980747764f6fe29b1b9b7676d1e41df7b05367afa6fa945e80901b7e604359588c353915fd0e57abd77f528f8ad7e2d41b504e98379d746b71176b721f3
+  metadata.gz: a69775fdbc1c12340e899ccbf18d6f9c8fa53340fbf7af47ffc0b279183c3ff25d4cfc762529081ccae6cc52e73ef72955e35d052b26f4349a260e3793a883e8
+  data.tar.gz: 298b879ef0206ca9522330b13ae1e4b1678c0aad1d2141185598986d76e1f3be42a0883bc2d8ff16d4b386385803bfeb39233b409e52d40b902bcaa57523818c

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.2.5"
+version = "0.3.0"
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8
@@ -23,6 +23,8 @@ dependencies {
     provided "org.embulk:embulk-core:0.9.7"
     compile 'com.domo:domo-java-sdk-all:0.4.0'
     compile 'junit:junit:4.12'
+    compile group: 'org.apache.commons', name: 'commons-lang3', version: '3.0'
+    compile group: 'commons-io', name: 'commons-io', version: '2.6'
     testCompile "junit:junit:4.+"
 }
 

data/src/main/java/org/embulk/output/embulk_output_domo/EmbulkOutputDomoOutputPlugin.java

@@ -1,3 +1,7 @@
+/**
+ * Embulk output plugin that can move really big input to domo stream
+ * We are going to use algorithm for uploading in parallel found in https://developer.domo.com/docs/stream/upload-in-parallel
+ */
 package org.embulk.output.embulk_output_domo;
 
 import java.lang.StringBuilder;
@@ -49,6 +53,24 @@ import java.util.HashMap;
 import java.util.ArrayList;
 import java.util.Date;
 import java.text.SimpleDateFormat;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.FileReader;
+import java.io.BufferedWriter;
+import java.io.BufferedReader;
+import java.io.FileOutputStream;
+
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.zip.GZIPOutputStream;
+import java.util.Collections;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.commons.io.FileUtils;
 
 
 public class EmbulkOutputDomoOutputPlugin
@@ -56,7 +78,7 @@ public class EmbulkOutputDomoOutputPlugin
 {
     private static DomoClient client = null;
     private static Execution execution = null;
-    private static StreamClient sdsClient = null;
+    private static StreamClient streamClient = null;
     private static Stream sds = null;
     private static TimestampFormatter[] timestampFormatters = null;
     private int partNum = 1;
@@ -67,6 +89,8 @@ public class EmbulkOutputDomoOutputPlugin
     private int currentPartCounter = 1;
     private static int totalBatches = 1;
     private static int pageReaderCount = 0;
+    private static String TEMP_DIR = "/tmp/csv/" +RandomStringUtils.randomAlphabetic(10)+"/";
+    public static int totalRecordsCounter = 0;
 
     public enum QuotePolicy
     {
@@ -86,12 +110,10 @@ public class EmbulkOutputDomoOutputPlugin
             return string;
         }
     }
-
     public interface TimestampColumnOption
             extends Task, TimestampFormatter.TimestampColumnOption
     {
     }
-
     public interface PluginTask
             extends Task, TimestampFormatter.Task
     {
@@ -140,7 +162,6 @@ public class EmbulkOutputDomoOutputPlugin
         @ConfigDefault("\"LF\"")
         Newline getNewlineInField();
     }
-
     public com.domo.sdk.datasets.model.Schema getDomoSchema(Schema schema){
         /**
          * We need to return domo Schema
@@ -211,16 +232,20 @@ public class EmbulkOutputDomoOutputPlugin
                         .build();
 
                 client = DomoClient.create(domoConfig);
-                sdsClient = client.streamClient();
+                streamClient = client.streamClient();
 
-                List<Stream> searchedSds = sdsClient.search("dataSource.name:" + task.getStreamName());
+                List<Stream> searchedSds = streamClient.search("dataSource.name:" + task.getStreamName());
                 sds = searchedSds.get(0);
                 logger.info("Stream "+ sds);
-                execution = sdsClient.createExecution(sds.getId());
+                execution = streamClient.createExecution(sds.getId());
                 logger.info("Created Execution: " + execution);
                 timestampFormatters = Timestamps.newTimestampColumnFormatters(task, schema, task.getColumnOptions());
                 totalBatches = task.getBatchSize();
+                File directory = new File(TEMP_DIR);
+                if(!directory.exists()) {
 
+                    directory.mkdirs();
+                }
             }
         }
         catch(Exception ex){
@@ -232,32 +257,59 @@ public class EmbulkOutputDomoOutputPlugin
         control.run(task.dump());
         return Exec.newConfigDiff();
     }
-
     @Override
     public ConfigDiff resume(TaskSource taskSource,
-            Schema schema, int taskCount,
-            OutputPlugin.Control control)
+                             Schema schema, int taskCount,
+                             OutputPlugin.Control control)
     {
         throw new UnsupportedOperationException("embulk_output_domo output plugin does not support resuming");
     }
-
     @Override
     public void cleanup(TaskSource taskSource,
-            Schema schema, int taskCount,
-            List<TaskReport> successTaskReports)
+                        Schema schema, int taskCount,
+                        List<TaskReport> successTaskReports)
     {
-//        List<List<StringBuilder>> batchLists = batches(allRecords, totalBatches);
-//        int i=1;
-//        for(List<StringBuilder> l : batchLists){
-//            sdsClient.uploadDataPart(sds.getId(), execution.getId(), i, stringifyList(l));
-//            i++;
-//        }
-//        logger.info("Finished Uploading");
+        try{
+            ArrayList<File> csvFiles = loadCSVFiles(TEMP_DIR);
+            File tempFolder = new File(TEMP_DIR);
+            List<File> compressedCsvFiles = toGzipFilesUTF8(csvFiles, tempFolder.getPath() + "/");
+            ExecutorService executorService = Executors.newCachedThreadPool();
+            List<Callable<Object>> uploadTasks = Collections.synchronizedList(new ArrayList<>());
+
+            // For each data part (csv gzip file), create a runnable upload task
+            long partNum = 1;
+            for (File compressedCsvFile : compressedCsvFiles){
+                long myPartNum = partNum;
+                // "uploadDataPart()" accepts csv strings, csv files, and compressed csv files
+                Runnable partUpload = () -> streamClient.uploadDataPart(sds.getId(), execution.getId(), myPartNum, compressedCsvFile);
+                uploadTasks.add(Executors.callable(partUpload));
+                partNum++;
+             }
+            // Asynchronously execute all uploading tasks
+            try {
+                executorService.invokeAll(uploadTasks);
+            }
+            catch (Exception e){
+                logger.error("Error uploading all data parts", e);
+            }
+
+        }catch(Exception e) {
+            logger.error("Exception on uploading!! "+e);
+            System.out.println(e.getMessage());
+            return;
+        }
         //Commit Execution
-        Execution committedExecution = sdsClient.commitExecution(sds.getId(), execution.getId());
+        Execution committedExecution = streamClient.commitExecution(sds.getId(), execution.getId());
         logger.info("Committed Execution: " + committedExecution);
-    }
+        try {
+            FileUtils.deleteDirectory(new File(TEMP_DIR));
+            logger.info("Delete temp directory");
+        }
+        catch (IOException ex){
+            logger.error("Delete temp directory Failed "+ ex);
+        }
 
+    }
     @Override
     public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int taskIndex)
     {
@@ -265,7 +317,6 @@ public class EmbulkOutputDomoOutputPlugin
         final PageReader reader = new PageReader(schema);
         return new DomoPageOutput(reader, client, task, schema);
     }
-
     public class DomoPageOutput
             implements TransactionalPageOutput
     {
@@ -274,36 +325,55 @@ public class EmbulkOutputDomoOutputPlugin
         private final PageReader pageReader;
         private DomoClient client;
         private PluginTask task;
+        private int partPageNum;
 
         private Schema schema;
-        ArrayList<StringBuilder> recordsPage = new ArrayList<StringBuilder>();
+        ArrayList<StringBuilder> recordsPage = null;
+        private char delimiter = ',';
+        private String delimiterString = ",";
+        private String nullString = "";
+        private QuotePolicy quotePolicy = null;
+        private char quote = '"';
+        private char escape =  quote;
+        private String newlineInField;
 
         public DomoPageOutput(final PageReader pageReader,
-                                    DomoClient client, PluginTask task, Schema schema)
+                              DomoClient client, PluginTask task, Schema schema)
         {
-            //logger.info("NEW PAGE CONSTRUCTOR!!");
+            logger.info("NEW PAGE CONSTRUCTOR!!");
             this.pageReader = pageReader;
             this.client = client;
             this.task = task;
             this.schema = schema;
-            partNum++;
+            this.partPageNum = partNum++;
+
+            try {
+                File directory = new File(TEMP_DIR);
+                if (!directory.exists()) {
+                    directory.mkdir();
+                }
+            }
+            catch(Exception ex){
+                System.out.println(ex.getMessage());
+            }
+            this.partPageNum = partNum++;
+            this.quotePolicy = this.task.getQuotePolicy();
+            this.quote = this.task.getQuoteChar() != '\0' ? this.task.getQuoteChar() : '"';
+            this.escape = this.task.getEscapeChar().or(this.quotePolicy == QuotePolicy.NONE ? '\\' : this.quote);
+            this.newlineInField = this.task.getNewlineInField().getString();
+            this.delimiter = ',';
+            this.delimiterString = ",";
+            this.nullString = "";
         }
 
         @Override
         public void add(Page page)
         {
+            final StringBuilder pageBuilder = new StringBuilder();
+            //logger.info("New page");
+            this.recordsPage = new ArrayList<StringBuilder>();
             try {
                 pageReader.setPage(page);
-
-                final char delimiter = ',';
-                final String delimiterString = ",";
-                final String nullString = "";
-                final QuotePolicy quotePolicy = this.task.getQuotePolicy();
-                final char quote = this.task.getQuoteChar() != '\0' ? this.task.getQuoteChar() : '"';
-                final char escape = this.task.getEscapeChar().or(quotePolicy == QuotePolicy.NONE ? '\\' : quote);
-                final String newlineInField = this.task.getNewlineInField().getString();
-
-
                 while (pageReader.nextRecord()) {
                     StringBuilder lineBuilder = new StringBuilder();
                     pageReader.getSchema().visitColumns(new ColumnVisitor() {
@@ -380,16 +450,17 @@ public class EmbulkOutputDomoOutputPlugin
                                 addNullString();
                             }
                         }
-
                     });
-
                     recordsPage.add(lineBuilder);
-                    totalRecords++;
                 }
-
-
-
-
+                try {
+                    //save as csv
+                    WriteToFile(stringify(recordsPage), RandomStringUtils.randomNumeric(10)+RandomStringUtils.randomAlphabetic(20).toString()+".csv");
+                }
+                catch (IOException e){
+                    logger.error("Exception on closing page!");
+                    logger.error(e.getMessage());
+                }
             }
             catch (Exception ex) {
                 throw new RuntimeException(ex);
@@ -404,10 +475,6 @@ public class EmbulkOutputDomoOutputPlugin
         @Override
         public void close()
         {
-            logger.info("Uplaod records count= "+recordsPage.size() +" partNum = "+partNum);
-            sdsClient.uploadDataPart(sds.getId(), execution.getId(), partNum, stringify(recordsPage));
-            // allRecords.addAll(recordsPage);
-            recordsPage = null;
         }
 
         @Override
@@ -422,7 +489,6 @@ public class EmbulkOutputDomoOutputPlugin
         }
 
     }
-
     private String setEscapeAndQuoteValue(String v, char delimiter, QuotePolicy policy, char quote, char escape, String newline, String nullString)
     {
         StringBuilder escapedValue = new StringBuilder();
@@ -469,13 +535,11 @@ public class EmbulkOutputDomoOutputPlugin
             return escapedValue.toString();
         }
     }
-
     private String setQuoteValue(String v, char quote)
     {
 
         return String.valueOf(quote) + v + quote;
     }
-
     private String stringifyList(List<StringBuilder> records){
         StringBuilder sb = new StringBuilder();
         for (StringBuilder s : records)
@@ -490,7 +554,6 @@ public class EmbulkOutputDomoOutputPlugin
         }
         return sb.toString();
     }
-
     private String stringify(ArrayList<StringBuilder> records) {
         StringBuilder sb = new StringBuilder();
         for (StringBuilder s : records)
@@ -521,4 +584,94 @@ public class EmbulkOutputDomoOutputPlugin
 
         return chunks;
     }
-}
+    public static String readFileAsString(String fileName)throws Exception
+    {
+        String data = "";
+        data = new String(Files.readAllBytes(Paths.get(fileName)));
+        return data;
+    }
+    public static List<File> toGzipFilesUTF8( List<File> sourceFiles, String path){
+        List<File> files = new ArrayList<>();
+        int batchMaxCount = 1000;
+        int currentCount = 0;
+        int remaining = sourceFiles.size();
+        ArrayList<File> batchFiles = new ArrayList<File>();
+        //System.out.println("All source files are "+remaining);
+        for (File sourceFile : sourceFiles) {
+            currentCount++;
+            batchFiles.add(sourceFile);
+            if(currentCount>=batchMaxCount || currentCount>=remaining){
+                remaining=remaining-batchMaxCount;
+                String zipFileName = sourceFile.getName().replace(".csv", ".zip");
+                files.add(toGzipFileUTF8(batchFiles, path + zipFileName));
+                // System.out.println("Add file "+sourceFile.getName()+"to zip file name = "+zipFileName+". Current count = "+currentCount +" Total records counter = "+totalRecordsCounter);
+                batchFiles.clear();
+                currentCount = 0;
+            }
+            //System.out.println("Add file "+sourceFile.getName()+ ". Current count = "+currentCount);
+
+        }
+        return files;
+    }
+    public static File toGzipFileUTF8(ArrayList<File> csvFiles, String zipFilePath){
+        File outputFile = new File(zipFilePath);
+        try {
+            GZIPOutputStream gzos = new GZIPOutputStream(new FileOutputStream(outputFile));
+            for (File csvFile : csvFiles){
+                BufferedReader reader = new BufferedReader(new FileReader(csvFile));
+
+                String currentLine;
+                while ((currentLine = reader.readLine()) != null){
+                    currentLine += System.lineSeparator();
+                    totalRecordsCounter++;
+                    // Specifying UTF-8 encoding is critical; getBytes() uses ISO-8859-1 by default
+                    gzos.write(currentLine.getBytes("UTF-8"));
+                }
+            }
+            gzos.flush();
+            gzos.finish();
+            gzos.close();
+
+        }
+        catch(IOException e) {
+            logger.error("Error compressing a string to gzip", e);
+        }
+
+        return outputFile;
+    }
+    public static void WriteToFile(String fileContent, String fileName) throws IOException {
+        //logger.info("writing csv file to "+fileName);
+
+        String tempFile = TEMP_DIR + fileName;
+        File file = new File(tempFile);
+        // if file does exists, then delete and create a new file
+        if (file.exists()) {
+            try {
+                File newFileName = new File(TEMP_DIR + "backup_" + fileName);
+                file.renameTo(newFileName);
+                file.createNewFile();
+            } catch (IOException e) {
+                e.printStackTrace();
+            }
+        }
+        FileWriter fw = new FileWriter(file.getAbsoluteFile());
+        BufferedWriter bw = new BufferedWriter(fw);
+        bw.write(fileContent);
+        bw.close();
+    }
+
+    public static ArrayList<File> loadCSVFiles (String searchFolder) {
+        File folder = new File(searchFolder);
+        File[] listOfFiles = folder.listFiles();
+        ArrayList<File> csvFiles = new ArrayList<File>();
+
+            for (File file : listOfFiles) {
+                if (file.isFile() && file.getName().indexOf(".csv")>0) {
+                    csvFiles.add(file);
+                }
+            }
+
+        //System.out.println("All csv files are "+ csvFiles);
+        return csvFiles;
+    }
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-embulk_output_domo
 version: !ruby/object:Gem::Version
-  version: 0.2.5
+  version: 0.3.0
 platform: ruby
 authors:
 - Angelos Alexopoulos
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-10-19 00:00:00.000000000 Z
+date: 2018-10-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -64,9 +64,10 @@ files:
 - classpath/junit-4.12.jar
 - classpath/okhttp-3.7.0.jar
 - classpath/logging-interceptor-3.7.0.jar
-- classpath/embulk-output-embulk_output_domo-0.2.5.jar
+- classpath/commons-io-2.6.jar
 - classpath/hamcrest-core-1.3.jar
 - classpath/okio-1.12.0.jar
+- classpath/embulk-output-embulk_output_domo-0.3.0.jar
 homepage: https://github.com/alexopoulos7/embulk-output-embulk_output_domo
 licenses:
 - MIT