embulk-input-marketo_extended 0.6.20 → 0.6.25

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1688ab62e23160e81799499798c02db8b509bf93
-  data.tar.gz: 6c32a444547d345af6c87522182199a04aff60d7
+  metadata.gz: 32159ad8f9fad3a8131bb16fd62735b23cfe8e42
+  data.tar.gz: adfe704d4f79971110e3d83aebb801ea52fa35ba
 SHA512:
-  metadata.gz: df3fead3cee56763900cff8f45a20ac804556c097aa283995aa5f8bc533248448ef06008b971783bc08d745b77cea5c5e04a5be5ac80fbdda9034a70f1480cff
-  data.tar.gz: 7fc26e3dba96dc1e5851b3f52e81a29ac31cef3ffdc5d38e9f16318e27e69c7b18ee5f14bcddb89a8ff3bbdb2d1c5344253720fecc172c3004e23240d1a822f7
+  metadata.gz: 8d24bcd0780d4c3081cc7b8592fc8c854f0f33ddc6ec786ee33a31b5fe3d2e91a35e1514583892bc1348d4da51177564854bdf176413559ba4e61e00530a8b29
+  data.tar.gz: fc11f862364a6915c6266a464559afc5362e56a05e171175e73783b1fe7fed848f1794e7f5552257ed275d9e220d80a595a65fa4d3665cc63adb5db015d79ef6

data/build.gradle

@@ -16,11 +16,13 @@ repositories {
 configurations {
     provided
 }
-version = "0.6.20"
+version = "0.6.25"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
+
 dependencies {
+
     compile  "org.embulk:embulk-core:0.8.+"
     provided "org.embulk:embulk-core:0.8.+"
     compile  "org.embulk.base.restclient:embulk-base-restclient:0.5.3"

data/src/main/java/org/embulk/input/marketo/CsvTokenizer.java

@@ -380,7 +380,7 @@ public class CsvTokenizer
                     else if (isQuote(c)) {
                         char next = peekNextChar();
                         final char nextNext = peekNextNextChar();
-//                        Exec.getLogger(CsvTokenizer.class).info("Character is {}. Next: {}, NextNext:{}", c,next,nextNext);
+                        Exec.getLogger(CsvTokenizer.class).info("Character is {}. Next: {}, NextNext:{}", c,next,nextNext);
                         if (isQuote(next)
                                 && (quotesInQuotedFields != "ACCEPT_STRAY_QUOTES_ASSUMING_NO_DELIMITERS_IN_FIELDS"
                                 || (!isDelimiter(nextNext) && !isEndOfLine(nextNext)))) {
@@ -399,14 +399,14 @@ public class CsvTokenizer
                             }
                         } else {
                             quotedValue.append(line.substring(valueStartPos, linePos - 1));
-//                            Exec.getLogger(CsvTokenizer.class).info("Current column state is {}. Quoted Value : {}", columnState, quotedValue);
+                            Exec.getLogger(CsvTokenizer.class).info("Current column state is {}. Quoted Value : {}", columnState, quotedValue);
                             columnState = ColumnState.AFTER_QUOTED_VALUE;
                         }
                     }
                     else if (isEscape(c)) {  // isQuote must be checked first in case of quote == escape
                         // In RFC 4180, CSV's escape char is '\"'. But '\\' is often used.
                         char next = peekNextChar();
-//                        Exec.getLogger(CsvTokenizer.class).info("Esc Character is {}. Next: {}, column state{}", c,next, columnState);
+                        Exec.getLogger(CsvTokenizer.class).info("Esc Character is {}. Next: {}, column state{}", c,next, columnState);
                         if (isEndOfLine(c)) {
                             // escape end of line. TODO assuming multi-line quoted value without newline?
                             quotedValue.append(line.substring(valueStartPos, linePos));

data/src/main/java/org/embulk/input/marketo/MarketoService.java

@@ -16,7 +16,7 @@ public interface MarketoService
 
     File extractLead(Date startTime, Date endTime, List<String> extractedFields, String filterField, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond);
 
-    File extractAllActivity(List<Integer> activityTypeIds, Date startTime, Date endTime, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond);
+    File extractAllActivity(List<Integer> activityTypeIds, Date startTime, Date endTime, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond, String landingZone);
 
     Iterable<ObjectNode> getAllListLead(List<String> extractFields);
 

data/src/main/java/org/embulk/input/marketo/MarketoServiceImpl.java

@@ -21,6 +21,11 @@ import java.io.OutputStream;
 import org.apache.commons.io.FileUtils;
 import java.util.Date;
 import java.util.List;
+import java.time.format.DateTimeFormatter;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+
+//import com.amazonaws.services.s3.AmazonS3;
 
 /**
  * Created by tai.khuu on 9/6/17.
@@ -37,6 +42,12 @@ public class MarketoServiceImpl implements MarketoService
 
     private MarketoRestClient marketoRestClient;
 
+    private static Date startTime;
+
+    private DateFormat dateFormat = new SimpleDateFormat("dd-MM-yyyy");
+
+//    private AmazonS3 s3;
+
     public MarketoServiceImpl(MarketoRestClient marketoRestClient)
     {
         this.marketoRestClient = marketoRestClient;
@@ -45,6 +56,7 @@ public class MarketoServiceImpl implements MarketoService
     @Override
     public File extractLead(final Date startTime, Date endTime, List<String> extractedFields, String filterField, int pollingTimeIntervalSecond, final int bulkJobTimeoutSecond)
     {
+
         final String exportID = marketoRestClient.createLeadBulkExtract(startTime, endTime, extractedFields, filterField);
         marketoRestClient.startLeadBulkExtract(exportID);
         try {
@@ -85,9 +97,59 @@ public class MarketoServiceImpl implements MarketoService
         return total;
     }
 
+//    public boolean exists(String bucketName, String key) {
+//        try {
+//            fullObject = s3Client.getObject(new GetObjectRequest(bucketName, key));
+//            System.out.println("s3 object Content-Type: " + fullObject.getObjectMetadata().getContentType());
+//            return fullObject;
+//        } catch(AmazonServiceException e) {
+//            return false;
+//        }
+//        return true;
+//    }
+//    public String getBucket(String s3_key){
+//        if(s3_key.lenght()) {
+//            return s3_key.split('/')[0];
+//        }
+//    }
+//
+//    public String getKey(String s3_key){
+//        if(s3_key.lenght()) {
+//            String [] key_parts = s3_key.split('/');
+//            String key = "";
+//            for (String k: key_parts){
+//                key = key + "/" + k ;
+//            }
+//            return s3_key.split('/')[0];
+//        }
+//    }
+
     @Override
-    public File extractAllActivity(List<Integer> activityTypeIds, Date startTime, Date endTime, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond)
+    public File extractAllActivity(List<Integer> activityTypeIds, Date startTime, Date endTime, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond, String landingZoneFile)
     {
+        String tempFile = "/tmp/marketo_" + this.dateFormat.format(startTime) +".csv";
+        this.startTime = startTime;
+
+        try{
+            if (landingZoneFile.length()>0){
+                LOGGER.info("Let's check for temp File = {}", tempFile);
+                File f = new File(tempFile);
+                if (f.exists() && f.isFile() && f.canRead()){
+                    return f;
+                }
+//                //  Lets read and return csv file from s3
+//                s3_object =  get_s3_object(getBucket(landingZoneFile), getKey(landingZoneFile));
+//                if(s3_object){
+//                    return s3_object.getObjectContent();
+//                }
+
+            }
+        }
+        catch (Exception e){
+            System.out.println("Exception on retrieving from landing zone");
+            System.out.println("Let's continue normal execution.");
+        }
+
         final String exportID = marketoRestClient.createActivityExtract(activityTypeIds, startTime, endTime);
         marketoRestClient.startActitvityBulkExtract(exportID);
         try {
@@ -110,10 +172,8 @@ public class MarketoServiceImpl implements MarketoService
 
     private File downloadBulkExtract(Function<BulkExtractRangeHeader, InputStream> getBulkExtractfunction)
     {
-        final File tempFile = Exec.getTempFileSpace().createTempFile(DEFAULT_FILE_FORMAT);
-        LOGGER.info("Temp File = {}", tempFile.getPath());
-//        File destination = new File("/tmp/example_test.csv");
 
+        final File tempFile = Exec.getTempFileSpace().createTempFile(DEFAULT_FILE_FORMAT);
 
         long startByte = 0;
         int resumeTime = 0;
@@ -122,13 +182,17 @@ public class MarketoServiceImpl implements MarketoService
             InputStream bulkExtractResult = getBulkExtractfunction.apply(bulkExtractRangeHeader);
             try {
                 saveExtractedFile(bulkExtractResult, tempFile);
-//                try{
-//                    FileUtils.copyFile(tempFile, destination);
-//                }
-//                catch (IOException e){
-//                    LOGGER.error("Exception when copying to file destination {}\nException:{}",destination.getPath(),e );
-//                }
-
+                if (this.startTime != null){
+                    String destinationFileName = "/tmp/marketo_" + this.dateFormat.format(this.startTime) +".csv";
+                    final File destination = new File(destinationFileName);
+                    try{
+                        FileUtils.copyFile(tempFile, destination);
+                        LOGGER.info("Save activities in file: "+destinationFileName);
+                    }
+                    catch (IOException e){
+                        LOGGER.error("Exception when copying to file destination {}\nException:{}",destination.getPath(),e );
+                    }
+                }
                 return tempFile;
             }
             catch (DownloadBulkExtractException e) {

data/src/main/java/org/embulk/input/marketo/delegate/ActivityBulkExtractInputPlugin.java

@@ -143,7 +143,11 @@ public class ActivityBulkExtractInputPlugin extends MarketoBaseBulkExtractInputP
     protected InputStream getExtractedStream(MarketoService service, PluginTask task, DateTime fromDate, DateTime toDate)
     {
         try {
-            return new FileInputStream(service.extractAllActivity(task.getActTypeIds(), fromDate.toDate(), toDate.toDate(), task.getPollingIntervalSecond(), task.getBulkJobTimeoutSecond()));
+            String landing_path = "";
+            if (task.getLandingPath().isPresent()) {
+                landing_path = task.getLandingPath().get();
+            }
+            return new FileInputStream(service.extractAllActivity(task.getActTypeIds(), fromDate.toDate(), toDate.toDate(), task.getPollingIntervalSecond(), task.getBulkJobTimeoutSecond(), landing_path));
         }
         catch (FileNotFoundException e) {
             throw new RuntimeException("Exception when trying to extract activity", e);

data/src/main/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPlugin.java

@@ -65,6 +65,10 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
         @ConfigDefault("1")
         Integer getFetchDays();
 
+        @Config("landing_zone")
+        @ConfigDefault("null")
+        Optional<String> getLandingPath();
+
         @Config("latest_fetch_time")
         @ConfigDefault("null")
         Optional<Long> getLatestFetchTime();
@@ -170,9 +174,9 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
                 int imported = 0;
                 while (csvRecords.hasNext()) {
                     Map<String, String> csvRecord = csvRecords.next();
-//                    if(csvRecord.containsKey("attributes")) {
-//                        System.out.println("Attributes Before");
-//                        System.out.println(csvRecord.get("attributes"));
+                    if(csvRecord.containsKey("attributes")) {
+                        System.out.println("Attributes Before");
+                        System.out.println(csvRecord.get("attributes"));
 //
 //                        csvRecord.put("attributes", csvRecord.get("attributes").replace("\\\"\"","\\\""));
 //                        System.out.println("Attributes After");
@@ -181,7 +185,7 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
 
 //                    for (Map.Entry<String, String> entry : csvRecord.entrySet()) {
 //                        System.out.println(entry.getKey() + ":" + entry.getValue().toString());
-//                    }
+                    }
                     ObjectNode objectNode = MarketoUtils.OBJECT_MAPPER.valueToTree(csvRecord);
                     recordImporter.importRecord(new AllStringJacksonServiceRecord(objectNode), pageBuilder);
                     imported = imported + 1;
@@ -409,9 +413,7 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
         public boolean hasNext()
         {
             if (currentCsvRecord == null) {
-                LOGGER.info("Current csv record: {}", currentCsvRecord);
                 currentCsvRecord = getNextCSVRecord();
-                LOGGER.info("Next csv record: {}", currentCsvRecord);
 
             }
             return currentCsvRecord != null;

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-marketo_extended
 version: !ruby/object:Gem::Version
-  version: 0.6.20
+  version: 0.6.25
 platform: ruby
 authors:
 - uu59
@@ -11,7 +11,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-11-05 00:00:00.000000000 Z
+date: 2020-11-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -140,7 +140,7 @@ files:
 - classpath/jetty-io-9.2.14.v20151106.jar
 - classpath/commons-io-2.8.0.jar
 - classpath/jetty-util-9.2.14.v20151106.jar
-- classpath/embulk-input-marketo_extended-0.6.20.jar
+- classpath/embulk-input-marketo_extended-0.6.25.jar
 - classpath/jetty-http-9.2.14.v20151106.jar
 - classpath/jetty-client-9.2.14.v20151106.jar
 - classpath/embulk-base-restclient-0.5.3.jar