embulk-input-marketo_extended 0.6.22 → 0.6.28

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a6efba1be34ca3cd08c0883805641173eb6259c0
-  data.tar.gz: d65d6decb9c9c8fb7b135d60a42488c8ddb7f5de
+  metadata.gz: dbd134238def5af6ba480e2c6a504ffd0da4869a
+  data.tar.gz: e396ca6875915874955aef4ea151831f6883d020
 SHA512:
-  metadata.gz: 0d3558a194417bec40a6943482d8e082c1ada38acb323c410f81672ab46f8ce59445334c0b89df6cc5ff7ab4279a6e9346f2c42119b8020460605530ea8a9068
-  data.tar.gz: 7210c0fcc06ad5df7d18285e18300973d249950df83641744acd72d5492d6b1aba2852500407893a127e48508241040537ace24edd47b2aa607bd420102f13e3
+  metadata.gz: bd7084e138f808b034e055ffaccb4b1f9e02b78a24997a68ce03948af5436107c98391e86b8b9b9f124102dbf8054dcc15ebfd2664237e88d3a6c1ab2b553daf
+  data.tar.gz: 4bb8624c8f0786fe7ceaa5f2a1ad581f6af20631962ddef131e4c71814ec2943c6c8b0406f0dcc92f9adbac692a4ecd95adeb3bdb6709cfa7fc6605fd891e8c0

data/build.gradle

@@ -16,11 +16,13 @@ repositories {
 configurations {
     provided
 }
-version = "0.6.22"
+version = "0.6.28"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
+
 dependencies {
+
     compile  "org.embulk:embulk-core:0.8.+"
     provided "org.embulk:embulk-core:0.8.+"
     compile  "org.embulk.base.restclient:embulk-base-restclient:0.5.3"

data/src/main/java/org/embulk/input/marketo/CsvTokenizer.java

@@ -380,7 +380,7 @@ public class CsvTokenizer
                     else if (isQuote(c)) {
                         char next = peekNextChar();
                         final char nextNext = peekNextNextChar();
-                        Exec.getLogger(CsvTokenizer.class).info("Character is {}. Next: {}, NextNext:{}", c,next,nextNext);
+//                        Exec.getLogger(CsvTokenizer.class).info("Character is {}. Next: {}, NextNext:{}", c,next,nextNext);
                         if (isQuote(next)
                                 && (quotesInQuotedFields != "ACCEPT_STRAY_QUOTES_ASSUMING_NO_DELIMITERS_IN_FIELDS"
                                 || (!isDelimiter(nextNext) && !isEndOfLine(nextNext)))) {
@@ -399,14 +399,14 @@ public class CsvTokenizer
                             }
                         } else {
                             quotedValue.append(line.substring(valueStartPos, linePos - 1));
-                            Exec.getLogger(CsvTokenizer.class).info("Current column state is {}. Quoted Value : {}", columnState, quotedValue);
+//                            Exec.getLogger(CsvTokenizer.class).info("Current column state is {}. Quoted Value : {}", columnState, quotedValue);
                             columnState = ColumnState.AFTER_QUOTED_VALUE;
                         }
                     }
                     else if (isEscape(c)) {  // isQuote must be checked first in case of quote == escape
                         // In RFC 4180, CSV's escape char is '\"'. But '\\' is often used.
                         char next = peekNextChar();
-                        Exec.getLogger(CsvTokenizer.class).info("Esc Character is {}. Next: {}, column state{}", c,next, columnState);
+//                        Exec.getLogger(CsvTokenizer.class).info("Esc Character is {}. Next: {}, column state{}", c,next, columnState);
                         if (isEndOfLine(c)) {
                             // escape end of line. TODO assuming multi-line quoted value without newline?
                             quotedValue.append(line.substring(valueStartPos, linePos));
@@ -416,11 +416,11 @@ public class CsvTokenizer
                             }
                             valueStartPos = 0;
                         }
-                        else if ( isEscape(next)) { // escaped quote
-                            quotedValue.append(line.substring(valueStartPos, linePos - 1));
-                            quotedValue.append(next);
-                            valueStartPos = ++linePos;
-                        }
+//                        else if ( isEscape(next)) { // escaped quote
+//                            quotedValue.append(line.substring(valueStartPos, linePos - 1));
+//                            quotedValue.append(next);
+//                            valueStartPos = ++linePos;
+//                        }
                     }
                     else {
                         if ((linePos - valueStartPos) + quotedValue.length() > maxQuotedSizeLimit) {
@@ -623,7 +623,7 @@ public class CsvTokenizer
                 throw new ConfigException("\"quote\" option accepts only 1 character.");
             }
             else if (str.isEmpty()) {
-                Exec.getLogger(CsvTokenizer.class).warn("Setting '' (empty string) to \"quote\" option is obsoleted. Currently it becomes '\"' automatically but this behavior will be removed. Please set '\"' explicitly.");
+//                Exec.getLogger(CsvTokenizer.class).warn("Setting '' (empty string) to \"quote\" option is obsoleted. Currently it becomes '\"' automatically but this behavior will be removed. Please set '\"' explicitly.");
                 return new QuoteCharacter('"');
             }
             else {
@@ -684,7 +684,7 @@ public class CsvTokenizer
                 throw new ConfigException("\"escape\" option accepts only 1 character.");
             }
             else if (str.isEmpty()) {
-                Exec.getLogger(CsvTokenizer.class).warn("Setting '' (empty string) to \"escape\" option is obsoleted. Currently it becomes null automatically but this behavior will be removed. Please set \"escape: null\" explicitly.");
+//                Exec.getLogger(CsvTokenizer.class).warn("Setting '' (empty string) to \"escape\" option is obsoleted. Currently it becomes null automatically but this behavior will be removed. Please set \"escape: null\" explicitly.");
                 return noEscape();
             }
             else {

data/src/main/java/org/embulk/input/marketo/MarketoService.java

@@ -16,7 +16,7 @@ public interface MarketoService
 
     File extractLead(Date startTime, Date endTime, List<String> extractedFields, String filterField, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond);
 
-    File extractAllActivity(List<Integer> activityTypeIds, Date startTime, Date endTime, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond);
+    File extractAllActivity(List<Integer> activityTypeIds, Date startTime, Date endTime, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond, String landingZone);
 
     Iterable<ObjectNode> getAllListLead(List<String> extractFields);
 

data/src/main/java/org/embulk/input/marketo/MarketoServiceImpl.java

@@ -21,6 +21,11 @@ import java.io.OutputStream;
 import org.apache.commons.io.FileUtils;
 import java.util.Date;
 import java.util.List;
+import java.time.format.DateTimeFormatter;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+
+//import com.amazonaws.services.s3.AmazonS3;
 
 /**
  * Created by tai.khuu on 9/6/17.
@@ -37,6 +42,12 @@ public class MarketoServiceImpl implements MarketoService
 
     private MarketoRestClient marketoRestClient;
 
+    private static Date startTime;
+
+    private DateFormat dateFormat = new SimpleDateFormat("dd-MM-yyyy");
+
+//    private AmazonS3 s3;
+
     public MarketoServiceImpl(MarketoRestClient marketoRestClient)
     {
         this.marketoRestClient = marketoRestClient;
@@ -45,6 +56,7 @@ public class MarketoServiceImpl implements MarketoService
     @Override
     public File extractLead(final Date startTime, Date endTime, List<String> extractedFields, String filterField, int pollingTimeIntervalSecond, final int bulkJobTimeoutSecond)
     {
+
         final String exportID = marketoRestClient.createLeadBulkExtract(startTime, endTime, extractedFields, filterField);
         marketoRestClient.startLeadBulkExtract(exportID);
         try {
@@ -85,9 +97,59 @@ public class MarketoServiceImpl implements MarketoService
         return total;
     }
 
+//    public boolean exists(String bucketName, String key) {
+//        try {
+//            fullObject = s3Client.getObject(new GetObjectRequest(bucketName, key));
+//            System.out.println("s3 object Content-Type: " + fullObject.getObjectMetadata().getContentType());
+//            return fullObject;
+//        } catch(AmazonServiceException e) {
+//            return false;
+//        }
+//        return true;
+//    }
+//    public String getBucket(String s3_key){
+//        if(s3_key.lenght()) {
+//            return s3_key.split('/')[0];
+//        }
+//    }
+//
+//    public String getKey(String s3_key){
+//        if(s3_key.lenght()) {
+//            String [] key_parts = s3_key.split('/');
+//            String key = "";
+//            for (String k: key_parts){
+//                key = key + "/" + k ;
+//            }
+//            return s3_key.split('/')[0];
+//        }
+//    }
+
     @Override
-    public File extractAllActivity(List<Integer> activityTypeIds, Date startTime, Date endTime, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond)
+    public File extractAllActivity(List<Integer> activityTypeIds, Date startTime, Date endTime, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond, String landingZoneFile)
     {
+        String tempFile = "/tmp/marketo_" + this.dateFormat.format(startTime) +".csv";
+        this.startTime = startTime;
+
+        try{
+            if (landingZoneFile.length()>0){
+                LOGGER.info("Let's check for temp File = {}", tempFile);
+                File f = new File(tempFile);
+                if (f.exists() && f.isFile() && f.canRead()){
+                    return f;
+                }
+//                //  Lets read and return csv file from s3
+//                s3_object =  get_s3_object(getBucket(landingZoneFile), getKey(landingZoneFile));
+//                if(s3_object){
+//                    return s3_object.getObjectContent();
+//                }
+
+            }
+        }
+        catch (Exception e){
+            System.out.println("Exception on retrieving from landing zone");
+            System.out.println("Let's continue normal execution.");
+        }
+
         final String exportID = marketoRestClient.createActivityExtract(activityTypeIds, startTime, endTime);
         marketoRestClient.startActitvityBulkExtract(exportID);
         try {
@@ -110,10 +172,8 @@ public class MarketoServiceImpl implements MarketoService
 
     private File downloadBulkExtract(Function<BulkExtractRangeHeader, InputStream> getBulkExtractfunction)
     {
-        final File tempFile = Exec.getTempFileSpace().createTempFile(DEFAULT_FILE_FORMAT);
-        LOGGER.info("Temp File = {}", tempFile.getPath());
-//        File destination = new File("/tmp/example_test.csv");
 
+        final File tempFile = Exec.getTempFileSpace().createTempFile(DEFAULT_FILE_FORMAT);
 
         long startByte = 0;
         int resumeTime = 0;
@@ -122,13 +182,17 @@ public class MarketoServiceImpl implements MarketoService
             InputStream bulkExtractResult = getBulkExtractfunction.apply(bulkExtractRangeHeader);
             try {
                 saveExtractedFile(bulkExtractResult, tempFile);
-//                try{
-//                    FileUtils.copyFile(tempFile, destination);
-//                }
-//                catch (IOException e){
-//                    LOGGER.error("Exception when copying to file destination {}\nException:{}",destination.getPath(),e );
-//                }
-
+                if (this.startTime != null){
+                    String destinationFileName = "/tmp/marketo_" + this.dateFormat.format(this.startTime) +".csv";
+                    final File destination = new File(destinationFileName);
+                    try{
+                        FileUtils.copyFile(tempFile, destination);
+                        LOGGER.info("Save activities in file: "+destinationFileName);
+                    }
+                    catch (IOException e){
+                        LOGGER.error("Exception when copying to file destination {}\nException:{}",destination.getPath(),e );
+                    }
+                }
                 return tempFile;
             }
             catch (DownloadBulkExtractException e) {

data/src/main/java/org/embulk/input/marketo/delegate/ActivityBulkExtractInputPlugin.java

@@ -143,7 +143,11 @@ public class ActivityBulkExtractInputPlugin extends MarketoBaseBulkExtractInputP
     protected InputStream getExtractedStream(MarketoService service, PluginTask task, DateTime fromDate, DateTime toDate)
     {
         try {
-            return new FileInputStream(service.extractAllActivity(task.getActTypeIds(), fromDate.toDate(), toDate.toDate(), task.getPollingIntervalSecond(), task.getBulkJobTimeoutSecond()));
+            String landing_path = "";
+            if (task.getLandingPath().isPresent()) {
+                landing_path = task.getLandingPath().get();
+            }
+            return new FileInputStream(service.extractAllActivity(task.getActTypeIds(), fromDate.toDate(), toDate.toDate(), task.getPollingIntervalSecond(), task.getBulkJobTimeoutSecond(), landing_path));
         }
         catch (FileNotFoundException e) {
             throw new RuntimeException("Exception when trying to extract activity", e);

data/src/main/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPlugin.java

@@ -65,6 +65,10 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
         @ConfigDefault("1")
         Integer getFetchDays();
 
+        @Config("landing_zone")
+        @ConfigDefault("null")
+        Optional<String> getLandingPath();
+
         @Config("latest_fetch_time")
         @ConfigDefault("null")
         Optional<Long> getLatestFetchTime();
@@ -444,7 +448,7 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
                 int i = 0;
                 while (tokenizer.hasNextColumn()) {
                     if (i == headers.size()){
-                        LOGGER.info("We have reached last column. {}", tokenizer.nextColumnOrNull());
+//                        LOGGER.info("We have reached last column. {}", tokenizer.nextColumnOrNull());
                         break;
                     }
                     kvMap.put(headers.get(i), tokenizer.nextColumnOrNull());

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-marketo_extended
 version: !ruby/object:Gem::Version
-  version: 0.6.22
+  version: 0.6.28
 platform: ruby
 authors:
 - uu59
@@ -11,7 +11,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-11-06 00:00:00.000000000 Z
+date: 2020-11-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -140,9 +140,9 @@ files:
 - classpath/jetty-io-9.2.14.v20151106.jar
 - classpath/commons-io-2.8.0.jar
 - classpath/jetty-util-9.2.14.v20151106.jar
-- classpath/embulk-input-marketo_extended-0.6.22.jar
 - classpath/jetty-http-9.2.14.v20151106.jar
 - classpath/jetty-client-9.2.14.v20151106.jar
+- classpath/embulk-input-marketo_extended-0.6.28.jar
 - classpath/embulk-base-restclient-0.5.3.jar
 - classpath/embulk-util-retryhelper-jetty92-0.5.3.jar
 homepage: https://github.com/treasure-data/embulk-input-marketo