embulk-input-marketo_extended 0.6.22 → 0.6.23

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a6efba1be34ca3cd08c0883805641173eb6259c0
-  data.tar.gz: d65d6decb9c9c8fb7b135d60a42488c8ddb7f5de
+  metadata.gz: 5b0c3ee50cdeedfe6defe0a8a9897d72215b2cf2
+  data.tar.gz: 1f5d6cc8b29e54d620a77e1e914669188b846b91
 SHA512:
-  metadata.gz: 0d3558a194417bec40a6943482d8e082c1ada38acb323c410f81672ab46f8ce59445334c0b89df6cc5ff7ab4279a6e9346f2c42119b8020460605530ea8a9068
-  data.tar.gz: 7210c0fcc06ad5df7d18285e18300973d249950df83641744acd72d5492d6b1aba2852500407893a127e48508241040537ace24edd47b2aa607bd420102f13e3
+  metadata.gz: c9a329a13abe4324ac2c662a1ed97d7bc9adda3b4e37da4d8d4428ba9d9d3d32600334093b38e4068b07e69918d1056e98468a293328ac89eab8c9ee88e5d0c5
+  data.tar.gz: a0f0fbf7645cf7915145f8f648f644b9a1f1fa2c1f57f87bb10404936020fdd18dd066103831134c381738d5822bbb618746141469def4b27d94de5c1faa7f91

data/build.gradle

@@ -16,11 +16,13 @@ repositories {
 configurations {
     provided
 }
-version = "0.6.22"
+version = "0.6.23"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
+
 dependencies {
+
     compile  "org.embulk:embulk-core:0.8.+"
     provided "org.embulk:embulk-core:0.8.+"
     compile  "org.embulk.base.restclient:embulk-base-restclient:0.5.3"

data/src/main/java/org/embulk/input/marketo/MarketoService.java

@@ -16,7 +16,7 @@ public interface MarketoService
 
     File extractLead(Date startTime, Date endTime, List<String> extractedFields, String filterField, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond);
 
-    File extractAllActivity(List<Integer> activityTypeIds, Date startTime, Date endTime, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond);
+    File extractAllActivity(List<Integer> activityTypeIds, Date startTime, Date endTime, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond, String landingZone);
 
     Iterable<ObjectNode> getAllListLead(List<String> extractFields);
 

data/src/main/java/org/embulk/input/marketo/MarketoServiceImpl.java

@@ -21,6 +21,11 @@ import java.io.OutputStream;
 import org.apache.commons.io.FileUtils;
 import java.util.Date;
 import java.util.List;
+import java.time.format.DateTimeFormatter;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+
+//import com.amazonaws.services.s3.AmazonS3;
 
 /**
  * Created by tai.khuu on 9/6/17.
@@ -37,6 +42,12 @@ public class MarketoServiceImpl implements MarketoService
 
     private MarketoRestClient marketoRestClient;
 
+    private static Date startTime;
+
+    private DateFormat dateFormat = new SimpleDateFormat("dd-MM-yyyy");
+
+//    private AmazonS3 s3;
+
     public MarketoServiceImpl(MarketoRestClient marketoRestClient)
     {
         this.marketoRestClient = marketoRestClient;
@@ -45,6 +56,7 @@ public class MarketoServiceImpl implements MarketoService
     @Override
     public File extractLead(final Date startTime, Date endTime, List<String> extractedFields, String filterField, int pollingTimeIntervalSecond, final int bulkJobTimeoutSecond)
     {
+
         final String exportID = marketoRestClient.createLeadBulkExtract(startTime, endTime, extractedFields, filterField);
         marketoRestClient.startLeadBulkExtract(exportID);
         try {
@@ -85,9 +97,59 @@ public class MarketoServiceImpl implements MarketoService
         return total;
     }
 
+//    public boolean exists(String bucketName, String key) {
+//        try {
+//            fullObject = s3Client.getObject(new GetObjectRequest(bucketName, key));
+//            System.out.println("s3 object Content-Type: " + fullObject.getObjectMetadata().getContentType());
+//            return fullObject;
+//        } catch(AmazonServiceException e) {
+//            return false;
+//        }
+//        return true;
+//    }
+//    public String getBucket(String s3_key){
+//        if(s3_key.lenght()) {
+//            return s3_key.split('/')[0];
+//        }
+//    }
+//
+//    public String getKey(String s3_key){
+//        if(s3_key.lenght()) {
+//            String [] key_parts = s3_key.split('/');
+//            String key = "";
+//            for (String k: key_parts){
+//                key = key + "/" + k ;
+//            }
+//            return s3_key.split('/')[0];
+//        }
+//    }
+
     @Override
-    public File extractAllActivity(List<Integer> activityTypeIds, Date startTime, Date endTime, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond)
+    public File extractAllActivity(List<Integer> activityTypeIds, Date startTime, Date endTime, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond, String landingZoneFile)
     {
+        String tempFile = "/tmp/marketo_" + this.dateFormat.format(startTime) +".csv";
+        this.startTime = startTime;
+
+        try{
+            if (landingZoneFile.length()>0){
+                LOGGER.info("Let's check for temp File = {}", tempFile);
+                File f = new File(tempFile);
+                if (f.exists() && f.isFile() && f.canRead()){
+                    return f;
+                }
+//                //  Lets read and return csv file from s3
+//                s3_object =  get_s3_object(getBucket(landingZoneFile), getKey(landingZoneFile));
+//                if(s3_object){
+//                    return s3_object.getObjectContent();
+//                }
+
+            }
+        }
+        catch (Exception e){
+            System.out.println("Exception on retrieving from landing zone");
+            System.out.println("Let's continue normal execution.");
+        }
+
         final String exportID = marketoRestClient.createActivityExtract(activityTypeIds, startTime, endTime);
         marketoRestClient.startActitvityBulkExtract(exportID);
         try {
@@ -110,10 +172,8 @@ public class MarketoServiceImpl implements MarketoService
 
     private File downloadBulkExtract(Function<BulkExtractRangeHeader, InputStream> getBulkExtractfunction)
     {
-        final File tempFile = Exec.getTempFileSpace().createTempFile(DEFAULT_FILE_FORMAT);
-        LOGGER.info("Temp File = {}", tempFile.getPath());
-//        File destination = new File("/tmp/example_test.csv");
 
+        final File tempFile = Exec.getTempFileSpace().createTempFile(DEFAULT_FILE_FORMAT);
 
         long startByte = 0;
         int resumeTime = 0;
@@ -122,13 +182,17 @@ public class MarketoServiceImpl implements MarketoService
             InputStream bulkExtractResult = getBulkExtractfunction.apply(bulkExtractRangeHeader);
             try {
                 saveExtractedFile(bulkExtractResult, tempFile);
-//                try{
-//                    FileUtils.copyFile(tempFile, destination);
-//                }
-//                catch (IOException e){
-//                    LOGGER.error("Exception when copying to file destination {}\nException:{}",destination.getPath(),e );
-//                }
-
+                if (this.startTime != null){
+                    String destinationFileName = "/tmp/marketo_" + this.dateFormat.format(this.startTime) +".csv";
+                    final File destination = new File(destinationFileName);
+                    try{
+                        FileUtils.copyFile(tempFile, destination);
+                        LOGGER.info("Save activities in file: "+destinationFileName);
+                    }
+                    catch (IOException e){
+                        LOGGER.error("Exception when copying to file destination {}\nException:{}",destination.getPath(),e );
+                    }
+                }
                 return tempFile;
             }
             catch (DownloadBulkExtractException e) {

data/src/main/java/org/embulk/input/marketo/delegate/ActivityBulkExtractInputPlugin.java

@@ -143,7 +143,7 @@ public class ActivityBulkExtractInputPlugin extends MarketoBaseBulkExtractInputP
     protected InputStream getExtractedStream(MarketoService service, PluginTask task, DateTime fromDate, DateTime toDate)
     {
         try {
-            return new FileInputStream(service.extractAllActivity(task.getActTypeIds(), fromDate.toDate(), toDate.toDate(), task.getPollingIntervalSecond(), task.getBulkJobTimeoutSecond()));
+            return new FileInputStream(service.extractAllActivity(task.getActTypeIds(), fromDate.toDate(), toDate.toDate(), task.getPollingIntervalSecond(), task.getBulkJobTimeoutSecond(), task.getLandingPath()));
         }
         catch (FileNotFoundException e) {
             throw new RuntimeException("Exception when trying to extract activity", e);

data/src/main/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPlugin.java

@@ -65,6 +65,10 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
         @ConfigDefault("1")
         Integer getFetchDays();
 
+        @Config("landing_zone")
+        @ConfigDefault("null")
+        String getLandingPath();
+
         @Config("latest_fetch_time")
         @ConfigDefault("null")
         Optional<Long> getLatestFetchTime();

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-marketo_extended
 version: !ruby/object:Gem::Version
-  version: 0.6.22
+  version: 0.6.23
 platform: ruby
 authors:
 - uu59
@@ -11,7 +11,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-11-06 00:00:00.000000000 Z
+date: 2020-11-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -140,7 +140,7 @@ files:
 - classpath/jetty-io-9.2.14.v20151106.jar
 - classpath/commons-io-2.8.0.jar
 - classpath/jetty-util-9.2.14.v20151106.jar
-- classpath/embulk-input-marketo_extended-0.6.22.jar
+- classpath/embulk-input-marketo_extended-0.6.23.jar
 - classpath/jetty-http-9.2.14.v20151106.jar
 - classpath/jetty-client-9.2.14.v20151106.jar
 - classpath/embulk-base-restclient-0.5.3.jar