embulk-input-marketo 0.5.7.alpha.4 → 0.5.7.alpha.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c42c9fcf28e826912b2e3c1f0952ba1d25b4652e
-  data.tar.gz: 87eb0b019feda052f4dd4c87cc8df852ffc2e33b
+  metadata.gz: eee177691c3a0ab66a962ff6a4d3fb6773e55217
+  data.tar.gz: 73c4017cbe372d9e5bac35c085750b7edf690030
 SHA512:
-  metadata.gz: a305f0e763086fb4cb6763840b3b7bffde0c475e9a0acf2e42a404341dc7dbe7a6bfdcce66727a9bd6febd48f55dd560b23cab9063c90a65dcdb8dd1887523e9
-  data.tar.gz: 250082fb8a23b96b9860fe7224b0b72ce49def3c791225acdb2a5262292b8d23a5c1958c712c2fe460168b498e34d383505b053a480b4c7b257e78440f7c9e64
+  metadata.gz: 103f84a50be5589e023d2d7ed353c20e2509deeece64257fcc31393d3d3e89c33208b3ea5779f2f69126abf0ac18ca1227bb1fb4859955d64a2d49d8041f9f59
+  data.tar.gz: f425bda7a48ad4aa94b50a039b7825ac8480ea4e568a2e8cd1ebbddfd0fcede9572226de5f3f17d1a63ab98d8112169b33631b8a751ffcb0aaff6225f165d170

data/build.gradle

@@ -16,7 +16,7 @@ repositories {
 configurations {
     provided
 }
-version = "0.5.7.alpha.4"
+version = "0.5.7.alpha.5"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 

data/src/main/java/org/embulk/input/marketo/MarketoInputPluginDelegate.java

@@ -2,10 +2,12 @@ package org.embulk.input.marketo;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.google.common.base.Optional;
 import org.embulk.base.restclient.DispatchingRestClientInputPluginDelegate;
 import org.embulk.base.restclient.RestClientInputPluginDelegate;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigException;
 import org.embulk.input.marketo.delegate.ActivityBulkExtractInputPlugin;
 import org.embulk.input.marketo.delegate.CampaignInputPlugin;
 import org.embulk.input.marketo.delegate.LeadBulkExtractInputPlugin;
@@ -13,6 +15,8 @@ import org.embulk.input.marketo.delegate.LeadWithListInputPlugin;
 import org.embulk.input.marketo.delegate.LeadWithProgramInputPlugin;
 import org.embulk.input.marketo.rest.MarketoRestClient;
 
+import java.util.Date;
+
 public class MarketoInputPluginDelegate
         extends DispatchingRestClientInputPluginDelegate<MarketoInputPluginDelegate.PluginTask>
 {
@@ -37,6 +41,17 @@ public class MarketoInputPluginDelegate
         @Config("maximum_retries_interval_milis")
         @ConfigDefault("120000")
         Integer getMaximumRetriesIntervalMilis();
+
+        @Config("hidden_from_date")
+        @ConfigDefault("\"2017-09-01\"")
+        @Override
+        Date getFromDate();
+
+        void setFromDate(Date fromDate);
+
+        @Config("from_date")
+        @ConfigDefault("null")
+        Optional<Date> getWrappedFromDate();
     }
 
     @SuppressWarnings("unchecked")
@@ -44,6 +59,16 @@ public class MarketoInputPluginDelegate
     protected RestClientInputPluginDelegate dispatchPerTask(PluginTask task)
     {
         Target target = task.getTarget();
+        switch (target) {
+            case LEAD:
+            case ACTIVITY:
+                if (!task.getWrappedFromDate().isPresent()) {
+                    throw new ConfigException("From date is required for target LEAD or ACTIVITY");
+                }
+                Date date = task.getWrappedFromDate().get();
+                task.setFromDate(date);
+                break;
+        }
         return target.getRestClientInputPluginDelegate();
     }
 

data/src/main/java/org/embulk/input/marketo/MarketoUtils.java

@@ -12,6 +12,7 @@ import org.embulk.base.restclient.record.ServiceRecord;
 import org.embulk.base.restclient.record.ValueLocator;
 import org.embulk.input.marketo.model.MarketoField;
 import org.embulk.spi.Exec;
+import org.joda.time.DateTime;
 import org.slf4j.Logger;
 
 import javax.annotation.Nullable;
@@ -90,14 +91,6 @@ public class MarketoUtils
         return kvMap;
     }
 
-    public static Date addDate(Date fromDate, Integer addDate)
-    {
-        Calendar c = Calendar.getInstance();
-        c.setTime(fromDate);
-        c.add(Calendar.DATE, addDate);
-        return c.getTime();
-    }
-
     public static String buildColumnName(String prefix, String columnName)
     {
         return prefix + "_" + columnName;

data/src/main/java/org/embulk/input/marketo/delegate/ActivityBulkExtractInputPlugin.java

@@ -37,8 +37,8 @@ public class ActivityBulkExtractInputPlugin extends MarketoBaseBulkExtractInputP
     {
         try (MarketoRestClient marketoRestClient = createMarketoRestClient(task)) {
             MarketoService marketoService = new MarketoServiceImpl(marketoRestClient);
-            Date fromDate = task.getFromDate().orNull();
-            return new FileInputStream(marketoService.extractAllActivity(fromDate, MarketoUtils.addDate(fromDate, task.getFetchDays()), task.getPollingIntervalSecond(), task.getBulkJobTimeoutSecond()));
+            Date fromDate = task.getFromDate();
+            return new FileInputStream(marketoService.extractAllActivity(fromDate, task.getToDate().get(), task.getPollingIntervalSecond(), task.getBulkJobTimeoutSecond()));
         }
         catch (FileNotFoundException e) {
             LOGGER.error("Exception when trying to extract activity", e);

data/src/main/java/org/embulk/input/marketo/delegate/LeadBulkExtractInputPlugin.java

@@ -42,8 +42,8 @@ public class LeadBulkExtractInputPlugin extends MarketoBaseBulkExtractInputPlugi
         try (MarketoRestClient marketoRestClient = createMarketoRestClient(task)) {
             MarketoService marketoService = new MarketoServiceImpl(marketoRestClient);
             List<String> fieldNames = task.getExtractedFields();
-            Date fromDate = task.getFromDate().orNull();
-            File file = marketoService.extractLead(fromDate, MarketoUtils.addDate(fromDate, task.getFetchDays()), fieldNames, task.getPollingIntervalSecond(), task.getBulkJobTimeoutSecond());
+            Date fromDate = task.getFromDate();
+            File file = marketoService.extractLead(fromDate, task.getToDate().get(), fieldNames, task.getPollingIntervalSecond(), task.getBulkJobTimeoutSecond());
             return new FileInputStream(file);
         }
         catch (FileNotFoundException e) {

data/src/main/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPlugin.java

@@ -52,11 +52,14 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
 
     private static final DateTimeFormatter ISO_DATETIME_FORMAT = ISODateTimeFormat.dateTimeParser();
 
+    private static final String IMPORTED_RECORD_COUNT = "imported";
+
+    private static final String FROM_DATE = "from_date";
+
     public interface PluginTask extends MarketoBaseInputPluginDelegate.PluginTask, CsvTokenizer.PluginTask
     {
         @Config("from_date")
-        @ConfigDefault("null")
-        Optional<Date> getFromDate();
+        Date getFromDate();
 
         @Config("fetch_days")
         @ConfigDefault("1")
@@ -80,6 +83,12 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
         @Config("latest_uids")
         @ConfigDefault("[]")
         Set<String> getPreviousUids();
+
+        @Config("to_date")
+        @ConfigDefault("null")
+        Optional<Date> getToDate();
+
+        void setToDate(Date toDate);
     }
 
     private String incrementalColumn;
@@ -95,13 +104,28 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
     @Override
     public void validateInputTask(T task)
     {
-        if (!task.getFromDate().isPresent()) {
+        super.validateInputTask(task);
+        if (task.getFromDate() == null) {
             throw new ConfigException("From date is required for Bulk Extract");
         }
         if (task.getFetchDays() > 30) {
             throw new ConfigException("Marketo bulk extract fetch days can't be more than 30");
         }
-        super.validateInputTask(task);
+        //Calculate to date
+        DateTime toDate = getToDate(task);
+        task.setToDate(toDate.toDate());
+    }
+
+    public DateTime getToDate(T task)
+    {
+        Date fromDate = task.getFromDate();
+        DateTime dateTime = new DateTime(fromDate);
+        DateTime toDate = dateTime.plusDays(task.getFetchDays());
+        if (toDate.isAfter(task.getJobStartTime())) {
+            //Minus 1 hour and lock down toDate
+            toDate = task.getJobStartTime().minusHours(1);
+        }
+        return toDate;
     }
 
     @Override
@@ -109,8 +133,10 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
     {
         ConfigDiff configDiff = super.buildConfigDiff(task, schema, taskCount, taskReports);
         Long currentLatestFetchTime = 0L;
-        Set<String> latestUIds = null;
+        Set latestUIds = null;
         if (incrementalColumn != null) {
+            int imported = 0;
+            DateFormat df = new SimpleDateFormat(MarketoUtils.MARKETO_DATE_SIMPLE_DATE_FORMAT);
             for (TaskReport taskReport : taskReports) {
                 Long latestFetchTime = taskReport.get(Long.class, LATEST_FETCH_TIME);
                 if (latestFetchTime == null) {
@@ -120,13 +146,12 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
                     currentLatestFetchTime = latestFetchTime;
                     latestUIds = taskReport.get(Set.class, LATEST_UID_LIST);
                 }
+                imported = imported + taskReport.get(Integer.class, IMPORTED_RECORD_COUNT);
             }
-            if (!currentLatestFetchTime.equals(0L)) {
-                configDiff.set(LATEST_FETCH_TIME, currentLatestFetchTime);
-                DateFormat df = new SimpleDateFormat(MarketoUtils.MARKETO_DATE_SIMPLE_DATE_FORMAT);
-                configDiff.set("from_date", df.format(new Date(currentLatestFetchTime)));
-                configDiff.set(LATEST_UID_LIST, latestUIds);
-            }
+            // in case of we didn't import anything but search range is entirely in the past. Then we should move the the range anyway.
+            configDiff.set(FROM_DATE, df.format(task.getToDate().get()));
+            configDiff.set(LATEST_FETCH_TIME, currentLatestFetchTime);
+            configDiff.set(LATEST_UID_LIST, latestUIds);
         }
         return configDiff;
     }
@@ -203,9 +228,9 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
         Set<String> latestUids = task.getPreviousUids();
         TaskReport taskReport = Exec.newTaskReport();
         int imported = 0;
-        DateTime currentTimestamp = null;
+        long currentTimestamp = 0L;
         if (task.getLatestFetchTime().isPresent()) {
-            currentTimestamp = new DateTime(task.getLatestFetchTime().get());
+            currentTimestamp = task.getLatestFetchTime().get();
         }
         try (LineDecoder lineDecoder = new LineDecoder(new InputStreamFileInput(task.getBufferAllocator(), inputStream), task)) {
             CsvTokenizer csvTokenizer = new CsvTokenizer(lineDecoder, task);
@@ -219,8 +244,13 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
             }
             while (csvTokenizer.nextRecord() && (imported < PREVIEW_RECORD_LIMIT || !Exec.isPreview())) {
                 List<String> values = new ArrayList<>();
-                while (csvTokenizer.hasNextColumn()) {
-                    values.add(csvTokenizer.nextColumnOrNull());
+                try {
+                    while (csvTokenizer.hasNextColumn()) {
+                        values.add(csvTokenizer.nextColumnOrNull());
+                    }
+                }
+                catch (CsvTokenizer.InvalidValueException ex) {
+                    throw new DataException("Encounter exception when parse csv file. Please check the quote or escape character", ex);
                 }
                 final Map<String, String> kvMap = MarketoUtils.zip(headers, values);
                 ObjectNode objectNode = MarketoUtils.OBJECT_MAPPER.valueToTree(kvMap);
@@ -236,31 +266,26 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
                     }
                 }
                 String incrementalTimeStamp = kvMap.get(incrementalColumn);
-                DateTime timestamp = ISO_DATETIME_FORMAT.parseDateTime(incrementalTimeStamp);
-                if (currentTimestamp == null) {
+                long timestamp = ISO_DATETIME_FORMAT.parseDateTime(incrementalTimeStamp).getMillis();
+                if (currentTimestamp < timestamp) {
                     currentTimestamp = timestamp;
+                    //switch timestamp
+                    latestUids.clear();
                 }
-                else {
-                    int compareTo = currentTimestamp.compareTo(timestamp);
-                    if (compareTo < 0) {
-                        currentTimestamp = timestamp;
-                        //switch timestamp
-                        latestUids.clear();
-                    }
-                    else if (compareTo == 0) {
-                        //timestamp is equal
-                        if (uidColumn != null) {
-                            JsonNode uidField = objectNode.get(uidColumn);
-                            latestUids.add(uidField.asText());
-                        }
+                else if (currentTimestamp == timestamp) {
+                    //timestamp is equal
+                    if (uidColumn != null) {
+                        JsonNode uidField = objectNode.get(uidColumn);
+                        latestUids.add(uidField.asText());
                     }
                 }
                 recordImporter.importRecord(new AllStringJacksonServiceRecord(objectNode), pageBuilder);
                 imported++;
             }
         }
-        taskReport.set(LATEST_FETCH_TIME, currentTimestamp == null ? 0L : currentTimestamp.getMillis());
+        taskReport.set(LATEST_FETCH_TIME, currentTimestamp);
         taskReport.set(LATEST_UID_LIST, latestUids);
+        taskReport.set(IMPORTED_RECORD_COUNT, imported);
         return taskReport;
     }
 

data/src/main/java/org/embulk/input/marketo/delegate/MarketoBaseInputPluginDelegate.java

@@ -14,6 +14,7 @@ import org.embulk.spi.Exec;
 import org.embulk.spi.Schema;
 import org.embulk.util.retryhelper.jetty92.DefaultJetty92ClientCreator;
 import org.embulk.util.retryhelper.jetty92.Jetty92RetryHelper;
+import org.joda.time.DateTime;
 
 import java.util.List;
 
@@ -50,6 +51,9 @@ public abstract class MarketoBaseInputPluginDelegate<T extends MarketoBaseInputP
 
         void setExtractedFields(List<String> extractedFields);
 
+        DateTime getJobStartTime();
+
+        void setJobStartTime(DateTime dateTime);
     }
 
     @Override
@@ -61,6 +65,7 @@ public abstract class MarketoBaseInputPluginDelegate<T extends MarketoBaseInputP
     @Override
     public void validateInputTask(T task)
     {
+        task.setJobStartTime(DateTime.now());
     }
 
     @VisibleForTesting

data/src/main/java/org/embulk/input/marketo/rest/MarketoBaseRestClient.java

@@ -128,6 +128,7 @@ public class MarketoBaseRestClient implements AutoCloseable
         if (accessTokenResponse.hasError()) {
             throw new DataException(accessTokenResponse.getErrorDescription());
         }
+        LOGGER.info("Acquired new access token");
         return accessTokenResponse.getAccessToken();
     }
 
@@ -194,6 +195,7 @@ public class MarketoBaseRestClient implements AutoCloseable
                     String code = error.getCode();
                     switch (code) {
                         case "602":
+                            LOGGER.info("Access token expired");
                             renewAccessToken();
                             return true;
                         case "606":

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-marketo
 version: !ruby/object:Gem::Version
-  version: 0.5.7.alpha.4
+  version: 0.5.7.alpha.5
 platform: ruby
 authors:
 - Tai Khuu
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-09-26 00:00:00.000000000 Z
+date: 2017-09-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -92,9 +92,9 @@ files:
 - src/test/java/org/embulk/input/marketo/rest/MarketoBaseRestClientTest.java
 - classpath/jetty-http-9.2.14.v20151106.jar
 - classpath/embulk-base-restclient-0.5.3.jar
+- classpath/embulk-input-marketo-0.5.7.alpha.5.jar
 - classpath/jetty-client-9.2.14.v20151106.jar
 - classpath/jetty-util-9.2.14.v20151106.jar
-- classpath/embulk-input-marketo-0.5.7.alpha.4.jar
 - classpath/embulk-util-retryhelper-jetty92-0.5.3.jar
 - classpath/jetty-io-9.2.14.v20151106.jar
 homepage: