embulk-input-marketo 0.5.7.alpha.6 → 0.6.0.alpha.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3c729920ccd16968b191ec5c12cf8f982f925f80
-  data.tar.gz: 1cf0acc73b4c5e90b8a11fd78770ee56791dc565
+  metadata.gz: 4581ee28de4fb94a896d4e18509f39086801c9eb
+  data.tar.gz: 352811dc47c6e42ff55bd1b0531fc83bc0c19949
 SHA512:
-  metadata.gz: 3f14afd21ae9c968c6c0e6f1ebaf324421ae4ff10c1ceb74020ea458e970f039c2b81e9cf026c40e73089986155ea116ee1d3d5cdb47b1455b529f7de5c51506
-  data.tar.gz: f54d8dff5fc370b8b6f4b2f793c3f90f9e9f57a510383edc83b4c996dc255951d8d6ee882b65d9146e609180a861cea793a1c2c8167d2905f44c8c479c0458c1
+  metadata.gz: 95d49092d1edee0454c4a4ed04c29c06c0ec44db9de8bc2aedccb06eb706e814a89317c9c55662d3e8cf6c967ac1e29209e3a27487a6429eb2743c416a7c9620
+  data.tar.gz: aea7947562f1d5b6753029a4de2c47f8edfb91c2f020863c8e916964591270219c09aeb3b1e85ab46f9b98b5c5a8c2976c14332d061bf9b1b14988736bf15598

data/build.gradle

@@ -16,7 +16,7 @@ repositories {
 configurations {
     provided
 }
-version = "0.5.7.alpha.6"
+version = "0.6.0.alpha.1"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
@@ -81,13 +81,13 @@ task gemspec {
 Gem::Specification.new do |spec|
   spec.name          = "${project.name}"
   spec.version       = "${project.version}"
-  spec.authors       = ["Tai Khuu"]
+  spec.authors       = ["uu59", "yoshihara", "taikhuu"]
   spec.summary       = %[Marketo input plugin for Embulk]
   spec.description   = %[Loads records from Marketo.]
-  spec.email         = ["tai@treasuredata.com"]
-  spec.licenses      = ["MIT"]
-  # TODO set this: spec.homepage      = "https://github.com/khuutantai/embulk-input-marketo"
-
+  spec.email         = ["k@uu59.org", "h.yoshihara@everyleaf.com", "tai@treasuredata.com"]
+  spec.licenses      = ["Apache2"]
+  spec.homepage      = "https://github.com/treasure-data/embulk-input-marketo"
+  
   spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
   spec.test_files    = spec.files.grep(%r"^(test|spec)/")
   spec.require_paths = ["lib"]

data/src/main/java/org/embulk/input/marketo/MarketoService.java

@@ -18,7 +18,7 @@ public interface MarketoService
 
     List<MarketoField> describeLeadByLists();
 
-    File extractLead(Date startTime, Date endTime, List<String> extractedFields, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond);
+    File extractLead(Date startTime, Date endTime, List<String> extractedFields, String filterField, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond);
 
     File extractAllActivity(Date startTime, Date endTime, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond);
 

data/src/main/java/org/embulk/input/marketo/MarketoServiceImpl.java

@@ -39,9 +39,9 @@ public class MarketoServiceImpl implements MarketoService
     }
 
     @Override
-    public File extractLead(Date startTime, Date endTime, List<String> extractedFields, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond)
+    public File extractLead(Date startTime, Date endTime, List<String> extractedFields, String filterField, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond)
     {
-        String exportID = marketoRestClient.createLeadBulkExtract(startTime, endTime, extractedFields);
+        String exportID = marketoRestClient.createLeadBulkExtract(startTime, endTime, extractedFields, filterField);
         marketoRestClient.startLeadBulkExtract(exportID);
         try {
             marketoRestClient.waitLeadExportJobComplete(exportID, pollingTimeIntervalSecond, bulkJobTimeoutSecond);
@@ -70,7 +70,7 @@ public class MarketoServiceImpl implements MarketoService
     @Override
     public File extractAllActivity(Date startTime, Date endTime, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond)
     {
-        String exportID = marketoRestClient.createActitvityExtract(startTime, endTime, null);
+        String exportID = marketoRestClient.createActivityExtract(startTime, endTime);
         marketoRestClient.startActitvityBulkExtract(exportID);
         try {
             marketoRestClient.waitActitvityExportJobComplete(exportID, pollingTimeIntervalSecond, bulkJobTimeoutSecond);
@@ -92,9 +92,8 @@ public class MarketoServiceImpl implements MarketoService
             final String id = node.get("id").asText();
             iterables.add(Iterables.transform(marketoRestClient.getLeadsByList(id, fieldNames), new Function<ObjectNode, ObjectNode>()
             {
-                @Nullable
                 @Override
-                public ObjectNode apply(@Nullable ObjectNode input)
+                public ObjectNode apply(ObjectNode input)
                 {
                     input.put(MarketoUtils.LIST_ID_COLUMN_NAME, id);
                     return input;

data/src/main/java/org/embulk/input/marketo/MarketoUtils.java

@@ -11,18 +11,12 @@ import org.embulk.base.restclient.jackson.JacksonTopLevelValueLocator;
 import org.embulk.base.restclient.record.ServiceRecord;
 import org.embulk.base.restclient.record.ValueLocator;
 import org.embulk.input.marketo.model.MarketoField;
-import org.embulk.spi.Exec;
 import org.joda.time.DateTime;
-import org.slf4j.Logger;
 
 import javax.annotation.Nullable;
 
 import java.util.ArrayList;
-import java.util.Calendar;
-import java.util.Date;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 
 /**
@@ -71,7 +65,7 @@ public class MarketoUtils
 
     public static List<String> getFieldNameFromMarketoFields(List<MarketoField> columns, String... excludedFields)
     {
-        Set<String> excludeFields= Sets.newHashSet(excludedFields);
+        Set<String> excludeFields = Sets.newHashSet(excludedFields);
         List<String> extractedFields = new ArrayList<>();
         for (MarketoField column : columns) {
             if (excludeFields.contains(column.getName())) {
@@ -82,17 +76,54 @@ public class MarketoUtils
         return extractedFields;
     }
 
-    public static  <K, V> Map<K, V> zip(List<K> keys, List<V> values)
+    public static String buildColumnName(String prefix, String columnName)
+    {
+        return prefix + "_" + columnName;
+    }
+
+    public static final List<DateRange> sliceRange(DateTime fromDate, DateTime toDate, int rangeSize)
     {
-        Map<K, V> kvMap = new HashMap<>();
-        for (int i = 0; i < keys.size(); i++) {
-            kvMap.put(keys.get(i), values.get(i));
+        List<DateRange> ranges = new ArrayList<>();
+        while (fromDate.isBefore(toDate)) {
+            DateTime nextToDate = fromDate.plusDays(rangeSize);
+            if (nextToDate.isAfter(toDate)) {
+                ranges.add(new DateRange(fromDate, toDate));
+                break;
+            }
+            ranges.add(new DateRange(fromDate, nextToDate));
+            fromDate = nextToDate.plusSeconds(1);
         }
-        return kvMap;
+        return ranges;
     }
 
-    public static String buildColumnName(String prefix, String columnName)
+    public static String getIdentityEndPoint(String accountId)
     {
-        return prefix + "_" + columnName;
+        return "https://" + accountId + ".mktorest.com/identity";
+    }
+
+    public static String getEndPoint(String accountID)
+    {
+        return "https://" + accountID + ".mktorest.com";
+    }
+
+    public static  final class DateRange
+    {
+        public final DateTime fromDate;
+        public final DateTime toDate;
+
+        public DateRange(DateTime fromDate, DateTime toDate)
+        {
+            this.fromDate = fromDate;
+            this.toDate = toDate;
+        }
+
+        @Override
+        public String toString()
+        {
+            return "DateRange{" +
+                    "fromDate=" + fromDate +
+                    ", toDate=" + toDate +
+                    '}';
+        }
     }
 }

data/src/main/java/org/embulk/input/marketo/delegate/ActivityBulkExtractInputPlugin.java

@@ -1,22 +1,20 @@
 package org.embulk.input.marketo.delegate;
 
+import com.google.common.base.Optional;
 import org.embulk.base.restclient.ServiceResponseMapper;
 import org.embulk.base.restclient.jackson.JacksonServiceResponseMapper;
 import org.embulk.base.restclient.record.ValueLocator;
 import org.embulk.input.marketo.MarketoService;
-import org.embulk.input.marketo.MarketoServiceImpl;
 import org.embulk.input.marketo.MarketoUtils;
-import org.embulk.input.marketo.rest.MarketoRestClient;
 import org.embulk.spi.DataException;
 import org.embulk.spi.Exec;
-import org.embulk.spi.Schema;
 import org.embulk.spi.type.Types;
+import org.joda.time.DateTime;
 import org.slf4j.Logger;
 
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.InputStream;
-import java.util.Date;
 
 /**
  * Created by tai.khuu on 9/18/17.
@@ -24,21 +22,24 @@ import java.util.Date;
 public class ActivityBulkExtractInputPlugin extends MarketoBaseBulkExtractInputPlugin<ActivityBulkExtractInputPlugin.PluginTask>
 {
     private static final Logger LOGGER = Exec.getLogger(ActivityBulkExtractInputPlugin.class);
+    public static final String INCREMENTAL_COLUMN = "activityDate";
+    public static final String UID_COLUMN = "marketoGUID";
 
     public interface PluginTask extends MarketoBaseBulkExtractInputPlugin.PluginTask {}
 
-    public ActivityBulkExtractInputPlugin()
+    @Override
+    public void validateInputTask(PluginTask task)
     {
-        super("activityDate", "marketoGUID");
+        task.setIncrementalColumn(Optional.of(INCREMENTAL_COLUMN));
+        task.setUidColumn(Optional.of(UID_COLUMN));
+        super.validateInputTask(task);
     }
 
     @Override
-    protected InputStream getExtractedStream(PluginTask task, Schema schema)
+    protected InputStream getExtractedStream(MarketoService service, PluginTask task, DateTime fromDate, DateTime toDate)
     {
-        try (MarketoRestClient marketoRestClient = createMarketoRestClient(task)) {
-            MarketoService marketoService = new MarketoServiceImpl(marketoRestClient);
-            Date fromDate = task.getFromDate();
-            return new FileInputStream(marketoService.extractAllActivity(fromDate, task.getToDate().orNull(), task.getPollingIntervalSecond(), task.getBulkJobTimeoutSecond()));
+        try {
+            return new FileInputStream(service.extractAllActivity(fromDate.toDate(), toDate.toDate(), task.getPollingIntervalSecond(), task.getBulkJobTimeoutSecond()));
         }
         catch (FileNotFoundException e) {
             LOGGER.error("Exception when trying to extract activity", e);

data/src/main/java/org/embulk/input/marketo/delegate/CampaignInputPlugin.java

@@ -3,19 +3,14 @@ package org.embulk.input.marketo.delegate;
 import com.google.common.collect.FluentIterable;
 import org.embulk.base.restclient.ServiceResponseMapper;
 import org.embulk.base.restclient.jackson.JacksonServiceResponseMapper;
-import org.embulk.base.restclient.record.RecordImporter;
 import org.embulk.base.restclient.record.ServiceRecord;
 import org.embulk.base.restclient.record.ValueLocator;
-import org.embulk.config.TaskReport;
 import org.embulk.input.marketo.MarketoService;
-import org.embulk.input.marketo.MarketoServiceImpl;
 import org.embulk.input.marketo.MarketoUtils;
-import org.embulk.input.marketo.model.MarketoField;
-import org.embulk.input.marketo.rest.MarketoRestClient;
-import org.embulk.spi.Exec;
-import org.embulk.spi.PageBuilder;
 import org.embulk.spi.type.Types;
 
+import java.util.Iterator;
+
 /**
  * Input plugin use to import Campaign
  * Created by tai.khuu on 9/18/17.
@@ -31,23 +26,9 @@ public class CampaignInputPlugin extends MarketoBaseInputPluginDelegate<Campaign
     }
 
     @Override
-    public TaskReport ingestServiceData(PluginTask task, RecordImporter recordImporter, int taskIndex, PageBuilder pageBuilder)
+    protected Iterator<ServiceRecord> getServiceRecords(MarketoService marketoService, PluginTask task)
     {
-        try (MarketoRestClient marketoRestClient = createMarketoRestClient(task)) {
-            MarketoService marketoService = new MarketoServiceImpl(marketoRestClient);
-            {
-                FluentIterable<ServiceRecord> serviceRecords = FluentIterable.from(marketoService.getCampaign()).transform(MarketoUtils.TRANSFORM_OBJECT_TO_JACKSON_SERVICE_RECORD_FUNCTION);
-                int imported = 0;
-                for (ServiceRecord serviceRecord : serviceRecords) {
-                    if (imported >= PREVIEW_RECORD_LIMIT && Exec.isPreview()) {
-                        break;
-                    }
-                    recordImporter.importRecord(serviceRecord, pageBuilder);
-                    imported++;
-                }
-                return Exec.newTaskReport();
-            }
-        }
+        return FluentIterable.from(marketoService.getCampaign()).transform(MarketoUtils.TRANSFORM_OBJECT_TO_JACKSON_SERVICE_RECORD_FUNCTION).iterator();
     }
 
     @Override

data/src/main/java/org/embulk/input/marketo/delegate/LeadBulkExtractInputPlugin.java

@@ -1,7 +1,10 @@
 package org.embulk.input.marketo.delegate;
 
+import com.google.common.base.Optional;
 import org.embulk.base.restclient.ServiceResponseMapper;
 import org.embulk.base.restclient.record.ValueLocator;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
 import org.embulk.input.marketo.MarketoService;
 import org.embulk.input.marketo.MarketoServiceImpl;
 import org.embulk.input.marketo.MarketoUtils;
@@ -9,14 +12,12 @@ import org.embulk.input.marketo.model.MarketoField;
 import org.embulk.input.marketo.rest.MarketoRestClient;
 import org.embulk.spi.DataException;
 import org.embulk.spi.Exec;
-import org.embulk.spi.Schema;
+import org.joda.time.DateTime;
 import org.slf4j.Logger;
 
-import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.InputStream;
-import java.util.Date;
 import java.util.List;
 
 /**
@@ -26,24 +27,30 @@ public class LeadBulkExtractInputPlugin extends MarketoBaseBulkExtractInputPlugi
 {
     private static final Logger LOGGER = Exec.getLogger(LeadBulkExtractInputPlugin.class);
 
+    private static final String UPDATED_AT = "updatedAt";
+
     public interface PluginTask extends MarketoBaseBulkExtractInputPlugin.PluginTask
     {
+        @Config("use_updated_at")
+        @ConfigDefault("false")
+        boolean getUseUpdatedAt();
     }
 
-    public LeadBulkExtractInputPlugin()
+    @Override
+    public void validateInputTask(PluginTask task)
     {
-        super("updatedAt", null);
+        if (task.getUseUpdatedAt()) {
+            task.setIncrementalColumn(Optional.of(UPDATED_AT));
+        }
+        super.validateInputTask(task);
     }
 
     @Override
-    protected InputStream getExtractedStream(PluginTask task, Schema schema)
+    protected InputStream getExtractedStream(MarketoService service, PluginTask task, DateTime fromDate, DateTime toDate)
     {
-        try (MarketoRestClient marketoRestClient = createMarketoRestClient(task)) {
-            MarketoService marketoService = new MarketoServiceImpl(marketoRestClient);
+        try {
             List<String> fieldNames = task.getExtractedFields();
-            Date fromDate = task.getFromDate();
-            File file = marketoService.extractLead(fromDate, task.getToDate().orNull(), fieldNames, task.getPollingIntervalSecond(), task.getBulkJobTimeoutSecond());
-            return new FileInputStream(file);
+            return new FileInputStream(service.extractLead(fromDate.toDate(), toDate.toDate(), fieldNames, task.getIncrementalColumn().orNull(), task.getPollingIntervalSecond(), task.getBulkJobTimeoutSecond()));
         }
         catch (FileNotFoundException e) {
             LOGGER.error("File not found", e);

data/src/main/java/org/embulk/input/marketo/delegate/LeadWithListInputPlugin.java

@@ -2,18 +2,15 @@ package org.embulk.input.marketo.delegate;
 
 import com.google.common.collect.FluentIterable;
 import org.embulk.base.restclient.ServiceResponseMapper;
-import org.embulk.base.restclient.record.RecordImporter;
 import org.embulk.base.restclient.record.ServiceRecord;
 import org.embulk.base.restclient.record.ValueLocator;
-import org.embulk.config.TaskReport;
 import org.embulk.input.marketo.MarketoService;
 import org.embulk.input.marketo.MarketoServiceImpl;
 import org.embulk.input.marketo.MarketoUtils;
 import org.embulk.input.marketo.model.MarketoField;
 import org.embulk.input.marketo.rest.MarketoRestClient;
-import org.embulk.spi.Exec;
-import org.embulk.spi.PageBuilder;
 
+import java.util.Iterator;
 import java.util.List;
 
 /**
@@ -30,21 +27,9 @@ public class LeadWithListInputPlugin extends MarketoBaseInputPluginDelegate<Lead
     }
 
     @Override
-    public TaskReport ingestServiceData(PluginTask task, RecordImporter recordImporter, int taskIndex, PageBuilder pageBuilder)
+    protected Iterator<ServiceRecord> getServiceRecords(MarketoService marketoService, PluginTask task)
     {
-        try (MarketoRestClient marketoRestClient = createMarketoRestClient(task)) {
-            MarketoService marketoService = new MarketoServiceImpl(marketoRestClient);
-            FluentIterable<ServiceRecord> serviceRecords = FluentIterable.from(marketoService.getAllListLead(task.getExtractedFields())).transform(MarketoUtils.TRANSFORM_OBJECT_TO_JACKSON_SERVICE_RECORD_FUNCTION);
-            int imported = 0;
-            for (ServiceRecord serviceRecord : serviceRecords) {
-                if (imported >= PREVIEW_RECORD_LIMIT && Exec.isPreview()) {
-                    break;
-                }
-                recordImporter.importRecord(serviceRecord, pageBuilder);
-                imported++;
-            }
-            return Exec.newTaskReport();
-        }
+        return FluentIterable.from(marketoService.getAllListLead(task.getExtractedFields())).transform(MarketoUtils.TRANSFORM_OBJECT_TO_JACKSON_SERVICE_RECORD_FUNCTION).iterator();
     }
 
     @Override

data/src/main/java/org/embulk/input/marketo/delegate/LeadWithProgramInputPlugin.java

@@ -2,18 +2,15 @@ package org.embulk.input.marketo.delegate;
 
 import com.google.common.collect.FluentIterable;
 import org.embulk.base.restclient.ServiceResponseMapper;
-import org.embulk.base.restclient.record.RecordImporter;
 import org.embulk.base.restclient.record.ServiceRecord;
 import org.embulk.base.restclient.record.ValueLocator;
-import org.embulk.config.TaskReport;
 import org.embulk.input.marketo.MarketoService;
 import org.embulk.input.marketo.MarketoServiceImpl;
 import org.embulk.input.marketo.MarketoUtils;
 import org.embulk.input.marketo.model.MarketoField;
 import org.embulk.input.marketo.rest.MarketoRestClient;
-import org.embulk.spi.Exec;
-import org.embulk.spi.PageBuilder;
 
+import java.util.Iterator;
 import java.util.List;
 
 /**
@@ -26,23 +23,11 @@ public class LeadWithProgramInputPlugin extends MarketoBaseInputPluginDelegate<L
     }
 
     @Override
-    public TaskReport ingestServiceData(PluginTask task, RecordImporter recordImporter, int taskIndex, PageBuilder pageBuilder)
+    protected Iterator<ServiceRecord> getServiceRecords(MarketoService marketoService, PluginTask task)
     {
-        try (MarketoRestClient marketoRestClient = createMarketoRestClient(task)) {
-            MarketoService marketoService = new MarketoServiceImpl(marketoRestClient);
-            List<String> fieldNames = task.getExtractedFields();
-            FluentIterable<ServiceRecord> serviceRecords = FluentIterable.from(marketoService.getAllProgramLead(fieldNames)).
-                    transform(MarketoUtils.TRANSFORM_OBJECT_TO_JACKSON_SERVICE_RECORD_FUNCTION);
-            int imported = 0;
-            for (ServiceRecord serviceRecord : serviceRecords) {
-                if (imported >= PREVIEW_RECORD_LIMIT && Exec.isPreview()) {
-                    break;
-                }
-                recordImporter.importRecord(serviceRecord, pageBuilder);
-                imported++;
-            }
-        }
-        return Exec.newTaskReport();
+        List<String> fieldNames = task.getExtractedFields();
+        return FluentIterable.from(marketoService.getAllProgramLead(fieldNames)).
+                transform(MarketoUtils.TRANSFORM_OBJECT_TO_JACKSON_SERVICE_RECORD_FUNCTION).iterator();
     }
 
     @Override

data/src/main/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPlugin.java

@@ -1,11 +1,13 @@
 package org.embulk.input.marketo.delegate;
 
-import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.base.Function;
 import com.google.common.base.Optional;
+import com.google.common.collect.Iterators;
 import org.embulk.base.restclient.jackson.JacksonServiceRecord;
 import org.embulk.base.restclient.jackson.JacksonServiceValue;
 import org.embulk.base.restclient.record.RecordImporter;
+import org.embulk.base.restclient.record.ServiceRecord;
 import org.embulk.base.restclient.record.ValueLocator;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
@@ -14,7 +16,10 @@ import org.embulk.config.ConfigException;
 import org.embulk.config.ConfigInject;
 import org.embulk.config.TaskReport;
 import org.embulk.input.marketo.CsvTokenizer;
+import org.embulk.input.marketo.MarketoService;
+import org.embulk.input.marketo.MarketoServiceImpl;
 import org.embulk.input.marketo.MarketoUtils;
+import org.embulk.input.marketo.rest.MarketoRestClient;
 import org.embulk.spi.BufferAllocator;
 import org.embulk.spi.Column;
 import org.embulk.spi.ColumnVisitor;
@@ -37,8 +42,11 @@ import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Date;
+import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.NoSuchElementException;
 import java.util.Set;
 
 /**
@@ -52,10 +60,10 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
 
     private static final DateTimeFormatter ISO_DATETIME_FORMAT = ISODateTimeFormat.dateTimeParser();
 
-    private static final String IMPORTED_RECORD_COUNT = "imported";
-
     private static final String FROM_DATE = "from_date";
 
+    private static final int MARKETO_MAX_RANGE_EXTRACT = 30;
+
     public interface PluginTask extends MarketoBaseInputPluginDelegate.PluginTask, CsvTokenizer.PluginTask
     {
         @Config("from_date")
@@ -92,17 +100,18 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
         @ConfigDefault("null")
         Optional<Date> getToDate();
 
-        void setToDate(Date toDate);
-    }
+        void setToDate(Optional<Date> toDate);
 
-    private String incrementalColumn;
+        @Config("incremental_column")
+        @ConfigDefault("\"createdAt\"")
+        Optional<String> getIncrementalColumn();
 
-    private String uidColumn;
+        void setIncrementalColumn(Optional<String> incrementalColumn);
 
-    public MarketoBaseBulkExtractInputPlugin(String incrementalColumn, String uidColumn)
-    {
-        this.incrementalColumn = incrementalColumn;
-        this.uidColumn = uidColumn;
+        @Config("uid_column")
+        @ConfigDefault("null")
+        Optional<String> getUidColumn();
+        void setUidColumn(Optional<String> uidColumn);
     }
 
     @Override
@@ -112,12 +121,9 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
         if (task.getFromDate() == null) {
             throw new ConfigException("From date is required for Bulk Extract");
         }
-        if (task.getFetchDays() > 30) {
-            throw new ConfigException("Marketo bulk extract fetch days can't be more than 30");
-        }
         //Calculate to date
         DateTime toDate = getToDate(task);
-        task.setToDate(toDate.toDate());
+        task.setToDate(Optional.of(toDate.toDate()));
     }
 
     public DateTime getToDate(T task)
@@ -138,8 +144,8 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
         ConfigDiff configDiff = super.buildConfigDiff(task, schema, taskCount, taskReports);
         Long currentLatestFetchTime = 0L;
         Set latestUIds = null;
+        String incrementalColumn = task.getIncrementalColumn().orNull();
         if (incrementalColumn != null && task.getIncremental()) {
-            int imported = 0;
             DateFormat df = new SimpleDateFormat(MarketoUtils.MARKETO_DATE_SIMPLE_DATE_FORMAT);
             for (TaskReport taskReport : taskReports) {
                 Long latestFetchTime = taskReport.get(Long.class, LATEST_FETCH_TIME);
@@ -150,7 +156,9 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
                     currentLatestFetchTime = latestFetchTime;
                     latestUIds = taskReport.get(Set.class, LATEST_UID_LIST);
                 }
-                imported = imported + taskReport.get(Integer.class, IMPORTED_RECORD_COUNT);
+                else if (currentLatestFetchTime == latestFetchTime) {
+                    latestUIds.addAll(taskReport.get(Set.class, LATEST_UID_LIST));
+                }
             }
             // in case of we didn't import anything but search range is entirely in the past. Then we should move the the range anyway.
             configDiff.set(FROM_DATE, df.format(task.getToDate().orNull()));
@@ -161,15 +169,66 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
     }
 
     @Override
-    public TaskReport ingestServiceData(T task, RecordImporter recordImporter, int taskIndex, PageBuilder pageBuilder)
+    public TaskReport ingestServiceData(final T task, RecordImporter recordImporter, int taskIndex, PageBuilder pageBuilder)
     {
-        InputStream extractedStream;
+        TaskReport taskReport = Exec.newTaskReport();
+        String incrementalColumn = task.getIncrementalColumn().orNull();
+        String uidColumn = task.getUidColumn().orNull();
         if (Exec.isPreview()) {
             return importMockPreviewData(pageBuilder);
         }
         else {
-            extractedStream = getExtractedStream(task, pageBuilder.getSchema());
-            return importRecordFromFile(task, extractedStream, recordImporter, pageBuilder);
+            try (LineDecoderIterator decoderIterator = getLineDecoderIterator(task)) {
+                Iterator<Map<String, String>> csvRecords = Iterators.concat(Iterators.transform(decoderIterator, new Function<LineDecoder, Iterator<Map<String, String>>>()
+                {
+                    @Override
+                    public Iterator<Map<String, String>> apply(LineDecoder input)
+                    {
+                        return new CsvRecordIterator(input, task);
+                    }
+                }));
+                long currentTimestamp = 0L;
+                Set<String> latestUids = task.getPreviousUids();
+                //Keep the preview code here when we can enable real preview
+                if (Exec.isPreview()) {
+                    csvRecords = Iterators.limit(csvRecords, PREVIEW_RECORD_LIMIT);
+                }
+                while (csvRecords.hasNext()) {
+                    Map<String, String> csvRecord = csvRecords.next();
+                    if (task.getIncremental()) {
+                        if (!csvRecord.containsKey(incrementalColumn)) {
+                            throw new DataException("Extracted record doesn't have incremental column " + incrementalColumn);
+                        }
+                        if (uidColumn != null) {
+                            String uid = csvRecord.get(uidColumn);
+                            if (latestUids.contains(uid)) {
+                                //Duplicate value
+                                continue;
+                            }
+                        }
+                        String incrementalTimeStamp = csvRecord.get(incrementalColumn);
+                        long timestamp = ISO_DATETIME_FORMAT.parseDateTime(incrementalTimeStamp).getMillis();
+                        if (currentTimestamp < timestamp) {
+                            currentTimestamp = timestamp;
+                            //switch timestamp
+                            latestUids.clear();
+                        }
+                        else if (currentTimestamp == timestamp) {
+                            //timestamp is equal
+                            if (uidColumn != null) {
+                                String uid = csvRecord.get(uidColumn);
+                                latestUids.add(uid);
+                            }
+                        }
+                    }
+
+                    ObjectNode objectNode = MarketoUtils.OBJECT_MAPPER.valueToTree(csvRecord);
+                    recordImporter.importRecord(new AllStringJacksonServiceRecord(objectNode), pageBuilder);
+                }
+                taskReport.set(LATEST_FETCH_TIME, currentTimestamp);
+                taskReport.set(LATEST_UID_LIST, latestUids);
+                return taskReport;
+            }
         }
     }
 
@@ -227,76 +286,20 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
         return Exec.newTaskReport();
     }
 
-    protected TaskReport importRecordFromFile(T task, InputStream inputStream, RecordImporter recordImporter, PageBuilder pageBuilder)
+    private LineDecoderIterator getLineDecoderIterator(T task)
     {
-        Set<String> latestUids = task.getPreviousUids();
-        TaskReport taskReport = Exec.newTaskReport();
-        int imported = 0;
-        long currentTimestamp = 0L;
-        if (task.getLatestFetchTime().isPresent()) {
-            currentTimestamp = task.getLatestFetchTime().get();
-        }
-        try (LineDecoder lineDecoder = new LineDecoder(new InputStreamFileInput(task.getBufferAllocator(), inputStream), task)) {
-            CsvTokenizer csvTokenizer = new CsvTokenizer(lineDecoder, task);
-            if (!csvTokenizer.nextFile()) {
-                throw new DataException("Can't read extract input stream");
-            }
-            csvTokenizer.nextRecord();
-            List<String> headers = new ArrayList<>();
-            while (csvTokenizer.hasNextColumn()) {
-                headers.add(csvTokenizer.nextColumn());
-            }
-            while (csvTokenizer.nextRecord() && (imported < PREVIEW_RECORD_LIMIT || !Exec.isPreview())) {
-                List<String> values = new ArrayList<>();
-                try {
-                    while (csvTokenizer.hasNextColumn()) {
-                        values.add(csvTokenizer.nextColumnOrNull());
-                    }
-                }
-                catch (CsvTokenizer.InvalidValueException ex) {
-                    throw new DataException("Encounter exception when parse csv file. Please check to see if you are using the correct" +
-                            "quote or escape character.", ex);
-                }
-                final Map<String, String> kvMap = MarketoUtils.zip(headers, values);
-                ObjectNode objectNode = MarketoUtils.OBJECT_MAPPER.valueToTree(kvMap);
+        List<MarketoUtils.DateRange> dateRanges = MarketoUtils.sliceRange(new DateTime(task.getFromDate()), new DateTime(task.getToDate().orNull()), MARKETO_MAX_RANGE_EXTRACT);
+        final Iterator<MarketoUtils.DateRange> iterator = dateRanges.iterator();
+        return new LineDecoderIterator(iterator, task);
+    }
 
-                if (task.getIncremental()) {
-                    if (!kvMap.containsKey(incrementalColumn)) {
-                        throw new DataException("Extracted record doesn't have incremental column " + incrementalColumn);
-                    }
-                    if (uidColumn != null) {
-                        String uid = kvMap.get(uidColumn);
-                        if (latestUids.contains(uid)) {
-                            //Duplicate value
-                            continue;
-                        }
-                    }
-                    String incrementalTimeStamp = kvMap.get(incrementalColumn);
-                    long timestamp = ISO_DATETIME_FORMAT.parseDateTime(incrementalTimeStamp).getMillis();
-                    if (currentTimestamp < timestamp) {
-                        currentTimestamp = timestamp;
-                        //switch timestamp
-                        latestUids.clear();
-                    }
-                    else if (currentTimestamp == timestamp) {
-                        //timestamp is equal
-                        if (uidColumn != null) {
-                            JsonNode uidField = objectNode.get(uidColumn);
-                            latestUids.add(uidField.asText());
-                        }
-                    }
-                }
-                recordImporter.importRecord(new AllStringJacksonServiceRecord(objectNode), pageBuilder);
-                imported++;
-            }
-        }
-        taskReport.set(LATEST_FETCH_TIME, currentTimestamp);
-        taskReport.set(LATEST_UID_LIST, latestUids);
-        taskReport.set(IMPORTED_RECORD_COUNT, imported);
-        return taskReport;
+    @Override
+    protected final Iterator<ServiceRecord> getServiceRecords(MarketoService marketoService, T task)
+    {
+        throw new UnsupportedOperationException();
     }
 
-    protected abstract InputStream getExtractedStream(T task, Schema schema);
+    protected abstract InputStream getExtractedStream(MarketoService service, T task, DateTime fromDate, DateTime toDate);
 
     private static class AllStringJacksonServiceRecord extends JacksonServiceRecord
     {
@@ -366,4 +369,127 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
             return timestampParser.parse(textValue);
         }
     }
+
+    private final class LineDecoderIterator implements Iterator<LineDecoder>, AutoCloseable
+    {
+        private LineDecoder currentLineDecoder;
+
+        private Iterator<MarketoUtils.DateRange> dateRangeIterator;
+
+        private MarketoService marketoService;
+
+        private MarketoRestClient marketoRestClient;
+        private T task;
+        public LineDecoderIterator(Iterator<MarketoUtils.DateRange> dateRangeIterator, T task)
+        {
+            marketoRestClient = createMarketoRestClient(task);
+            marketoService = new MarketoServiceImpl(marketoRestClient);
+            this.dateRangeIterator = dateRangeIterator;
+            this.task = task;
+        }
+
+        @Override
+        public void close()
+        {
+            if (currentLineDecoder != null) {
+                currentLineDecoder.close();
+            }
+            if (marketoRestClient != null) {
+                marketoRestClient.close();
+            }
+        }
+
+        @Override
+        public boolean hasNext()
+        {
+            return dateRangeIterator.hasNext();
+        }
+
+        @Override
+        public LineDecoder next()
+        {
+            if (hasNext()) {
+                MarketoUtils.DateRange next = dateRangeIterator.next();
+                InputStream extractedStream = getExtractedStream(marketoService, task, next.fromDate, next.toDate);
+                currentLineDecoder = new LineDecoder(new InputStreamFileInput(task.getBufferAllocator(), extractedStream), task);
+                return currentLineDecoder;
+            }
+            throw new NoSuchElementException();
+        }
+
+        @Override
+        public void remove()
+        {
+            throw new UnsupportedOperationException("Removed are not supported");
+        }
+    }
+
+    private class CsvRecordIterator implements Iterator<Map<String, String>>
+    {
+        private CsvTokenizer tokenizer;
+
+        private List<String> headers;
+
+        private Map<String, String> currentCsvRecord;
+        public CsvRecordIterator(LineDecoder lineDecoder, T task)
+        {
+            tokenizer = new CsvTokenizer(lineDecoder, task);
+            if (!tokenizer.nextFile()) {
+                throw new DataException("Can't read extract input stream");
+            }
+            headers = new ArrayList<>();
+            tokenizer.nextRecord();
+            while (tokenizer.hasNextColumn()) {
+                headers.add(tokenizer.nextColumn());
+            }
+        }
+
+        @Override
+        public boolean hasNext()
+        {
+            if (currentCsvRecord == null) {
+                currentCsvRecord = getNextCSVRecord();
+            }
+            return currentCsvRecord != null;
+        }
+
+        @Override
+        public Map<String, String> next()
+        {
+            try {
+                if (hasNext()) {
+                    return currentCsvRecord;
+                }
+            }
+            finally {
+                currentCsvRecord = null;
+            }
+            throw new NoSuchElementException();
+        }
+
+        @Override
+        public void remove()
+        {
+            throw new UnsupportedOperationException();
+        }
+        private Map<String, String> getNextCSVRecord()
+        {
+            if (!tokenizer.nextRecord()) {
+                return null;
+            }
+            Map<String, String> kvMap = new HashMap<>();
+            try {
+                int i = 0;
+                while (tokenizer.hasNextColumn()) {
+                    kvMap.put(headers.get(i), tokenizer.nextColumnOrNull());
+                    i++;
+                }
+            }
+            catch (CsvTokenizer.InvalidValueException ex) {
+                throw new DataException("Encounter exception when parse csv file. Please check to see if you are using the correct" +
+                        "quote or escape character.", ex);
+            }
+            return kvMap;
+        }
+    }
 }