embulk-input-marketo 0.6.10 → 0.6.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 368e0b80c346570f82c7c16c2e07135f0b73b543
-  data.tar.gz: e1edfbc1bf62ed3624cb7fc25cf317cc8664dd0a
+  metadata.gz: 1b375807158e07d67abc81e6319ef6ec9a087f9b
+  data.tar.gz: 596f26cb4804ce01819b27499ead25b35c567100
 SHA512:
-  metadata.gz: dd5b2dea8460522bf57c8a92a38a47f7030d3675a28a4bffb4cb48153af6d8679be4c28783ca7757265a3923cdb9197876dad87bac228c9a652e476531d539d2
-  data.tar.gz: 17475bdc7f4f3f89e9bb9e568ab85f74d0d38f4ec4a00afa5dbba58fb4999113061de2ebcbb9c485ebea7cedfe77d649851cd418a7da6fbee92f6af9dc9009c5
+  metadata.gz: d5743056090cbad09a2a75a6f460f2d843e721e254b2e898e7e809faad8f0309732244282c91f5dd6bbed9cafb95d0be519bba6305e8bee2726e49b0905867f8
+  data.tar.gz: ab9fe2a479f4ccc842efb62e80d14dc4bac8941b64dc6c7d589c35a412a6f782fc902edd75a014e48b410889dca3e63c99a184dd533675c0707c44f520da0696

data/.gitignore

@@ -12,3 +12,4 @@ build/
 .classpath
 .project
 
+/bin/

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.6.11 - 2018-09-10
+* [enhance] Implement Assets Programs [#89](https://github.com/treasure-data/embulk-input-marketo/pull/89)
+
 ## 0.6.10 - 2018-05-28
 * [fixed] Add included column option [#87](https://github.com/treasure-data/embulk-input-marketo/pull/87)
 

data/README.md

@@ -12,6 +12,7 @@ embulk-input-marketo is the gem preparing Embulk input plugins for [Marketo](htt
 - Lead by list(all_lead_with_list_id)
 - Lead by program(all_lead_with_program_id)
 - Campaign(campaign)
+- Assets Programs (program)
 
 This plugin uses Marketo REST API.
 
@@ -57,7 +58,7 @@ All bulk extract target use this configuration parameter
 | name                        | required | default value | description                                                                                                                   |
 |-----------------------------|----------|---------------|-------------------------------------------------------------------------------------------------------------------------------|
 | **from_date**               | true     |               | Import data since this date. Example: 2017-10-11T06:43:24+00:00                                                               |
-| **fetch_days**              | false    | 1             | Ammount of days to fetch since from_date                                                                                      |
+| **fetch_days**              | false    | 1             | Amount of days to fetch since from_date                                                                                      |
 | **polling_interval_second** | false    | 60            | Amount of time to wait between pooling job status in second                                                                   |
 | **bulk_job_timeout_second** | false    | 3600          | Amount of time to wait for bulk job to complete in second                                                                     |
 | **incremental**             | false    | true          | If incremental is set to true, next run will have from_date set to the previous to_date(calculated by from_date + fetch_days) |
@@ -74,7 +75,7 @@ Configuration:
 | name                | required | default value | description                                                                                                                                               |
 |---------------------|----------|---------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------|
 | **use_updated_at**  | false    | false         | Support filter with `updateAt` column, but not all Marketo Account have the feature to filter by updatedAt, updatedAt don't support incremental ingestion |
-| **included_fields** | false    | empty         | List of lead fields to included in export request sent to Marketo, can be used to reduce the size of BulkExtract file                                     |
+| **included_fields** | false    | null         | List of lead fields to included in export request sent to Marketo, can be used to reduce the size of BulkExtract file                                     |
 
 Schema type: Dynamic via describe lead endpoint.
 
@@ -84,7 +85,7 @@ Range ingestion: yes
 
 ### Activity
 
-Activity target extract all Marketo  actvity log. Configuration include all bulk extract configuraiont
+Activity target extract all Marketo  activity log. Configuration include all bulk extract configuration
 
 `target: activity`
 
@@ -116,7 +117,7 @@ Configuration:
 
 | name                | required | default value | description                                                                                                     |
 |---------------------|----------|---------------|-----------------------------------------------------------------------------------------------------------------|
-| **included_fields** | false    | empty         | List of lead fields to included in export request sent to Marketo, can be used to reduce request, response size |
+| **included_fields** | false    | null         | List of lead fields to included in export request sent to Marketo, can be used to reduce request, response size |
 
 Schema type: Dynamic via describe leads. Schema will have 1 addition column name listId that contain the id of the list the lead belong to
 
@@ -124,7 +125,7 @@ Incremental support: no
 
 Range ingestion: no
 
-### Laed by program
+### Lead by program
 
 Extract all Lead data including lead's program id
 
@@ -134,7 +135,7 @@ Configuration:
 
 | name                | required | default value | description                                                                                                     |
 |---------------------|----------|---------------|-----------------------------------------------------------------------------------------------------------------|
-| **included_fields** | false    | empty         | List of lead fields to included in export request sent to Marketo, can be used to reduce request, response size |
+| **included_fields** | false    | null         | List of lead fields to included in export request sent to Marketo, can be used to reduce request, response size |
 
 Schema type: Dynamic via describe leads. Schema will have 1 addition column name listId that contain the id of the list the lead belong to
 
@@ -142,6 +143,32 @@ Incremental support: no
 
 Range ingestion: no
 
+### Assets programs
+
+Get Assets Programs by Query Tag type, Date range or all if no query by specified.
+
+`target: program`
+
+Configuration:
+
+| name                        | required | default value | description                                                                                                                                                  |
+|-----------------------------|----------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| **query_by**                | false    |   null        | Get assets programs by query, supported values `date_range`, `tag_type` leave unset to fetch all programs                                                    |
+| **earliest_updated_at**     | false    |   null        | Required if query by `date_range` is selected. Exclude programs prior to this date. Must be valid ISO-8601 string                                            |
+| **latest_updated_at**       | false    |   null        | Required if query by `date_range` is selected. Exclude programs after this date. Must be valid ISO-8601 string                                               |
+| **filter_type**             | false    |   null        | Optional value send with query by `date_range` is selected to filter out the result from Marketo. Supported values `id`, `programId`, `folderId`, `workspace`|
+| **filter_values**           | false    |   null        | Set the values associated with `filter_type`                                                                                                                 |
+| **tag_type**                | false    |   null        | Required if query by `tag_type` is selected. Type of program tag                                                                                             |
+| **tag_value**               | false    |   null        | Required if query by `tag_type` is selected. Value of the tag                                                                                                |
+| **report_duration**         | false    |   null        | Amount of milliseconds to fetch from `earliest_updated_at`. If `incremental = true` this value will automatically calculated for the first run by `latest_updated_at` - `earliest_updated_at` |
+| **incremental**             | false    | true          | If incremental is set to true, next run will have `earliest_updated_at` set to the previous `latest_updated_at` + `report_duration`. Incremental import only support by query `date_range`     |
+
+Schema type: Static schema
+
+Incremental support: yes (Query by `date_range` only)
+
+Range ingestion: yes
+
 ## Example
 
 For lead, you have `partial-config.yml` like below:
@@ -163,3 +190,21 @@ You can run `embulk guess partial-config.yml -o lead-config.yml` and got `lead-c
 
 Next, you can run `embulk preview lead-config.yml` for preview and `embulk run lead-config.yml` for run.
 
+Example of Assets Programs config
+```yaml
+in:
+  account_id: ACCOUNT_ID
+  client_id: CLIENT_ID
+  client_secret: CLIENT_SECRET
+  target: program
+  type: marketo
+  query_by: date_range
+  filter_type: folderId
+  filter_values: 
+   - 2598
+   - 1001
+  earliest_updated_at: 2018-08-20T00:00:00.000Z
+  latest_updated_at: 2018-08-31T00:00:00.000Z
+  incremental: true
+  ```
+

data/build.gradle

@@ -16,7 +16,7 @@ repositories {
 configurations {
     provided
 }
-version = "0.6.10"
+version = "0.6.11"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 

data/src/main/java/org/embulk/input/marketo/CsvTokenizer.java

@@ -612,6 +612,15 @@ public class CsvTokenizer
             return new String(new char[] { character });
         }
 
+        @Override
+        public int hashCode()
+        {
+            final int prime = 31;
+            int result = 1;
+            result = prime * result + character;
+            return result;
+        }
+
         @Override
         public boolean equals(Object obj)
         {
@@ -673,5 +682,14 @@ public class CsvTokenizer
             EscapeCharacter o = (EscapeCharacter) obj;
             return character == o.character;
         }
+
+        @Override
+        public int hashCode()
+        {
+            final int prime = 31;
+            int result = 1;
+            result = prime * result + character;
+            return result;
+        }
     }
 }

data/src/main/java/org/embulk/input/marketo/MarketoInputPluginDelegate.java

@@ -3,6 +3,7 @@ package org.embulk.input.marketo;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.google.common.base.Optional;
+
 import org.embulk.base.restclient.DispatchingRestClientInputPluginDelegate;
 import org.embulk.base.restclient.RestClientInputPluginDelegate;
 import org.embulk.config.Config;
@@ -13,6 +14,7 @@ import org.embulk.input.marketo.delegate.CampaignInputPlugin;
 import org.embulk.input.marketo.delegate.LeadBulkExtractInputPlugin;
 import org.embulk.input.marketo.delegate.LeadWithListInputPlugin;
 import org.embulk.input.marketo.delegate.LeadWithProgramInputPlugin;
+import org.embulk.input.marketo.delegate.ProgramInputPlugin;
 import org.embulk.input.marketo.rest.MarketoRestClient;
 
 import java.util.Date;
@@ -25,7 +27,9 @@ public class MarketoInputPluginDelegate
             LeadBulkExtractInputPlugin.PluginTask,
             LeadWithProgramInputPlugin.PluginTask,
             ActivityBulkExtractInputPlugin.PluginTask,
-            CampaignInputPlugin.PluginTask, MarketoRestClient.PluginTask
+            CampaignInputPlugin.PluginTask,
+            ProgramInputPlugin.PluginTask,
+            MarketoRestClient.PluginTask
     {
         @Config("target")
         Target getTarget();
@@ -68,7 +72,8 @@ public class MarketoInputPluginDelegate
         ACTIVITY(new ActivityBulkExtractInputPlugin()),
         CAMPAIGN(new CampaignInputPlugin()),
         ALL_LEAD_WITH_LIST_ID(new LeadWithListInputPlugin()),
-        ALL_LEAD_WITH_PROGRAM_ID(new LeadWithProgramInputPlugin());
+        ALL_LEAD_WITH_PROGRAM_ID(new LeadWithProgramInputPlugin()),
+        PROGRAM(new ProgramInputPlugin());
 
         private RestClientInputPluginDelegate restClientInputPluginDelegate;
 

data/src/main/java/org/embulk/input/marketo/MarketoService.java

@@ -23,4 +23,10 @@ public interface MarketoService
     Iterable<ObjectNode> getAllProgramLead(List<String> extractFields);
 
     Iterable<ObjectNode> getCampaign();
+
+    Iterable<ObjectNode> getPrograms();
+
+    Iterable<ObjectNode> getProgramsByTag(String tagType, String tagValue);
+
+    Iterable<ObjectNode> getProgramsByDateRange(Date earliestUpdatedAt, Date latestUpdatedAt, String filterType, List<String> filterValues);
 }

data/src/main/java/org/embulk/input/marketo/MarketoServiceImpl.java

@@ -206,4 +206,22 @@ public class MarketoServiceImpl implements MarketoService
             return byteWritten;
         }
     }
+
+    @Override
+    public Iterable<ObjectNode> getPrograms()
+    {
+        return marketoRestClient.getPrograms();
+    }
+
+    @Override
+    public Iterable<ObjectNode> getProgramsByTag(String tagType, String tagValue)
+    {
+        return marketoRestClient.getProgramsByTag(tagType, tagValue);
+    }
+
+    @Override
+    public Iterable<ObjectNode> getProgramsByDateRange(Date earliestUpdatedAt, Date latestUpdatedAt, String filterType, List<String> filterValues)
+    {
+        return marketoRestClient.getProgramsByDateRange(earliestUpdatedAt, latestUpdatedAt, filterType, filterValues);
+    }
 }

data/src/main/java/org/embulk/input/marketo/delegate/LeadBulkExtractInputPlugin.java

@@ -1,6 +1,7 @@
 package org.embulk.input.marketo.delegate;
 
 import com.google.common.base.Optional;
+
 import org.embulk.base.restclient.ServiceResponseMapper;
 import org.embulk.base.restclient.record.ValueLocator;
 import org.embulk.config.Config;

data/src/main/java/org/embulk/input/marketo/delegate/LeadWithListInputPlugin.java

@@ -1,6 +1,7 @@
 package org.embulk.input.marketo.delegate;
 
 import com.google.common.collect.FluentIterable;
+
 import org.embulk.base.restclient.ServiceResponseMapper;
 import org.embulk.base.restclient.record.ServiceRecord;
 import org.embulk.base.restclient.record.ValueLocator;
@@ -10,7 +11,6 @@ import org.embulk.input.marketo.MarketoUtils;
 import org.embulk.input.marketo.model.MarketoField;
 import org.embulk.input.marketo.rest.MarketoRestClient;
 
-import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
 
@@ -48,7 +48,6 @@ public class LeadWithListInputPlugin extends MarketoBaseInputPluginDelegate<Lead
 
     private static class LeadWithListServiceResponseMapper extends LeadServiceResponseMapperBuilder<PluginTask>
     {
-
         public LeadWithListServiceResponseMapper(LeadWithListInputPlugin.PluginTask task, MarketoService marketoService)
         {
             super(task, marketoService);

data/src/main/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPlugin.java

@@ -79,10 +79,6 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
         @ConfigDefault("3600")
         Integer getBulkJobTimeoutSecond();
 
-        @Config("incremental")
-        @ConfigDefault("true")
-        Boolean getIncremental();
-
         @Config("to_date")
         @ConfigDefault("null")
         Optional<Date> getToDate();

data/src/main/java/org/embulk/input/marketo/delegate/MarketoBaseInputPluginDelegate.java

@@ -1,22 +1,23 @@
 package org.embulk.input.marketo.delegate;
 
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Optional;
+
 import org.embulk.base.restclient.DefaultServiceDataSplitter;
 import org.embulk.base.restclient.RestClientInputPluginDelegate;
 import org.embulk.base.restclient.RestClientInputTaskBase;
 import org.embulk.base.restclient.ServiceDataSplitter;
 import org.embulk.base.restclient.record.RecordImporter;
 import org.embulk.base.restclient.record.ServiceRecord;
-import org.embulk.config.*;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.TaskReport;
 import org.embulk.input.marketo.MarketoService;
 import org.embulk.input.marketo.MarketoServiceImpl;
 import org.embulk.input.marketo.rest.MarketoRestClient;
 import org.embulk.spi.Exec;
 import org.embulk.spi.PageBuilder;
 import org.embulk.spi.Schema;
-import org.embulk.util.retryhelper.jetty92.DefaultJetty92ClientCreator;
-import org.embulk.util.retryhelper.jetty92.Jetty92RetryHelper;
 import org.joda.time.DateTime;
 
 import java.util.Iterator;
@@ -35,6 +36,10 @@ public abstract class MarketoBaseInputPluginDelegate<T extends MarketoBaseInputP
         @ConfigDefault("\"mk\"")
         String getSchemaColumnPrefix();
 
+        @Config("incremental")
+        @ConfigDefault("true")
+        Boolean getIncremental();
+
         DateTime getJobStartTime();
 
         void setJobStartTime(DateTime dateTime);

data/src/main/java/org/embulk/input/marketo/delegate/ProgramInputPlugin.java

@@ -0,0 +1,229 @@
+package org.embulk.input.marketo.delegate;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.base.Optional;
+import com.google.common.collect.FluentIterable;
+
+import org.embulk.base.restclient.ServiceResponseMapper;
+import org.embulk.base.restclient.jackson.JacksonServiceResponseMapper;
+import org.embulk.base.restclient.record.RecordImporter;
+import org.embulk.base.restclient.record.ServiceRecord;
+import org.embulk.base.restclient.record.ValueLocator;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigException;
+import org.embulk.config.TaskReport;
+import org.embulk.input.marketo.MarketoService;
+import org.embulk.input.marketo.MarketoUtils;
+import org.embulk.spi.Exec;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.Schema;
+import org.embulk.spi.type.Types;
+import org.joda.time.DateTime;
+import org.joda.time.Duration;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+import org.slf4j.Logger;
+
+import java.util.Date;
+import java.util.Iterator;
+import java.util.List;
+
+public class ProgramInputPlugin extends MarketoBaseInputPluginDelegate<ProgramInputPlugin.PluginTask>
+{
+    private static final DateTimeFormatter DATE_FORMATTER = DateTimeFormat.forPattern(MarketoUtils.MARKETO_DATE_SIMPLE_DATE_FORMAT);
+    private final Logger logger = Exec.getLogger(getClass());
+
+    public interface PluginTask extends MarketoBaseInputPluginDelegate.PluginTask
+    {
+        @Config("query_by")
+        @ConfigDefault("null")
+        Optional<QueryBy> getQueryBy();
+
+        @Config("tag_type")
+        @ConfigDefault("null")
+        Optional<String> getTagType();
+
+        @Config("tag_value")
+        @ConfigDefault("null")
+        Optional<String> getTagVallue();
+
+        @Config("earliest_updated_at")
+        @ConfigDefault("null")
+        Optional<Date> getEarliestUpdatedAt();
+
+        @Config("latest_updated_at")
+        @ConfigDefault("null")
+        Optional<Date> getLatestUpdatedAt();
+
+        @Config("filter_type")
+        @ConfigDefault("null")
+        Optional<String> getFilterType();
+
+        @Config("filter_values")
+        @ConfigDefault("null")
+        Optional<List<String>> getFilterValues();
+
+        @Config("report_duration")
+        @ConfigDefault("null")
+        Optional<Long> getReportDuration();
+
+        void setLatestUpdatedAt(Optional<Date> latestUpdatedAt);
+    }
+
+    public ProgramInputPlugin()
+    {
+    }
+
+    @Override
+    public void validateInputTask(PluginTask task)
+    {
+        super.validateInputTask(task);
+        // validate if query_by is selected
+        if (task.getQueryBy().isPresent()) {
+            switch(task.getQueryBy().get()) {
+            case TAG_TYPE:
+                //make sure tag type and tag value are not empty
+                if (!task.getTagType().isPresent() || !task.getTagVallue().isPresent()) {
+                    throw new ConfigException("tag_type and tag_value are required when query by Tag Type");
+                }
+                break;
+            case DATE_RANGE:
+                // make sure earliest_updated_at is not empty
+                if (!task.getEarliestUpdatedAt().isPresent()) {
+                    throw new ConfigException("`earliest_updated_at` is required when query by Date Range");
+                }
+
+                DateTime earliest = new DateTime(task.getEarliestUpdatedAt().get());
+                if (task.getReportDuration().isPresent()) {
+                    logger.info("`report_duration` is present, Prefer `report_duration` over `latest_updated_at`");
+                    // Update the latestUpdatedAt for the config
+                    DateTime latest = earliest.plus(task.getReportDuration().get());
+                    task.setLatestUpdatedAt(Optional.of(latest.toDate()));
+                }
+
+                // latest_updated_at is required calculate time range
+                if (!task.getLatestUpdatedAt().isPresent()) {
+                    throw new ConfigException("`latest_updated_at` is required when query by Date Range");
+                }
+
+                DateTime latest = new DateTime(task.getLatestUpdatedAt().get());
+                if (earliest.isAfter(DateTime.now())) {
+                    throw new ConfigException(String.format("`earliest_updated_at` (%s) cannot precede the current date (%s)",
+                                    earliest.toString(DATE_FORMATTER),
+                                    (DateTime.now().toString(DATE_FORMATTER))));
+                }
+
+                if (earliest.isAfter(latest)) {
+                    throw new ConfigException(String.format("Invalid date range. `earliest_updated_at` (%s) cannot precede the `latest_updated_at` (%s).",
+                                    earliest.toString(DATE_FORMATTER),
+                                    latest.toString(DATE_FORMATTER)));
+                }
+                // if filter type is selected, filter value must be presented
+                if (task.getFilterType().isPresent() && (!task.getFilterValues().isPresent() || task.getFilterValues().get().isEmpty())) {
+                    throw new ConfigException("filter_value is required when selected filter_type");
+                }
+            }
+        }
+    }
+
+    @Override
+    public TaskReport ingestServiceData(PluginTask task, RecordImporter recordImporter, int taskIndex, PageBuilder pageBuilder)
+    {
+        // query by date range and incremental import and not preview
+        if (task.getQueryBy().isPresent() && task.getQueryBy().get() == QueryBy.DATE_RANGE && task.getIncremental() && !Exec.isPreview()) {
+            DateTime latestUpdateAt = new DateTime(task.getLatestUpdatedAt().get());
+            DateTime now = DateTime.now();
+            // Do not run incremental import if latest_updated_at precede current time
+            if (latestUpdateAt.isAfter(now)) {
+                logger.warn("`latest_updated_at` ({}) preceded current time ({}). Will try to import next run", latestUpdateAt.toString(DATE_FORMATTER), now.toString(DATE_FORMATTER));
+
+                TaskReport taskReport = Exec.newTaskReport();
+                taskReport.set("earliest_updated_at", new DateTime(task.getEarliestUpdatedAt().get()).toString(DATE_FORMATTER));
+                if (task.getReportDuration().isPresent()) {
+                    taskReport.set("report_duration", task.getReportDuration().get());
+                }
+                return taskReport;
+            }
+        }
+        return super.ingestServiceData(task, recordImporter, taskIndex, pageBuilder);
+    }
+
+    @Override
+    protected Iterator<ServiceRecord> getServiceRecords(MarketoService marketoService, PluginTask task)
+    {
+        Iterable<ObjectNode> nodes = null;
+        if (task.getQueryBy().isPresent()) {
+            switch (task.getQueryBy().get()) {
+            case TAG_TYPE:
+                nodes = marketoService.getProgramsByTag(task.getTagType().get(), task.getTagVallue().get());
+                break;
+            case DATE_RANGE:
+                nodes = marketoService.getProgramsByDateRange(task.getEarliestUpdatedAt().get(),
+                                task.getLatestUpdatedAt().get(),
+                                task.getFilterType().orNull(),
+                                task.getFilterValues().orNull());
+            }
+        }
+        else {
+            nodes = marketoService.getPrograms();
+        }
+        return FluentIterable.from(nodes).transform(MarketoUtils.TRANSFORM_OBJECT_TO_JACKSON_SERVICE_RECORD_FUNCTION).iterator();
+    }
+
+    @Override
+    public ConfigDiff buildConfigDiff(PluginTask task, Schema schema, int taskCount, List<TaskReport> taskReports)
+    {
+        ConfigDiff configDiff = super.buildConfigDiff(task, schema, taskCount, taskReports);
+        // set next next earliestUpdatedAt, latestUpdatedAt
+        if (task.getQueryBy().isPresent() && task.getQueryBy().get() == QueryBy.DATE_RANGE && task.getIncremental()) {
+
+            DateTime earliest = new DateTime(task.getEarliestUpdatedAt().get());
+            DateTime latest = new DateTime(task.getLatestUpdatedAt().get());
+
+            Duration d = new Duration(earliest, latest);
+            DateTime nextEarliestUpdatedAt = latest.plusSeconds(1);
+
+            configDiff.set("earliest_updated_at", nextEarliestUpdatedAt.toString(DATE_FORMATTER));
+            configDiff.set("report_duration", task.getReportDuration().or(d.getMillis()));
+        }
+        return configDiff;
+    }
+
+    @Override
+    public ServiceResponseMapper<? extends ValueLocator> buildServiceResponseMapper(PluginTask task)
+    {
+        JacksonServiceResponseMapper.Builder builder = JacksonServiceResponseMapper.builder();
+        builder.add("id", Types.LONG)
+                .add("name", Types.STRING)
+                .add("sfdcId", Types.STRING)
+                .add("sfdcName", Types.STRING)
+                .add("description", Types.STRING)
+                .add("createdAt", Types.TIMESTAMP, MarketoUtils.MARKETO_DATE_TIME_FORMAT)
+                .add("updatedAt", Types.TIMESTAMP, MarketoUtils.MARKETO_DATE_TIME_FORMAT)
+                .add("startDate", Types.TIMESTAMP, MarketoUtils.MARKETO_DATE_TIME_FORMAT)
+                .add("endDate", Types.TIMESTAMP, MarketoUtils.MARKETO_DATE_TIME_FORMAT)
+                .add("url", Types.STRING)
+                .add("type", Types.STRING)
+                .add("channel", Types.STRING)
+                .add("folder", Types.JSON)
+                .add("status", Types.STRING)
+                .add("costs", Types.JSON)
+                .add("tags", Types.JSON)
+                .add("workspace", Types.STRING);
+        return builder.build();
+    }
+
+    public enum QueryBy {
+        TAG_TYPE,
+        DATE_RANGE;
+
+        @JsonCreator
+        public static QueryBy of(String value)
+        {
+            return QueryBy.valueOf(value.toUpperCase());
+        }
+    }
+}

data/src/main/java/org/embulk/input/marketo/rest/MarketoBaseRestClient.java

@@ -184,6 +184,7 @@ public class MarketoBaseRestClient implements AutoCloseable
                         }
                     }
                 }
+                LOGGER.info("CALLING {} -> {} - params: {}", method, target, params);
                 if (contentProvider != null) {
                     request.content(contentProvider);
                 }

data/src/main/java/org/embulk/input/marketo/rest/MarketoRESTEndpoint.java

@@ -23,7 +23,8 @@ public enum MarketoRESTEndpoint
     GET_LEADS_BY_LIST("/rest/v1/lists/${list_id}/leads.json"),
     GET_PROGRAMS("/rest/asset/v1/programs.json"),
     GET_LEADS_BY_PROGRAM("/rest/v1/leads/programs/${program_id}.json"),
-    GET_CAMPAIGN("/rest/v1/campaigns.json");
+    GET_CAMPAIGN("/rest/v1/campaigns.json"),
+    GET_PROGRAMS_BY_TAG("/rest/asset/v1/program/byTag.json");
 
     private String endpoint;
 

data/src/main/java/org/embulk/input/marketo/rest/MarketoRestClient.java

@@ -6,6 +6,7 @@ import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.ImmutableListMultimap;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Multimap;
+
 import org.eclipse.jetty.client.util.FormContentProvider;
 import org.eclipse.jetty.util.Fields;
 import org.embulk.config.Config;
@@ -341,6 +342,7 @@ public class MarketoRestClient extends MarketoBaseRestClient
     {
         return getRecordWithTokenPagination(endPoint + MarketoRESTEndpoint.GET_CAMPAIGN.getEndpoint(), null, ObjectNode.class);
     }
+
     private <T> RecordPagingIterable<T> getRecordWithOffsetPagination(final String endPoint, final Multimap<String, String> parameters, final Class<T> recordClass)
     {
         return new RecordPagingIterable<>(new RecordPagingIterable.PagingFunction<RecordPagingIterable.OffsetPage<T>>()
@@ -411,4 +413,26 @@ public class MarketoRestClient extends MarketoBaseRestClient
             }
         });
     }
+
+    public Iterable<ObjectNode> getProgramsByTag(String tagType, String tagValue)
+    {
+        Multimap<String, String> multimap = ArrayListMultimap.create();
+        multimap.put("tagType", tagType);
+        multimap.put("tagValue", tagValue);
+        return getRecordWithOffsetPagination(endPoint + MarketoRESTEndpoint.GET_PROGRAMS_BY_TAG.getEndpoint(), multimap, ObjectNode.class);
+    }
+
+    public Iterable<ObjectNode> getProgramsByDateRange(Date earliestUpdatedAt, Date latestUpdatedAt, String filterType, List<String> filterValues)
+    {
+        SimpleDateFormat timeFormat = new SimpleDateFormat(MarketoUtils.MARKETO_DATE_SIMPLE_DATE_FORMAT);
+        Multimap<String, String> multimap = ArrayListMultimap.create();
+        multimap.put("earliestUpdatedAt", timeFormat.format(earliestUpdatedAt));
+        multimap.put("latestUpdatedAt", timeFormat.format(latestUpdatedAt));
+        // put filter params if exist.
+        if (filterType != null) {
+            multimap.put("filterType", filterType);
+            multimap.put("filterValues", String.join(",", filterValues));
+        }
+        return getRecordWithOffsetPagination(endPoint + MarketoRESTEndpoint.GET_PROGRAMS.getEndpoint(), multimap, ObjectNode.class);
+    }
 }

data/src/test/java/org/embulk/input/marketo/delegate/ActivityBulkExtractInputPluginTest.java

@@ -26,10 +26,7 @@ import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.util.Arrays;
 import java.util.Date;
-import java.util.List;
-import java.util.Set;
 
-import static org.junit.Assert.assertEquals;
 import static org.mockito.ArgumentMatchers.any;
 
 /**

data/src/test/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPluginTest.java

@@ -1,7 +1,7 @@
 package org.embulk.input.marketo.delegate;
 
 import com.google.common.base.Optional;
-import com.google.common.collect.Sets;
+
 import org.embulk.EmbulkTestRuntime;
 import org.embulk.config.ConfigDiff;
 import org.embulk.config.ConfigException;
@@ -23,7 +23,6 @@ import java.text.DateFormat;
 import java.text.SimpleDateFormat;
 import java.util.Arrays;
 import java.util.Date;
-import java.util.Set;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.fail;

data/src/test/java/org/embulk/input/marketo/delegate/ProgramInputPluginTest.java

@@ -0,0 +1,345 @@
+package org.embulk.input.marketo.delegate;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import com.fasterxml.jackson.databind.JavaType;
+import com.fasterxml.jackson.databind.JsonMappingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.base.Optional;
+import com.google.common.base.Predicate;
+
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.base.restclient.ServiceResponseMapper;
+import org.embulk.base.restclient.record.RecordImporter;
+import org.embulk.base.restclient.record.ValueLocator;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigLoader;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskReport;
+import org.embulk.input.marketo.MarketoUtils;
+import org.embulk.input.marketo.delegate.ProgramInputPlugin.PluginTask;
+import org.embulk.input.marketo.delegate.ProgramInputPlugin.QueryBy;
+import org.embulk.input.marketo.rest.MarketoRestClient;
+import org.embulk.input.marketo.rest.RecordPagingIterable;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.Schema;
+import org.joda.time.DateTime;
+import org.joda.time.Duration;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TestRule;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mockito;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Date;
+import java.util.List;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+
+public class ProgramInputPluginTest
+{
+    private static final DateTimeFormatter DATE_FORMATER = DateTimeFormat.forPattern(MarketoUtils.MARKETO_DATE_SIMPLE_DATE_FORMAT);
+
+    private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    private ExpectedException thrown = ExpectedException.none();
+
+    @Rule
+    @SuppressFBWarnings("URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD")
+    public TestRule chain = RuleChain.outerRule(runtime).around(thrown);
+
+    private ConfigSource baseConfig;
+
+    private ProgramInputPlugin mockPlugin;
+
+    private MarketoRestClient mockRestClient;
+
+    @Before
+    public void setUp() throws Exception
+    {
+        mockPlugin = Mockito.spy(new ProgramInputPlugin());
+        baseConfig = config();
+        mockRestClient = Mockito.mock(MarketoRestClient.class);
+        Mockito.doReturn(mockRestClient).when(mockPlugin).createMarketoRestClient(Mockito.any(ProgramInputPlugin.PluginTask.class));
+    }
+
+//    -----------Verify configs --------------
+    @Test
+    public void testQueryByTagTypeConfigMissingTagType()
+    {
+        thrown.expect(ConfigException.class);
+        thrown.expectMessage("tag_type and tag_value are required when query by Tag Type");
+        ConfigSource config = baseConfig.set("query_by", Optional.of(QueryBy.TAG_TYPE)).set("tag_value", Optional.of("dummy"));
+        mockPlugin.validateInputTask(config.loadConfig(PluginTask.class));
+    }
+
+    @Test
+    public void testQueryByTagTypeConfigMissingTagValue()
+    {
+        thrown.expect(ConfigException.class);
+        thrown.expectMessage("tag_type and tag_value are required when query by Tag Type");
+        ConfigSource config = baseConfig.set("query_by", Optional.of(QueryBy.TAG_TYPE)).set("tag_type", Optional.of("dummy"));
+        mockPlugin.validateInputTask(config.loadConfig(PluginTask.class));
+    }
+
+    @Test
+    public void testQueryByDateRangeConfigMissingEarliestUpatedAt()
+    {
+        thrown.expect(ConfigException.class);
+        thrown.expectMessage("`earliest_updated_at` is required when query by Date Range");
+        ConfigSource config = baseConfig.set("query_by", Optional.of(QueryBy.DATE_RANGE)).set("latest_updated_at", Optional.of(DateTime.now().minusDays(10).toDate()));
+        mockPlugin.validateInputTask(config.loadConfig(PluginTask.class));
+    }
+
+    @Test
+    public void testQueryByDateRangeConfigMissingLatestUpdatedAt()
+    {
+        thrown.expect(ConfigException.class);
+        thrown.expectMessage("`latest_updated_at` is required when query by Date Range");
+        ConfigSource config = baseConfig.set("query_by", Optional.of(QueryBy.DATE_RANGE)).set("earliest_updated_at", Optional.of(DateTime.now().minusDays(10).toDate()));
+        mockPlugin.validateInputTask(config.loadConfig(PluginTask.class));
+    }
+
+    @Test
+    public void testQueryByDateRangeConfigMissingLatestUpdatedAtNonIncremental()
+    {
+        thrown.expect(ConfigException.class);
+        thrown.expectMessage("`latest_updated_at` is required when query by Date Range");
+        ConfigSource config = baseConfig
+                        .set("query_by", Optional.of(QueryBy.DATE_RANGE))
+                        .set("incremental", Boolean.FALSE)
+                        .set("earliest_updated_at", Optional.of(DateTime.now().minusDays(10).toDate()));
+        mockPlugin.validateInputTask(config.loadConfig(PluginTask.class));
+    }
+
+    @Test
+    public void testNoErrorQueryByDateRangeConfigHasReportDurationNonIncremental()
+    {
+        ConfigSource config = baseConfig
+                        .set("query_by", Optional.of(QueryBy.DATE_RANGE))
+                        .set("report_duration", Optional.of(60L * 1000))
+                        .set("incremental", Boolean.FALSE)
+                        .set("earliest_updated_at", Optional.of(DateTime.now().minusDays(10).toDate()));
+        mockPlugin.validateInputTask(config.loadConfig(PluginTask.class));
+    }
+
+    @Test
+    public void testNoErrorQueryByDateRangeConfigHasReportDuration()
+    {
+        ConfigSource config = baseConfig
+                        .set("report_duration", Optional.of(60L * 1000))
+                        .set("query_by", Optional.of(QueryBy.DATE_RANGE))
+                        .set("earliest_updated_at", Optional.of(DateTime.now().minusDays(10).toDate()));
+        mockPlugin.validateInputTask(config.loadConfig(PluginTask.class));
+    }
+
+    @Test
+    public void testQueryByDateRangeConfigHasEarliestUpdatedAtExceededNow()
+    {
+        DateTime earliestUpdatedAt = DateTime.now().plusDays(1);
+        DateTime latestUpdatedAt = DateTime.now().minusDays(2);
+        thrown.expect(ConfigException.class);
+        thrown.expectMessage(String.format("`earliest_updated_at` (%s) cannot precede the current date ", earliestUpdatedAt.toString(DATE_FORMATER)));
+        ConfigSource config = baseConfig
+                        .set("query_by", Optional.of(QueryBy.DATE_RANGE))
+                        .set("earliest_updated_at", Optional.of(earliestUpdatedAt))
+                        .set("latest_updated_at", Optional.of(latestUpdatedAt));
+        mockPlugin.validateInputTask(config.loadConfig(PluginTask.class));
+    }
+
+    @Test
+    public void testQueryByDateRangeConfigHasEarliestUpdatedAtExceededLatestUpdatedAt()
+    {
+        DateTime earliestUpdatedAt = DateTime.now().minusDays(10);
+        DateTime latestUpdatedAt = DateTime.now().minusDays(20);
+        thrown.expect(ConfigException.class);
+        thrown.expectMessage(String.format("Invalid date range. `earliest_updated_at` (%s) cannot precede the `latest_updated_at` (%s).",
+                        earliestUpdatedAt.toString(DATE_FORMATER),
+                        latestUpdatedAt.toString(DATE_FORMATER)));
+
+        ConfigSource config = baseConfig
+                        .set("query_by", Optional.of(QueryBy.DATE_RANGE))
+                        .set("earliest_updated_at", Optional.of(earliestUpdatedAt))
+                        .set("latest_updated_at", Optional.of(latestUpdatedAt));
+        mockPlugin.validateInputTask(config.loadConfig(PluginTask.class));
+    }
+
+    @Test
+    public void testHasFilterTypeButMissingFilterValue()
+    {
+        DateTime earliestUpdatedAt = DateTime.now().minusDays(20);
+        DateTime latestUpdatedAt = DateTime.now().minusDays(10);
+        thrown.expect(ConfigException.class);
+        thrown.expectMessage("filter_value is required when selected filter_type");
+
+        ConfigSource config = baseConfig
+                        .set("query_by", Optional.of(QueryBy.DATE_RANGE))
+                        .set("earliest_updated_at", Optional.of(earliestUpdatedAt))
+                        .set("latest_updated_at", Optional.of(latestUpdatedAt))
+                        .set("filter_type", Optional.of("dummy"));
+        mockPlugin.validateInputTask(config.loadConfig(PluginTask.class));
+    }
+
+    @Test
+    public void testSkipIncrementalRunIfLastUpdatedAtExceedsNow()
+    {
+        DateTime earliestUpdatedAt = DateTime.now().minusDays(20);
+        DateTime latestUpdatedAt = earliestUpdatedAt.plusDays(21);
+        //21 days
+        long reportDuration = 21 * 24 * 60 * 60 * 1000;
+
+        ConfigSource config = baseConfig
+                        .set("query_by", Optional.of(QueryBy.DATE_RANGE))
+                        .set("earliest_updated_at", Optional.of(earliestUpdatedAt))
+                        .set("latest_updated_at", Optional.of(latestUpdatedAt))
+                        .set("report_duration", reportDuration)
+                        .set("incremental", true);
+        ServiceResponseMapper<? extends ValueLocator> mapper = mockPlugin.buildServiceResponseMapper(config.loadConfig(PluginTask.class));
+        RecordImporter recordImporter = mapper.createRecordImporter();
+        PageBuilder mockPageBuilder = Mockito.mock(PageBuilder.class);
+        TaskReport taskReport = mockPlugin.ingestServiceData(config.loadConfig(PluginTask.class), recordImporter, 1, mockPageBuilder);
+        // page builder object should never get called.
+        Mockito.verify(mockPageBuilder, Mockito.never()).addRecord();
+
+        String earliestUpdatedAtStr = taskReport.get(String.class, "earliest_updated_at");
+        long duration = taskReport.get(Long.class, "report_duration");
+        assertEquals(duration, reportDuration);
+        assertEquals(earliestUpdatedAtStr, earliestUpdatedAt.toString(DATE_FORMATER));
+    }
+
+    @Test
+    public void testBuildConfigDiff() throws Exception
+    {
+        TaskReport taskReport1 = Mockito.mock(TaskReport.class);
+        DateTime earliestUpdatedAt = DateTime.now().minusDays(20);
+        DateTime latestUpdatedAt = DateTime.now().minusDays(10);
+        ConfigSource config = baseConfig
+                        .set("query_by", Optional.of(QueryBy.DATE_RANGE))
+                        .set("earliest_updated_at", Optional.of(earliestUpdatedAt))
+                        .set("latest_updated_at", Optional.of(latestUpdatedAt))
+                        .set("incremental", true);
+
+        ConfigDiff diff = mockPlugin.buildConfigDiff(config.loadConfig(PluginTask.class), Mockito.mock(Schema.class), 1, Arrays.asList(taskReport1));
+
+        long reportDuration = diff.get(Long.class, "report_duration");
+        String nextErliestUpdatedAt = diff.get(String.class, "earliest_updated_at");
+
+        assertEquals(reportDuration, new Duration(earliestUpdatedAt, latestUpdatedAt).getMillis());
+        assertEquals(nextErliestUpdatedAt, latestUpdatedAt.plusSeconds(1).toString(DATE_FORMATER));
+    }
+
+    @SuppressWarnings("unchecked")
+    private void testRun(ConfigSource config, Predicate<MarketoRestClient> expectedCall) throws JsonParseException, JsonMappingException, IOException
+    {
+        // Mock response data
+        RecordPagingIterable<ObjectNode> mockRecordPagingIterable = Mockito.mock(RecordPagingIterable.class);
+        JavaType javaType = OBJECT_MAPPER.getTypeFactory().constructParametrizedType(List.class, List.class, ObjectNode.class);
+        List<ObjectNode> objectNodeList = OBJECT_MAPPER.readValue(this.getClass().getResourceAsStream("/fixtures/all_program_full.json"), javaType);
+        Mockito.when(mockRecordPagingIterable.iterator()).thenReturn(objectNodeList.iterator());
+        Mockito.when(mockRestClient.getProgramsByTag(Mockito.anyString(), Mockito.anyString())).thenReturn(mockRecordPagingIterable);
+        Mockito.when(mockRestClient.getPrograms()).thenReturn(mockRecordPagingIterable);
+        Mockito.when(mockRestClient.getProgramsByDateRange(
+                      Mockito.any(Date.class),
+                      Mockito.any(Date.class),
+                      Mockito.nullable(String.class),
+                      Mockito.nullable(List.class))).thenReturn(mockRecordPagingIterable);
+
+        ServiceResponseMapper<? extends ValueLocator> mapper = mockPlugin.buildServiceResponseMapper(config.loadConfig(PluginTask.class));
+        RecordImporter recordImporter = mapper.createRecordImporter();
+        PageBuilder mockPageBuilder = Mockito.mock(PageBuilder.class);
+        mockPlugin.ingestServiceData(config.loadConfig(PluginTask.class), recordImporter, 1, mockPageBuilder);
+
+        // The method getProgramByTag is called 1 time
+//        Mockito.verify(mockRestClient, Mockito.times(1)).getProgramsByTag(Mockito.anyString(), Mockito.anyString());
+        expectedCall.apply(mockRestClient);
+
+        Schema embulkSchema = mapper.getEmbulkSchema();
+        // 17 columns
+        assertEquals(embulkSchema.size(), 17);
+        // verify 3 times the method setLong for column id has been called
+        ArgumentCaptor<Long> longArgumentCaptor = ArgumentCaptor.forClass(Long.class);
+        Mockito.verify(mockPageBuilder, Mockito.times(3)).setLong(Mockito.eq(embulkSchema.lookupColumn("id")), longArgumentCaptor.capture());
+        List<Long> allValues = longArgumentCaptor.getAllValues();
+        assertArrayEquals(new Long[]{1004L, 1001L, 1003L}, allValues.toArray());
+    }
+
+    @Test
+    public void testRunQueryByTagType() throws JsonParseException, JsonMappingException, IOException
+    {
+        ConfigSource config = baseConfig
+                        .set("query_by", Optional.of(QueryBy.TAG_TYPE))
+                        .set("tag_type", Optional.of("dummy"))
+                        .set("tag_value", Optional.of("dummy"));
+        Predicate<MarketoRestClient> expectedCall = new Predicate<MarketoRestClient>()
+        {
+            @Override
+            public boolean apply(MarketoRestClient mockRest)
+            {
+                Mockito.verify(mockRest, Mockito.times(1)).getProgramsByTag(Mockito.anyString(), Mockito.anyString());
+                return true;
+            }
+        };
+        testRun(config, expectedCall);
+    }
+
+    @Test
+    public void testRunWithoutQueryBy() throws JsonParseException, JsonMappingException, IOException
+    {
+        Predicate<MarketoRestClient> expectedCall = new Predicate<MarketoRestClient>()
+        {
+            @Override
+            public boolean apply(MarketoRestClient input)
+            {
+                Mockito.verify(input, Mockito.times(1)).getPrograms();
+                return true;
+            }
+        };
+        testRun(baseConfig, expectedCall);
+    }
+
+    @Test
+    public void testRunQueryByDateRange() throws JsonParseException, JsonMappingException, IOException
+    {
+        DateTime earliestUpdatedAt = DateTime.now().minusDays(20);
+        DateTime latestUpdatedAt = DateTime.now().minusDays(10);
+        ConfigSource config = baseConfig
+                        .set("query_by", Optional.of(QueryBy.DATE_RANGE))
+                        .set("earliest_updated_at", Optional.of(earliestUpdatedAt))
+                        .set("latest_updated_at", Optional.of(latestUpdatedAt));
+        Predicate<MarketoRestClient> expectedCall = new Predicate<MarketoRestClient>()
+        {
+            @SuppressWarnings("unchecked")
+            @Override
+            public boolean apply(MarketoRestClient input)
+            {
+                Mockito.verify(input, Mockito.times(1)).getProgramsByDateRange(
+                                Mockito.any(Date.class),
+                                Mockito.any(Date.class),
+                                Mockito.nullable(String.class),
+                                Mockito.nullable(List.class));
+                return true;
+            }
+        };
+        testRun(config, expectedCall);
+    }
+
+    public ConfigSource config() throws IOException
+    {
+        ConfigLoader configLoader = runtime.getInjector().getInstance(ConfigLoader.class);
+        return configLoader.fromYaml(this.getClass().getResourceAsStream("/config/rest_config.yaml"));
+    }
+}

data/src/test/java/org/embulk/input/marketo/rest/MarketoRestClientTest.java

@@ -20,11 +20,15 @@ import org.embulk.input.marketo.model.MarketoResponse;
 import org.embulk.spi.DataException;
 import org.embulk.util.retryhelper.jetty92.Jetty92ResponseReader;
 import org.embulk.util.retryhelper.jetty92.Jetty92RetryHelper;
+import org.joda.time.DateTime;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
 import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.mockito.ArgumentCaptor;
+import org.mockito.ArgumentMatchers;
 import org.mockito.Mockito;
 
 import java.io.IOException;
@@ -443,4 +447,101 @@ public class MarketoRestClientTest
         Assert.assertEquals("GET", params1.get("_method").iterator().next());
         Assert.assertEquals("nextPageToken=z4MgsIiC5C%3D%3D%3D%3D%3D%3D&batchSize=300", content);
     }
+
+    @Test
+    public void testGetProgramsByTagType() throws Exception
+    {
+        ArrayNode listPages = (ArrayNode) OBJECT_MAPPER.readTree(new String(ByteStreams.toByteArray(this.getClass().getResourceAsStream("/fixtures/program_response.json")))).get("responses");
+        MarketoResponse<ObjectNode> page1 = OBJECT_MAPPER.readValue(listPages.get(0).toString(), RESPONSE_TYPE);
+        MarketoResponse<ObjectNode> page2 = OBJECT_MAPPER.readValue(listPages.get(1).toString(), RESPONSE_TYPE);
+
+        final String tagType = "dummy_tag";
+        final String tagValue = "dummy_value";
+
+        Mockito.doReturn(page1).doReturn(page2).when(marketoRestClient).doGet(
+                        Mockito.eq(END_POINT + MarketoRESTEndpoint.GET_PROGRAMS_BY_TAG.getEndpoint()),
+                        (Map) ArgumentMatchers.isNull(),
+                        Mockito.any(Multimap.class),
+                        Mockito.any(MarketoResponseJetty92EntityReader.class));
+        Iterable<ObjectNode> lists = marketoRestClient.getProgramsByTag(tagType, tagValue);
+        Iterator<ObjectNode> iterator = lists.iterator();
+        ObjectNode program1 = iterator.next();
+        ObjectNode program2 = iterator.next();
+        ObjectNode program3 = iterator.next();
+        Assert.assertFalse(iterator.hasNext());
+        Assert.assertEquals("MB_Sep_25_test_program", program1.get("name").asText());
+        Assert.assertEquals("TD Output Test Program", program2.get("name").asText());
+        Assert.assertEquals("Bill_progream", program3.get("name").asText());
+
+        ArgumentCaptor<ImmutableListMultimap> immutableListMultimapArgumentCaptor = ArgumentCaptor.forClass(ImmutableListMultimap.class);
+        Mockito.verify(marketoRestClient, Mockito.times(2)).doGet(
+                        Mockito.eq(END_POINT + MarketoRESTEndpoint.GET_PROGRAMS_BY_TAG.getEndpoint()),
+                        (Map) ArgumentMatchers.isNull(),
+                        immutableListMultimapArgumentCaptor.capture(),
+                        Mockito.any(MarketoResponseJetty92EntityReader.class));
+        List<ImmutableListMultimap> params = immutableListMultimapArgumentCaptor.getAllValues();
+
+        ImmutableListMultimap params1 = params.get(0);
+        Assert.assertEquals("0", params1.get("offset").get(0));
+        Assert.assertEquals("2", params1.get("maxReturn").get(0));
+        Assert.assertEquals("dummy_tag", params1.get("tagType").get(0));
+        Assert.assertEquals("dummy_value", params1.get("tagValue").get(0));
+
+        ImmutableListMultimap params2 = params.get(1);
+        Assert.assertEquals("2", params2.get("offset").get(0));
+        Assert.assertEquals("dummy_tag", params2.get("tagType").get(0));
+        Assert.assertEquals("dummy_value", params2.get("tagValue").get(0));
+    }
+
+    @Test
+    public void TestProgramsByDateRange() throws Exception
+    {
+        ArrayNode listPages = (ArrayNode) OBJECT_MAPPER.readTree(new String(ByteStreams.toByteArray(this.getClass().getResourceAsStream("/fixtures/program_response.json")))).get("responses");
+        MarketoResponse<ObjectNode> page1 = OBJECT_MAPPER.readValue(listPages.get(0).toString(), RESPONSE_TYPE);
+        MarketoResponse<ObjectNode> page2 = OBJECT_MAPPER.readValue(listPages.get(1).toString(), RESPONSE_TYPE);
+
+        Mockito.doReturn(page1).doReturn(page2).when(marketoRestClient).doGet(
+                        Mockito.eq(END_POINT + MarketoRESTEndpoint.GET_PROGRAMS.getEndpoint()),
+                        (Map) ArgumentMatchers.isNull(),
+                        Mockito.any(Multimap.class),
+                        Mockito.any(MarketoResponseJetty92EntityReader.class));
+        DateTime earliestUpdatedAt = DateTime.now().minusDays(10);
+        DateTime latestUpdatedAt = earliestUpdatedAt.plusDays(5);
+        String filterType = "filter1";
+        List<String> filterValues = Arrays.asList("value1", "value2");
+
+        Iterable<ObjectNode> lists = marketoRestClient.getProgramsByDateRange(earliestUpdatedAt.toDate(), latestUpdatedAt.toDate(), filterType, filterValues);
+        Iterator<ObjectNode> iterator = lists.iterator();
+        ObjectNode program1 = iterator.next();
+        ObjectNode program2 = iterator.next();
+        ObjectNode program3 = iterator.next();
+        Assert.assertFalse(iterator.hasNext());
+        Assert.assertEquals("MB_Sep_25_test_program", program1.get("name").asText());
+        Assert.assertEquals("TD Output Test Program", program2.get("name").asText());
+        Assert.assertEquals("Bill_progream", program3.get("name").asText());
+
+        ArgumentCaptor<ImmutableListMultimap> immutableListMultimapArgumentCaptor = ArgumentCaptor.forClass(ImmutableListMultimap.class);
+        Mockito.verify(marketoRestClient, Mockito.times(2)).doGet(
+                        Mockito.eq(END_POINT + MarketoRESTEndpoint.GET_PROGRAMS.getEndpoint()),
+                        (Map) ArgumentMatchers.isNull(),
+                        immutableListMultimapArgumentCaptor.capture(),
+                        Mockito.any(MarketoResponseJetty92EntityReader.class));
+        List<ImmutableListMultimap> params = immutableListMultimapArgumentCaptor.getAllValues();
+        DateTimeFormatter fmt = DateTimeFormat.forPattern(MarketoUtils.MARKETO_DATE_SIMPLE_DATE_FORMAT);
+
+        ImmutableListMultimap params1 = params.get(0);
+        Assert.assertEquals("0", params1.get("offset").get(0));
+        Assert.assertEquals("2", params1.get("maxReturn").get(0));
+        Assert.assertEquals(earliestUpdatedAt.toString(fmt), params1.get("earliestUpdatedAt").get(0));
+        Assert.assertEquals(latestUpdatedAt.toString(fmt), params1.get("latestUpdatedAt").get(0));
+        Assert.assertEquals("filter1", params1.get("filterType").get(0));
+        Assert.assertEquals(String.join(",", filterValues), params1.get("filterValues").get(0));
+
+        ImmutableListMultimap params2 = params.get(1);
+        Assert.assertEquals("2", params2.get("offset").get(0));
+        Assert.assertEquals(earliestUpdatedAt.toString(fmt), params2.get("earliestUpdatedAt").get(0));
+        Assert.assertEquals(latestUpdatedAt.toString(fmt), params2.get("latestUpdatedAt").get(0));
+        Assert.assertEquals("filter1", params2.get("filterType").get(0));
+        Assert.assertEquals(String.join(",", filterValues), params2.get("filterValues").get(0));
+    }
 }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-marketo
 version: !ruby/object:Gem::Version
-  version: 0.6.10
+  version: 0.6.11
 platform: ruby
 authors:
 - uu59
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-05-28 00:00:00.000000000 Z
+date: 2018-09-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -78,6 +78,7 @@ files:
 - src/main/java/org/embulk/input/marketo/delegate/LeadWithProgramInputPlugin.java
 - src/main/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPlugin.java
 - src/main/java/org/embulk/input/marketo/delegate/MarketoBaseInputPluginDelegate.java
+- src/main/java/org/embulk/input/marketo/delegate/ProgramInputPlugin.java
 - src/main/java/org/embulk/input/marketo/exception/MarketoAPIException.java
 - src/main/java/org/embulk/input/marketo/model/BulkExtractRangeHeader.java
 - src/main/java/org/embulk/input/marketo/model/MarketoAccessTokenResponse.java
@@ -102,6 +103,7 @@ files:
 - src/test/java/org/embulk/input/marketo/delegate/LeadWithListInputPluginTest.java
 - src/test/java/org/embulk/input/marketo/delegate/LeadWithProgramInputPluginTest.java
 - src/test/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPluginTest.java
+- src/test/java/org/embulk/input/marketo/delegate/ProgramInputPluginTest.java
 - src/test/java/org/embulk/input/marketo/rest/MarketoBaseRestClientTest.java
 - src/test/java/org/embulk/input/marketo/rest/MarketoRestClientTest.java
 - src/test/resources/config/activity_bulk_extract_config.yaml
@@ -124,13 +126,13 @@ files:
 - src/test/resources/fixtures/list_reponse_full.json
 - src/test/resources/fixtures/lists_response.json
 - src/test/resources/fixtures/program_response.json
+- classpath/jetty-io-9.2.14.v20151106.jar
+- classpath/jetty-util-9.2.14.v20151106.jar
 - classpath/jetty-http-9.2.14.v20151106.jar
-- classpath/embulk-base-restclient-0.5.3.jar
 - classpath/jetty-client-9.2.14.v20151106.jar
-- classpath/jetty-util-9.2.14.v20151106.jar
-- classpath/embulk-input-marketo-0.6.10.jar
+- classpath/embulk-base-restclient-0.5.3.jar
+- classpath/embulk-input-marketo-0.6.11.jar
 - classpath/embulk-util-retryhelper-jetty92-0.5.3.jar
-- classpath/jetty-io-9.2.14.v20151106.jar
 homepage: https://github.com/treasure-data/embulk-input-marketo
 licenses:
 - Apache2