embulk-input-marketo-through-proxy 0.6.20

data/src/main/java/org/embulk/input/marketo/MarketoInputPlugin.java

@@ -0,0 +1,15 @@
+package org.embulk.input.marketo;
+
+import org.embulk.base.restclient.RestClientInputPluginBase;
+
+/**
+ * Created by tai.khuu on 8/22/17.
+ */
+public class MarketoInputPlugin
+        extends RestClientInputPluginBase<MarketoInputPluginDelegate.PluginTask>
+{
+    public MarketoInputPlugin()
+    {
+        super(MarketoInputPluginDelegate.PluginTask.class, new MarketoInputPluginDelegate());
+    }
+}

data/src/main/java/org/embulk/input/marketo/MarketoInputPluginDelegate.java

@@ -0,0 +1,100 @@
+package org.embulk.input.marketo;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.google.common.base.Optional;
+
+import org.embulk.base.restclient.DispatchingRestClientInputPluginDelegate;
+import org.embulk.base.restclient.RestClientInputPluginDelegate;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigException;
+import org.embulk.input.marketo.delegate.ActivityBulkExtractInputPlugin;
+import org.embulk.input.marketo.delegate.CampaignInputPlugin;
+import org.embulk.input.marketo.delegate.CustomObjectInputPlugin;
+import org.embulk.input.marketo.delegate.LeadBulkExtractInputPlugin;
+import org.embulk.input.marketo.delegate.LeadWithListInputPlugin;
+import org.embulk.input.marketo.delegate.LeadWithProgramInputPlugin;
+import org.embulk.input.marketo.delegate.ProgramInputPlugin;
+import org.embulk.input.marketo.rest.MarketoRestClient;
+
+import java.util.Date;
+
+public class MarketoInputPluginDelegate
+        extends DispatchingRestClientInputPluginDelegate<MarketoInputPluginDelegate.PluginTask>
+{
+    public interface PluginTask
+            extends LeadWithListInputPlugin.PluginTask,
+            LeadBulkExtractInputPlugin.PluginTask,
+            LeadWithProgramInputPlugin.PluginTask,
+            ActivityBulkExtractInputPlugin.PluginTask,
+            CampaignInputPlugin.PluginTask,
+            ProgramInputPlugin.PluginTask,
+            MarketoRestClient.PluginTask,
+            CustomObjectInputPlugin.PluginTask
+    {
+        @Config("target")
+        Target getTarget();
+
+        //We don't need to let the internal plugin know that it being dispatched and force it to set require field optional
+        //We will hide the real from_date, and set it when validating task
+        @Config("hidden_from_date")
+        @ConfigDefault("\"1970-01-01\"")
+        @Override
+        Date getFromDate();
+
+        void setFromDate(Date fromDate);
+
+        @Config("from_date")
+        @ConfigDefault("null")
+        Optional<Date> getWrappedFromDate();
+    }
+
+    @SuppressWarnings("unchecked")
+    @Override
+    protected RestClientInputPluginDelegate dispatchPerTask(PluginTask task)
+    {
+        Target target = task.getTarget();
+        switch (target) {
+            case LEAD:
+            case ACTIVITY:
+                if (!task.getWrappedFromDate().isPresent()) {
+                    throw new ConfigException("From date is required for target LEAD or ACTIVITY");
+                }
+                Date date = task.getWrappedFromDate().get();
+                task.setFromDate(date);
+                break;
+        }
+        return target.getRestClientInputPluginDelegate();
+    }
+
+    public enum Target
+    {
+        LEAD(new LeadBulkExtractInputPlugin()),
+        ACTIVITY(new ActivityBulkExtractInputPlugin()),
+        CAMPAIGN(new CampaignInputPlugin()),
+        ALL_LEAD_WITH_LIST_ID(new LeadWithListInputPlugin()),
+        ALL_LEAD_WITH_PROGRAM_ID(new LeadWithProgramInputPlugin()),
+        PROGRAM(new ProgramInputPlugin()),
+        CUSTOM_OBJECT(new CustomObjectInputPlugin());
+
+        private RestClientInputPluginDelegate restClientInputPluginDelegate;
+
+        Target(RestClientInputPluginDelegate restClientInputPluginDelegate)
+        {
+            this.restClientInputPluginDelegate = restClientInputPluginDelegate;
+        }
+
+        @JsonIgnore
+        public RestClientInputPluginDelegate getRestClientInputPluginDelegate()
+        {
+            return restClientInputPluginDelegate;
+        }
+
+        @JsonCreator
+        public static Target of(String value)
+        {
+            return Target.valueOf(value.toUpperCase());
+        }
+    }
+}

data/src/main/java/org/embulk/input/marketo/MarketoService.java

@@ -0,0 +1,47 @@
+package org.embulk.input.marketo;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.embulk.input.marketo.model.MarketoField;
+
+import java.io.File;
+import java.util.Date;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Created by tai.khuu on 9/6/17.
+ */
+public interface MarketoService
+{
+    List<MarketoField> describeLead();
+
+    File extractLead(Date startTime, Date endTime, List<String> extractedFields, String filterField, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond);
+
+    File extractAllActivity(List<Integer> activityTypeIds, Date startTime, Date endTime, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond);
+
+    Iterable<ObjectNode> getAllListLead(List<String> extractFields, Iterable<ObjectNode> inputListIds);
+
+    Iterable<ObjectNode> getAllProgramLead(List<String> extractFields, Iterable<ObjectNode> requestProgs);
+
+    Iterable<ObjectNode> getCampaign();
+
+    Iterable<ObjectNode> getPrograms();
+
+    Iterable<ObjectNode> getProgramsByIds(Set<String> ids);
+
+    Iterable<ObjectNode> getProgramsByTag(String tagType, String tagValue);
+
+    Iterable<ObjectNode> getProgramsByDateRange(Date earliestUpdatedAt, Date latestUpdatedAt, String filterType, List<String> filterValues);
+
+    Iterable<ObjectNode> getCustomObject(String customObjectApiName, String filterType, Set<String> filterValues, String returnFields);
+
+    Iterable<ObjectNode> getCustomObject(String customObjectAPIName, String customObjectFilterType, String customObjectFields, Integer fromValue, Integer toValue);
+
+    List<MarketoField> describeCustomObject(String customObjectAPIName);
+
+    Iterable<ObjectNode> getActivityTypes();
+
+    Iterable<ObjectNode> getListsByIds(Set<String> ids);
+
+    Iterable<ObjectNode> getLists();
+}

data/src/main/java/org/embulk/input/marketo/MarketoServiceImpl.java

@@ -0,0 +1,258 @@
+package org.embulk.input.marketo;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import org.apache.commons.lang3.StringUtils;
+import org.embulk.input.marketo.model.BulkExtractRangeHeader;
+import org.embulk.input.marketo.model.MarketoField;
+import org.embulk.input.marketo.rest.MarketoRestClient;
+import org.embulk.input.marketo.rest.RecordPagingIterable;
+import org.embulk.spi.DataException;
+import org.embulk.spi.Exec;
+import org.slf4j.Logger;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Date;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+/**
+ * Created by tai.khuu on 9/6/17.
+ */
+public class MarketoServiceImpl implements MarketoService
+{
+    private static final Logger LOGGER = Exec.getLogger(MarketoServiceImpl.class);
+
+    private static final String DEFAULT_FILE_FORMAT = "csv";
+
+    private static final int BUF_SIZE = 0x1000;
+
+    private static final int MAX_RESUME_TIME = 50;
+
+    private MarketoRestClient marketoRestClient;
+
+    public MarketoServiceImpl(MarketoRestClient marketoRestClient)
+    {
+        this.marketoRestClient = marketoRestClient;
+    }
+
+    @Override
+    public File extractLead(final Date startTime, Date endTime, List<String> extractedFields, String filterField, int pollingTimeIntervalSecond, final int bulkJobTimeoutSecond)
+    {
+        final String exportID = marketoRestClient.createLeadBulkExtract(startTime, endTime, extractedFields, filterField);
+        marketoRestClient.startLeadBulkExtract(exportID);
+        try {
+            marketoRestClient.waitLeadExportJobComplete(exportID, pollingTimeIntervalSecond, bulkJobTimeoutSecond);
+        }
+        catch (InterruptedException e) {
+            LOGGER.error("Exception when waiting for export job id: {}", exportID, e);
+            throw new DataException("Error when wait for bulk extract");
+        }
+        return downloadBulkExtract(new Function<BulkExtractRangeHeader, InputStream>()
+        {
+            @Override
+            public InputStream apply(BulkExtractRangeHeader bulkExtractRangeHeader)
+            {
+                return marketoRestClient.getLeadBulkExtractResult(exportID, bulkExtractRangeHeader);
+            }
+        });
+    }
+
+    private long saveExtractedFile(InputStream extractResult, File tempFile) throws DownloadBulkExtractException
+    {
+        long total = 0;
+        try (OutputStream fileOuputStream = new FileOutputStream(tempFile, true)) {
+            byte[] buf = new byte[BUF_SIZE];
+            while (true) {
+                int r = extractResult.read(buf);
+                if (r == -1) {
+                    break;
+                }
+                fileOuputStream.write(buf, 0, r);
+                total += r;
+            }
+        }
+        catch (IOException e) {
+            LOGGER.error("Encounter exception when download bulk extract file", e);
+            throw new DownloadBulkExtractException("Encounter exception when download bulk extract file", e, total);
+        }
+        return total;
+    }
+
+    @Override
+    public File extractAllActivity(List<Integer> activityTypeIds, Date startTime, Date endTime, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond)
+    {
+        final String exportID = marketoRestClient.createActivityExtract(activityTypeIds, startTime, endTime);
+        marketoRestClient.startActitvityBulkExtract(exportID);
+        try {
+            marketoRestClient.waitActitvityExportJobComplete(exportID, pollingTimeIntervalSecond, bulkJobTimeoutSecond);
+        }
+        catch (InterruptedException e) {
+            LOGGER.error("Exception when waiting for export job id: {}", exportID, e);
+            throw new DataException("Error when wait for bulk extract");
+        }
+        return downloadBulkExtract(new Function<BulkExtractRangeHeader, InputStream>()
+        {
+            @Override
+            public InputStream apply(BulkExtractRangeHeader bulkExtractRangeHeader)
+            {
+                return marketoRestClient.getActivitiesBulkExtractResult(exportID, bulkExtractRangeHeader);
+            }
+        });
+    }
+
+    private File downloadBulkExtract(Function<BulkExtractRangeHeader, InputStream> getBulkExtractfunction)
+    {
+        final File tempFile = Exec.getTempFileSpace().createTempFile(DEFAULT_FILE_FORMAT);
+        long startByte = 0;
+        int resumeTime = 0;
+        while (resumeTime < MAX_RESUME_TIME) {
+            BulkExtractRangeHeader bulkExtractRangeHeader = new BulkExtractRangeHeader(startByte);
+            InputStream bulkExtractResult = getBulkExtractfunction.apply(bulkExtractRangeHeader);
+            try {
+                saveExtractedFile(bulkExtractResult, tempFile);
+                return tempFile;
+            }
+            catch (DownloadBulkExtractException e) {
+                startByte = startByte + e.getByteWritten();
+                LOGGER.warn("will resume activity bulk extract at byte [{}]", startByte);
+            }
+            resumeTime = resumeTime + 1;
+        }
+        //Too many resume we still can't get the file
+        throw new DataException("Can't down load bulk extract");
+    }
+
+    @Override
+    public Iterable<ObjectNode> getAllListLead(List<String> fieldNames, Iterable<ObjectNode> inputListIds)
+    {
+        final String fieldNameString = StringUtils.join(fieldNames, ",");
+        return MarketoUtils.flatMap(inputListIds, input -> {
+            final String id = input.get("id").asText();
+            return Iterables.transform(marketoRestClient.getLeadsByList(id, fieldNameString), input1 -> input1.put(MarketoUtils.LIST_ID_COLUMN_NAME, id));
+        });
+    }
+
+    @Override
+    public Iterable<ObjectNode> getAllProgramLead(List<String> fieldNames, Iterable<ObjectNode> requestProgs)
+    {
+        final String fieldNameString = StringUtils.join(fieldNames, ",");
+        return MarketoUtils.flatMap(requestProgs, input -> {
+            final String id = input.get("id").asText();
+            return Iterables.transform(marketoRestClient.getLeadsByProgram(id, fieldNameString), input1 -> input1.put(MarketoUtils.PROGRAM_ID_COLUMN_NAME, id));
+        });
+    }
+
+    @Override
+    public RecordPagingIterable<ObjectNode> getCampaign()
+    {
+        return marketoRestClient.getCampaign();
+    }
+
+    @Override
+    public List<MarketoField> describeLead()
+    {
+        return marketoRestClient.describeLead();
+    }
+
+    private static class DownloadBulkExtractException extends Exception
+    {
+        private final long byteWritten;
+
+        public DownloadBulkExtractException(String message, Throwable cause, long byteWritten)
+        {
+            super(message, cause);
+            this.byteWritten = byteWritten;
+        }
+
+        public DownloadBulkExtractException(Throwable cause, long byteWritten)
+        {
+            super(cause);
+            this.byteWritten = byteWritten;
+        }
+
+        public long getByteWritten()
+        {
+            return byteWritten;
+        }
+    }
+
+    @Override
+    public Iterable<ObjectNode> getPrograms()
+    {
+        return marketoRestClient.getPrograms();
+    }
+
+    @Override
+    public Iterable<ObjectNode> getProgramsByIds(Set<String> ids)
+    {
+        return marketoRestClient.getProgramsByIds(ids);
+    }
+
+    @Override
+    public Iterable<ObjectNode> getProgramsByTag(String tagType, String tagValue)
+    {
+        return marketoRestClient.getProgramsByTag(tagType, tagValue);
+    }
+
+    @Override
+    public Iterable<ObjectNode> getProgramsByDateRange(Date earliestUpdatedAt, Date latestUpdatedAt, String filterType, List<String> filterValues)
+    {
+        return marketoRestClient.getProgramsByDateRange(earliestUpdatedAt, latestUpdatedAt, filterType, filterValues);
+    }
+
+    @Override
+    public List<MarketoField> describeCustomObject(String customObjectAPIName)
+    {
+        return marketoRestClient.describeCustomObject(customObjectAPIName);
+    }
+
+    @Override
+    public Iterable<ObjectNode> getCustomObject(String customObjectAPIName, String customObjectFilterType, String customObjectFields, Integer fromValue, Integer toValue)
+    {
+        if (toValue == null) {
+            return marketoRestClient.getCustomObject(customObjectAPIName, customObjectFilterType, customObjectFields, fromValue, toValue);
+        }
+
+        // make sure to import values in the whole range
+        Set<String> filterValues = IntStream.rangeClosed(fromValue, toValue.intValue()).mapToObj(String::valueOf).collect(Collectors.toSet());
+        return getCustomObject(customObjectAPIName, customObjectFilterType, filterValues, customObjectFields);
+    }
+
+    @Override
+    public Iterable<ObjectNode> getCustomObject(String customObjectApiName, String filterType, Set<String> filterValues, String returnFields)
+    {
+        // Marketo allows maximum 300 (comma separated) filterValues per request. Split the input and process by chunk.
+        List<List<String>> partitionedFilterValues = Lists.partition(Lists.newArrayList(filterValues), 300);
+        return MarketoUtils.flatMap(partitionedFilterValues, (part) -> {
+            String strFilterValues = StringUtils.join(part, ",");
+            return marketoRestClient.getCustomObject(customObjectApiName, filterType, strFilterValues, returnFields);
+        });
+    }
+
+    @Override
+    public Iterable<ObjectNode> getActivityTypes()
+    {
+        return marketoRestClient.getActivityTypes();
+    }
+
+    @Override
+    public Iterable<ObjectNode> getListsByIds(Set<String> ids)
+    {
+        return marketoRestClient.getListsByIds(ids);
+    }
+
+    @Override
+    public Iterable<ObjectNode> getLists()
+    {
+        return marketoRestClient.getLists();
+    }
+}

data/src/main/java/org/embulk/input/marketo/MarketoUtils.java

@@ -0,0 +1,212 @@
+package org.embulk.input.marketo;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.base.Function;
+import com.google.common.collect.Sets;
+import org.embulk.base.restclient.ServiceResponseMapper;
+import org.embulk.base.restclient.jackson.JacksonServiceRecord;
+import org.embulk.base.restclient.jackson.JacksonServiceResponseMapper;
+import org.embulk.base.restclient.jackson.JacksonTopLevelValueLocator;
+import org.embulk.base.restclient.record.ServiceRecord;
+import org.embulk.base.restclient.record.ValueLocator;
+import org.embulk.input.marketo.model.MarketoField;
+import org.embulk.spi.Exec;
+import org.embulk.spi.util.RetryExecutor;
+import org.slf4j.Logger;
+
+import javax.annotation.Nullable;
+
+import java.time.OffsetDateTime;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.Set;
+
+/**
+ * Created by tai.khuu on 9/18/17.
+ */
+public class MarketoUtils
+{
+    public static final String MARKETO_DATE_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S%z";
+    public static final String MARKETO_DATE_FORMAT = "%Y-%m-%d";
+    public static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+    public static final Function<ObjectNode, ServiceRecord> TRANSFORM_OBJECT_TO_JACKSON_SERVICE_RECORD_FUNCTION = new Function<ObjectNode, ServiceRecord>()
+    {
+        @Nullable
+        @Override
+        public JacksonServiceRecord apply(@Nullable ObjectNode input)
+        {
+            return new JacksonServiceRecord(input);
+        }
+    };
+
+    public static final String MARKETO_DATE_SIMPLE_DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ssZ";
+
+    public static final String LIST_ID_COLUMN_NAME = "listId";
+
+    public static final String PROGRAM_ID_COLUMN_NAME = "programId";
+
+    private MarketoUtils()
+    {
+    }
+
+    public static ServiceResponseMapper<? extends ValueLocator> buildDynamicResponseMapper(String prefix, List<MarketoField> columns)
+    {
+        JacksonServiceResponseMapper.Builder builder = JacksonServiceResponseMapper.builder();
+        for (MarketoField column : columns) {
+            String columName = buildColumnName(prefix, column.getName());
+            MarketoField.MarketoDataType marketoDataType = column.getMarketoDataType();
+            if (marketoDataType.getFormat().isPresent()) {
+                builder.add(new JacksonTopLevelValueLocator(column.getName()), columName, marketoDataType.getType(), marketoDataType.getFormat().get());
+            }
+            else {
+                builder.add(new JacksonTopLevelValueLocator(column.getName()), columName, marketoDataType.getType());
+            }
+        }
+        return builder.build();
+    }
+
+    public static List<String> getFieldNameFromMarketoFields(List<MarketoField> columns, String... excludedFields)
+    {
+        Set<String> excludeFields = Sets.newHashSet(excludedFields);
+        List<String> extractedFields = new ArrayList<>();
+        for (MarketoField column : columns) {
+            if (excludeFields.contains(column.getName())) {
+                continue;
+            }
+            extractedFields.add(column.getName());
+        }
+        return extractedFields;
+    }
+
+    public static String buildColumnName(String prefix, String columnName)
+    {
+        return prefix + "_" + columnName;
+    }
+
+    public static final List<DateRange> sliceRange(OffsetDateTime fromDate, OffsetDateTime toDate, int rangeSize)
+    {
+        List<DateRange> ranges = new ArrayList<>();
+        while (fromDate.isBefore(toDate)) {
+            OffsetDateTime nextToDate = fromDate.plusDays(rangeSize);
+            if (nextToDate.isAfter(toDate)) {
+                ranges.add(new DateRange(fromDate, toDate));
+                break;
+            }
+            ranges.add(new DateRange(fromDate, nextToDate));
+            fromDate = nextToDate.plusSeconds(1);
+        }
+        return ranges;
+    }
+
+    public static String getIdentityEndPoint(String accountId)
+    {
+        return "https://" + accountId + ".mktorest.com/identity";
+    }
+
+    public static String getEndPoint(String accountID)
+    {
+        return "https://" + accountID + ".mktorest.com";
+    }
+
+    public static  final class DateRange
+    {
+        public final OffsetDateTime fromDate;
+        public final OffsetDateTime toDate;
+
+        public DateRange(OffsetDateTime fromDate, OffsetDateTime toDate)
+        {
+            this.fromDate = fromDate;
+            this.toDate = toDate;
+        }
+
+        @Override
+        public String toString()
+        {
+            return "DateRange{" +
+                    "fromDate=" + fromDate +
+                    ", toDate=" + toDate +
+                    '}';
+        }
+    }
+
+    public static <T> T executeWithRetry(int maximumRetries, int initialRetryIntervalMillis, int maximumRetryIntervalMillis, AlwaysRetryRetryable<T> alwaysRetryRetryable) throws RetryExecutor.RetryGiveupException, InterruptedException
+    {
+        return RetryExecutor
+                .retryExecutor()
+                .withRetryLimit(maximumRetries)
+                .withInitialRetryWait(initialRetryIntervalMillis)
+                .withMaxRetryWait(maximumRetryIntervalMillis)
+                .runInterruptible(alwaysRetryRetryable);
+    }
+
+    public abstract static class AlwaysRetryRetryable<T> implements  RetryExecutor.Retryable<T>
+    {
+        private static final Logger LOGGER = Exec.getLogger(AlwaysRetryRetryable.class);
+
+        @Override
+        public abstract T call() throws Exception;
+
+        @Override
+        public boolean isRetryableException(Exception exception)
+        {
+            return true;
+        }
+
+        @Override
+        public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait) throws RetryExecutor.RetryGiveupException
+        {
+            LOGGER.info("Retry [{}]/[{}] with retryWait [{}] on exception {}", retryCount, retryLimit, retryWait, exception.getMessage());
+        }
+
+        @Override
+        public void onGiveup(Exception firstException, Exception lastException) throws RetryExecutor.RetryGiveupException
+        {
+            LOGGER.info("Giving up execution on exception", lastException);
+        }
+    }
+    public static <T, R> Iterable<R> flatMap(final Iterable<T> iterable, final Function<T, Iterable<R>> function)
+    {
+        final Iterator<T> iterator = iterable.iterator();
+        return new Iterable<R>()
+        {
+            @Override
+            public Iterator<R> iterator()
+            {
+                return new Iterator<R>()
+                {
+                    Iterator<R> currentIterator;
+                    @Override
+                    public boolean hasNext()
+                    {
+                        if (currentIterator != null && currentIterator.hasNext()) {
+                            return true;
+                        }
+                        while (iterator.hasNext()) {
+                            currentIterator = function.apply(iterator.next()).iterator();
+                            if (currentIterator.hasNext()) {
+                                return true;
+                            }
+                        }
+                        return false;
+                    }
+                    @Override
+                    public R next()
+                    {
+                        if (hasNext()) {
+                            return currentIterator.next();
+                        }
+                        throw new NoSuchElementException();
+                    }
+                    @Override
+                    public void remove()
+                    {
+                        throw new UnsupportedOperationException();
+                    }
+                };
+            }
+        };
+    }
+}