embulk-input-marketo-through-proxy 0.6.20

data/src/main/java/org/embulk/input/marketo/delegate/MarketoBaseInputPluginDelegate.java

@@ -0,0 +1,160 @@
+package org.embulk.input.marketo.delegate;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Lists;
+import org.apache.commons.lang3.StringUtils;
+import org.embulk.base.restclient.DefaultServiceDataSplitter;
+import org.embulk.base.restclient.RestClientInputPluginDelegate;
+import org.embulk.base.restclient.RestClientInputTaskBase;
+import org.embulk.base.restclient.ServiceDataSplitter;
+import org.embulk.base.restclient.record.RecordImporter;
+import org.embulk.base.restclient.record.ServiceRecord;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigException;
+import org.embulk.config.TaskReport;
+import org.embulk.input.marketo.MarketoService;
+import org.embulk.input.marketo.MarketoServiceImpl;
+import org.embulk.input.marketo.rest.MarketoRestClient;
+import org.embulk.spi.Exec;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.Schema;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.time.OffsetDateTime;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+/**
+ * Created by tai.khuu on 9/18/17.
+ */
+public abstract class MarketoBaseInputPluginDelegate<T extends MarketoBaseInputPluginDelegate.PluginTask> implements RestClientInputPluginDelegate<T>
+{
+    private final Logger logger = LoggerFactory.getLogger(getClass());
+    public static final int PREVIEW_RECORD_LIMIT = 15;
+    public static final String ID_LIST_SEPARATOR_CHAR = ",";
+
+    public interface PluginTask
+            extends RestClientInputTaskBase, MarketoRestClient.PluginTask
+    {
+        @Config("schema_column_prefix")
+        @ConfigDefault("\"mk\"")
+        String getSchemaColumnPrefix();
+
+        @Config("incremental")
+        @ConfigDefault("true")
+        Boolean getIncremental();
+
+        String getJobStartTime();
+
+        void setJobStartTime(String dateTime);
+    }
+
+    @Override
+    public ConfigDiff buildConfigDiff(T task, Schema schema, int taskCount, List<TaskReport> taskReports)
+    {
+        return Exec.newConfigDiff();
+    }
+
+    @Override
+    public void validateInputTask(T task)
+    {
+        task.setJobStartTime(OffsetDateTime.now(ZoneOffset.UTC).format(DateTimeFormatter.ISO_OFFSET_DATE_TIME));
+    }
+
+    @Override
+    public TaskReport ingestServiceData(T task, RecordImporter recordImporter, int taskIndex, PageBuilder pageBuilder)
+    {
+        if (Exec.isPreview()) {
+            task.setBatchSize(PREVIEW_RECORD_LIMIT);
+        }
+        try (MarketoRestClient restClient = createMarketoRestClient(task)) {
+            MarketoService marketoService = new MarketoServiceImpl(restClient);
+            Iterator<ServiceRecord> serviceRecords = getServiceRecords(marketoService, task);
+            int imported = 0;
+            while (serviceRecords.hasNext() && (imported < PREVIEW_RECORD_LIMIT || !Exec.isPreview())) {
+                ServiceRecord next = serviceRecords.next();
+                recordImporter.importRecord(next, pageBuilder);
+                imported++;
+            }
+            return Exec.newTaskReport();
+        }
+    }
+
+    protected abstract Iterator<ServiceRecord> getServiceRecords(MarketoService marketoService, T task);
+
+    @VisibleForTesting
+    public MarketoRestClient createMarketoRestClient(PluginTask task)
+    {
+        return new MarketoRestClient(task);
+    }
+
+    @Override
+    public ServiceDataSplitter<T> buildServiceDataSplitter(T task)
+    {
+        return new DefaultServiceDataSplitter();
+    }
+
+    protected Iterable<ObjectNode> getObjectsByIds(String[] inputIds, Function<Set<String>, Iterable<ObjectNode>> getByIdFunction)
+    {
+        final Set<String> ids = new HashSet<>();
+        final List<String> invalidIds = new ArrayList<>();
+
+        for (int i = 0; i < inputIds.length; i++) {
+            String currentId = StringUtils.trimToNull(inputIds[i]);
+            // ignore null or empty ids
+            if (currentId == null) {
+                continue;
+            }
+
+            // ignore or throw for NaN ids
+            if (StringUtils.isNumeric(currentId)) {
+                ids.add(currentId);
+            }
+            else {
+                invalidIds.add(inputIds[i]);
+            }
+        }
+
+        if (ids.isEmpty()) {
+            throw new ConfigException("No valid Id specified");
+        }
+
+        if (!invalidIds.isEmpty()) {
+            logger.warn("Ignore invalid Id(s): {}", invalidIds);
+        }
+
+        List<ObjectNode> actualList = Lists.newArrayList(getByIdFunction.apply(ids));
+        if (actualList.isEmpty()) {
+            throw new ConfigException("No valid Id found");
+        }
+
+        if (actualList.size() != ids.size()) {
+            logNoneExistedIds(ids, actualList);
+        }
+
+        return actualList;
+    }
+
+    private void logNoneExistedIds(Set<String> ids, List<ObjectNode> actualList)
+    {
+        List<String> actualIds = actualList.parallelStream().map(n -> String.valueOf(n.get("id").asInt())).collect(Collectors.toList());
+        List<String> missingIds = new ArrayList<>();
+        for (String id : ids) {
+            if (!actualIds.contains(id)) {
+                missingIds.add(id);
+            }
+        }
+        logger.warn("Ignore not exists Id(s): {}", missingIds);
+    }
+}

data/src/main/java/org/embulk/input/marketo/delegate/ProgramInputPlugin.java

@@ -0,0 +1,234 @@
+package org.embulk.input.marketo.delegate;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.base.Optional;
+import com.google.common.collect.FluentIterable;
+import org.embulk.base.restclient.ServiceResponseMapper;
+import org.embulk.base.restclient.jackson.JacksonServiceResponseMapper;
+import org.embulk.base.restclient.record.RecordImporter;
+import org.embulk.base.restclient.record.ServiceRecord;
+import org.embulk.base.restclient.record.ValueLocator;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigException;
+import org.embulk.config.TaskReport;
+import org.embulk.input.marketo.MarketoService;
+import org.embulk.input.marketo.MarketoUtils;
+import org.embulk.spi.Exec;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.Schema;
+import org.embulk.spi.type.Types;
+import org.slf4j.Logger;
+
+import java.time.Duration;
+import java.time.OffsetDateTime;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.time.temporal.ChronoUnit;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.List;
+
+public class ProgramInputPlugin extends MarketoBaseInputPluginDelegate<ProgramInputPlugin.PluginTask>
+{
+    private static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern(MarketoUtils.MARKETO_DATE_SIMPLE_DATE_FORMAT);
+    private final Logger logger = Exec.getLogger(getClass());
+
+    public interface PluginTask extends MarketoBaseInputPluginDelegate.PluginTask
+    {
+        @Config("query_by")
+        @ConfigDefault("null")
+        Optional<QueryBy> getQueryBy();
+
+        @Config("tag_type")
+        @ConfigDefault("null")
+        Optional<String> getTagType();
+
+        @Config("tag_value")
+        @ConfigDefault("null")
+        Optional<String> getTagVallue();
+
+        @Config("earliest_updated_at")
+        @ConfigDefault("null")
+        Optional<Date> getEarliestUpdatedAt();
+
+        @Config("latest_updated_at")
+        @ConfigDefault("null")
+        Optional<Date> getLatestUpdatedAt();
+
+        @Config("filter_type")
+        @ConfigDefault("null")
+        Optional<String> getFilterType();
+
+        @Config("filter_values")
+        @ConfigDefault("null")
+        Optional<List<String>> getFilterValues();
+
+        @Config("report_duration")
+        @ConfigDefault("null")
+        Optional<Long> getReportDuration();
+
+        void setLatestUpdatedAt(Optional<Date> latestUpdatedAt);
+    }
+
+    public ProgramInputPlugin()
+    {
+    }
+
+    @Override
+    public void validateInputTask(PluginTask task)
+    {
+        super.validateInputTask(task);
+        // validate if query_by is selected
+        if (task.getQueryBy().isPresent()) {
+            switch(task.getQueryBy().get()) {
+            case TAG_TYPE:
+                //make sure tag type and tag value are not empty
+                if (!task.getTagType().isPresent() || !task.getTagVallue().isPresent()) {
+                    throw new ConfigException("tag_type and tag_value are required when query by Tag Type");
+                }
+                break;
+            case DATE_RANGE:
+                // make sure earliest_updated_at is not empty
+                if (!task.getEarliestUpdatedAt().isPresent()) {
+                    throw new ConfigException("`earliest_updated_at` is required when query by Date Range");
+                }
+
+                OffsetDateTime earliest = OffsetDateTime.ofInstant(task.getEarliestUpdatedAt().get().toInstant(), ZoneOffset.UTC);
+                if (task.getReportDuration().isPresent()) {
+                    logger.info("`report_duration` is present, Prefer `report_duration` over `latest_updated_at`");
+                    // Update the latestUpdatedAt for the config
+                    OffsetDateTime latest = earliest.plus(task.getReportDuration().get(), ChronoUnit.MILLIS);
+                    task.setLatestUpdatedAt(Optional.of(Date.from(latest.toInstant())));
+                }
+
+                // latest_updated_at is required calculate time range
+                if (!task.getLatestUpdatedAt().isPresent()) {
+                    throw new ConfigException("`latest_updated_at` is required when query by Date Range");
+                }
+
+                OffsetDateTime latest = OffsetDateTime.ofInstant(task.getLatestUpdatedAt().get().toInstant(), ZoneOffset.UTC);
+                if (earliest.isAfter(OffsetDateTime.now(ZoneOffset.UTC))) {
+                    throw new ConfigException(String.format("`earliest_updated_at` (%s) cannot precede the current date (%s)",
+                                    earliest.format(DATE_FORMATTER),
+                                    (OffsetDateTime.now(ZoneOffset.UTC).format(DATE_FORMATTER))));
+                }
+
+                if (earliest.isAfter(latest)) {
+                    throw new ConfigException(String.format("Invalid date range. `earliest_updated_at` (%s) cannot precede the `latest_updated_at` (%s).",
+                                    earliest.format(DATE_FORMATTER),
+                                    latest.format(DATE_FORMATTER)));
+                }
+                // if filter type is selected, filter value must be presented
+                if (task.getFilterType().isPresent() && (!task.getFilterValues().isPresent() || task.getFilterValues().get().isEmpty())) {
+                    throw new ConfigException("filter_value is required when selected filter_type");
+                }
+            }
+        }
+    }
+
+    @Override
+    public TaskReport ingestServiceData(PluginTask task, RecordImporter recordImporter, int taskIndex, PageBuilder pageBuilder)
+    {
+        // query by date range and incremental import and not preview
+        if (task.getQueryBy().isPresent() && task.getQueryBy().get() == QueryBy.DATE_RANGE && task.getIncremental() && !Exec.isPreview()) {
+            OffsetDateTime latestUpdateAt = OffsetDateTime.ofInstant(task.getLatestUpdatedAt().get().toInstant(), ZoneOffset.UTC);
+            OffsetDateTime now = OffsetDateTime.now(ZoneOffset.UTC);
+            // Do not run incremental import if latest_updated_at precede current time
+            if (latestUpdateAt.isAfter(now)) {
+                logger.warn("`latest_updated_at` ({}) preceded current time ({}). Will try to import next run",
+                        latestUpdateAt.format(DATE_FORMATTER), now.format(DATE_FORMATTER));
+
+                OffsetDateTime earliest = OffsetDateTime.ofInstant(task.getEarliestUpdatedAt().get().toInstant(), ZoneOffset.UTC);
+                TaskReport taskReport = Exec.newTaskReport();
+                taskReport.set("earliest_updated_at", earliest.format(DATE_FORMATTER));
+                if (task.getReportDuration().isPresent()) {
+                    taskReport.set("report_duration", task.getReportDuration().get());
+                }
+                return taskReport;
+            }
+        }
+        return super.ingestServiceData(task, recordImporter, taskIndex, pageBuilder);
+    }
+
+    @Override
+    protected Iterator<ServiceRecord> getServiceRecords(MarketoService marketoService, PluginTask task)
+    {
+        Iterable<ObjectNode> nodes = null;
+        if (task.getQueryBy().isPresent()) {
+            switch (task.getQueryBy().get()) {
+            case TAG_TYPE:
+                nodes = marketoService.getProgramsByTag(task.getTagType().get(), task.getTagVallue().get());
+                break;
+            case DATE_RANGE:
+                nodes = marketoService.getProgramsByDateRange(task.getEarliestUpdatedAt().get(),
+                                task.getLatestUpdatedAt().get(),
+                                task.getFilterType().orNull(),
+                                task.getFilterValues().orNull());
+            }
+        }
+        else {
+            nodes = marketoService.getPrograms();
+        }
+        return FluentIterable.from(nodes).transform(MarketoUtils.TRANSFORM_OBJECT_TO_JACKSON_SERVICE_RECORD_FUNCTION).iterator();
+    }
+
+    @Override
+    public ConfigDiff buildConfigDiff(PluginTask task, Schema schema, int taskCount, List<TaskReport> taskReports)
+    {
+        ConfigDiff configDiff = super.buildConfigDiff(task, schema, taskCount, taskReports);
+        // set next next earliestUpdatedAt, latestUpdatedAt
+        if (task.getQueryBy().isPresent() && task.getQueryBy().get() == QueryBy.DATE_RANGE && task.getIncremental()) {
+            OffsetDateTime earliest = task.getEarliestUpdatedAt().isPresent() ?
+                    OffsetDateTime.ofInstant(task.getEarliestUpdatedAt().get().toInstant(), ZoneOffset.UTC) :
+                    OffsetDateTime.now(ZoneOffset.UTC);
+            OffsetDateTime latest = task.getLatestUpdatedAt().isPresent() ?
+                    OffsetDateTime.ofInstant(task.getLatestUpdatedAt().get().toInstant(), ZoneOffset.UTC) :
+                    OffsetDateTime.now(ZoneOffset.UTC);
+
+            Duration d = Duration.between(earliest, latest);
+            OffsetDateTime nextEarliestUpdatedAt = latest.plusSeconds(1);
+
+            configDiff.set("earliest_updated_at", nextEarliestUpdatedAt.format(DATE_FORMATTER));
+            configDiff.set("report_duration", task.getReportDuration().or(d.toMillis()));
+        }
+        return configDiff;
+    }
+
+    @Override
+    public ServiceResponseMapper<? extends ValueLocator> buildServiceResponseMapper(PluginTask task)
+    {
+        JacksonServiceResponseMapper.Builder builder = JacksonServiceResponseMapper.builder();
+        builder.add("id", Types.LONG)
+                .add("name", Types.STRING)
+                .add("sfdcId", Types.STRING)
+                .add("sfdcName", Types.STRING)
+                .add("description", Types.STRING)
+                .add("createdAt", Types.TIMESTAMP, MarketoUtils.MARKETO_DATE_TIME_FORMAT)
+                .add("updatedAt", Types.TIMESTAMP, MarketoUtils.MARKETO_DATE_TIME_FORMAT)
+                .add("startDate", Types.TIMESTAMP, MarketoUtils.MARKETO_DATE_TIME_FORMAT)
+                .add("endDate", Types.TIMESTAMP, MarketoUtils.MARKETO_DATE_TIME_FORMAT)
+                .add("url", Types.STRING)
+                .add("type", Types.STRING)
+                .add("channel", Types.STRING)
+                .add("folder", Types.JSON)
+                .add("status", Types.STRING)
+                .add("costs", Types.JSON)
+                .add("tags", Types.JSON)
+                .add("workspace", Types.STRING);
+        return builder.build();
+    }
+
+    public enum QueryBy {
+        TAG_TYPE,
+        DATE_RANGE;
+
+        @JsonCreator
+        public static QueryBy of(String value)
+        {
+            return QueryBy.valueOf(value.toUpperCase());
+        }
+    }
+}

data/src/main/java/org/embulk/input/marketo/exception/MarketoAPIException.java

@@ -0,0 +1,30 @@
+package org.embulk.input.marketo.exception;
+
+import org.embulk.input.marketo.model.MarketoError;
+
+import java.util.List;
+
+/**
+ * Exception class for all API Exception
+ * Created by tai.khuu on 9/5/17.
+ */
+public class MarketoAPIException extends Exception
+{
+    private final List<MarketoError> marketoErrors;
+    public MarketoAPIException(List<MarketoError> marketoErrors)
+    {
+        this.marketoErrors = marketoErrors;
+    }
+
+    public List<MarketoError> getMarketoErrors()
+    {
+        return marketoErrors;
+    }
+
+    @Override
+    public String getMessage()
+    {
+        MarketoError error = getMarketoErrors().get(0);
+        return "Marketo API Error, code: " + error.getCode() + ", message: " + error.getMessage();
+    }
+}

data/src/main/java/org/embulk/input/marketo/model/BulkExtractRangeHeader.java

@@ -0,0 +1,26 @@
+package org.embulk.input.marketo.model;
+
+/**
+ * Created by tai.khuu on 10/12/17.
+ */
+public class BulkExtractRangeHeader
+{
+    private Long start;
+    private Long end;
+
+    public BulkExtractRangeHeader(long start)
+    {
+        this.start = start;
+    }
+
+    public BulkExtractRangeHeader(long start, long end)
+    {
+        this.start = start;
+        this.end = end;
+    }
+
+    public String toRangeHeaderValue()
+    {
+        return "bytes=" + start + "-" + (end != null ? end : "");
+    }
+}