embulk-input-marketo-through-proxy 0.6.20

data/src/main/java/org/embulk/input/marketo/delegate/ActivityBulkExtractInputPlugin.java

@@ -0,0 +1,169 @@
+package org.embulk.input.marketo.delegate;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Optional;
+import org.apache.commons.lang3.StringUtils;
+import org.embulk.base.restclient.ServiceResponseMapper;
+import org.embulk.base.restclient.jackson.JacksonServiceResponseMapper;
+import org.embulk.base.restclient.record.ValueLocator;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigException;
+import org.embulk.input.marketo.MarketoService;
+import org.embulk.input.marketo.MarketoServiceImpl;
+import org.embulk.input.marketo.MarketoUtils;
+import org.embulk.input.marketo.rest.MarketoRestClient;
+import org.embulk.spi.type.Types;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.InputStream;
+import java.text.MessageFormat;
+import java.time.OffsetDateTime;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * Created by tai.khuu on 9/18/17.
+ */
+public class ActivityBulkExtractInputPlugin extends MarketoBaseBulkExtractInputPlugin<ActivityBulkExtractInputPlugin.PluginTask>
+{
+    public static final String INCREMENTAL_COLUMN = "activityDate";
+    public static final String UID_COLUMN = "marketoGUID";
+
+    public interface PluginTask extends MarketoBaseBulkExtractInputPlugin.PluginTask
+    {
+        @Config("activity_type_ids")
+        @ConfigDefault("[]")
+        List<String> getActivityTypeIds();
+
+        @Config("act_type_ids")
+        @ConfigDefault("[]")
+        List<Integer> getActTypeIds();
+
+        void setActTypeIds(List<Integer> activityIds);
+    }
+
+    @Override
+    public void validateInputTask(PluginTask task)
+    {
+        task.setIncrementalColumn(Optional.of(INCREMENTAL_COLUMN));
+        task.setUidColumn(Optional.of(UID_COLUMN));
+        if (!task.getActivityTypeIds().isEmpty()) {
+            List<Integer> activityIds = checkValidActivityTypeIds(task);
+
+            // check input with values from server
+            try (MarketoRestClient restClient = createMarketoRestClient(task)) {
+                MarketoService marketoService = new MarketoServiceImpl(restClient);
+                Iterable<ObjectNode> nodes = marketoService.getActivityTypes();
+                if (nodes != null) {
+                    checkValidActivityTypeIds(nodes, activityIds);
+                }
+                // ignorable if unable to get activity type ids. If thing gone wrong, the bulk extract will throw errors
+            }
+
+            // task will use getActTypeIds instead of getActivityTypeIds method
+            task.setActTypeIds(activityIds);
+        }
+        super.validateInputTask(task);
+    }
+
+    /**
+     * Check if user input activity_type_ids valid
+     * @param task
+     * @return values transformed to array of Integer
+     */
+    private List<Integer> checkValidActivityTypeIds(PluginTask task)
+    {
+        List<String> invalidIds = new ArrayList<>();
+        for (String id : task.getActivityTypeIds()) {
+            if (StringUtils.isBlank(id) || !StringUtils.isNumeric(StringUtils.trimToEmpty(id))) {
+                invalidIds.add(id);
+            }
+        }
+
+        if (!invalidIds.isEmpty()) {
+            throw new ConfigException(MessageFormat.format("Invalid activity type id: [{0}]", StringUtils.join(invalidIds, ", ")));
+        }
+
+        // transform and set
+        List<Integer> activityIds = new ArrayList<>();
+        for (String id : task.getActivityTypeIds()) {
+            activityIds.add(Integer.valueOf(StringUtils.trimToEmpty(id)));
+        }
+
+        return activityIds;
+    }
+
+    @VisibleForTesting
+    protected void checkValidActivityTypeIds(Iterable<ObjectNode> nodes, List<Integer> activityIds)
+    {
+        Iterator<ObjectNode> it = nodes.iterator();
+
+        List<Integer> inputIds = new ArrayList<>(activityIds);
+
+        while (it.hasNext()) {
+            ObjectNode node = it.next();
+            int id = node.get("id").asInt(0);
+            if (id > 0) {
+                inputIds.remove(Integer.valueOf(id));
+            }
+        }
+
+        if (!inputIds.isEmpty()) {
+            throw new ConfigException(MessageFormat.format("Invalid activity type ids: [{0}], Available activity types: \n{1}",
+                    StringUtils.join(inputIds, ", "),
+                    buildActivityIdNameInfo(nodes)));
+        }
+    }
+
+    private String buildActivityIdNameInfo(Iterable<ObjectNode> nodes)
+    {
+        Iterator<ObjectNode> it = nodes.iterator();
+        StringBuilder messageBuilder = new StringBuilder();
+        while (it.hasNext()) {
+            ObjectNode node = it.next();
+            int id = node.get("id").asInt(0);
+            String name = node.get("name").asText("");
+            if (id > 0) {
+                messageBuilder.append("- activity id: ");
+                messageBuilder.append(String.valueOf(id));
+                messageBuilder.append(", name: ");
+                messageBuilder.append(name);
+                messageBuilder.append("\n");
+            }
+        }
+
+        return messageBuilder.toString();
+    }
+
+    @Override
+    protected InputStream getExtractedStream(MarketoService service, PluginTask task, OffsetDateTime fromDate, OffsetDateTime toDate)
+    {
+        try {
+            return new FileInputStream(service.extractAllActivity(task.getActTypeIds(), Date.from(fromDate.toInstant()),
+                    Date.from(toDate.toInstant()), task.getPollingIntervalSecond(), task.getBulkJobTimeoutSecond()));
+        }
+        catch (FileNotFoundException e) {
+            throw new RuntimeException("Exception when trying to extract activity", e);
+        }
+    }
+
+    @Override
+    public ServiceResponseMapper<? extends ValueLocator> buildServiceResponseMapper(PluginTask task)
+    {
+        JacksonServiceResponseMapper.Builder builder = JacksonServiceResponseMapper.builder();
+        builder.add("marketoGUID", Types.STRING)
+                .add("leadId", Types.STRING)
+                .add("activityDate", Types.TIMESTAMP, MarketoUtils.MARKETO_DATE_TIME_FORMAT)
+                .add("activityTypeId", Types.STRING)
+                .add("campaignId", Types.STRING)
+                .add("primaryAttributeValueId", Types.STRING)
+                .add("primaryAttributeValue", Types.STRING)
+                .add("attributes", Types.JSON);
+        return builder.build();
+    }
+}

data/src/main/java/org/embulk/input/marketo/delegate/CampaignInputPlugin.java

@@ -0,0 +1,48 @@
+package org.embulk.input.marketo.delegate;
+
+import com.google.common.collect.FluentIterable;
+import org.embulk.base.restclient.ServiceResponseMapper;
+import org.embulk.base.restclient.jackson.JacksonServiceResponseMapper;
+import org.embulk.base.restclient.record.ServiceRecord;
+import org.embulk.base.restclient.record.ValueLocator;
+import org.embulk.input.marketo.MarketoService;
+import org.embulk.input.marketo.MarketoUtils;
+import org.embulk.spi.type.Types;
+
+import java.util.Iterator;
+
+/**
+ * Input plugin use to import Campaign
+ * Created by tai.khuu on 9/18/17.
+ */
+public class CampaignInputPlugin extends MarketoBaseInputPluginDelegate<CampaignInputPlugin.PluginTask>
+{
+    public interface PluginTask extends MarketoBaseInputPluginDelegate.PluginTask
+    {
+    }
+
+    public CampaignInputPlugin()
+    {
+    }
+
+    @Override
+    protected Iterator<ServiceRecord> getServiceRecords(MarketoService marketoService, PluginTask task)
+    {
+        return FluentIterable.from(marketoService.getCampaign()).transform(MarketoUtils.TRANSFORM_OBJECT_TO_JACKSON_SERVICE_RECORD_FUNCTION).iterator();
+    }
+
+    @Override
+    public ServiceResponseMapper<? extends ValueLocator> buildServiceResponseMapper(PluginTask task)
+    {
+        JacksonServiceResponseMapper.Builder builder = JacksonServiceResponseMapper.builder();
+        builder.add("id", Types.LONG)
+                .add("name", Types.STRING)
+                .add("description", Types.STRING)
+                .add("type", Types.STRING)
+                .add("workspaceName", Types.STRING)
+                .add("createdAt", Types.TIMESTAMP, MarketoUtils.MARKETO_DATE_TIME_FORMAT)
+                .add("updatedAt", Types.TIMESTAMP, MarketoUtils.MARKETO_DATE_TIME_FORMAT)
+                .add("active", Types.BOOLEAN);
+        return builder.build();
+    }
+}

data/src/main/java/org/embulk/input/marketo/delegate/CustomObjectInputPlugin.java

@@ -0,0 +1,124 @@
+package org.embulk.input.marketo.delegate;
+
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.base.Optional;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.Iterables;
+import org.apache.commons.lang3.StringUtils;
+import org.embulk.base.restclient.ServiceResponseMapper;
+import org.embulk.base.restclient.record.ServiceRecord;
+import org.embulk.base.restclient.record.ValueLocator;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigException;
+import org.embulk.input.marketo.MarketoService;
+import org.embulk.input.marketo.MarketoServiceImpl;
+import org.embulk.input.marketo.MarketoUtils;
+import org.embulk.input.marketo.rest.MarketoRestClient;
+import org.embulk.spi.Exec;
+import org.slf4j.Logger;
+
+import java.util.Iterator;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+public class CustomObjectInputPlugin extends MarketoBaseInputPluginDelegate<CustomObjectInputPlugin.PluginTask>
+{
+    private final Logger logger = Exec.getLogger(getClass());
+
+    public interface PluginTask extends MarketoBaseInputPluginDelegate.PluginTask, CustomObjectResponseMapperBuilder.PluginTask
+    {
+        @Config("custom_object_filter_type")
+        @ConfigDefault("\"\"")
+        String getCustomObjectFilterType();
+
+        @Config("custom_object_filter_from_value")
+        @ConfigDefault("1")
+        Integer getFromValue();
+
+        @Config("custom_object_filter_to_value")
+        @ConfigDefault("null")
+        Optional<Integer> getToValue();
+
+        @Config("custom_object_filter_values")
+        @ConfigDefault("null")
+        Optional<String> getCustomObjectFilterValues();
+    }
+
+    public CustomObjectInputPlugin()
+    {
+    }
+
+    @Override
+    public void validateInputTask(PluginTask task)
+    {
+        super.validateInputTask(task);
+        if (StringUtils.isBlank(task.getCustomObjectFilterType())) {
+            throw new ConfigException("`custom_object_filter_type` cannot be empty");
+        }
+        if (StringUtils.isBlank(task.getCustomObjectAPIName())) {
+            throw new ConfigException("`custom_object_api_name` cannot be empty");
+        }
+        if (!task.getCustomObjectFilterValues().isPresent()) {
+            if (task.getToValue().isPresent() && !isValidFilterRange(task)) {
+                throw new ConfigException(String.format("`to_value` (%s) cannot be less than the `from_value` (%s)", task.getToValue().get(), task.getFromValue()));
+            }
+        }
+        else if (refineFilterValues(task.getCustomObjectFilterValues().get()).isEmpty()) {
+            throw new ConfigException("`custom_object_filter_values` cannot contain empty values only");
+        }
+    }
+
+    private Set<String> refineFilterValues(String filterValues)
+    {
+        return Stream.of(StringUtils.split(filterValues, ",")).map(StringUtils::trimToEmpty).filter(StringUtils::isNotBlank).collect(Collectors.toSet());
+    }
+
+    private boolean isValidFilterRange(PluginTask task)
+    {
+        return task.getToValue().get() > task.getFromValue();
+    }
+
+    @Override
+    protected Iterator<ServiceRecord> getServiceRecords(MarketoService marketoService, PluginTask task)
+    {
+        Iterable<ObjectNode> responseObj;
+        if (task.getCustomObjectFilterValues().isPresent()) {
+            Set<String> refinedValues = refineFilterValues(task.getCustomObjectFilterValues().get());
+            responseObj = marketoService.getCustomObject(task.getCustomObjectAPIName(), task.getCustomObjectFilterType(), refinedValues, task.getCustomObjectFields().orNull());
+        }
+        else {
+            // When `to_value` is not set, will try to import all consecutive custom objects started from `from_value`
+            responseObj = marketoService.getCustomObject(task.getCustomObjectAPIName(), task.getCustomObjectFilterType(), task.getCustomObjectFields().orNull(), task.getFromValue(), task.getToValue().orNull());
+        }
+
+        return FluentIterable.from(filterInvalidRecords(responseObj)).transform(MarketoUtils.TRANSFORM_OBJECT_TO_JACKSON_SERVICE_RECORD_FUNCTION).iterator();
+    }
+
+    /**
+     * Marketo include error in invalid records. This method will filter those records and print it to output log
+     * @param resultIt
+     * @return
+     */
+    private Iterable<ObjectNode> filterInvalidRecords(Iterable<ObjectNode> resultIt)
+    {
+        return Iterables.filter(resultIt, (obj) -> {
+            if (obj.has("reasons")) {
+                logger.warn(obj.get("reasons").toString());
+                return false;
+            }
+            return true;
+        });
+    }
+
+    @Override
+    public ServiceResponseMapper<? extends ValueLocator> buildServiceResponseMapper(PluginTask task)
+    {
+        try (MarketoRestClient marketoRestClient = createMarketoRestClient(task)) {
+            MarketoService marketoService = new MarketoServiceImpl(marketoRestClient);
+            CustomObjectResponseMapperBuilder<PluginTask> customObjectResponseMapperBuilder = new CustomObjectResponseMapperBuilder<>(task, marketoService);
+            return customObjectResponseMapperBuilder.buildServiceResponseMapper(task);
+        }
+    }
+}

data/src/main/java/org/embulk/input/marketo/delegate/CustomObjectResponseMapperBuilder.java

@@ -0,0 +1,81 @@
+package org.embulk.input.marketo.delegate;
+
+import com.google.common.base.Optional;
+import org.apache.commons.lang3.StringUtils;
+import org.embulk.base.restclient.ServiceResponseMapper;
+import org.embulk.base.restclient.ServiceResponseMapperBuildable;
+import org.embulk.base.restclient.record.ValueLocator;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.input.marketo.MarketoService;
+import org.embulk.input.marketo.MarketoUtils;
+import org.embulk.input.marketo.model.MarketoField;
+import org.embulk.spi.Exec;
+import org.slf4j.Logger;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class CustomObjectResponseMapperBuilder<T extends CustomObjectResponseMapperBuilder.PluginTask> implements ServiceResponseMapperBuildable<T>
+{
+    private static final Logger LOGGER = Exec.getLogger(CustomObjectResponseMapperBuilder.class);
+    private MarketoService marketoService;
+
+    private T pluginTask;
+
+    public interface PluginTask extends MarketoBaseInputPluginDelegate.PluginTask
+    {
+        @Config("custom_object_api_name")
+        @ConfigDefault("\"\"")
+        String getCustomObjectAPIName();
+
+        @Config("custom_object_fields")
+        @ConfigDefault("null")
+        Optional<String> getCustomObjectFields();
+    }
+
+    public CustomObjectResponseMapperBuilder(T task, MarketoService marketoService)
+    {
+        this.pluginTask = task;
+        this.marketoService = marketoService;
+    }
+
+    protected List<MarketoField> getCustomObjectColumns()
+    {
+        List<MarketoField> columns = marketoService.describeCustomObject(pluginTask.getCustomObjectAPIName());
+        if (pluginTask.getCustomObjectFields().isPresent() && StringUtils.isNotBlank(pluginTask.getCustomObjectFields().get())) {
+            List<MarketoField> filteredColumns = new ArrayList<>();
+            List<String> includedFields = Arrays.asList(pluginTask.getCustomObjectFields().get().split(","));
+            for (String fieldName : includedFields) {
+                Optional<MarketoField> includedField = lookupFieldIgnoreCase(columns, fieldName);
+                if (includedField.isPresent()) {
+                    filteredColumns.add(includedField.get());
+                }
+                else {
+                    LOGGER.warn("Included field [{}] not found in Marketo Custom Object [{}] field", fieldName, pluginTask.getCustomObjectAPIName());
+                }
+            }
+            columns = filteredColumns;
+            LOGGER.info("Included Fields option is set, included columns: [{}]", columns);
+        }
+        return columns;
+    }
+
+    private static Optional<MarketoField> lookupFieldIgnoreCase(List<MarketoField> inputList, String lookupFieldName)
+    {
+        for (MarketoField marketoField : inputList) {
+            if (marketoField.getName().equalsIgnoreCase(lookupFieldName)) {
+                return Optional.of(marketoField);
+            }
+        }
+        return Optional.absent();
+    }
+
+    @Override
+    public ServiceResponseMapper<? extends ValueLocator> buildServiceResponseMapper(T task)
+    {
+        List<MarketoField> customObjectColumns = getCustomObjectColumns();
+        return MarketoUtils.buildDynamicResponseMapper(pluginTask.getSchemaColumnPrefix(), customObjectColumns);
+    }
+}

data/src/main/java/org/embulk/input/marketo/delegate/LeadBulkExtractInputPlugin.java

@@ -0,0 +1,68 @@
+package org.embulk.input.marketo.delegate;
+
+import com.google.common.base.Optional;
+import org.embulk.base.restclient.ServiceResponseMapper;
+import org.embulk.base.restclient.record.ValueLocator;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.input.marketo.MarketoService;
+import org.embulk.input.marketo.MarketoServiceImpl;
+import org.embulk.input.marketo.rest.MarketoRestClient;
+import org.embulk.spi.Exec;
+import org.slf4j.Logger;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.InputStream;
+import java.time.OffsetDateTime;
+import java.util.Date;
+import java.util.List;
+
+/**
+ * Created by tai.khuu on 9/18/17.
+ */
+public class LeadBulkExtractInputPlugin extends MarketoBaseBulkExtractInputPlugin<LeadBulkExtractInputPlugin.PluginTask>
+{
+    private static final Logger LOGGER = Exec.getLogger(LeadBulkExtractInputPlugin.class);
+
+    private static final String UPDATED_AT = "updatedAt";
+
+    public interface PluginTask extends MarketoBaseBulkExtractInputPlugin.PluginTask, LeadServiceResponseMapperBuilder.PluginTask
+    {
+        @Config("use_updated_at")
+        @ConfigDefault("false")
+        boolean getUseUpdatedAt();
+    }
+
+    @Override
+    public void validateInputTask(PluginTask task)
+    {
+        if (task.getUseUpdatedAt()) {
+            task.setIncrementalColumn(Optional.of(UPDATED_AT));
+        }
+        super.validateInputTask(task);
+    }
+
+    @Override
+    protected InputStream getExtractedStream(MarketoService service, PluginTask task, OffsetDateTime fromDate, OffsetDateTime toDate)
+    {
+        try {
+            List<String> fieldNames = task.getExtractedFields();
+            return new FileInputStream(service.extractLead(Date.from(fromDate.toInstant()), Date.from(toDate.toInstant()),
+                    fieldNames, task.getIncrementalColumn().orNull(), task.getPollingIntervalSecond(), task.getBulkJobTimeoutSecond()));
+        }
+        catch (FileNotFoundException e) {
+            throw new RuntimeException("File not found", e);
+        }
+    }
+
+    @Override
+    public ServiceResponseMapper<? extends ValueLocator> buildServiceResponseMapper(PluginTask task)
+    {
+        try (MarketoRestClient marketoRestClient = createMarketoRestClient(task)) {
+            MarketoService marketoService = new MarketoServiceImpl(marketoRestClient);
+            LeadServiceResponseMapperBuilder<PluginTask> leadServiceResponseMapperBuilder = new LeadServiceResponseMapperBuilder<>(task, marketoService);
+            return leadServiceResponseMapperBuilder.buildServiceResponseMapper(task);
+        }
+    }
+}