embulk-input-marketo 0.5.6 → 0.5.7.alpha.1

data/src/main/java/org/embulk/input/marketo/model/filter/DateRangeFilter.java

@@ -0,0 +1,31 @@
+package org.embulk.input.marketo.model.filter;
+
+/**
+ * Created by tai.khuu on 8/27/17.
+ */
+public class DateRangeFilter implements MarketoFilter
+{
+    private String startAt;
+
+    private String endAt;
+
+    public String getStartAt()
+    {
+        return startAt;
+    }
+
+    public void setStartAt(String startAt)
+    {
+        this.startAt = startAt;
+    }
+
+    public String getEndAt()
+    {
+        return endAt;
+    }
+
+    public void setEndAt(String endAt)
+    {
+        this.endAt = endAt;
+    }
+}

data/src/main/java/org/embulk/input/marketo/model/filter/ListFilter.java

@@ -0,0 +1,10 @@
+package org.embulk.input.marketo.model.filter;
+
+import java.util.ArrayList;
+
+/**
+ * Created by tai.khuu on 8/27/17.
+ */
+public class ListFilter extends ArrayList<String> implements MarketoFilter
+{
+}

data/src/main/java/org/embulk/input/marketo/model/filter/MarketoFilter.java

@@ -0,0 +1,8 @@
+package org.embulk.input.marketo.model.filter;
+
+/**
+ * Created by tai.khuu on 8/27/17.
+ */
+public interface MarketoFilter
+{
+}

data/src/main/java/org/embulk/input/marketo/rest/MarketoBaseRestClient.java

@@ -0,0 +1,226 @@
+package org.embulk.input.marketo.rest;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ImmutableListMultimap;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
+import org.eclipse.jetty.client.HttpClient;
+import org.eclipse.jetty.client.api.ContentProvider;
+import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.client.api.Response;
+import org.eclipse.jetty.client.util.StringContentProvider;
+import org.eclipse.jetty.http.HttpMethod;
+import org.embulk.input.marketo.exception.MarketoAPIException;
+import org.embulk.input.marketo.model.MarketoAccessTokenResponse;
+import org.embulk.input.marketo.model.MarketoError;
+import org.embulk.spi.DataException;
+import org.embulk.spi.Exec;
+import org.embulk.util.retryhelper.jetty92.Jetty92ResponseReader;
+import org.embulk.util.retryhelper.jetty92.Jetty92RetryHelper;
+import org.embulk.util.retryhelper.jetty92.Jetty92SingleRequester;
+import org.embulk.util.retryhelper.jetty92.StringJetty92ResponseEntityReader;
+import org.slf4j.Logger;
+
+import java.io.IOException;
+import java.net.SocketTimeoutException;
+import java.nio.charset.StandardCharsets;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeoutException;
+
+import static com.fasterxml.jackson.core.JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS;
+import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES;
+
+/**
+ * Marketo base rest client that provide
+ * Created by tai.khuu on 9/7/17.
+ */
+public class MarketoBaseRestClient implements AutoCloseable
+{
+    private static final Logger LOGGER = Exec.getLogger(MarketoBaseRestClient.class);
+
+    private static final String APPLICATION_JSON = "application/json";
+
+    private static final String AUTHORIZATION_HEADER = "Authorization";
+
+    protected static final long READ_TIMEOUT_MILLIS = 30000;
+
+    private String identityEndPoint;
+
+    private String clientId;
+
+    private String clientSecret;
+
+    private String accessToken;
+
+    private int marketoLimitIntervalMilis;
+
+    private Jetty92RetryHelper retryHelper;
+
+    protected static final ObjectMapper OBJECT_MAPPER = new ObjectMapper().configure(FAIL_ON_UNKNOWN_PROPERTIES, false).configure(ALLOW_UNQUOTED_CONTROL_CHARS, false);
+
+    MarketoBaseRestClient(String identityEndPoint, String clientId, String clientSecret, int marketoLimitIntervalMilis, Jetty92RetryHelper retryHelper)
+    {
+        this.identityEndPoint = identityEndPoint;
+        this.clientId = clientId;
+        this.clientSecret = clientSecret;
+        this.retryHelper = retryHelper;
+        this.marketoLimitIntervalMilis = marketoLimitIntervalMilis;
+    }
+
+    private void renewAccessToken()
+    {
+        accessToken = requestAccessToken();
+    }
+
+    @VisibleForTesting
+    public String getAccessToken()
+    {
+        if (accessToken == null) {
+            synchronized (this) {
+                if (accessToken == null) {
+                    accessToken = requestAccessToken();
+                }
+            }
+        }
+        return accessToken;
+    }
+
+    private String requestAccessToken()
+    {
+        final Multimap<String, String> params = ArrayListMultimap.create();
+        params.put("client_id", clientId);
+        params.put("client_secret", clientSecret);
+        params.put("grant_type", "client_credentials");
+        String response = retryHelper.requestWithRetry(new StringJetty92ResponseEntityReader(READ_TIMEOUT_MILLIS), new Jetty92SingleRequester()
+        {
+            @Override
+            public void requestOnce(HttpClient client, Response.Listener responseListener)
+            {
+                Request request = client.newRequest(identityEndPoint + MarketoRESTEndpoint.ACCESS_TOKEN.getEndpoint()).method(HttpMethod.GET);
+                for (String key : params.keySet()) {
+                    for (String value : params.get(key)) {
+                        request.param(key, value);
+                    }
+                }
+                request.send(responseListener);
+            }
+
+            @Override
+            protected boolean isResponseStatusToRetry(Response response)
+            {
+                return response.getStatus() == 502;
+            }
+        });
+
+        MarketoAccessTokenResponse accessTokenResponse;
+
+        try {
+            accessTokenResponse = OBJECT_MAPPER.readValue(response, MarketoAccessTokenResponse.class);
+        }
+        catch (IOException e) {
+            LOGGER.error("Exception when parse access token response", e);
+            throw new DataException("Can't parse access token response");
+        }
+        if (accessTokenResponse.hasError()) {
+            throw new DataException(accessTokenResponse.getErrorDescription());
+        }
+        return accessTokenResponse.getAccessToken();
+    }
+
+    protected <T> T doGet(final String target, final Map<String, String> headers, final Multimap<String, String> params, Jetty92ResponseReader<T> responseReader)
+    {
+        return doRequest(target, HttpMethod.GET, headers, params, null, responseReader);
+    }
+
+    protected <T> T doPost(final String target, final Map<String, String> headers, final Multimap<String, String> params, final String content, Jetty92ResponseReader<T> responseReader)
+    {
+        StringContentProvider contentProvider = null;
+        if (content != null) {
+            contentProvider = new StringContentProvider(APPLICATION_JSON, content, StandardCharsets.UTF_8);
+        }
+        return doRequest(target, HttpMethod.POST, headers, params, contentProvider, responseReader);
+    }
+
+    protected <T> T doRequest(final String target, final HttpMethod method, final Map<String, String> headers, final Multimap<String, String> params, final ContentProvider contentProvider, Jetty92ResponseReader<T> responseReader)
+    {
+        return retryHelper.requestWithRetry(responseReader, new Jetty92SingleRequester()
+        {
+            @Override
+            public void requestOnce(HttpClient client, Response.Listener responseListener)
+            {
+                Request request = client.newRequest(target).method(method);
+
+                if (headers != null) {
+                    for (String key : headers.keySet()) {
+                        request.header(key, headers.get(key));
+                    }
+                }
+                request.header(AUTHORIZATION_HEADER, "Bearer " + getAccessToken());
+                if (params != null) {
+                    for (String key : params.keySet()) {
+                        for (String value : params.get(key)) {
+                            request.param(key, value);
+                        }
+                    }
+                }
+                if (contentProvider != null) {
+                    request.content(contentProvider, APPLICATION_JSON);
+                }
+                request.send(responseListener);
+            }
+
+            @Override
+            protected boolean isResponseStatusToRetry(Response response)
+            {
+                //413 failed job
+                //414 failed job
+                //502 retry
+                return response.getStatus() / 4 != 100;
+            }
+
+            @Override
+            protected boolean isExceptionToRetry(Exception exception)
+            {
+                if (exception instanceof ExecutionException) {
+                    this.toRetry((Exception) exception.getCause());
+                }
+                if (exception instanceof MarketoAPIException) {
+                    //Retry Authenticate Exception
+                    MarketoError error = ((MarketoAPIException) exception).getMarketoErrors().get(0);
+                    String code = error.getCode();
+                    switch (code) {
+                        case "602":
+                            renewAccessToken();
+                            return true;
+                        case "606":
+                            try {
+                                Thread.sleep(marketoLimitIntervalMilis);
+                            }
+                            catch (InterruptedException e) {
+                                LOGGER.error("Encounter exception when waiting for interval limit", e);
+                                throw new DataException("Exception when wait for interval limit");
+                            }
+                            return true;
+                        case "615":
+                            return true;
+                        default:
+                            return false;
+                    }
+                }
+                return exception instanceof TimeoutException || exception instanceof SocketTimeoutException || super.isExceptionToRetry(exception);
+            }
+        });
+    }
+
+    @Override
+    public void close()
+    {
+        if (retryHelper != null) {
+            retryHelper.close();
+        }
+    }
+}

data/src/main/java/org/embulk/input/marketo/rest/MarketoFileResponseEntityReader.java

@@ -0,0 +1,69 @@
+package org.embulk.input.marketo.rest;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.io.CharStreams;
+import org.eclipse.jetty.client.api.Response;
+import org.eclipse.jetty.client.util.InputStreamResponseListener;
+import org.eclipse.jetty.http.HttpHeader;
+import org.embulk.input.marketo.exception.MarketoAPIException;
+import org.embulk.input.marketo.model.MarketoResponse;
+import org.embulk.util.retryhelper.jetty92.Jetty92ResponseReader;
+
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Created by tai.khuu on 9/5/17.
+ */
+public class MarketoFileResponseEntityReader implements Jetty92ResponseReader<InputStream>
+{
+    private static final ObjectReader OBJECT_READER = new ObjectMapper().readerFor(new TypeReference<MarketoResponse<ObjectNode>>(){ });
+
+    private InputStreamResponseListener listener;
+
+    private long timeout;
+
+    public MarketoFileResponseEntityReader(long timeout)
+    {
+        this.listener = new InputStreamResponseListener();
+        this.timeout = timeout;
+    }
+
+    @Override
+    public Response.Listener getListener()
+    {
+        return this.listener;
+    }
+
+    @Override
+    public Response getResponse() throws Exception
+    {
+        return this.listener.get(timeout, TimeUnit.MILLISECONDS);
+    }
+
+    @Override
+    public InputStream readResponseContent() throws Exception
+    {
+        if (!getResponse().getHeaders().getField(HttpHeader.CONTENT_TYPE).getValue().equals("text/csv")) {
+            String errorString = readResponseContentInString();
+
+            MarketoResponse<ObjectNode> errorResponse = OBJECT_READER.readValue(errorString);
+            if (!errorResponse.isSuccess()) {
+                throw new MarketoAPIException(errorResponse.getErrors());
+            }
+        }
+        return this.listener.getInputStream();
+    }
+
+    @Override
+    public String readResponseContentInString() throws Exception
+    {
+        try (InputStreamReader inputStreamReader = new InputStreamReader(this.listener.getInputStream())) {
+            return CharStreams.toString(inputStreamReader);
+        }
+    }
+}

data/src/main/java/org/embulk/input/marketo/rest/MarketoRESTEndpoint.java

@@ -0,0 +1,44 @@
+package org.embulk.input.marketo.rest;
+
+import org.apache.commons.lang3.text.StrSubstitutor;
+
+import java.util.Map;
+
+/**
+ * Created by tai.khuu on 9/5/17.
+ */
+public enum MarketoRESTEndpoint
+{
+    ACCESS_TOKEN("/oauth/token"),
+    CREATE_LEAD_EXTRACT("/bulk/v1/leads/export/create.json"),
+    CREATE_ACTIVITY_EXTRACT("/bulk/v1/activities/export/create.json"),
+    DESCRIBE_LEAD("/rest/v1/leads/describe.json"),
+    START_LEAD_EXPORT_JOB("/bulk/v1/leads/export/${export_id}/enqueue.json"),
+    START_ACTIVITY_EXPORT_JOB("/bulk/v1/activities/export/${export_id}/enqueue.json"),
+    GET_ACTIVITY_EXPORT_STATUS("/bulk/v1/activities/export/${export_id}/status.json"),
+    GET_LEAD_EXPORT_STATUS("/bulk/v1/leads/export/${export_id}/status.json"),
+    GET_LEAD_EXPORT_RESULT("/bulk/v1/leads/export/${export_id}/file.json"),
+    GET_ACTIVITY_EXPORT_RESULT("/bulk/v1/activities/export/${export_id}/file.json"),
+    GET_LISTS("/rest/v1/lists.json"),
+    GET_LEADS_BY_LIST("/rest/v1/lists/${list_id}/leads.json"),
+    GET_PROGRAMS("/rest/asset/v1/programs.json"),
+    GET_LEADS_BY_PROGRAM("/rest/v1/leads/programs/${program_id}.json"),
+    GET_CAMPAIGN("/rest/v1/campaigns.json");
+
+    private String endpoint;
+
+    MarketoRESTEndpoint(String endpoint)
+    {
+        this.endpoint = endpoint;
+    }
+
+    public String getEndpoint()
+    {
+        return endpoint;
+    }
+
+    public String getEndpoint(Map<String, String> pathParams)
+    {
+        return StrSubstitutor.replace(endpoint, pathParams);
+    }
+}

data/src/main/java/org/embulk/input/marketo/rest/MarketoResponseJetty92EntityReader.java

@@ -0,0 +1,88 @@
+package org.embulk.input.marketo.rest;
+
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.JavaType;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.io.CharStreams;
+import org.eclipse.jetty.client.api.Response;
+import org.eclipse.jetty.client.util.InputStreamResponseListener;
+import org.embulk.input.marketo.exception.MarketoAPIException;
+import org.embulk.input.marketo.model.MarketoResponse;
+import org.embulk.spi.DataException;
+import org.embulk.spi.Exec;
+import org.embulk.util.retryhelper.jetty92.Jetty92ResponseReader;
+import org.slf4j.Logger;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Created by tai.khuu on 9/1/17.
+ */
+public class MarketoResponseJetty92EntityReader<T> implements Jetty92ResponseReader<MarketoResponse<T>>
+{
+    private InputStreamResponseListener listener;
+
+    private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+
+    private static final Logger LOGGER = Exec.getLogger(MarketoResponseJetty92EntityReader.class);
+    private Long timeout;
+
+    private JavaType javaType;
+
+    public MarketoResponseJetty92EntityReader(long timeout)
+    {
+        this.listener = new InputStreamResponseListener();
+        this.timeout = timeout;
+        javaType = OBJECT_MAPPER.getTypeFactory().constructParametrizedType(MarketoResponse.class, MarketoResponse.class, ObjectNode.class);
+    }
+
+    public MarketoResponseJetty92EntityReader(long timeout, Class<T> resultClass)
+    {
+        this.listener = new InputStreamResponseListener();
+        this.timeout = timeout;
+        this.javaType = OBJECT_MAPPER.getTypeFactory().constructParametrizedType(MarketoResponse.class, MarketoResponse.class, resultClass);
+    }
+
+    @Override
+    public Response.Listener getListener()
+    {
+        return listener;
+    }
+
+    @Override
+    public Response getResponse() throws Exception
+    {
+        return this.listener.get(timeout, TimeUnit.MILLISECONDS);
+    }
+
+    @Override
+    public MarketoResponse<T> readResponseContent() throws Exception
+    {
+        String response = readResponseContentInString();
+        try {
+            MarketoResponse<T> marketoResponse = OBJECT_MAPPER.readValue(response, javaType);
+            if (!marketoResponse.isSuccess()) {
+                throw new MarketoAPIException(marketoResponse.getErrors());
+            }
+            return marketoResponse;
+        }
+        catch (IOException ex) {
+            LOGGER.error("Can't parse json content", ex);
+            throw new DataException("Exception when parse json content");
+        }
+    }
+
+    @Override
+    public String readResponseContentInString() throws Exception
+    {
+        InputStream inputStream = this.listener.getInputStream();
+        try (InputStreamReader inputStreamReader = new InputStreamReader(inputStream, StandardCharsets.UTF_8)) {
+            return CharStreams.toString(inputStreamReader);
+        }
+    }
+}