embulk-input-zendesk-all 0.3.7

data/src/main/java/org/embulk/input/zendesk/clients/ZendeskRestClient.java

@@ -0,0 +1,291 @@
+package org.embulk.input.zendesk.clients;
+
+import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableMap.Builder;
+import com.google.common.util.concurrent.RateLimiter;
+import com.google.common.util.concurrent.Uninterruptibles;
+import org.apache.http.Header;
+import org.apache.http.HttpResponse;
+import org.apache.http.HttpStatus;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpRequestBase;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.util.EntityUtils;
+import org.embulk.config.ConfigException;
+import org.embulk.input.zendesk.ZendeskInputPlugin.PluginTask;
+import org.embulk.input.zendesk.models.ZendeskException;
+import org.embulk.input.zendesk.utils.ZendeskConstants;
+import org.embulk.input.zendesk.utils.ZendeskUtils;
+import org.embulk.spi.DataException;
+import org.embulk.spi.Exec;
+import org.embulk.spi.util.RetryExecutor;
+import org.slf4j.Logger;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.http.HttpHeaders.AUTHORIZATION;
+import static org.apache.http.protocol.HTTP.CONTENT_TYPE;
+import static org.embulk.spi.util.RetryExecutor.retryExecutor;
+
+public class ZendeskRestClient
+{
+    private static final int CONNECTION_TIME_OUT = 240000;
+
+    private static final Logger logger = Exec.getLogger(ZendeskRestClient.class);
+    private static final ObjectMapper objectMapper = new ObjectMapper();
+    private static RateLimiter rateLimiter;
+
+    static {
+        objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+    }
+
+    public ZendeskRestClient()
+    {
+    }
+
+    public String doGet(final String url, final PluginTask task, final boolean isPreview)
+    {
+        try {
+            return retryExecutor().withRetryLimit(task.getRetryLimit())
+                    .withInitialRetryWait(task.getRetryInitialWaitSec() * 1000)
+                    .withMaxRetryWait(task.getMaxRetryWaitSec() * 1000)
+                    .runInterruptible(new RetryExecutor.Retryable<String>()
+                    {
+                        @Override
+                        public String call()
+                                throws Exception
+                        {
+                            return sendGetRequest(url, task);
+                        }
+
+                        @Override
+                        public boolean isRetryableException(final Exception exception)
+                        {
+                            if (exception instanceof ZendeskException) {
+                                final int statusCode = ((ZendeskException) exception).getStatusCode();
+                                return isResponseStatusToRetry(statusCode, exception.getMessage(), ((ZendeskException) exception).getRetryAfter(), isPreview);
+                            }
+                            return false;
+                        }
+
+                        @Override
+                        public void onRetry(final Exception exception, final int retryCount, final int retryLimit, final int retryWait)
+                        {
+                            if (exception instanceof ZendeskException) {
+                                final int retryAfter = ((ZendeskException) exception).getRetryAfter();
+                                final String message;
+                                if (retryAfter > 0 && retryAfter > (retryWait / 1000)) {
+                                    message = String
+                                            .format("Retrying '%d'/'%d' after '%d' seconds. HTTP status code: '%s'",
+                                                    retryCount, retryLimit,
+                                                    retryAfter,
+                                                    ((ZendeskException) exception).getStatusCode());
+                                    logger.warn(message);
+                                    Uninterruptibles.sleepUninterruptibly(retryAfter - (retryWait / 1000), TimeUnit.SECONDS);
+                                }
+                                else {
+                                    message = String
+                                            .format("Retrying '%d'/'%d' after '%d' seconds. HTTP status code: '%s'",
+                                                    retryCount, retryLimit,
+                                                    retryWait / 1000,
+                                                    ((ZendeskException) exception).getStatusCode());
+                                    logger.warn(message);
+                                }
+                            }
+                            else {
+                                final String message = String
+                                        .format("Retrying '%d'/'%d' after '%d' seconds. Message: '%s'",
+                                                retryCount, retryLimit,
+                                                retryWait / 1000,
+                                                exception.getMessage());
+                                logger.warn(message, exception);
+                            }
+                        }
+
+                        @Override
+                        public void onGiveup(final Exception firstException, final Exception lastException)
+                        {
+                        }
+                    });
+        }
+        catch (final RetryExecutor.RetryGiveupException | InterruptedException e) {
+            if (e instanceof RetryExecutor.RetryGiveupException && e.getCause() != null && e.getCause() instanceof ZendeskException) {
+                throw new ConfigException("Status: '" + ((ZendeskException) (e.getCause())).getStatusCode() + "', error message: '" + e.getCause().getMessage() + "'",
+                        e.getCause());
+            }
+            throw new ConfigException(e);
+        }
+    }
+
+    @VisibleForTesting
+    protected HttpClient createHttpClient()
+    {
+        final RequestConfig config = RequestConfig.custom()
+                .setConnectTimeout(CONNECTION_TIME_OUT)
+                .setConnectionRequestTimeout(CONNECTION_TIME_OUT)
+                .build();
+        return HttpClientBuilder.create().setDefaultRequestConfig(config).build();
+    }
+
+    private String sendGetRequest(final String url, final PluginTask task)
+            throws ZendeskException
+    {
+        try {
+            final HttpClient client = createHttpClient();
+            final HttpRequestBase request = createGetRequest(url, task);
+
+            if (rateLimiter != null) {
+                rateLimiter.acquire();
+            }
+            logger.info(">>> {}{}", request.getURI().getPath(),
+                    request.getURI().getQuery() != null ? "?" + request.getURI().getQuery() : "");
+            final HttpResponse response = client.execute(request);
+            if (rateLimiter == null) {
+                initRateLimiter(response);
+            }
+            final int statusCode = response.getStatusLine().getStatusCode();
+            if (statusCode != HttpStatus.SC_OK) {
+                if (statusCode == ZendeskConstants.HttpStatus.TOO_MANY_REQUEST || statusCode == HttpStatus.SC_INTERNAL_SERVER_ERROR
+                        || statusCode == HttpStatus.SC_SERVICE_UNAVAILABLE) {
+                    final Header retryHeader = response.getFirstHeader("Retry-After");
+                    if (retryHeader != null) {
+                        throw new ZendeskException(statusCode, EntityUtils.toString(response.getEntity()), Integer.parseInt(retryHeader.getValue()));
+                    }
+                }
+                throw new ZendeskException(statusCode, EntityUtils.toString(response.getEntity()), 0);
+            }
+            return EntityUtils.toString(response.getEntity());
+        }
+        catch (final IOException ex) {
+            throw new ZendeskException(-1, ex.getMessage(), 0);
+        }
+    }
+
+    private boolean isResponseStatusToRetry(final int status, final String message, final int retryAfter, final boolean isPreview)
+    {
+        if (status == HttpStatus.SC_NOT_FOUND) {
+            ObjectMapper objectMapper = new ObjectMapper();
+            try {
+                JsonNode jsonNode = objectMapper.readTree(message);
+                if (jsonNode.has("error") && jsonNode.get("error").has("title") && jsonNode.get("error").get("title").asText().startsWith("No help desk at ")) {
+                    throw new ConfigException("This address is not registered in Zendesk. Please check the login_url again");
+                }
+            }
+            catch (IOException e) {
+                // In case we can't parse the message, error should not be show here
+            }
+
+            // 404 would be returned e.g. ticket comments are empty (on fetchRelatedObjects method)
+            return false;
+        }
+
+        if (status == HttpStatus.SC_CONFLICT) {
+            logger.warn(String.format("'%s' temporally failure.", status));
+            return true;
+        }
+
+        if (status == HttpStatus.SC_UNPROCESSABLE_ENTITY) {
+            if (message.startsWith(ZendeskConstants.Misc.TOO_RECENT_START_TIME)) {
+                //That means "No records from start_time". We can recognize it same as 200.
+                return false;
+            }
+            throw new ConfigException("Status: '" + status + "', error message: '" + message + "'");
+        }
+
+        if (status == ZendeskConstants.HttpStatus.TOO_MANY_REQUEST || status == HttpStatus.SC_INTERNAL_SERVER_ERROR
+                || status == HttpStatus.SC_SERVICE_UNAVAILABLE) {
+            if (!isPreview) {
+                if (retryAfter > 0) {
+                    logger.warn("Reached API limitation, wait for at least '{}' '{}'", retryAfter, TimeUnit.SECONDS.name());
+                }
+                else if (status != ZendeskConstants.HttpStatus.TOO_MANY_REQUEST) {
+                    logger.warn(String.format("'%s' temporally failure.", status));
+                }
+                return true;
+            }
+            throw new DataException("Rate Limited. Waiting '" + retryAfter + "' seconds to re-run");
+        }
+
+        // Won't retry for 4xx range errors except above. Almost they should be ConfigError e.g. 403 Forbidden
+        if (status / 100 == 4) {
+            if (status == HttpStatus.SC_UNAUTHORIZED) {
+                throw new ConfigException("Cannot authenticate due to invalid login credentials");
+            }
+
+            if (status == HttpStatus.SC_FORBIDDEN) {
+                throw new ConfigException("You do not have access to this resource. Please contact the account owner of this help desk for further help.");
+            }
+
+            throw new ConfigException("Status '" + status + "', message '" + message + "'");
+        }
+
+        logger.warn("Server returns unknown status code '" + status + "' message '" + message + "'");
+        return true;
+    }
+
+    private HttpRequestBase createGetRequest(final String url, final PluginTask task)
+    {
+        final HttpGet request = new HttpGet(url);
+        final ImmutableMap<String, String> headers = buildAuthHeader(task);
+        headers.forEach(request::setHeader);
+        return request;
+    }
+
+    private ImmutableMap<String, String> buildAuthHeader(final PluginTask task)
+    {
+        final Builder<String, String> builder = new Builder<>();
+        builder.put(AUTHORIZATION, buildCredential(task));
+        addCommonHeader(builder, task);
+        return builder.build();
+    }
+
+    private String buildCredential(final PluginTask task)
+    {
+        switch (task.getAuthenticationMethod()) {
+            case BASIC:
+                return "Basic " + ZendeskUtils.convertBase64(String.format("%s:%s", task.getUsername().get(), task.getPassword().get()));
+            case TOKEN:
+                return "Basic " + ZendeskUtils.convertBase64(String.format("%s/token:%s", task.getUsername().get(), task.getToken().get()));
+            case OAUTH:
+                return "Bearer " + task.getAccessToken().get();
+        }
+        return "";
+    }
+
+    private void addCommonHeader(final Builder<String, String> builder, final PluginTask task)
+    {
+        task.getAppMarketPlaceIntegrationName().ifPresent(s -> builder.put(ZendeskConstants.Header.ZENDESK_MARKETPLACE_NAME, s));
+        task.getAppMarketPlaceAppId().ifPresent(s -> builder.put(ZendeskConstants.Header.ZENDESK_MARKETPLACE_APP_ID, s));
+        task.getAppMarketPlaceOrgId().ifPresent(s -> builder.put(ZendeskConstants.Header.ZENDESK_MARKETPLACE_ORGANIZATION_ID, s));
+
+        builder.put(CONTENT_TYPE, ZendeskConstants.Header.APPLICATION_JSON);
+    }
+
+    private synchronized void initRateLimiter(final HttpResponse response)
+    {
+        if (response.containsHeader("x-rate-limit")) {
+            double permits = 0.0;
+            final String rateLimit = response.getFirstHeader("x-rate-limit").getValue();
+            try {
+                permits = Double.parseDouble(rateLimit);
+            }
+            catch (final NumberFormatException e) {
+                throw new DataException("Error when parse x-rate-limit: '" + response.getFirstHeader("x-rate-limit").getValue() + "'");
+            }
+
+            if (permits > 0) {
+                permits = permits / 60;
+                logger.info("Permits per second " + permits);
+
+                rateLimiter = RateLimiter.create(permits);
+            }
+        }
+    }
+}

data/src/main/java/org/embulk/input/zendesk/models/AuthenticationMethod.java

@@ -0,0 +1,23 @@
+package org.embulk.input.zendesk.models;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import org.embulk.config.ConfigException;
+
+import java.util.Arrays;
+
+public enum AuthenticationMethod
+{
+    BASIC, OAUTH, TOKEN;
+
+    @JsonCreator
+    public static AuthenticationMethod fromString(final String value)
+    {
+        try {
+            return AuthenticationMethod.valueOf(value.trim().toUpperCase());
+        }
+        catch (IllegalArgumentException e) {
+            throw new ConfigException("Unsupported Authentication mode '" + value + "', supported values: '"
+                    + Arrays.toString(Target.values()) + "'");
+        }
+    }
+}

data/src/main/java/org/embulk/input/zendesk/models/Target.java

@@ -0,0 +1,47 @@
+package org.embulk.input.zendesk.models;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import org.embulk.config.ConfigException;
+
+import java.util.Arrays;
+
+public enum Target
+{
+    /** For ticket_metrics - we fetch by using include metric_sets with ticket target
+    *    so the jsonName is different comparing to the target name
+    */
+    TICKETS("tickets"), USERS("users"), ORGANIZATIONS("organizations"), TICKET_EVENTS("ticket_events"),
+    TICKET_METRICS("metric_sets"), TICKET_FIELDS("ticket_fields"), TICKET_FORMS("ticket_forms"),
+    TICKET_METRIC_EVENTS("ticket_metric_events"), SATISFACTION_RATINGS("satisfaction_ratings"),
+    RECIPIENTS("recipients"), SCORES("responses"), OBJECT_RECORDS("data"), RELATIONSHIP_RECORDS("data"), USER_EVENTS("data");
+
+    String jsonName;
+
+    Target(String jsonName)
+    {
+        this.jsonName = jsonName;
+    }
+
+    @JsonCreator
+    public static Target fromString(final String value)
+    {
+        try {
+            return Target.valueOf(value.trim().toUpperCase());
+        }
+        catch (IllegalArgumentException e) {
+            throw new ConfigException("Unsupported target '" + value + "', supported values: '"
+                    + Arrays.toString(Target.values()) + "'");
+        }
+    }
+
+    @Override
+    public String toString()
+    {
+        return this.name().trim().toLowerCase();
+    }
+
+    public String getJsonName()
+    {
+        return this.jsonName;
+    }
+}

data/src/main/java/org/embulk/input/zendesk/models/ZendeskException.java

@@ -0,0 +1,25 @@
+package org.embulk.input.zendesk.models;
+
+public class ZendeskException extends Exception
+{
+    private static final long serialVersionUID = -256731723520584046L;
+    private final int statusCode;
+    private final int retryAfter;
+
+    public ZendeskException(final int statusCode, final String message, final int retryAfter)
+    {
+        super(message);
+        this.statusCode = statusCode;
+        this.retryAfter = retryAfter;
+    }
+
+    public int getStatusCode()
+    {
+        return statusCode;
+    }
+
+    public int getRetryAfter()
+    {
+        return retryAfter;
+    }
+}

data/src/main/java/org/embulk/input/zendesk/services/ZendeskCustomObjectService.java

@@ -0,0 +1,110 @@
+package org.embulk.input.zendesk.services;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.http.HttpStatus;
+import org.embulk.config.ConfigException;
+import org.embulk.config.TaskReport;
+import org.embulk.input.zendesk.RecordImporter;
+import org.embulk.input.zendesk.ZendeskInputPlugin;
+import org.embulk.input.zendesk.clients.ZendeskRestClient;
+import org.embulk.input.zendesk.models.Target;
+import org.embulk.input.zendesk.models.ZendeskException;
+import org.embulk.input.zendesk.stream.paginator.sunshine.CustomObjectSpliterator;
+import org.embulk.input.zendesk.utils.ZendeskConstants;
+import org.embulk.input.zendesk.utils.ZendeskUtils;
+import org.embulk.spi.Exec;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+
+public class ZendeskCustomObjectService implements ZendeskService
+{
+    protected ZendeskInputPlugin.PluginTask task;
+
+    private ZendeskRestClient zendeskRestClient;
+
+    public ZendeskCustomObjectService(final ZendeskInputPlugin.PluginTask task)
+    {
+        this.task = task;
+    }
+
+    public boolean isSupportIncremental()
+    {
+        return false;
+    }
+
+    @Override
+    public TaskReport addRecordToImporter(final int taskIndex, final RecordImporter recordImporter)
+    {
+        final List<String> paths = getListPathByTarget();
+
+        paths.parallelStream().forEach(path -> StreamSupport.stream(new CustomObjectSpliterator(path, getZendeskRestClient(), task, Exec.isPreview()), !Exec.isPreview())
+                .forEach(recordImporter::addRecord));
+
+        return Exec.newTaskReport();
+    }
+
+    @Override
+    public JsonNode getDataFromPath(final String path, final int page, final boolean isPreview, final long startTime)
+    {
+        Preconditions.checkArgument(isPreview, "IsPreview should be true to use this method");
+
+        Optional<String> response = Optional.empty();
+        final List<String> paths = path.isEmpty()
+                ? getListPathByTarget()
+                : Collections.singletonList(path);
+
+        for (final String temp : paths) {
+            try {
+                response = Optional.ofNullable(getZendeskRestClient().doGet(temp, task, true));
+
+                if (response.isPresent()) {
+                    break;
+                }
+            }
+            catch (final ConfigException e) {
+                // Sometimes we get 404 when having invalid endpoint, so ignore when we get 404
+                if (!(e.getCause() instanceof ZendeskException && ((ZendeskException) e.getCause()).getStatusCode() == HttpStatus.SC_NOT_FOUND)) {
+                    throw e;
+                }
+            }
+        }
+
+        return response.map(ZendeskUtils::parseJsonObject).orElse(new ObjectMapper().createObjectNode());
+    }
+
+    @VisibleForTesting
+    protected ZendeskRestClient getZendeskRestClient()
+    {
+        if (zendeskRestClient == null) {
+            zendeskRestClient = new ZendeskRestClient();
+        }
+        return zendeskRestClient;
+    }
+
+    private List<String> getListPathByTarget()
+    {
+        return task.getTarget().equals(Target.OBJECT_RECORDS)
+                ? task.getObjectTypes().stream().map(this::buildPath).collect(Collectors.toList())
+                : task.getRelationshipTypes().stream().map(this::buildPath).collect(Collectors.toList());
+    }
+
+    private String buildPath(final String value)
+    {
+        final String perPage = Exec.isPreview() ? "1" : "1000";
+
+        return ZendeskUtils.getURIBuilder(task.getLoginUrl())
+                .setPath(task.getTarget().equals(Target.OBJECT_RECORDS)
+                        ? ZendeskConstants.Url.API_OBJECT_RECORD
+                        : ZendeskConstants.Url.API_RELATIONSHIP_RECORD)
+                .setParameter("type", value)
+                .setParameter("per_page", perPage)
+                .toString();
+    }
+}