embulk-input-zendesk-all 0.3.7

data/src/main/java/org/embulk/input/zendesk/services/ZendeskUserEventService.java

@@ -0,0 +1,158 @@
+package org.embulk.input.zendesk.services;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.http.client.utils.URIBuilder;
+import org.embulk.config.TaskReport;
+import org.embulk.input.zendesk.RecordImporter;
+import org.embulk.input.zendesk.ZendeskInputPlugin;
+import org.embulk.input.zendesk.clients.ZendeskRestClient;
+import org.embulk.input.zendesk.models.Target;
+import org.embulk.input.zendesk.stream.paginator.sunshine.UserEventSpliterator;
+import org.embulk.input.zendesk.stream.paginator.support.OrganizationSpliterator;
+import org.embulk.input.zendesk.stream.paginator.support.UserSpliterator;
+import org.embulk.input.zendesk.utils.ZendeskConstants;
+import org.embulk.input.zendesk.utils.ZendeskDateUtils;
+import org.embulk.input.zendesk.utils.ZendeskUtils;
+import org.embulk.spi.DataException;
+import org.embulk.spi.Exec;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+
+public class ZendeskUserEventService implements ZendeskService
+{
+    protected ZendeskInputPlugin.PluginTask task;
+
+    private ZendeskRestClient zendeskRestClient;
+
+    public ZendeskUserEventService(final ZendeskInputPlugin.PluginTask task)
+    {
+        this.task = task;
+    }
+
+    public boolean isSupportIncremental()
+    {
+        return false;
+    }
+
+    @Override
+    public TaskReport addRecordToImporter(final int taskIndex, final RecordImporter recordImporter)
+    {
+        final TaskReport taskReport = Exec.newTaskReport();
+
+        if (Exec.isPreview()) {
+            JsonNode jsonNode = mockJsonNode();
+            recordImporter.addRecord(jsonNode.get(0));
+            return taskReport;
+        }
+
+        final List<JsonNode> organizations = StreamSupport.stream(new OrganizationSpliterator(buildOrganizationURI(), getZendeskRestClient(), task), false)
+                .collect(Collectors.toList());
+        final Set<String> knownUserIds = ConcurrentHashMap.newKeySet();
+        organizations.parallelStream().forEach(
+                organization -> {
+                    Stream<JsonNode> stream = StreamSupport.stream(new UserSpliterator(buildOrganizationWithUserURI(organization.get("url").asText()),
+                            getZendeskRestClient(), task, Exec.isPreview()), true);
+
+                    if (task.getDedup()) {
+                        stream = stream.filter(item -> knownUserIds.add(item.get("id").asText()));
+                    }
+
+                    stream.forEach(s ->
+                    {
+                        Stream<JsonNode> userEventStream = StreamSupport.stream(new UserEventSpliterator(s.get("id").asText(), buildUserEventURI(s.get("id").asText()),
+                                getZendeskRestClient(), task, Exec.isPreview()), true);
+                        userEventStream.forEach(recordImporter::addRecord);
+                    });
+                }
+        );
+        return taskReport;
+    }
+
+    @Override
+    public JsonNode getDataFromPath(final String path, final int page, final boolean isPreview, final long startTime)
+    {
+        return new ObjectMapper().createObjectNode().set(task.getTarget().getJsonName(), mockJsonNode());
+    }
+
+    @VisibleForTesting
+    protected ZendeskRestClient getZendeskRestClient()
+    {
+        if (zendeskRestClient == null) {
+            zendeskRestClient = new ZendeskRestClient();
+        }
+        return zendeskRestClient;
+    }
+
+    private String buildOrganizationURI()
+    {
+        return ZendeskUtils.getURIBuilder(task.getLoginUrl())
+                .setPath(ZendeskConstants.Url.API + "/" + Target.ORGANIZATIONS.getJsonName())
+                .setParameter("per_page", "100")
+                .setParameter("page", "1")
+                .toString();
+    }
+
+    private String buildOrganizationWithUserURI(final String path)
+    {
+        return path.replace(".json", "")
+                + "/users.json?per_page=100&page=1";
+    }
+
+    private String buildUserEventURI(final String userID)
+    {
+        final URIBuilder uriBuilder = ZendeskUtils.getURIBuilder(task.getLoginUrl())
+                .setPath(ZendeskConstants.Url.API_USER_EVENT)
+                .setParameter("identifier", task.getProfileSource().get() + ":user_id:" + userID);
+
+        task.getUserEventSource().ifPresent(eventSource -> uriBuilder.setParameter("source", eventSource));
+        task.getUserEventType().ifPresent(eventType -> uriBuilder.setParameter("type", eventType));
+        task.getStartTime().ifPresent(startTime -> {
+            try {
+                uriBuilder.setParameter("start_time", ZendeskDateUtils.convertToDateTimeFormat(startTime, ZendeskConstants.Misc.ISO_INSTANT));
+            }
+            catch (DataException e) {
+                uriBuilder.setParameter("start_time", ZendeskDateUtils.convertToDateTimeFormat(Instant.EPOCH.toString(), ZendeskConstants.Misc.ISO_INSTANT));
+            }
+        });
+
+        task.getEndTime().ifPresent(endTime -> uriBuilder.setParameter("end_time", ZendeskDateUtils.convertToDateTimeFormat(endTime, ZendeskConstants.Misc.ISO_INSTANT)));
+
+        return uriBuilder.toString();
+    }
+
+    private JsonNode mockJsonNode()
+    {
+        try {
+            String mockData = "[\n" +
+                    "    {\n" +
+                    "      \"id\": \"5c7f31aef8df240001e60bbf\",\n" +
+                    "      \"type\": \"remove_from_cart\",\n" +
+                    "      \"source\": \"shopify\",\n" +
+                    "      \"description\": \"\",\n" +
+                    "      \"authenticated\": true,\n" +
+                    "      \"created_at\": \"2019-03-06T02:34:22Z\",\n" +
+                    "      \"received_at\": \"2019-03-06T02:34:22Z\",\n" +
+                    "      \"properties\": {\n" +
+                    "        \"model\": 221,\n" +
+                    "        \"size\": 6\n" +
+                    "      },\n" +
+                    "      \"user_id\": \"12312354234\"\n" +
+                    "    }\n" +
+                    "]";
+
+            return new ObjectMapper().readTree(mockData);
+        }
+        catch (IOException ex) {
+            throw new RuntimeException("Can not create sample data " + ex.getMessage());
+        }
+    }
+}

data/src/main/java/org/embulk/input/zendesk/stream/PagingSpliterator.java

@@ -0,0 +1,40 @@
+package org.embulk.input.zendesk.stream;
+
+import org.embulk.input.zendesk.ZendeskInputPlugin;
+import org.embulk.input.zendesk.clients.ZendeskRestClient;
+
+import java.util.Spliterator;
+
+public abstract class PagingSpliterator<E> implements Spliterator<E>
+{
+    protected ZendeskRestClient zendeskRestClient;
+    protected boolean isPreview;
+    protected ZendeskInputPlugin.PluginTask task;
+    protected String path;
+
+    protected PagingSpliterator(final String path, final ZendeskRestClient zendeskRestClient, final ZendeskInputPlugin.PluginTask task, final boolean isPreview)
+    {
+        this.path = path;
+        this.zendeskRestClient = zendeskRestClient;
+        this.task = task;
+        this.isPreview = isPreview;
+    }
+
+    @Override
+    public Spliterator<E> trySplit()
+    {
+        return null;
+    }
+
+    @Override
+    public int characteristics()
+    {
+        return DISTINCT | NONNULL | IMMUTABLE;
+    }
+
+    @Override
+    public long estimateSize()
+    {
+        return Long.MAX_VALUE;
+    }
+}

data/src/main/java/org/embulk/input/zendesk/stream/paginator/sunshine/CustomObjectSpliterator.java

@@ -0,0 +1,42 @@
+package org.embulk.input.zendesk.stream.paginator.sunshine;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import org.embulk.input.zendesk.ZendeskInputPlugin;
+import org.embulk.input.zendesk.clients.ZendeskRestClient;
+import org.embulk.input.zendesk.utils.ZendeskUtils;
+
+import java.util.Iterator;
+import java.util.function.Consumer;
+
+public class CustomObjectSpliterator extends SunshineSpliterator
+{
+    public CustomObjectSpliterator(final String path, final ZendeskRestClient zendeskRestClient, final ZendeskInputPlugin.PluginTask task, final boolean isPreview)
+    {
+        super(path, zendeskRestClient, task, isPreview);
+    }
+
+    @Override
+    protected boolean isContinue(final JsonNode jsonNode, final Consumer<? super JsonNode> action)
+    {
+        final Iterator<JsonNode> iterator = ZendeskUtils.getListRecords(jsonNode, task.getTarget().getJsonName());
+
+        if (isPreview) {
+            if (iterator.hasNext()) {
+                JsonNode item = iterator.next();
+                // we have data for preview, no need to continue
+                if (item != null && !item.isNull()) {
+                    action.accept(item);
+                    return false;
+                }
+            }
+        }
+        handleRunIterator(iterator, action);
+
+        if (jsonNode.has("links") && !ZendeskUtils.isNull(jsonNode.get("links"))
+                && jsonNode.get("links").has("next") && !ZendeskUtils.isNull(jsonNode.get("links").get("next"))) {
+            path = task.getLoginUrl() + jsonNode.get("links").get("next");
+            return true;
+        }
+        return false;
+    }
+}

data/src/main/java/org/embulk/input/zendesk/stream/paginator/sunshine/SunshineSpliterator.java

@@ -0,0 +1,66 @@
+package org.embulk.input.zendesk.stream.paginator.sunshine;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import org.apache.http.HttpStatus;
+import org.embulk.config.ConfigException;
+import org.embulk.input.zendesk.ZendeskInputPlugin;
+import org.embulk.input.zendesk.clients.ZendeskRestClient;
+import org.embulk.input.zendesk.models.ZendeskException;
+import org.embulk.input.zendesk.stream.PagingSpliterator;
+import org.embulk.input.zendesk.utils.ZendeskUtils;
+
+import java.util.Iterator;
+import java.util.function.Consumer;
+
+public class SunshineSpliterator extends PagingSpliterator<JsonNode>
+{
+    public SunshineSpliterator(final String path, final ZendeskRestClient zendeskRestClient, final ZendeskInputPlugin.PluginTask task, final boolean isPreview)
+    {
+        super(path, zendeskRestClient, task, isPreview);
+    }
+
+    @Override
+    public boolean tryAdvance(final Consumer<? super JsonNode> action)
+    {
+        try {
+            final String result = zendeskRestClient.doGet(path, task, isPreview);
+
+            if (result != null && !result.isEmpty()) {
+                final JsonNode jsonNode = ZendeskUtils.parseJsonObject(result);
+                final JsonNode targetJsonNode = jsonNode.get(task.getTarget().getJsonName());
+                if (!ZendeskUtils.isNull(targetJsonNode)) {
+                    return isContinue(jsonNode, action);
+                }
+            }
+        }
+        catch (final ConfigException e) {
+            if (!(e.getCause() instanceof ZendeskException && ((ZendeskException) e.getCause()).getStatusCode() == HttpStatus.SC_NOT_FOUND)) {
+                throw e;
+            }
+        }
+        return false;
+    }
+
+    protected void handleRunIterator(Iterator<JsonNode> iterator,  final Consumer<? super JsonNode> action)
+    {
+        iterator.forEachRemaining(
+                item -> {
+                    if (item != null && !item.isNull()) {
+                        action.accept(iterator.next());
+                    }
+                });
+    }
+
+    protected boolean isContinue(final JsonNode jsonNode, final Consumer<? super JsonNode> action)
+    {
+        final Iterator<JsonNode> iterator = ZendeskUtils.getListRecords(jsonNode, task.getTarget().getJsonName());
+        handleRunIterator(iterator, action);
+
+        if (jsonNode.has("links") && !ZendeskUtils.isNull(jsonNode.get("links"))
+                && jsonNode.get("links").has("next") && !ZendeskUtils.isNull(jsonNode.get("links").get("next"))) {
+            path = task.getLoginUrl() + jsonNode.get("links").get("next");
+            return true;
+        }
+        return false;
+    }
+}

data/src/main/java/org/embulk/input/zendesk/stream/paginator/sunshine/UserEventSpliterator.java

@@ -0,0 +1,35 @@
+package org.embulk.input.zendesk.stream.paginator.sunshine;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.embulk.input.zendesk.ZendeskInputPlugin;
+import org.embulk.input.zendesk.clients.ZendeskRestClient;
+import org.embulk.input.zendesk.utils.ZendeskUtils;
+
+import java.util.Iterator;
+import java.util.function.Consumer;
+
+public class UserEventSpliterator extends SunshineSpliterator
+{
+    private String userID;
+
+    public UserEventSpliterator(final String userID, final String path, final ZendeskRestClient zendeskRestClient, final ZendeskInputPlugin.PluginTask task, final boolean isPreview)
+    {
+        super(path, zendeskRestClient, task, isPreview);
+        this.userID = userID;
+    }
+
+    @Override
+    protected void handleRunIterator(final Iterator<JsonNode> iterator, final Consumer<? super JsonNode> action)
+    {
+        iterator.forEachRemaining(
+                item -> {
+                    if (!ZendeskUtils.isNull(item)) {
+                        JsonNode temp = iterator.next();
+                        // Because in the returned json doesn't have user_id, so we try to add to it
+                        ((ObjectNode) temp).put("user_id", this.userID);
+                        action.accept(temp);
+                    }
+                });
+    }
+}

data/src/main/java/org/embulk/input/zendesk/stream/paginator/support/OrganizationSpliterator.java

@@ -0,0 +1,13 @@
+package org.embulk.input.zendesk.stream.paginator.support;
+
+import org.embulk.input.zendesk.ZendeskInputPlugin;
+import org.embulk.input.zendesk.clients.ZendeskRestClient;
+import org.embulk.input.zendesk.models.Target;
+
+public class OrganizationSpliterator extends SupportSpliterator
+{
+    public OrganizationSpliterator(final String path, final ZendeskRestClient zendeskRestClient, final ZendeskInputPlugin.PluginTask task)
+    {
+        super(Target.ORGANIZATIONS, path, zendeskRestClient, task, false);
+    }
+}

data/src/main/java/org/embulk/input/zendesk/stream/paginator/support/SupportSpliterator.java

@@ -0,0 +1,44 @@
+package org.embulk.input.zendesk.stream.paginator.support;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import org.embulk.input.zendesk.ZendeskInputPlugin;
+import org.embulk.input.zendesk.clients.ZendeskRestClient;
+import org.embulk.input.zendesk.models.Target;
+import org.embulk.input.zendesk.stream.PagingSpliterator;
+import org.embulk.input.zendesk.utils.ZendeskUtils;
+
+import java.util.Iterator;
+import java.util.function.Consumer;
+
+public class SupportSpliterator extends PagingSpliterator<JsonNode>
+{
+    Target target;
+
+    public SupportSpliterator(final Target target, final String path, final ZendeskRestClient zendeskRestClient, final ZendeskInputPlugin.PluginTask task, final boolean isPreview)
+    {
+        super(path, zendeskRestClient, task, isPreview);
+        this.target = target;
+    }
+
+    @Override
+    public boolean tryAdvance(final Consumer<? super JsonNode> action)
+    {
+        final String result = zendeskRestClient.doGet(path, task, isPreview);
+        if (result != null && !result.isEmpty()) {
+            final JsonNode jsonNode = ZendeskUtils.parseJsonObject(result);
+            final Iterator<JsonNode> iterator = ZendeskUtils.getListRecords(jsonNode, target.getJsonName());
+            iterator.forEachRemaining(
+                    item -> {
+                        if (!ZendeskUtils.isNull(item)) {
+                            action.accept(item);
+                        }
+                    });
+            if (jsonNode.get("next_page") != null && !jsonNode.get("next_page").isNull()) {
+                path = jsonNode.get("next_page").asText();
+                return true;
+            }
+        }
+
+        return false;
+    }
+}

data/src/main/java/org/embulk/input/zendesk/stream/paginator/support/UserSpliterator.java

@@ -0,0 +1,13 @@
+package org.embulk.input.zendesk.stream.paginator.support;
+
+import org.embulk.input.zendesk.ZendeskInputPlugin;
+import org.embulk.input.zendesk.clients.ZendeskRestClient;
+import org.embulk.input.zendesk.models.Target;
+
+public class UserSpliterator extends SupportSpliterator
+{
+    public UserSpliterator(final String path, final ZendeskRestClient zendeskRestClient, final ZendeskInputPlugin.PluginTask task, final boolean isPreview)
+    {
+        super(Target.USERS, path, zendeskRestClient, task, isPreview);
+    }
+}

data/src/main/java/org/embulk/input/zendesk/utils/ZendeskConstants.java

@@ -0,0 +1,72 @@
+package org.embulk.input.zendesk.utils;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.Arrays;
+import java.util.List;
+
+public class ZendeskConstants
+{
+    private ZendeskConstants()
+    {
+    }
+
+    public static class Header
+    {
+        public static final String APPLICATION_JSON = "application/json";
+
+        public static final String ZENDESK_MARKETPLACE_NAME = "X-Zendesk-Marketplace-Name";
+        public static final String ZENDESK_MARKETPLACE_ORGANIZATION_ID = "X-Zendesk-Marketplace-Organization-Id";
+        public static final String ZENDESK_MARKETPLACE_APP_ID = "X-Zendesk-Marketplace-App-Id";
+    }
+
+    public static class Field
+    {
+        public static final String START_TIME = "start_time";
+        public static final String END_TIME = "end_time";
+        public static final String COUNT = "count";
+        public static final String GENERATED_TIMESTAMP = "generated_timestamp";
+        public static final String UPDATED_AT = "updated_at";
+        public static final String ID = "id";
+        public static final String NEXT_PAGE="next_page";
+    }
+
+    public static class Url
+    {
+        public static final String API = "/api/v2";
+        public static final String API_INCREMENTAL = API + "/incremental";
+        public static final String API_NPS_INCREMENTAL = API + "/nps/incremental";
+        public static final String API_OBJECT_RECORD = "api/sunshine/objects/records";
+        public static final String API_RELATIONSHIP_RECORD = "api/sunshine/relationships/records";
+        public static final String API_USER_EVENT = "api/sunshine/events";
+    }
+
+    public static class Misc
+    {
+        public static final String RUBY_TIMESTAMP_FORMAT = "%Y-%m-%dT%H:%M:%S%z";
+        public static final String RUBY_TIMESTAMP_FORMAT_INPUT = "uuuu-MM-dd HH:mm:ss Z";
+        public static final String JAVA_TIMESTAMP_FORMAT = "uuuu-MM-dd'T'HH:mm:ss.SSS'Z'";
+        public static final String ISO_TIMESTAMP_FORMAT = "uuuu-MM-dd'T'HH:mm:ssXXX";
+        public static final String ISO_INSTANT = "uuuu-MM-dd'T'HH:mm:ss'Z'";
+        public static final String RUBY_TIMESTAMP_FORMAT_INPUT_NO_SPACE = "uuuu-MM-dd HH:mm:ssZ";
+        public static final String TOO_RECENT_START_TIME = "Too recent start_time.";
+        public static final int RECORDS_SIZE_PER_PAGE = 100;
+        public static final int MAXIMUM_RECORDS_INCREMENTAL = 1000;
+
+        // 1 MB
+        public static final int GUESS_BUFFER_SIZE = 1024 * 1024;
+        public static final List<String> SUPPORT_DATE_TIME_FORMAT = ImmutableList.copyOf(Arrays.asList(ZendeskConstants.Misc.ISO_INSTANT, ZendeskConstants.Misc.RUBY_TIMESTAMP_FORMAT_INPUT,
+                ZendeskConstants.Misc.JAVA_TIMESTAMP_FORMAT, ZendeskConstants.Misc.ISO_TIMESTAMP_FORMAT,
+                ZendeskConstants.Misc.RUBY_TIMESTAMP_FORMAT_INPUT_NO_SPACE));
+    }
+
+    public static class Regex
+    {
+        public static final String ID = "_id$";
+    }
+
+    public static class HttpStatus
+    {
+        public static final int TOO_MANY_REQUEST = 429;
+    }
+}