embulk-input-zendesk 0.3.4 → 0.3.5

data/src/main/java/org/embulk/input/zendesk/services/ZendeskService.java

@@ -0,0 +1,14 @@
+package org.embulk.input.zendesk.services;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import org.embulk.config.TaskReport;
+import org.embulk.input.zendesk.RecordImporter;
+
+public interface ZendeskService
+{
+    boolean isSupportIncremental();
+
+    TaskReport addRecordToImporter(int taskIndex, RecordImporter recordImporter);
+
+    JsonNode getDataFromPath(String path, int page, boolean isPreview, long startTime);
+}

data/src/main/java/org/embulk/input/zendesk/services/ZendeskSupportAPIService.java

@@ -1,110 +1,52 @@
 package org.embulk.input.zendesk.services;
 
-import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Throwables;
+import org.apache.http.client.utils.URIBuilder;
 import org.embulk.input.zendesk.ZendeskInputPlugin.PluginTask;
-import org.embulk.input.zendesk.clients.ZendeskRestClient;
 import org.embulk.input.zendesk.models.Target;
 import org.embulk.input.zendesk.utils.ZendeskConstants;
 import org.embulk.input.zendesk.utils.ZendeskUtils;
-import org.embulk.spi.DataException;
 
-import java.io.IOException;
-
-public class ZendeskSupportAPIService
+public class ZendeskSupportAPIService extends ZendeskNormalServices
 {
-    private ZendeskRestClient zendeskRestClient;
-
-    private PluginTask task;
-
-    private static ObjectMapper mapper = new ObjectMapper();
-
-    static {
-        mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-        mapper.configure(com.fasterxml.jackson.core.JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS, false);
-    }
-
     public ZendeskSupportAPIService(final PluginTask task)
     {
-        this.task = task;
+        super(task);
     }
 
-    public void setTask(PluginTask task)
+    public boolean isSupportIncremental()
     {
-        this.task = task;
+        return !(task.getTarget().equals(Target.TICKET_FORMS) || task.getTarget().equals(Target.TICKET_FIELDS));
     }
 
-    public JsonNode getData(String path, final int page, final boolean isPreview, long startTime)
+    @Override
+    protected String buildURI(final int page, long startTime)
     {
-        if (path.isEmpty()) {
-            path = buildPath(page, startTime);
-        }
-        try {
-            final String response = getZendeskRestClient().doGet(path, task, isPreview);
-            return parseJsonObject(response);
-        }
-        catch (final Exception e) {
-            throw Throwables.propagate(e);
-        }
-    }
+        final URIBuilder uriBuilder = ZendeskUtils.getURIBuilder(task.getLoginUrl()).setPath(buildPath());
 
-    @VisibleForTesting
-    protected ZendeskRestClient getZendeskRestClient()
-    {
-        if (zendeskRestClient == null) {
-            zendeskRestClient = new ZendeskRestClient();
+        if (isSupportIncremental()) {
+            uriBuilder.setParameter(ZendeskConstants.Field.START_TIME, String.valueOf(startTime));
+            if (Target.TICKET_METRICS.equals(task.getTarget())) {
+                uriBuilder.setParameter("include", "metric_sets");
+            }
         }
-        return zendeskRestClient;
-    }
-
-    private ObjectNode parseJsonObject(final String jsonText)
-    {
-        JsonNode node = parseJsonNode(jsonText);
-        if (node.isObject()) {
-            return (ObjectNode) node;
+        else {
+            uriBuilder.setParameter("sort_by", "id")
+                    .setParameter("per_page", String.valueOf(100))
+                    .setParameter("page", String.valueOf(page));
         }
 
-        throw new DataException("Expected object node to parse but doesn't get");
-    }
-
-    private JsonNode parseJsonNode(final String jsonText)
-    {
-        try {
-            return mapper.readTree(jsonText);
-        }
-        catch (final IOException e) {
-            throw Throwables.propagate(e);
-        }
+        return uriBuilder.toString();
     }
 
-    private String buildPath(final int page, long startTime)
+    private String buildPath()
     {
-        boolean isSupportIncremental = ZendeskUtils.isSupportAPIIncremental(task.getTarget());
-
-        StringBuilder urlBuilder = new StringBuilder(task.getLoginUrl())
-                .append("/")
-                .append(isSupportIncremental
-                        ? ZendeskConstants.Url.API_INCREMENTAL
-                        : ZendeskConstants.Url.API)
-                .append("/")
-                .append(Target.TICKET_METRICS.equals(task.getTarget())
+        return (isSupportIncremental()
+                ? ZendeskConstants.Url.API_INCREMENTAL
+                : ZendeskConstants.Url.API) +
+                "/" +
+                (Target.TICKET_METRICS.equals(task.getTarget())
                         ? Target.TICKETS.toString()
                         : task.getTarget().toString())
-                .append(".json?");
-
-        urlBuilder
-                .append(isSupportIncremental
-                        ? "start_time=" + startTime
-                        : "sort_by=id&per_page=100&page=" + page);
-
-        if (Target.TICKET_METRICS.equals(task.getTarget())) {
-            urlBuilder.append("&include=metric_sets");
-        }
-
-        return urlBuilder.toString();
+                + ".json";
     }
 }

data/src/main/java/org/embulk/input/zendesk/services/ZendeskUserEventService.java

@@ -0,0 +1,158 @@
+package org.embulk.input.zendesk.services;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.http.client.utils.URIBuilder;
+import org.embulk.config.TaskReport;
+import org.embulk.input.zendesk.RecordImporter;
+import org.embulk.input.zendesk.ZendeskInputPlugin;
+import org.embulk.input.zendesk.clients.ZendeskRestClient;
+import org.embulk.input.zendesk.models.Target;
+import org.embulk.input.zendesk.stream.paginator.sunshine.UserEventSpliterator;
+import org.embulk.input.zendesk.stream.paginator.support.OrganizationSpliterator;
+import org.embulk.input.zendesk.stream.paginator.support.UserSpliterator;
+import org.embulk.input.zendesk.utils.ZendeskConstants;
+import org.embulk.input.zendesk.utils.ZendeskDateUtils;
+import org.embulk.input.zendesk.utils.ZendeskUtils;
+import org.embulk.spi.DataException;
+import org.embulk.spi.Exec;
+
+import java.io.IOException;
+import java.time.Instant;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+
+public class ZendeskUserEventService implements ZendeskService
+{
+    protected ZendeskInputPlugin.PluginTask task;
+
+    private ZendeskRestClient zendeskRestClient;
+
+    public ZendeskUserEventService(final ZendeskInputPlugin.PluginTask task)
+    {
+        this.task = task;
+    }
+
+    public boolean isSupportIncremental()
+    {
+        return false;
+    }
+
+    @Override
+    public TaskReport addRecordToImporter(final int taskIndex, final RecordImporter recordImporter)
+    {
+        final TaskReport taskReport = Exec.newTaskReport();
+
+        if (Exec.isPreview()) {
+            JsonNode jsonNode = mockJsonNode();
+            recordImporter.addRecord(jsonNode.get(0));
+            return taskReport;
+        }
+
+        final List<JsonNode> organizations = StreamSupport.stream(new OrganizationSpliterator(buildOrganizationURI(), getZendeskRestClient(), task), false)
+                .collect(Collectors.toList());
+        final Set<String> knownUserIds = ConcurrentHashMap.newKeySet();
+        organizations.parallelStream().forEach(
+                organization -> {
+                    Stream<JsonNode> stream = StreamSupport.stream(new UserSpliterator(buildOrganizationWithUserURI(organization.get("url").asText()),
+                            getZendeskRestClient(), task, Exec.isPreview()), true);
+
+                    if (task.getDedup()) {
+                        stream = stream.filter(item -> knownUserIds.add(item.get("id").asText()));
+                    }
+
+                    stream.forEach(s ->
+                    {
+                        Stream<JsonNode> userEventStream = StreamSupport.stream(new UserEventSpliterator(s.get("id").asText(), buildUserEventURI(s.get("id").asText()),
+                                getZendeskRestClient(), task, Exec.isPreview()), true);
+                        userEventStream.forEach(recordImporter::addRecord);
+                    });
+                }
+        );
+        return taskReport;
+    }
+
+    @Override
+    public JsonNode getDataFromPath(final String path, final int page, final boolean isPreview, final long startTime)
+    {
+        return new ObjectMapper().createObjectNode().set(task.getTarget().getJsonName(), mockJsonNode());
+    }
+
+    @VisibleForTesting
+    protected ZendeskRestClient getZendeskRestClient()
+    {
+        if (zendeskRestClient == null) {
+            zendeskRestClient = new ZendeskRestClient();
+        }
+        return zendeskRestClient;
+    }
+
+    private String buildOrganizationURI()
+    {
+        return ZendeskUtils.getURIBuilder(task.getLoginUrl())
+                .setPath(ZendeskConstants.Url.API + "/" + Target.ORGANIZATIONS.getJsonName())
+                .setParameter("per_page", "100")
+                .setParameter("page", "1")
+                .toString();
+    }
+
+    private String buildOrganizationWithUserURI(final String path)
+    {
+        return path.replace(".json", "")
+                + "/users.json?per_page=100&page=1";
+    }
+
+    private String buildUserEventURI(final String userID)
+    {
+        final URIBuilder uriBuilder = ZendeskUtils.getURIBuilder(task.getLoginUrl())
+                .setPath(ZendeskConstants.Url.API_USER_EVENT)
+                .setParameter("identifier", task.getProfileSource().get() + ":user_id:" + userID);
+
+        task.getUserEventSource().ifPresent(eventSource -> uriBuilder.setParameter("source", eventSource));
+        task.getUserEventType().ifPresent(eventType -> uriBuilder.setParameter("type", eventType));
+        task.getStartTime().ifPresent(startTime -> {
+            try {
+                uriBuilder.setParameter("start_time", ZendeskDateUtils.convertToDateTimeFormat(startTime, ZendeskConstants.Misc.ISO_INSTANT));
+            }
+            catch (DataException e) {
+                uriBuilder.setParameter("start_time", ZendeskDateUtils.convertToDateTimeFormat(Instant.EPOCH.toString(), ZendeskConstants.Misc.ISO_INSTANT));
+            }
+        });
+
+        task.getEndTime().ifPresent(endTime -> uriBuilder.setParameter("end_time", ZendeskDateUtils.convertToDateTimeFormat(endTime, ZendeskConstants.Misc.ISO_INSTANT)));
+
+        return uriBuilder.toString();
+    }
+
+    private JsonNode mockJsonNode()
+    {
+        try {
+            String mockData = "[\n" +
+                    "    {\n" +
+                    "      \"id\": \"5c7f31aef8df240001e60bbf\",\n" +
+                    "      \"type\": \"remove_from_cart\",\n" +
+                    "      \"source\": \"shopify\",\n" +
+                    "      \"description\": \"\",\n" +
+                    "      \"authenticated\": true,\n" +
+                    "      \"created_at\": \"2019-03-06T02:34:22Z\",\n" +
+                    "      \"received_at\": \"2019-03-06T02:34:22Z\",\n" +
+                    "      \"properties\": {\n" +
+                    "        \"model\": 221,\n" +
+                    "        \"size\": 6\n" +
+                    "      },\n" +
+                    "      \"user_id\": \"12312354234\"\n" +
+                    "    }\n" +
+                    "]";
+
+            return new ObjectMapper().readTree(mockData);
+        }
+        catch (IOException ex) {
+            throw new RuntimeException("Can not create sample data " + ex.getMessage());
+        }
+    }
+}

data/src/main/java/org/embulk/input/zendesk/stream/PagingSpliterator.java

@@ -0,0 +1,40 @@
+package org.embulk.input.zendesk.stream;
+
+import org.embulk.input.zendesk.ZendeskInputPlugin;
+import org.embulk.input.zendesk.clients.ZendeskRestClient;
+
+import java.util.Spliterator;
+
+public abstract class PagingSpliterator<E> implements Spliterator<E>
+{
+    protected ZendeskRestClient zendeskRestClient;
+    protected boolean isPreview;
+    protected ZendeskInputPlugin.PluginTask task;
+    protected String path;
+
+    protected PagingSpliterator(final String path, final ZendeskRestClient zendeskRestClient, final ZendeskInputPlugin.PluginTask task, final boolean isPreview)
+    {
+        this.path = path;
+        this.zendeskRestClient = zendeskRestClient;
+        this.task = task;
+        this.isPreview = isPreview;
+    }
+
+    @Override
+    public Spliterator<E> trySplit()
+    {
+        return null;
+    }
+
+    @Override
+    public int characteristics()
+    {
+        return DISTINCT | NONNULL | IMMUTABLE;
+    }
+
+    @Override
+    public long estimateSize()
+    {
+        return Long.MAX_VALUE;
+    }
+}

data/src/main/java/org/embulk/input/zendesk/stream/paginator/sunshine/CustomObjectSpliterator.java

@@ -0,0 +1,42 @@
+package org.embulk.input.zendesk.stream.paginator.sunshine;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import org.embulk.input.zendesk.ZendeskInputPlugin;
+import org.embulk.input.zendesk.clients.ZendeskRestClient;
+import org.embulk.input.zendesk.utils.ZendeskUtils;
+
+import java.util.Iterator;
+import java.util.function.Consumer;
+
+public class CustomObjectSpliterator extends SunshineSpliterator
+{
+    public CustomObjectSpliterator(final String path, final ZendeskRestClient zendeskRestClient, final ZendeskInputPlugin.PluginTask task, final boolean isPreview)
+    {
+        super(path, zendeskRestClient, task, isPreview);
+    }
+
+    @Override
+    protected boolean isContinue(final JsonNode jsonNode, final Consumer<? super JsonNode> action)
+    {
+        final Iterator<JsonNode> iterator = ZendeskUtils.getListRecords(jsonNode, task.getTarget().getJsonName());
+
+        if (isPreview) {
+            if (iterator.hasNext()) {
+                JsonNode item = iterator.next();
+                // we have data for preview, no need to continue
+                if (item != null && !item.isNull()) {
+                    action.accept(item);
+                    return false;
+                }
+            }
+        }
+        handleRunIterator(iterator, action);
+
+        if (jsonNode.has("links") && !ZendeskUtils.isNull(jsonNode.get("links"))
+                && jsonNode.get("links").has("next") && !ZendeskUtils.isNull(jsonNode.get("links").get("next"))) {
+            path = task.getLoginUrl() + jsonNode.get("links").get("next");
+            return true;
+        }
+        return false;
+    }
+}

data/src/main/java/org/embulk/input/zendesk/stream/paginator/sunshine/SunshineSpliterator.java

@@ -0,0 +1,66 @@
+package org.embulk.input.zendesk.stream.paginator.sunshine;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import org.apache.http.HttpStatus;
+import org.embulk.config.ConfigException;
+import org.embulk.input.zendesk.ZendeskInputPlugin;
+import org.embulk.input.zendesk.clients.ZendeskRestClient;
+import org.embulk.input.zendesk.models.ZendeskException;
+import org.embulk.input.zendesk.stream.PagingSpliterator;
+import org.embulk.input.zendesk.utils.ZendeskUtils;
+
+import java.util.Iterator;
+import java.util.function.Consumer;
+
+public class SunshineSpliterator extends PagingSpliterator<JsonNode>
+{
+    public SunshineSpliterator(final String path, final ZendeskRestClient zendeskRestClient, final ZendeskInputPlugin.PluginTask task, final boolean isPreview)
+    {
+        super(path, zendeskRestClient, task, isPreview);
+    }
+
+    @Override
+    public boolean tryAdvance(final Consumer<? super JsonNode> action)
+    {
+        try {
+            final String result = zendeskRestClient.doGet(path, task, isPreview);
+
+            if (result != null && !result.isEmpty()) {
+                final JsonNode jsonNode = ZendeskUtils.parseJsonObject(result);
+                final JsonNode targetJsonNode = jsonNode.get(task.getTarget().getJsonName());
+                if (!ZendeskUtils.isNull(targetJsonNode)) {
+                    return isContinue(jsonNode, action);
+                }
+            }
+        }
+        catch (final ConfigException e) {
+            if (!(e.getCause() instanceof ZendeskException && ((ZendeskException) e.getCause()).getStatusCode() == HttpStatus.SC_NOT_FOUND)) {
+                throw e;
+            }
+        }
+        return false;
+    }
+
+    protected void handleRunIterator(Iterator<JsonNode> iterator,  final Consumer<? super JsonNode> action)
+    {
+        iterator.forEachRemaining(
+                item -> {
+                    if (item != null && !item.isNull()) {
+                        action.accept(iterator.next());
+                    }
+                });
+    }
+
+    protected boolean isContinue(final JsonNode jsonNode, final Consumer<? super JsonNode> action)
+    {
+        final Iterator<JsonNode> iterator = ZendeskUtils.getListRecords(jsonNode, task.getTarget().getJsonName());
+        handleRunIterator(iterator, action);
+
+        if (jsonNode.has("links") && !ZendeskUtils.isNull(jsonNode.get("links"))
+                && jsonNode.get("links").has("next") && !ZendeskUtils.isNull(jsonNode.get("links").get("next"))) {
+            path = task.getLoginUrl() + jsonNode.get("links").get("next");
+            return true;
+        }
+        return false;
+    }
+}