embulk-input-zendesk 0.3.4 → 0.3.5

data/src/main/java/org/embulk/input/zendesk/clients/ZendeskRestClient.java

@@ -7,7 +7,6 @@ import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableMap.Builder;
 import com.google.common.util.concurrent.RateLimiter;
 import com.google.common.util.concurrent.Uninterruptibles;
-
 import org.apache.http.Header;
 import org.apache.http.HttpResponse;
 import org.apache.http.HttpStatus;
@@ -26,23 +25,21 @@ import org.embulk.spi.DataException;
 import org.embulk.spi.Exec;
 import org.embulk.spi.util.RetryExecutor;
 import org.slf4j.Logger;
-import static org.apache.http.HttpHeaders.AUTHORIZATION;
-import static org.apache.http.protocol.HTTP.CONTENT_TYPE;
-import static org.embulk.spi.util.RetryExecutor.retryExecutor;
 
 import java.io.IOException;
-
 import java.util.concurrent.TimeUnit;
 
+import static org.apache.http.HttpHeaders.AUTHORIZATION;
+import static org.apache.http.protocol.HTTP.CONTENT_TYPE;
+import static org.embulk.spi.util.RetryExecutor.retryExecutor;
+
 public class ZendeskRestClient
 {
     private static final int CONNECTION_TIME_OUT = 240000;
 
     private static final Logger logger = Exec.getLogger(ZendeskRestClient.class);
-
-    private static RateLimiter rateLimiter;
-
     private static final ObjectMapper objectMapper = new ObjectMapper();
+    private static RateLimiter rateLimiter;
 
     static {
         objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
@@ -52,35 +49,37 @@ public class ZendeskRestClient
     {
     }
 
-    public String doGet(String url, PluginTask task, boolean isPreview)
+    public String doGet(final String url, final PluginTask task, final boolean isPreview)
     {
         try {
             return retryExecutor().withRetryLimit(task.getRetryLimit())
                     .withInitialRetryWait(task.getRetryInitialWaitSec() * 1000)
                     .withMaxRetryWait(task.getMaxRetryWaitSec() * 1000)
-                    .runInterruptible(new RetryExecutor.Retryable<String>() {
+                    .runInterruptible(new RetryExecutor.Retryable<String>()
+                    {
                         @Override
-                        public String call() throws Exception
+                        public String call()
+                                throws Exception
                         {
                             return sendGetRequest(url, task);
                         }
 
                         @Override
-                        public boolean isRetryableException(Exception exception)
+                        public boolean isRetryableException(final Exception exception)
                         {
                             if (exception instanceof ZendeskException) {
-                                int statusCode = ((ZendeskException) exception).getStatusCode();
+                                final int statusCode = ((ZendeskException) exception).getStatusCode();
                                 return isResponseStatusToRetry(statusCode, exception.getMessage(), ((ZendeskException) exception).getRetryAfter(), isPreview);
                             }
                             return false;
                         }
 
                         @Override
-                        public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
+                        public void onRetry(final Exception exception, final int retryCount, final int retryLimit, final int retryWait)
                         {
                             if (exception instanceof ZendeskException) {
-                                int retryAfter = ((ZendeskException) exception).getRetryAfter();
-                                String message;
+                                final int retryAfter = ((ZendeskException) exception).getRetryAfter();
+                                final String message;
                                 if (retryAfter > 0 && retryAfter > (retryWait / 1000)) {
                                     message = String
                                             .format("Retrying '%d'/'%d' after '%d' seconds. HTTP status code: '%s'",
@@ -100,7 +99,7 @@ public class ZendeskRestClient
                                 }
                             }
                             else {
-                                String message = String
+                                final String message = String
                                         .format("Retrying '%d'/'%d' after '%d' seconds. Message: '%s'",
                                                 retryCount, retryLimit,
                                                 retryWait / 1000,
@@ -110,15 +109,15 @@ public class ZendeskRestClient
                         }
 
                         @Override
-                        public void onGiveup(Exception firstException, Exception lastException)
+                        public void onGiveup(final Exception firstException, final Exception lastException)
                         {
                         }
                     });
         }
-        catch (RetryExecutor.RetryGiveupException | InterruptedException e) {
+        catch (final RetryExecutor.RetryGiveupException | InterruptedException e) {
             if (e instanceof RetryExecutor.RetryGiveupException && e.getCause() != null && e.getCause() instanceof ZendeskException) {
                 throw new ConfigException("Status: '" + ((ZendeskException) (e.getCause())).getStatusCode() + "', error message: '" + e.getCause().getMessage() + "'",
-                                          e.getCause());
+                        e.getCause());
             }
             throw new ConfigException(e);
         }
@@ -127,27 +126,34 @@ public class ZendeskRestClient
     @VisibleForTesting
     protected HttpClient createHttpClient()
     {
-        RequestConfig config = RequestConfig.custom()
+        final RequestConfig config = RequestConfig.custom()
                 .setConnectTimeout(CONNECTION_TIME_OUT)
                 .setConnectionRequestTimeout(CONNECTION_TIME_OUT)
                 .build();
         return HttpClientBuilder.create().setDefaultRequestConfig(config).build();
     }
 
-    private String sendGetRequest(final String url, final PluginTask task) throws ZendeskException
+    private String sendGetRequest(final String url, final PluginTask task)
+            throws ZendeskException
     {
         try {
-            HttpClient client = createHttpClient();
-            HttpRequestBase request = createGetRequest(url, task);
+            final HttpClient client = createHttpClient();
+            final HttpRequestBase request = createGetRequest(url, task);
+
+            if (rateLimiter != null) {
+                rateLimiter.acquire();
+            }
             logger.info(">>> {}{}", request.getURI().getPath(),
                     request.getURI().getQuery() != null ? "?" + request.getURI().getQuery() : "");
-            HttpResponse response = client.execute(request);
-            getRateLimiter(response).acquire();
-            int statusCode = response.getStatusLine().getStatusCode();
+            final HttpResponse response = client.execute(request);
+            if (rateLimiter == null) {
+                initRateLimiter(response);
+            }
+            final int statusCode = response.getStatusLine().getStatusCode();
             if (statusCode != HttpStatus.SC_OK) {
                 if (statusCode == ZendeskConstants.HttpStatus.TOO_MANY_REQUEST || statusCode == HttpStatus.SC_INTERNAL_SERVER_ERROR
                         || statusCode == HttpStatus.SC_SERVICE_UNAVAILABLE) {
-                    Header retryHeader = response.getFirstHeader("Retry-After");
+                    final Header retryHeader = response.getFirstHeader("Retry-After");
                     if (retryHeader != null) {
                         throw new ZendeskException(statusCode, EntityUtils.toString(response.getEntity()), Integer.parseInt(retryHeader.getValue()));
                     }
@@ -156,12 +162,12 @@ public class ZendeskRestClient
             }
             return EntityUtils.toString(response.getEntity());
         }
-        catch (IOException ex) {
+        catch (final IOException ex) {
             throw new ZendeskException(-1, ex.getMessage(), 0);
         }
     }
 
-    private boolean isResponseStatusToRetry(final int status, final String message, int retryAfter, final boolean isPreview)
+    private boolean isResponseStatusToRetry(final int status, final String message, final int retryAfter, final boolean isPreview)
     {
         if (status == HttpStatus.SC_NOT_FOUND) {
             // 404 would be returned e.g. ticket comments are empty (on fetchRelatedObjects method)
@@ -204,23 +210,23 @@ public class ZendeskRestClient
         return true;
     }
 
-    private HttpRequestBase createGetRequest(String url, PluginTask task)
+    private HttpRequestBase createGetRequest(final String url, final PluginTask task)
     {
-        HttpGet request = new HttpGet(url);
-        ImmutableMap<String, String> headers = buildAuthHeader(task);
+        final HttpGet request = new HttpGet(url);
+        final ImmutableMap<String, String> headers = buildAuthHeader(task);
         headers.forEach(request::setHeader);
         return request;
     }
 
-    private ImmutableMap<String, String> buildAuthHeader(PluginTask task)
+    private ImmutableMap<String, String> buildAuthHeader(final PluginTask task)
     {
-        Builder<String, String> builder = new Builder<>();
+        final Builder<String, String> builder = new Builder<>();
         builder.put(AUTHORIZATION, buildCredential(task));
         addCommonHeader(builder, task);
         return builder.build();
     }
 
-    private String buildCredential(PluginTask task)
+    private String buildCredential(final PluginTask task)
     {
         switch (task.getAuthenticationMethod()) {
             case BASIC:
@@ -233,7 +239,7 @@ public class ZendeskRestClient
         return "";
     }
 
-    private void addCommonHeader(final Builder<String, String> builder, PluginTask task)
+    private void addCommonHeader(final Builder<String, String> builder, final PluginTask task)
     {
         task.getAppMarketPlaceIntegrationName().ifPresent(s -> builder.put(ZendeskConstants.Header.ZENDESK_MARKETPLACE_NAME, s));
         task.getAppMarketPlaceAppId().ifPresent(s -> builder.put(ZendeskConstants.Header.ZENDESK_MARKETPLACE_APP_ID, s));
@@ -242,28 +248,24 @@ public class ZendeskRestClient
         builder.put(CONTENT_TYPE, ZendeskConstants.Header.APPLICATION_JSON);
     }
 
-    private RateLimiter getRateLimiter(final HttpResponse response)
+    private synchronized void initRateLimiter(final HttpResponse response)
     {
-        if (rateLimiter == null) {
-            rateLimiter = initRateLimiter(response);
-        }
-        return rateLimiter;
-    }
-
-    private static synchronized RateLimiter initRateLimiter(final HttpResponse response)
-    {
-        double permits = 0.0;
         if (response.containsHeader("x-rate-limit")) {
-            String rateLimit = response.getFirstHeader("x-rate-limit").getValue();
+            double permits = 0.0;
+            final String rateLimit = response.getFirstHeader("x-rate-limit").getValue();
             try {
                 permits = Double.parseDouble(rateLimit);
             }
-            catch (NumberFormatException e) {
+            catch (final NumberFormatException e) {
                 throw new DataException("Error when parse x-rate-limit: '" + response.getFirstHeader("x-rate-limit").getValue() + "'");
             }
-            permits = permits / 60;
+
+            if (permits > 0) {
+                permits = permits / 60;
+                logger.info("Permits per second " + permits);
+
+                rateLimiter = RateLimiter.create(permits);
+            }
         }
-        logger.info("Permits per second " + permits);
-        return RateLimiter.create(permits);
     }
 }

data/src/main/java/org/embulk/input/zendesk/models/Target.java

@@ -10,9 +10,9 @@ public enum Target
     /** For ticket_metrics - we fetch by using include metric_sets with ticket target
     *    so the jsonName is different comparing to the target name
     */
-    TICKETS("tickets"), USERS("users"), ORGANIZATIONS("organizations"),
-    TICKET_EVENTS("ticket_events"), TICKET_METRICS("metric_sets"),
-    TICKET_FIELDS("ticket_fields"), TICKET_FORMS("ticket_forms");
+    TICKETS("tickets"), USERS("users"), ORGANIZATIONS("organizations"), TICKET_EVENTS("ticket_events"),
+    TICKET_METRICS("metric_sets"), TICKET_FIELDS("ticket_fields"), TICKET_FORMS("ticket_forms"),
+    RECIPIENTS("recipients"), SCORES("responses"), OBJECT_RECORDS("data"), RELATIONSHIP_RECORDS("data"), USER_EVENTS("data");
 
     String jsonName;
 

data/src/main/java/org/embulk/input/zendesk/models/ZendeskException.java

@@ -6,7 +6,7 @@ public class ZendeskException extends Exception
     private final int statusCode;
     private final int retryAfter;
 
-    public ZendeskException(int statusCode, String message, int retryAfter)
+    public ZendeskException(final int statusCode, final String message, final int retryAfter)
     {
         super(message);
         this.statusCode = statusCode;

data/src/main/java/org/embulk/input/zendesk/services/ZendeskCustomObjectService.java

@@ -0,0 +1,110 @@
+package org.embulk.input.zendesk.services;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.http.HttpStatus;
+import org.embulk.config.ConfigException;
+import org.embulk.config.TaskReport;
+import org.embulk.input.zendesk.RecordImporter;
+import org.embulk.input.zendesk.ZendeskInputPlugin;
+import org.embulk.input.zendesk.clients.ZendeskRestClient;
+import org.embulk.input.zendesk.models.Target;
+import org.embulk.input.zendesk.models.ZendeskException;
+import org.embulk.input.zendesk.stream.paginator.sunshine.CustomObjectSpliterator;
+import org.embulk.input.zendesk.utils.ZendeskConstants;
+import org.embulk.input.zendesk.utils.ZendeskUtils;
+import org.embulk.spi.Exec;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+
+public class ZendeskCustomObjectService implements ZendeskService
+{
+    protected ZendeskInputPlugin.PluginTask task;
+
+    private ZendeskRestClient zendeskRestClient;
+
+    public ZendeskCustomObjectService(final ZendeskInputPlugin.PluginTask task)
+    {
+        this.task = task;
+    }
+
+    public boolean isSupportIncremental()
+    {
+        return false;
+    }
+
+    @Override
+    public TaskReport addRecordToImporter(final int taskIndex, final RecordImporter recordImporter)
+    {
+        final List<String> paths = getListPathByTarget();
+
+        paths.parallelStream().forEach(path -> StreamSupport.stream(new CustomObjectSpliterator(path, getZendeskRestClient(), task, Exec.isPreview()), !Exec.isPreview())
+                .forEach(recordImporter::addRecord));
+
+        return Exec.newTaskReport();
+    }
+
+    @Override
+    public JsonNode getDataFromPath(final String path, final int page, final boolean isPreview, final long startTime)
+    {
+        Preconditions.checkArgument(isPreview, "IsPreview should be true to use this method");
+
+        Optional<String> response = Optional.empty();
+        final List<String> paths = path.isEmpty()
+                ? getListPathByTarget()
+                : Collections.singletonList(path);
+
+        for (final String temp : paths) {
+            try {
+                response = Optional.ofNullable(getZendeskRestClient().doGet(temp, task, true));
+
+                if (response.isPresent()) {
+                    break;
+                }
+            }
+            catch (final ConfigException e) {
+                // Sometimes we get 404 when having invalid endpoint, so ignore when we get 404
+                if (!(e.getCause() instanceof ZendeskException && ((ZendeskException) e.getCause()).getStatusCode() == HttpStatus.SC_NOT_FOUND)) {
+                    throw e;
+                }
+            }
+        }
+
+        return response.map(ZendeskUtils::parseJsonObject).orElse(new ObjectMapper().createObjectNode());
+    }
+
+    @VisibleForTesting
+    protected ZendeskRestClient getZendeskRestClient()
+    {
+        if (zendeskRestClient == null) {
+            zendeskRestClient = new ZendeskRestClient();
+        }
+        return zendeskRestClient;
+    }
+
+    private List<String> getListPathByTarget()
+    {
+        return task.getTarget().equals(Target.OBJECT_RECORDS)
+                ? task.getObjectTypes().stream().map(this::buildPath).collect(Collectors.toList())
+                : task.getRelationshipTypes().stream().map(this::buildPath).collect(Collectors.toList());
+    }
+
+    private String buildPath(final String value)
+    {
+        final String perPage = Exec.isPreview() ? "1" : "1000";
+
+        return ZendeskUtils.getURIBuilder(task.getLoginUrl())
+                .setPath(task.getTarget().equals(Target.OBJECT_RECORDS)
+                        ? ZendeskConstants.Url.API_OBJECT_RECORD
+                        : ZendeskConstants.Url.API_RELATIONSHIP_RECORD)
+                .setParameter("type", value)
+                .setParameter("per_page", perPage)
+                .toString();
+    }
+}

data/src/main/java/org/embulk/input/zendesk/services/ZendeskNPSService.java

@@ -0,0 +1,30 @@
+package org.embulk.input.zendesk.services;
+
+import org.embulk.input.zendesk.ZendeskInputPlugin;
+import org.embulk.input.zendesk.utils.ZendeskConstants;
+import org.embulk.input.zendesk.utils.ZendeskUtils;
+
+public class ZendeskNPSService extends ZendeskNormalServices
+{
+    public ZendeskNPSService(final ZendeskInputPlugin.PluginTask task)
+    {
+        super(task);
+    }
+
+    public boolean isSupportIncremental()
+    {
+        return true;
+    }
+
+    @Override
+    protected String buildURI(final int page, final long startTime)
+    {
+        return ZendeskUtils.getURIBuilder(task.getLoginUrl())
+                .setPath(ZendeskConstants.Url.API_NPS_INCREMENTAL
+                        + "/"
+                        + task.getTarget().getJsonName()
+                        + ".json")
+                .setParameter(ZendeskConstants.Field.START_TIME, String.valueOf(startTime))
+                .toString();
+    }
+}

data/src/main/java/org/embulk/input/zendesk/services/ZendeskNormalServices.java

@@ -0,0 +1,239 @@
+package org.embulk.input.zendesk.services;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Throwables;
+import org.apache.http.HttpStatus;
+import org.apache.http.client.utils.URIBuilder;
+import org.embulk.config.ConfigException;
+import org.embulk.config.TaskReport;
+import org.embulk.input.zendesk.RecordImporter;
+import org.embulk.input.zendesk.ZendeskInputPlugin;
+import org.embulk.input.zendesk.clients.ZendeskRestClient;
+import org.embulk.input.zendesk.models.ZendeskException;
+import org.embulk.input.zendesk.utils.ZendeskConstants;
+import org.embulk.input.zendesk.utils.ZendeskDateUtils;
+import org.embulk.input.zendesk.utils.ZendeskUtils;
+import org.embulk.spi.Exec;
+import org.slf4j.Logger;
+
+import java.time.Instant;
+
+import java.util.Iterator;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+public abstract class ZendeskNormalServices implements ZendeskService
+{
+    private static final Logger logger = Exec.getLogger(ZendeskNormalServices.class);
+
+    protected ZendeskInputPlugin.PluginTask task;
+
+    private ZendeskRestClient zendeskRestClient;
+
+    protected ZendeskNormalServices(final ZendeskInputPlugin.PluginTask task)
+    {
+        this.task = task;
+    }
+
+    public TaskReport addRecordToImporter(final int taskIndex, final RecordImporter recordImporter)
+    {
+        TaskReport taskReport = Exec.newTaskReport();
+
+        if (isSupportIncremental()) {
+            importDataForIncremental(task, recordImporter, taskReport);
+        }
+        else {
+            importDataForNonIncremental(task, taskIndex, recordImporter);
+        }
+
+        return taskReport;
+    }
+
+    public JsonNode getDataFromPath(String path, final int page, final boolean isPreview, final long startTime)
+    {
+        if (path.isEmpty()) {
+            path = buildURI(page, startTime);
+        }
+
+        final String response = getZendeskRestClient().doGet(path, task, isPreview);
+        return ZendeskUtils.parseJsonObject(response);
+    }
+
+    protected abstract String buildURI(int page, long startTime);
+
+    @VisibleForTesting
+    protected ZendeskRestClient getZendeskRestClient()
+    {
+        if (zendeskRestClient == null) {
+            zendeskRestClient = new ZendeskRestClient();
+        }
+        return zendeskRestClient;
+    }
+
+    private void importDataForIncremental(final ZendeskInputPlugin.PluginTask task, final RecordImporter recordImporter, final TaskReport taskReport)
+    {
+        long startTime = 0;
+
+        if (task.getStartTime().isPresent()) {
+            startTime = ZendeskDateUtils.getStartTime(task.getStartTime().get());
+        }
+
+        // For incremental target, we will run in one task but split in multiple threads inside for data deduplication.
+        // Run with incremental will contain duplicated data.
+        ThreadPoolExecutor pool = null;
+        try {
+            final Set<String> knownIds = ConcurrentHashMap.newKeySet();
+            pool = new ThreadPoolExecutor(
+                    10, 100, 60, TimeUnit.SECONDS, new LinkedBlockingQueue<>()
+            );
+
+            while (true) {
+                int recordCount = 0;
+
+                // Page argument isn't used in incremental API so we just set it to 0
+                final JsonNode result = getDataFromPath("", 0, false, startTime);
+                final Iterator<JsonNode> iterator = ZendeskUtils.getListRecords(result, task.getTarget().getJsonName());
+
+                int numberOfRecords = 0;
+                if (result.has(ZendeskConstants.Field.COUNT)) {
+                    numberOfRecords = result.get(ZendeskConstants.Field.COUNT).asInt();
+                }
+
+                while (iterator.hasNext()) {
+                    final JsonNode recordJsonNode = iterator.next();
+
+                    if (isUpdatedBySystem(recordJsonNode, startTime)) {
+                        continue;
+                    }
+
+                    if (task.getDedup()) {
+                        final String recordID = recordJsonNode.get(ZendeskConstants.Field.ID).asText();
+
+                        // add success -> no duplicate
+                        if (!knownIds.add(recordID)) {
+                            continue;
+                        }
+                    }
+
+                    pool.submit(() -> fetchSubResourceAndAddToImporter(recordJsonNode, task, recordImporter));
+                    recordCount++;
+                    if (Exec.isPreview()) {
+                        return;
+                    }
+                }
+
+                logger.info("Fetched '{}' records from start_time '{}'", recordCount, startTime);
+
+                // https://developer.zendesk.com/rest_api/docs/support/incremental_export#pagination
+                // When there are more than 1000 records share the same time stamp, the count > 1000
+                long apiEndTime = result.get(ZendeskConstants.Field.END_TIME).asLong();
+                startTime = startTime == apiEndTime
+                        ? apiEndTime + 1
+                        : apiEndTime;
+
+                if (numberOfRecords < ZendeskConstants.Misc.MAXIMUM_RECORDS_INCREMENTAL) {
+                    break;
+                }
+            }
+
+            if (!Exec.isPreview()) {
+                storeStartTimeForConfigDiff(taskReport, startTime);
+            }
+        }
+        finally {
+            if (pool != null) {
+                pool.shutdown();
+                try {
+                    pool.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
+                }
+                catch (final InterruptedException e) {
+                    logger.warn("Error when wait pool to finish");
+                    throw Throwables.propagate(e);
+                }
+            }
+        }
+    }
+
+    private void storeStartTimeForConfigDiff(final TaskReport taskReport, final long resultEndTime)
+    {
+        if (task.getIncremental()) {
+            // resultEndTime = 0 mean no records, we should store now time for next run
+            if (resultEndTime == 0) {
+                taskReport.set(ZendeskConstants.Field.START_TIME, Instant.now().getEpochSecond());
+            }
+            else {
+                // NOTE: start_time compared as "=>", not ">".
+                // If we will use end_time for next start_time, we got the same records that are fetched
+                // end_time + 1 is workaround for that
+                taskReport.set(ZendeskConstants.Field.START_TIME, resultEndTime + 1);
+            }
+        }
+    }
+
+    private void fetchSubResourceAndAddToImporter(final JsonNode jsonNode, final ZendeskInputPlugin.PluginTask task, final RecordImporter recordImporter)
+    {
+        task.getIncludes().forEach(include -> {
+            final String relatedObjectName = include.trim();
+
+            final URIBuilder uriBuilder = ZendeskUtils.getURIBuilder(task.getLoginUrl())
+                    .setPath(ZendeskConstants.Url.API
+                            + "/" + task.getTarget().toString()
+                            + "/" + jsonNode.get(ZendeskConstants.Field.ID).asText()
+                            + "/" + relatedObjectName + ".json");
+            try {
+                final JsonNode result = getDataFromPath(uriBuilder.toString(), 0, false, 0);
+                if (result != null && result.has(relatedObjectName)) {
+                    ((ObjectNode) jsonNode).set(include, result.get(relatedObjectName));
+                }
+            }
+            catch (final ConfigException e) {
+                // Sometimes we get 404 when having invalid endpoint, so ignore when we get 404 InvalidEndpoint
+                if (!(e.getCause() instanceof ZendeskException && ((ZendeskException) e.getCause()).getStatusCode() == HttpStatus.SC_NOT_FOUND)) {
+                    throw e;
+                }
+            }
+        });
+
+        recordImporter.addRecord(jsonNode);
+    }
+
+    private boolean isUpdatedBySystem(final JsonNode recordJsonNode, final long startTime)
+    {
+        /*
+         * https://developer.zendesk.com/rest_api/docs/core/incremental_export#excluding-system-updates
+         * "generated_timestamp" will be updated when Zendesk internal changing
+         * "updated_at" will be updated when ticket data was changed
+         * start_time for query parameter will be processed on Zendesk with generated_timestamp,
+         * but it was calculated by record' updated_at time.
+         * So the doesn't changed record from previous import would be appear by Zendesk internal changes.
+         */
+        if (recordJsonNode.has(ZendeskConstants.Field.GENERATED_TIMESTAMP) && recordJsonNode.has(ZendeskConstants.Field.UPDATED_AT)) {
+            final String recordUpdatedAtTime = recordJsonNode.get(ZendeskConstants.Field.UPDATED_AT).asText();
+            final long recordUpdatedAtToEpochSecond = ZendeskDateUtils.isoToEpochSecond(recordUpdatedAtTime);
+
+            return recordUpdatedAtToEpochSecond <= startTime;
+        }
+
+        return false;
+    }
+
+    private void importDataForNonIncremental(final ZendeskInputPlugin.PluginTask task, final int taskIndex, RecordImporter recordImporter)
+    {
+        // Page start from 1 => page = taskIndex + 1
+        final JsonNode result = getDataFromPath("", taskIndex + 1, false, 0);
+        final Iterator<JsonNode> iterator = ZendeskUtils.getListRecords(result, task.getTarget().getJsonName());
+
+        while (iterator.hasNext()) {
+            fetchSubResourceAndAddToImporter(iterator.next(), task, recordImporter);
+
+            if (Exec.isPreview()) {
+                break;
+            }
+        }
+    }
+}