embulk-input-zendesk 0.3.4 → 0.3.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 87550faf658f946b3304fc3e19db5903dea08cc0
-  data.tar.gz: be6b8435c7650e8113ed7a36757e61ba0b4ff986
+  metadata.gz: 2dab4909dba68349742e03eef6be0ff02014caf1
+  data.tar.gz: 05bee8746fae83639eeb0daacb4b38ac5f765868
 SHA512:
-  metadata.gz: cc547390d1c61746c3bca7c6e6483da976f0aadb9fb0fdd389a94fea0f79f237192edb39e55a523d3d5fb685355210f225b6b65014c37b83bf421c101575a3a3
-  data.tar.gz: 2b029eb64f6c10bb082aa00c132a2e1dfe2df86c30151b6a133c9460df7f84c94a06bfa48d1e134f43e16c0052effbc64358a4b881d62b17367d39da7f3b5101
+  metadata.gz: 0b0831ce1c1a40d1d9a3b2b3e8a73d8a2fa14a6e5144d264e1e210f2289c092fed6778c7ccea823fcf4464cc8b8ef61182306426730f4a82c85a50b262783bdf
+  data.tar.gz: e1706af5499ff07076761b0aa591920abf867a2a95d38c4b5e18a47dd61e73ecb653cf5f3d4cca8beb631ddf3f6b3de9018ebb92fecf8558a4f6e3713c4b0715

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.3.5 - 2019-06-03
+* [enhancement] Add new targets #58 [#58](https://github.com/treasure-data/embulk-input-zendesk/pull/58)
+
 ## 0.3.4 - 2019-04-11
 * [enhancement] Add new time format #56 [#56](https://github.com/treasure-data/embulk-input-zendesk/pull/56)
 

data/README.md

@@ -22,7 +22,7 @@ Required Embulk version >= 0.9.6.
 
 - **login_url**: Login URL for Zendesk (string, required)
 - **auth_method**: `basic`, `token`, or `oauth`. For more detail on [zendesk document](https://developer.zendesk.com/rest_api/docs/core/introduction#security-and-authentication). (string, required)
-- **target**: Which export Zendesk resource. Currently supported are `tickets`, `ticket_events`, `users`, `organizations`, `ticket_fields`, `ticket_forms` or `ticket_metrics`. (string, required)
+- **target**: Which export Zendesk resource. Currently supported are `tickets`, `ticket_events`, `users`, `organizations`, `ticket_fields`, `ticket_forms`, `ticket_metrics`, `scores`, `recipients`, `object_records`, `relationship_records` or `user_events`. (string, required)
 - **includes**: Will fetch sub resources. For example, ticket has ticket_audits, ticket_comments. See below example config. (array, default: `[]`)
 - **username**: The user name a.k.a. email. Required if `auth_method` is `basic` or `token`. (string, default: `null`)
 - **password**: Password. required if `auth_method` is `basic`. (string, default: `null`)
@@ -36,7 +36,12 @@ Required Embulk version >= 0.9.6.
 - **app_marketplace_integration_name**: Invisible to user, only requires to be a part of the Zendesk Apps Marketplace. This should be used to name of the integration.
 - **app_marketplace_org_id**: Invisible to user, only requires to be a part of the Zendesk Apps Marketplace. This should be the Organization ID for your organization from the new developer portal.
 - **app_marketplace_app_id**: Invisible to user, only requires to be a part of the Zendesk Apps Marketplace. This is the “App ID” that will be assigned to you when you submit your app.
-  
+- **object_types**: List custom object types, required if `target` is `object_records`.
+- **relationship_types**: List custom relationship types, required if `target` is `relationship_records`.
+- **profile_source**: Profile source of user event, required if `target` is `user_events`.
+- **user_event_source**: Source of user event, required if `target` is `user_events`.
+- **user_event_type**: Type of user event, required if `target` is `user_events`.   
+
 ## Example
 
 ```yaml
@@ -59,3 +64,28 @@ in:
 ```
 $ ./gradlew package
 ```
+
+
+@Config("object_types")
+        @ConfigDefault("[]")
+        List<String> getObjectTypes();
+
+        @Config("relationship_types")
+        @ConfigDefault("[]")
+        List<String> getRelationshipTypes();
+
+        @Config("profile_source")
+        @ConfigDefault("null")
+        Optional<String> getProfileSource();
+
+        @Config("end_time")
+        @ConfigDefault("null")
+        Optional<String> getEndTime();
+
+        @Config("user_event_type")
+        @ConfigDefault("null")
+        Optional<String> getUserEventType();
+
+        @Config("user_event_source")
+        @ConfigDefault("null")
+        Optional<String> getUserEventSource();

data/build.gradle

@@ -15,7 +15,7 @@ configurations {
     provided
 }
 
-version = "0.3.4"
+version = "0.3.5"
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8

data/src/main/java/org/embulk/input/zendesk/RecordImporter.java

@@ -0,0 +1,134 @@
+package org.embulk.input.zendesk;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import org.embulk.input.zendesk.utils.ZendeskDateUtils;
+import org.embulk.input.zendesk.utils.ZendeskUtils;
+import org.embulk.spi.Column;
+import org.embulk.spi.ColumnVisitor;
+import org.embulk.spi.Exec;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.Schema;
+import org.embulk.spi.json.JsonParser;
+import org.embulk.spi.time.Timestamp;
+import org.slf4j.Logger;
+
+import java.util.function.Function;
+
+public class RecordImporter
+{
+    private Schema schema;
+    private PageBuilder pageBuilder;
+
+    private static final Logger logger = Exec.getLogger(RecordImporter.class);
+
+    public RecordImporter(Schema schema, PageBuilder pageBuilder)
+    {
+        this.schema = schema;
+        this.pageBuilder = pageBuilder;
+    }
+
+    public synchronized void addRecord(final JsonNode record)
+    {
+        schema.visitColumns(new ColumnVisitor()
+        {
+            @Override
+            public void jsonColumn(final Column column)
+            {
+                final JsonNode data = record.get(column.getName());
+
+                setColumn(column, data, (value) -> {
+                    pageBuilder.setJson(column, new JsonParser().parse(value.toString()));
+                    return null;
+                });
+            }
+
+            @Override
+            public void stringColumn(final Column column)
+            {
+                final JsonNode data = record.get(column.getName());
+
+                setColumn(column, data, (value) -> {
+                    pageBuilder.setString(column, value.asText());
+                    return null;
+                });
+            }
+
+            @Override
+            public void timestampColumn(final Column column)
+            {
+                final JsonNode data = record.get(column.getName());
+                setColumn(column, data, (value) -> {
+                    final Timestamp timestamp = getTimestampValue(value.asText());
+                    if (timestamp == null) {
+                        pageBuilder.setNull(column);
+                    }
+                    else {
+                        pageBuilder.setTimestamp(column, timestamp);
+                    }
+                    return null;
+                });
+            }
+
+            @Override
+            public void booleanColumn(final Column column)
+            {
+                final JsonNode data = record.get(column.getName());
+
+                setColumn(column, data, (value) -> {
+                    pageBuilder.setBoolean(column, value.asBoolean());
+                    return null;
+                });
+            }
+
+            @Override
+            public void longColumn(final Column column)
+            {
+                final JsonNode data = record.get(column.getName());
+
+                setColumn(column, data, (value) -> {
+                    pageBuilder.setLong(column, value.asLong());
+                    return null;
+                });
+            }
+
+            @Override
+            public void doubleColumn(final Column column)
+            {
+                final JsonNode data = record.get(column.getName());
+
+                setColumn(column, data, (value) -> {
+                    pageBuilder.setDouble(column, value.asDouble());
+                    return null;
+                });
+            }
+
+            private void setColumn(final Column column, final JsonNode data, final Function<JsonNode, Void> setter)
+            {
+                if (ZendeskUtils.isNull(data)) {
+                    pageBuilder.setNull(column);
+                    return;
+                }
+                setter.apply(data);
+            }
+        });
+
+        pageBuilder.addRecord();
+    }
+
+    /*
+     * For getting the timestamp value of the node
+     * Sometime if the parser could not parse the value then return null
+     * */
+    private Timestamp getTimestampValue(final String value)
+    {
+        Timestamp result = null;
+        try {
+            final long timeStamp = ZendeskDateUtils.isoToEpochSecond(value);
+            result = Timestamp.ofEpochSecond(timeStamp);
+        }
+        catch (final Exception e) {
+            logger.warn("Error when parse time stamp data " + value);
+        }
+        return result;
+    }
+}

data/src/main/java/org/embulk/input/zendesk/ZendeskInputPlugin.java

@@ -5,10 +5,8 @@ import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Throwables;
 import com.google.common.collect.ImmutableList;
 
-import org.apache.http.HttpStatus;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigDiff;
@@ -20,14 +18,15 @@ import org.embulk.config.TaskSource;
 import org.embulk.exec.GuessExecutor;
 import org.embulk.input.zendesk.models.AuthenticationMethod;
 import org.embulk.input.zendesk.models.Target;
-import org.embulk.input.zendesk.models.ZendeskException;
+import org.embulk.input.zendesk.services.ZendeskCustomObjectService;
+import org.embulk.input.zendesk.services.ZendeskNPSService;
+import org.embulk.input.zendesk.services.ZendeskService;
 import org.embulk.input.zendesk.services.ZendeskSupportAPIService;
+import org.embulk.input.zendesk.services.ZendeskUserEventService;
 import org.embulk.input.zendesk.utils.ZendeskConstants;
 import org.embulk.input.zendesk.utils.ZendeskDateUtils;
 import org.embulk.input.zendesk.utils.ZendeskUtils;
-import org.embulk.input.zendesk.utils.ZendeskValidatorUtils;
 import org.embulk.spi.Buffer;
-import org.embulk.spi.DataException;
 import org.embulk.spi.Exec;
 import org.embulk.spi.InputPlugin;
 import org.embulk.spi.PageBuilder;
@@ -45,14 +44,10 @@ import java.time.Instant;
 import java.time.OffsetDateTime;
 import java.time.ZoneOffset;
 import java.time.format.DateTimeFormatter;
+import java.util.Arrays;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Optional;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 import java.util.stream.StreamSupport;
@@ -133,27 +128,54 @@ public class ZendeskInputPlugin implements InputPlugin
         @ConfigDefault("null")
         Optional<String> getAppMarketPlaceAppId();
 
+        @Config("object_types")
+        @ConfigDefault("[]")
+        List<String> getObjectTypes();
+
+        @Config("relationship_types")
+        @ConfigDefault("[]")
+        List<String> getRelationshipTypes();
+
+        @Config("profile_source")
+        @ConfigDefault("null")
+        Optional<String> getProfileSource();
+
+        @Config("end_time")
+        @ConfigDefault("null")
+        Optional<String> getEndTime();
+
+        @Config("user_event_type")
+        @ConfigDefault("null")
+        Optional<String> getUserEventType();
+
+        @Config("user_event_source")
+        @ConfigDefault("null")
+        Optional<String> getUserEventSource();
+
         @Config("columns")
         SchemaConfig getColumns();
     }
 
-    private static final Logger logger = Exec.getLogger(ZendeskInputPlugin.class);
+    private ZendeskService zendeskService;
+
+    private RecordImporter recordImporter;
 
-    private ZendeskSupportAPIService zendeskSupportAPIService;
+    private static final Logger logger = Exec.getLogger(ZendeskInputPlugin.class);
 
     @Override
     public ConfigDiff transaction(final ConfigSource config, final Control control)
     {
         final PluginTask task = config.loadConfig(PluginTask.class);
-        ZendeskValidatorUtils.validateInputTask(task, getZendeskSupportAPIService(task));
+        validateInputTask(task);
         final Schema schema = task.getColumns().toSchema();
         int taskCount = 1;
 
         // For non-incremental target, we will split records based on number of pages. 100 records per page
         // In preview, run with taskCount = 1
-        if (!ZendeskUtils.isSupportAPIIncremental(task.getTarget()) && !Exec.isPreview()) {
-            final JsonNode result = getZendeskSupportAPIService(task).getData("", 0, false, 0);
-            if (result.has(ZendeskConstants.Field.COUNT) && result.get(ZendeskConstants.Field.COUNT).isInt()) {
+        if (!Exec.isPreview() && !getZendeskService(task).isSupportIncremental() && getZendeskService(task) instanceof ZendeskSupportAPIService) {
+            final JsonNode result = getZendeskService(task).getDataFromPath("", 0, false, 0);
+            if (result.has(ZendeskConstants.Field.COUNT) && !result.get(ZendeskConstants.Field.COUNT).isNull()
+                    && result.get(ZendeskConstants.Field.COUNT).isInt()) {
                 taskCount = ZendeskUtils.numberToSplitWithHintingInTask(result.get(ZendeskConstants.Field.COUNT).asInt());
             }
         }
@@ -178,7 +200,7 @@ public class ZendeskInputPlugin implements InputPlugin
     {
         final PluginTask task = taskSource.loadTask(PluginTask.class);
         try (final PageBuilder pageBuilder = getPageBuilder(schema, output)) {
-            final TaskReport taskReport = ingestServiceData(task, taskIndex, schema, pageBuilder);
+            final TaskReport taskReport = getZendeskService(task).addRecordToImporter(taskIndex, getRecordImporter(schema, pageBuilder));
             pageBuilder.finish();
             return taskReport;
         }
@@ -189,20 +211,10 @@ public class ZendeskInputPlugin implements InputPlugin
     {
         config.set("columns", new ObjectMapper().createArrayNode());
         final PluginTask task = config.loadConfig(PluginTask.class);
-        ZendeskValidatorUtils.validateInputTask(task, getZendeskSupportAPIService(task));
+        validateInputTask(task);
         return Exec.newConfigDiff().set("columns", buildColumns(task));
     }
 
-    @VisibleForTesting
-    protected ZendeskSupportAPIService getZendeskSupportAPIService(final PluginTask task)
-    {
-        if (this.zendeskSupportAPIService == null) {
-            this.zendeskSupportAPIService = new ZendeskSupportAPIService(task);
-        }
-        this.zendeskSupportAPIService.setTask(task);
-        return this.zendeskSupportAPIService;
-    }
-
     @VisibleForTesting
     protected PageBuilder getPageBuilder(final Schema schema, final PageOutput output)
     {
@@ -220,153 +232,26 @@ public class ZendeskInputPlugin implements InputPlugin
                         taskReport.get(JsonNode.class, ZendeskConstants.Field.START_TIME).asLong()), ZoneOffset.UTC);
 
                 configDiff.set(ZendeskConstants.Field.START_TIME,
-                            offsetDateTime.format(DateTimeFormatter.ofPattern(ZendeskConstants.Misc.RUBY_TIMESTAMP_FORMAT_INPUT)));
+                        offsetDateTime.format(DateTimeFormatter.ofPattern(ZendeskConstants.Misc.RUBY_TIMESTAMP_FORMAT_INPUT)));
             }
         }
         return configDiff;
     }
 
-    private TaskReport ingestServiceData(final PluginTask task, final int taskIndex,
-                                         final Schema schema, final PageBuilder pageBuilder)
-    {
-        final TaskReport taskReport = Exec.newTaskReport();
-
-        if (ZendeskUtils.isSupportAPIIncremental(task.getTarget())) {
-            importDataForIncremental(task, schema, pageBuilder, taskReport);
-        }
-        else {
-            importDataForNonIncremental(task, taskIndex, schema, pageBuilder);
-        }
-
-        return taskReport;
-    }
-
-    private void importDataForIncremental(final PluginTask task, final Schema schema,
-                                          final PageBuilder pageBuilder, final TaskReport taskReport)
-    {
-        long startTime = 0;
-
-        if (ZendeskUtils.isSupportAPIIncremental(task.getTarget()) && task.getStartTime().isPresent()) {
-                startTime = ZendeskDateUtils.isoToEpochSecond(task.getStartTime().get());
-        }
-
-        // For incremental target, we will run in one task but split in multiple threads inside for data deduplication.
-        // Run with incremental will contain duplicated data.
-        ThreadPoolExecutor pool = null;
-        try {
-            Set<String> knownIds = ConcurrentHashMap.newKeySet();
-            pool = new ThreadPoolExecutor(
-                    10, 100, 60, TimeUnit.SECONDS, new LinkedBlockingQueue<>()
-            );
-
-            while (true) {
-                int recordCount = 0;
-
-                // Page argument isn't used in incremental API so we just set it to 0
-                final JsonNode result = getZendeskSupportAPIService(task).getData("", 0, false, startTime);
-                final Iterator<JsonNode> iterator = getListRecords(result, task.getTarget().getJsonName());
-
-                int numberOfRecords = 0;
-                if (result.has(ZendeskConstants.Field.COUNT)) {
-                    numberOfRecords = result.get(ZendeskConstants.Field.COUNT).asInt();
-                }
-
-                while (iterator.hasNext()) {
-                    final JsonNode recordJsonNode = iterator.next();
-
-                    if (isUpdatedBySystem(recordJsonNode, startTime)) {
-                       continue;
-                    }
-
-                    if (task.getDedup()) {
-                        String recordID = recordJsonNode.get(ZendeskConstants.Field.ID).asText();
-                        if (knownIds.contains(recordID)) {
-                            continue;
-                        }
-                        knownIds.add(recordID);
-                    }
-
-                    pool.submit(() -> fetchData(recordJsonNode, task, schema, pageBuilder));
-                    recordCount++;
-                    if (Exec.isPreview()) {
-                        return;
-                    }
-                }
-                logger.info("Fetched '{}' records from start_time '{}'", recordCount, startTime);
-
-                if (task.getIncremental()) {
-                    if (result.has(ZendeskConstants.Field.END_TIME) && !result.get(ZendeskConstants.Field.END_TIME).isNull()
-                            && result.has(task.getTarget().getJsonName())) {
-                        // NOTE: start_time compared as "=>", not ">".
-                        // If we will use end_time for next start_time, we got the same record that is last fetched
-                        // end_time + 1 is workaround for that
-                        taskReport.set(ZendeskConstants.Field.START_TIME, result.get(ZendeskConstants.Field.END_TIME).asLong() + 1);
-                    }
-                    else {
-                        // Sometimes no record and no end_time fetched on the job, but we should generate start_time on config_diff.
-                        taskReport.set(ZendeskConstants.Field.START_TIME, Instant.now().getEpochSecond());
-                    }
-                }
-
-                if (numberOfRecords < ZendeskConstants.Misc.MAXIMUM_RECORDS_INCREMENTAL) {
-                    break;
-                }
-                else {
-                    startTime = result.get(ZendeskConstants.Field.END_TIME).asLong();
-                }
-            }
-        }
-        finally {
-            if (pool != null) {
-                pool.shutdown();
-                try {
-                    pool.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
-                }
-                catch (final InterruptedException e) {
-                    logger.warn("Error when wait pool to finish");
-                    throw Throwables.propagate(e);
-                }
-            }
-        }
-    }
-
-    private void importDataForNonIncremental(final PluginTask task, final int taskIndex, final Schema schema,
-                                             final PageBuilder pageBuilder)
-    {
-        // Page start from 1 => page = taskIndex + 1
-        final JsonNode result = getZendeskSupportAPIService(task).getData("", taskIndex + 1, false, 0);
-        final Iterator<JsonNode> iterator = getListRecords(result, task.getTarget().getJsonName());
-
-        while (iterator.hasNext()) {
-            fetchData(iterator.next(), task, schema, pageBuilder);
-
-            if (Exec.isPreview()) {
-                break;
-            }
-        }
-    }
-
-    private Iterator<JsonNode> getListRecords(JsonNode result, String targetJsonName)
-    {
-        if (!result.has(targetJsonName) || !result.get(targetJsonName).isArray()) {
-            throw new DataException(String.format("Missing '%s' from Zendesk API response", targetJsonName));
-        }
-        return result.get(targetJsonName).elements();
-    }
-
     private JsonNode buildColumns(final PluginTask task)
     {
-        JsonNode jsonNode = getZendeskSupportAPIService(task).getData("", 0, true, 0);
+        JsonNode jsonNode = getZendeskService(task).getDataFromPath("", 0, true, 0);
 
         String targetName = task.getTarget().getJsonName();
 
-        if (jsonNode.has(targetName) && jsonNode.get(targetName).isArray() && jsonNode.get(targetName).size() > 0) {
+        if (jsonNode.has(targetName) && !jsonNode.get(targetName).isNull() && jsonNode.get(targetName).isArray() && jsonNode.get(targetName).size() > 0) {
             return addAllColumnsToSchema(jsonNode, task.getTarget(), task.getIncludes());
         }
         throw new ConfigException("Could not guess schema due to empty data set");
     }
 
     private final Pattern idPattern = Pattern.compile(ZendeskConstants.Regex.ID);
+
     private JsonNode addAllColumnsToSchema(final JsonNode jsonNode, final Target target, final List<String> includes)
     {
         final JsonNode sample = new ObjectMapper().valueToTree(StreamSupport.stream(
@@ -394,6 +279,11 @@ public class ZendeskInputPlugin implements InputPlugin
                     }
                     entry.put("type", Types.LONG.getName());
                 }
+
+                // Id of User Events target is more suitable for String
+                if (target.equals(Target.USER_EVENTS)) {
+                    entry.put("type", Types.STRING.getName());
+                }
             }
             else if (idPattern.matcher(name).find()) {
                 if (type.equals(Types.TIMESTAMP.getName())) {
@@ -417,60 +307,150 @@ public class ZendeskInputPlugin implements InputPlugin
                 .forEach(arrayNode::add);
     }
 
-    private void fetchData(final JsonNode jsonNode, final PluginTask task, final Schema schema,
-                                     final PageBuilder pageBuilder)
+    private ConfigSource createGuessConfig()
+    {
+        return Exec.newConfigSource()
+                .set("guess_plugins", ImmutableList.of("zendesk"))
+                .set("guess_sample_buffer_bytes", ZendeskConstants.Misc.GUESS_BUFFER_SIZE);
+    }
+
+    private ZendeskService getZendeskService(PluginTask task)
+    {
+        if (zendeskService == null) {
+            zendeskService = dispatchPerTarget(task);
+        }
+        return zendeskService;
+    }
+
+    @VisibleForTesting
+    protected ZendeskService dispatchPerTarget(ZendeskInputPlugin.PluginTask task)
+    {
+        switch (task.getTarget()) {
+            case TICKETS:
+            case USERS:
+            case ORGANIZATIONS:
+            case TICKET_METRICS:
+            case TICKET_EVENTS:
+            case TICKET_FORMS:
+            case TICKET_FIELDS:
+                return new ZendeskSupportAPIService(task);
+            case RECIPIENTS:
+            case SCORES:
+                return new ZendeskNPSService(task);
+            case OBJECT_RECORDS:
+            case RELATIONSHIP_RECORDS:
+                return new ZendeskCustomObjectService(task);
+            case USER_EVENTS:
+                return new ZendeskUserEventService(task);
+            default:
+                throw new ConfigException("Unsupported " + task.getTarget() + ", supported values: '" + Arrays.toString(Target.values()) + "'");
+        }
+    }
+
+    private RecordImporter getRecordImporter(Schema schema, PageBuilder pageBuilder)
+    {
+        if (recordImporter == null) {
+            recordImporter = new RecordImporter(schema, pageBuilder);
+        }
+        return recordImporter;
+    }
+
+    private void validateInputTask(PluginTask task)
+    {
+        validateAppMarketPlace(task.getAppMarketPlaceIntegrationName().isPresent(),
+                task.getAppMarketPlaceAppId().isPresent(),
+                task.getAppMarketPlaceOrgId().isPresent());
+        validateCredentials(task);
+        validateIncremental(task);
+        validateCustomObject(task);
+        validateUserEvent(task);
+    }
+
+    private void validateCredentials(PluginTask task)
     {
-        // FIXME:  if include is not contained in schema, data should be ignore
-        task.getIncludes().forEach(include -> {
-            String relatedObjectName = include.trim();
-            final String url = task.getLoginUrl()
-                    + "/"
-                    + ZendeskConstants.Url.API
-                    + "/" + task.getTarget().toString()
-                    + "/" + jsonNode.get(ZendeskConstants.Field.ID).asText()
-                    + "/" + relatedObjectName + ".json";
-
-            try {
-                final JsonNode result = getZendeskSupportAPIService(task).getData(url, 0, false, 0);
-                if (result != null && result.has(relatedObjectName)) {
-                    ((ObjectNode) jsonNode).set(include, result.get(relatedObjectName));
+        switch (task.getAuthenticationMethod()) {
+            case OAUTH:
+                if (!task.getAccessToken().isPresent()) {
+                    throw new ConfigException(String.format("access_token is required for authentication method '%s'",
+                            task.getAuthenticationMethod().name().toLowerCase()));
                 }
-            }
-            catch (final ConfigException e) {
-                // Sometimes we get 404 when having invalid endpoint, so ignore when we get 404 InvalidEndpoint
-                if (!(e.getCause() instanceof ZendeskException && ((ZendeskException) e.getCause()).getStatusCode() == HttpStatus.SC_NOT_FOUND)) {
-                    throw e;
+                break;
+            case TOKEN:
+                if (!task.getUsername().isPresent() || !task.getToken().isPresent()) {
+                    throw new ConfigException(String.format("username and token are required for authentication method '%s'",
+                            task.getAuthenticationMethod().name().toLowerCase()));
                 }
-            }
-        });
+                break;
+            case BASIC:
+                if (!task.getUsername().isPresent() || !task.getPassword().isPresent()) {
+                    throw new ConfigException(String.format("username and password are required for authentication method '%s'",
+                            task.getAuthenticationMethod().name().toLowerCase()));
+                }
+                break;
+            default:
+                throw new ConfigException("Unknown authentication method");
+        }
+    }
 
-        ZendeskUtils.addRecord(jsonNode, schema, pageBuilder);
+    private void validateAppMarketPlace(final boolean isAppMarketIntegrationNamePresent,
+            final boolean isAppMarketAppIdPresent,
+            final boolean isAppMarketOrgIdPresent)
+    {
+        final boolean isAllAvailable =
+                isAppMarketIntegrationNamePresent && isAppMarketAppIdPresent && isAppMarketOrgIdPresent;
+        final boolean isAllUnAvailable =
+                !isAppMarketIntegrationNamePresent && !isAppMarketAppIdPresent && !isAppMarketOrgIdPresent;
+        // All or nothing needed
+        if (!(isAllAvailable || isAllUnAvailable)) {
+            throw new ConfigException("All of app_marketplace_integration_name, app_marketplace_org_id, " +
+                    "app_marketplace_app_id " +
+                    "are required to fill out for Apps Marketplace API header");
+        }
     }
 
-    private ConfigSource createGuessConfig()
+    private void validateIncremental(PluginTask task)
     {
-        return Exec.newConfigSource()
-                .set("guess_plugins", ImmutableList.of("zendesk"))
-                .set("guess_sample_buffer_bytes", ZendeskConstants.Misc.GUESS_BUFFER_SIZE);
+        if (task.getIncremental() && getZendeskService(task).isSupportIncremental())  {
+            if (!task.getDedup()) {
+                logger.warn("You've selected to skip de-duplicating records, result may contain duplicated data");
+            }
+
+            if (!getZendeskService(task).isSupportIncremental() && task.getStartTime().isPresent()) {
+                logger.warn(String.format("Target: '%s' doesn't support incremental export API. Will be ignored start_time option",
+                        task.getTarget()));
+            }
+        }
     }
 
-    private boolean isUpdatedBySystem(JsonNode recordJsonNode, long startTime)
+    private void validateCustomObject(PluginTask task)
     {
-        /*
-         * https://developer.zendesk.com/rest_api/docs/core/incremental_export#excluding-system-updates
-         * "generated_timestamp" will be updated when Zendesk internal changing
-         * "updated_at" will be updated when ticket data was changed
-         * start_time for query parameter will be processed on Zendesk with generated_timestamp,
-         * but it was calculated by record' updated_at time.
-         * So the doesn't changed record from previous import would be appear by Zendesk internal changes.
-         * We ignore record that has updated_at <= start_time
-         */
-        if (recordJsonNode.has(ZendeskConstants.Field.GENERATED_TIMESTAMP) && recordJsonNode.has(ZendeskConstants.Field.UPDATED_AT)) {
-            String recordUpdatedAtTime = recordJsonNode.get(ZendeskConstants.Field.UPDATED_AT).asText();
-            long recordUpdatedAtToEpochSecond = ZendeskDateUtils.isoToEpochSecond(recordUpdatedAtTime);
-            return recordUpdatedAtToEpochSecond <= startTime;
+        if (task.getTarget().equals(Target.OBJECT_RECORDS) && task.getObjectTypes().isEmpty()) {
+            throw new ConfigException("Should have at least one Object Type");
+        }
+
+        if (task.getTarget().equals(Target.RELATIONSHIP_RECORDS) && task.getRelationshipTypes().isEmpty()) {
+            throw new ConfigException("Should have at least one Relationship Type");
         }
+    }
 
-        return false;
+    private void validateUserEvent(PluginTask task)
+    {
+        if (task.getTarget().equals(Target.USER_EVENTS)) {
+            if (!task.getProfileSource().isPresent()) {
+                throw new ConfigException("Profile Source is required for User Event Target");
+            }
+
+            // Can't set end_time to 0, so it should be valid
+            task.getEndTime().ifPresent(time -> {
+                if (!ZendeskDateUtils.supportedTimeFormat(task.getEndTime().get()).isPresent()) {
+                    throw new ConfigException("End Time should follow these format " + ZendeskConstants.Misc.SUPPORT_DATE_TIME_FORMAT.toString());
+                }
+            });
+
+            if (task.getStartTime().isPresent() && task.getEndTime().isPresent()
+                    && ZendeskDateUtils.getStartTime(task.getStartTime().get()) > ZendeskDateUtils.isoToEpochSecond(task.getEndTime().get())) {
+                throw new ConfigException("End Time should be later or equal than Start Time");
+            }
+        }
     }
 }