embulk-input-zendesk 0.3.5 → 0.3.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2dab4909dba68349742e03eef6be0ff02014caf1
-  data.tar.gz: 05bee8746fae83639eeb0daacb4b38ac5f765868
+  metadata.gz: 59c7819359a24e229366275644b6d543809d7c08
+  data.tar.gz: 818fe70947171c2c96d7951f54316f8b13a357e9
 SHA512:
-  metadata.gz: 0b0831ce1c1a40d1d9a3b2b3e8a73d8a2fa14a6e5144d264e1e210f2289c092fed6778c7ccea823fcf4464cc8b8ef61182306426730f4a82c85a50b262783bdf
-  data.tar.gz: e1706af5499ff07076761b0aa591920abf867a2a95d38c4b5e18a47dd61e73ecb653cf5f3d4cca8beb631ddf3f6b3de9018ebb92fecf8558a4f6e3713c4b0715
+  metadata.gz: 477172f0826c449a67af48191f29796887c9c8a8b9d8e9550cbfd613b194c406f8047c80e2ab31d0cdcff70410e3f8b4feda7e94316ffbaad3db050033b90b9a
+  data.tar.gz: 7a24e2b1bbe4ad103bf5e4c86055e1c069b9f0ca38898b2a1bbbcafc2afa9dad6ccb25bc0d5e45136adf7f8a669c16df3b293fb5a65f32d7799b1f42989d4f9e

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.3.6 - 2019-07-02
+* [enhancement] Improve error message #61 [#61](https://github.com/treasure-data/embulk-input-zendesk/pull/61)
+* [enhancement] Support `end_time` field for incremental #60 [#60](https://github.com/treasure-data/embulk-input-zendesk/pull/60)
+
 ## 0.3.5 - 2019-06-03
 * [enhancement] Add new targets #58 [#58](https://github.com/treasure-data/embulk-input-zendesk/pull/58)
 

data/build.gradle

@@ -15,7 +15,7 @@ configurations {
     provided
 }
 
-version = "0.3.5"
+version = "0.3.6"
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8

data/src/main/java/org/embulk/input/zendesk/ZendeskInputPlugin.java

@@ -167,6 +167,7 @@ public class ZendeskInputPlugin implements InputPlugin
     {
         final PluginTask task = config.loadConfig(PluginTask.class);
         validateInputTask(task);
+
         final Schema schema = task.getColumns().toSchema();
         int taskCount = 1;
 
@@ -199,6 +200,21 @@ public class ZendeskInputPlugin implements InputPlugin
     public TaskReport run(final TaskSource taskSource, final Schema schema, final int taskIndex, final PageOutput output)
     {
         final PluginTask task = taskSource.loadTask(PluginTask.class);
+
+        if (getZendeskService(task).isSupportIncremental() && !isValidTimeRange(task)) {
+            if (Exec.isPreview()) {
+                throw new ConfigException("Invalid End time. End time is greater than current time");
+            }
+
+            logger.warn("The end time, '" + task.getEndTime().get() + "', is greater than the current time. No records will be imported");
+
+            // we just need to store config_diff when incremental_mode is enable
+            if (task.getIncremental()) {
+                return buildTaskReportKeepOldStartAndEndTime(task);
+            }
+            return Exec.newTaskReport();
+        }
+
         try (final PageBuilder pageBuilder = getPageBuilder(schema, output)) {
             final TaskReport taskReport = getZendeskService(task).addRecordToImporter(taskIndex, getRecordImporter(schema, pageBuilder));
             pageBuilder.finish();
@@ -212,6 +228,9 @@ public class ZendeskInputPlugin implements InputPlugin
         config.set("columns", new ObjectMapper().createArrayNode());
         final PluginTask task = config.loadConfig(PluginTask.class);
         validateInputTask(task);
+        if (!isValidTimeRange(task)) {
+            throw new ConfigException("Invalid End time. End time is greater than current time");
+        }
         return Exec.newConfigDiff().set("columns", buildColumns(task));
     }
 
@@ -234,6 +253,14 @@ public class ZendeskInputPlugin implements InputPlugin
                 configDiff.set(ZendeskConstants.Field.START_TIME,
                         offsetDateTime.format(DateTimeFormatter.ofPattern(ZendeskConstants.Misc.RUBY_TIMESTAMP_FORMAT_INPUT)));
             }
+
+            if (taskReport.has(ZendeskConstants.Field.END_TIME)) {
+                final OffsetDateTime offsetDateTime = OffsetDateTime.ofInstant(Instant.ofEpochSecond(
+                        taskReport.get(JsonNode.class, ZendeskConstants.Field.END_TIME).asLong()), ZoneOffset.UTC);
+
+                configDiff.set(ZendeskConstants.Field.END_TIME,
+                        offsetDateTime.format(DateTimeFormatter.ofPattern(ZendeskConstants.Misc.RUBY_TIMESTAMP_FORMAT_INPUT)));
+            }
         }
         return configDiff;
     }
@@ -364,6 +391,7 @@ public class ZendeskInputPlugin implements InputPlugin
         validateIncremental(task);
         validateCustomObject(task);
         validateUserEvent(task);
+        validateTime(task);
     }
 
     private void validateCredentials(PluginTask task)
@@ -439,7 +467,12 @@ public class ZendeskInputPlugin implements InputPlugin
             if (!task.getProfileSource().isPresent()) {
                 throw new ConfigException("Profile Source is required for User Event Target");
             }
+        }
+    }
 
+    private void validateTime(PluginTask task)
+    {
+        if (getZendeskService(task).isSupportIncremental()) {
             // Can't set end_time to 0, so it should be valid
             task.getEndTime().ifPresent(time -> {
                 if (!ZendeskDateUtils.supportedTimeFormat(task.getEndTime().get()).isPresent()) {
@@ -453,4 +486,24 @@ public class ZendeskInputPlugin implements InputPlugin
             }
         }
     }
+
+    private boolean isValidTimeRange(PluginTask task)
+    {
+        return !task.getEndTime().isPresent() || ZendeskDateUtils.isoToEpochSecond(task.getEndTime().get()) <= Instant.now().getEpochSecond();
+    }
+
+    private TaskReport buildTaskReportKeepOldStartAndEndTime(PluginTask task)
+    {
+        final TaskReport taskReport = Exec.newTaskReport();
+
+        if (task.getStartTime().isPresent()) {
+            taskReport.set(ZendeskConstants.Field.START_TIME, ZendeskDateUtils.isoToEpochSecond(task.getStartTime().get()));
+        }
+
+        if (task.getEndTime().isPresent()) {
+            taskReport.set(ZendeskConstants.Field.END_TIME, ZendeskDateUtils.isoToEpochSecond(task.getEndTime().get()));
+        }
+
+        return taskReport;
+    }
 }

data/src/main/java/org/embulk/input/zendesk/clients/ZendeskRestClient.java

@@ -1,6 +1,7 @@
 package org.embulk.input.zendesk.clients;
 
 import com.fasterxml.jackson.databind.DeserializationFeature;
+import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableMap;
@@ -170,6 +171,17 @@ public class ZendeskRestClient
     private boolean isResponseStatusToRetry(final int status, final String message, final int retryAfter, final boolean isPreview)
     {
         if (status == HttpStatus.SC_NOT_FOUND) {
+            ObjectMapper objectMapper = new ObjectMapper();
+            try {
+                JsonNode jsonNode = objectMapper.readTree(message);
+                if (jsonNode.has("error") && jsonNode.get("error").has("title") && jsonNode.get("error").get("title").asText().startsWith("No help desk at ")) {
+                    throw new ConfigException("This address is not registered in Zendesk. Please check the login_url again");
+                }
+            }
+            catch (IOException e) {
+                // In case we can't parse the message, error should not be show here
+            }
+
             // 404 would be returned e.g. ticket comments are empty (on fetchRelatedObjects method)
             return false;
         }
@@ -203,6 +215,14 @@ public class ZendeskRestClient
 
         // Won't retry for 4xx range errors except above. Almost they should be ConfigError e.g. 403 Forbidden
         if (status / 100 == 4) {
+            if (status == HttpStatus.SC_UNAUTHORIZED) {
+                throw new ConfigException("Cannot authenticate due to invalid login credentials");
+            }
+
+            if (status == HttpStatus.SC_FORBIDDEN) {
+                throw new ConfigException("You do not have access to this resource. Please contact the account owner of this help desk for further help.");
+            }
+
             throw new ConfigException("Status '" + status + "', message '" + message + "'");
         }
 

data/src/main/java/org/embulk/input/zendesk/services/ZendeskNormalServices.java

@@ -11,6 +11,7 @@ import org.embulk.config.TaskReport;
 import org.embulk.input.zendesk.RecordImporter;
 import org.embulk.input.zendesk.ZendeskInputPlugin;
 import org.embulk.input.zendesk.clients.ZendeskRestClient;
+import org.embulk.input.zendesk.models.Target;
 import org.embulk.input.zendesk.models.ZendeskException;
 import org.embulk.input.zendesk.utils.ZendeskConstants;
 import org.embulk.input.zendesk.utils.ZendeskDateUtils;
@@ -77,10 +78,17 @@ public abstract class ZendeskNormalServices implements ZendeskService
 
     private void importDataForIncremental(final ZendeskInputPlugin.PluginTask task, final RecordImporter recordImporter, final TaskReport taskReport)
     {
+        long initStartTime = 0;
         long startTime = 0;
+        long endTime = Long.MAX_VALUE;
 
         if (task.getStartTime().isPresent()) {
             startTime = ZendeskDateUtils.getStartTime(task.getStartTime().get());
+            initStartTime = startTime;
+        }
+
+        if (task.getEndTime().isPresent()) {
+            endTime = ZendeskDateUtils.isoToEpochSecond(task.getEndTime().get());
         }
 
         // For incremental target, we will run in one task but split in multiple threads inside for data deduplication.
@@ -92,12 +100,14 @@ public abstract class ZendeskNormalServices implements ZendeskService
                     10, 100, 60, TimeUnit.SECONDS, new LinkedBlockingQueue<>()
             );
 
+            long apiEndTime = 0;
             while (true) {
                 int recordCount = 0;
 
                 // Page argument isn't used in incremental API so we just set it to 0
                 final JsonNode result = getDataFromPath("", 0, false, startTime);
                 final Iterator<JsonNode> iterator = ZendeskUtils.getListRecords(result, task.getTarget().getJsonName());
+                apiEndTime = result.get(ZendeskConstants.Field.END_TIME).asLong();
 
                 int numberOfRecords = 0;
                 if (result.has(ZendeskConstants.Field.COUNT)) {
@@ -111,6 +121,29 @@ public abstract class ZendeskNormalServices implements ZendeskService
                         continue;
                     }
 
+                    // Contain some records  that later than end_time. Checked and don't add.
+                    // Because the api already sorted by updated_at or timestamp for ticket_events, we just need to break no need to check further.
+                    if (apiEndTime > endTime) {
+                        long checkedTime = 0;
+                        if (recordJsonNode.has(ZendeskConstants.Field.UPDATED_AT) && !recordJsonNode.get(ZendeskConstants.Field.UPDATED_AT).isNull()) {
+                            checkedTime = ZendeskDateUtils.isoToEpochSecond(recordJsonNode.get(ZendeskConstants.Field.UPDATED_AT).textValue());
+                        }
+
+                        // ticket events is updated by system not user's action so it only has timestamp field
+                        if (task.getTarget().equals(Target.TICKET_EVENTS) && recordJsonNode.has("timestamp") && !recordJsonNode.get("timestamp").isNull()) {
+                            checkedTime = recordJsonNode.get("timestamp").asLong();
+                        }
+
+                        // scores (or response) is only store rated_at time
+                        if (task.getTarget().equals(Target.SCORES) && recordJsonNode.has("rated_at") && !recordJsonNode.get("rated_at").isNull()) {
+                            checkedTime = ZendeskDateUtils.isoToEpochSecond(recordJsonNode.get("rated_at").textValue());
+                        }
+
+                        if (checkedTime > endTime) {
+                            break;
+                        }
+                    }
+
                     if (task.getDedup()) {
                         final String recordID = recordJsonNode.get(ZendeskConstants.Field.ID).asText();
 
@@ -131,18 +164,17 @@ public abstract class ZendeskNormalServices implements ZendeskService
 
                 // https://developer.zendesk.com/rest_api/docs/support/incremental_export#pagination
                 // When there are more than 1000 records share the same time stamp, the count > 1000
-                long apiEndTime = result.get(ZendeskConstants.Field.END_TIME).asLong();
                 startTime = startTime == apiEndTime
                         ? apiEndTime + 1
                         : apiEndTime;
 
-                if (numberOfRecords < ZendeskConstants.Misc.MAXIMUM_RECORDS_INCREMENTAL) {
+                if (numberOfRecords < ZendeskConstants.Misc.MAXIMUM_RECORDS_INCREMENTAL || startTime > endTime) {
                     break;
                 }
             }
 
             if (!Exec.isPreview()) {
-                storeStartTimeForConfigDiff(taskReport, startTime);
+                storeStartTimeForConfigDiff(taskReport, initStartTime, startTime);
             }
         }
         finally {
@@ -159,19 +191,34 @@ public abstract class ZendeskNormalServices implements ZendeskService
         }
     }
 
-    private void storeStartTimeForConfigDiff(final TaskReport taskReport, final long resultEndTime)
+    private void storeStartTimeForConfigDiff(final TaskReport taskReport, final long initStartTime, final long resultEndTime)
     {
         if (task.getIncremental()) {
-            // resultEndTime = 0 mean no records, we should store now time for next run
+            long nextStartTime;
+            long now = Instant.now().getEpochSecond();
+            // no record to add
             if (resultEndTime == 0) {
-                taskReport.set(ZendeskConstants.Field.START_TIME, Instant.now().getEpochSecond());
+                nextStartTime = now;
             }
             else {
-                // NOTE: start_time compared as "=>", not ">".
-                // If we will use end_time for next start_time, we got the same records that are fetched
-                // end_time + 1 is workaround for that
-                taskReport.set(ZendeskConstants.Field.START_TIME, resultEndTime + 1);
+                if (task.getEndTime().isPresent()) {
+                    long endTime = ZendeskDateUtils.isoToEpochSecond(task.getEndTime().get());
+                    nextStartTime = endTime + 1;
+                }
+                else {
+                    // NOTE: start_time compared as "=>", not ">".
+                    // If we will use end_time for next start_time, we got the same records that are fetched
+                    // end_time + 1 is workaround for that
+                    nextStartTime = resultEndTime + 1;
+                }
+            }
+
+            if (task.getEndTime().isPresent()) {
+                long endTime = ZendeskDateUtils.isoToEpochSecond(task.getEndTime().get());
+                taskReport.set(ZendeskConstants.Field.END_TIME, nextStartTime + endTime - initStartTime);
             }
+
+            taskReport.set(ZendeskConstants.Field.START_TIME, nextStartTime);
         }
     }
 

data/src/main/java/org/embulk/input/zendesk/utils/ZendeskConstants.java

@@ -43,11 +43,11 @@ public class ZendeskConstants
     public static class Misc
     {
         public static final String RUBY_TIMESTAMP_FORMAT = "%Y-%m-%dT%H:%M:%S%z";
-        public static final String RUBY_TIMESTAMP_FORMAT_INPUT = "yyyy-MM-dd HH:mm:ss Z";
-        public static final String JAVA_TIMESTAMP_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'";
-        public static final String ISO_TIMESTAMP_FORMAT = "yyyy-MM-dd'T'HH:mm:ssXXX";
-        public static final String ISO_INSTANT = "yyyy-MM-dd'T'HH:mm:ss'Z'";
-        public static final String RUBY_TIMESTAMP_FORMAT_INPUT_NO_SPACE = "yyyy-MM-dd HH:mm:ssZ";
+        public static final String RUBY_TIMESTAMP_FORMAT_INPUT = "uuuu-MM-dd HH:mm:ss Z";
+        public static final String JAVA_TIMESTAMP_FORMAT = "uuuu-MM-dd'T'HH:mm:ss.SSS'Z'";
+        public static final String ISO_TIMESTAMP_FORMAT = "uuuu-MM-dd'T'HH:mm:ssXXX";
+        public static final String ISO_INSTANT = "uuuu-MM-dd'T'HH:mm:ss'Z'";
+        public static final String RUBY_TIMESTAMP_FORMAT_INPUT_NO_SPACE = "uuuu-MM-dd HH:mm:ssZ";
         public static final String TOO_RECENT_START_TIME = "Too recent start_time.";
         public static final int RECORDS_SIZE_PER_PAGE = 100;
         public static final int MAXIMUM_RECORDS_INCREMENTAL = 1000;

data/src/main/java/org/embulk/input/zendesk/utils/ZendeskDateUtils.java

@@ -9,6 +9,7 @@ import java.time.ZoneOffset;
 import java.time.format.DateTimeFormatter;
 import java.time.format.DateTimeParseException;
 
+import java.time.format.ResolverStyle;
 import java.util.Optional;
 
 public class ZendeskDateUtils
@@ -21,9 +22,14 @@ public class ZendeskDateUtils
     {
         final Optional<String> pattern = supportedTimeFormat(time);
         if (pattern.isPresent()) {
-            final DateTimeFormatter formatter = DateTimeFormatter.ofPattern(pattern.get()).withZone(ZoneOffset.UTC);
-            final OffsetDateTime offsetDateTime = LocalDateTime.parse(time, formatter).atOffset(ZoneOffset.UTC);
-            return offsetDateTime.toInstant().getEpochSecond();
+            final DateTimeFormatter formatter = DateTimeFormatter.ofPattern(pattern.get()).withZone(ZoneOffset.UTC).withResolverStyle(ResolverStyle.STRICT);
+            try {
+                final OffsetDateTime offsetDateTime = LocalDateTime.parse(time, formatter).atOffset(ZoneOffset.UTC);
+                return offsetDateTime.toInstant().getEpochSecond();
+            }
+            catch (DateTimeParseException e) {
+                throw new DataException(e.getMessage());
+            }
         }
 
         throw new DataException("Fail to parse value '" + time + "' follow formats " + ZendeskConstants.Misc.SUPPORT_DATE_TIME_FORMAT.toString());
@@ -46,8 +52,7 @@ public class ZendeskDateUtils
 
     public static String convertToDateTimeFormat(String datetime, String dateTimeFormat)
     {
-        return OffsetDateTime.ofInstant(Instant.ofEpochSecond(ZendeskDateUtils.isoToEpochSecond(datetime)), ZoneOffset.UTC)
-                .format(DateTimeFormatter.ofPattern(dateTimeFormat));
+        return Instant.ofEpochSecond(ZendeskDateUtils.isoToEpochSecond(datetime)).atOffset(ZoneOffset.UTC).format(DateTimeFormatter.ofPattern(dateTimeFormat));
     }
 
     // start_time should be start from 0

data/src/test/java/org/embulk/input/zendesk/TestZendeskInputPlugin.java

@@ -14,6 +14,7 @@ import org.embulk.input.zendesk.services.ZendeskService;
 import org.embulk.input.zendesk.services.ZendeskSupportAPIService;
 import org.embulk.input.zendesk.services.ZendeskUserEventService;
 import org.embulk.input.zendesk.utils.ZendeskConstants;
+import org.embulk.input.zendesk.utils.ZendeskDateUtils;
 import org.embulk.input.zendesk.utils.ZendeskPluginTestRuntime;
 import org.embulk.input.zendesk.utils.ZendeskTestHelper;
 
@@ -143,19 +144,23 @@ public class TestZendeskInputPlugin
     }
 
     @Test
-    public void testRunIncrementalStoreStartTime()
+    public void testRunIncrementalStoreStartTimeAndEndTime()
     {
-        final ConfigSource src = ZendeskTestHelper.getConfigSource("incremental.yml");
+        final ConfigSource src = ZendeskTestHelper.getConfigSource("incremental.yml")
+                .set("end_time", "2019-04-12 06:51:50 +0000");
         TaskReport taskReport = Exec.newTaskReport();
         taskReport.set(ZendeskConstants.Field.START_TIME, 1557026576);
+        taskReport.set(ZendeskConstants.Field.END_TIME, 1560309776);
 
         when(zendeskSupportAPIService.isSupportIncremental()).thenReturn(true);
         when(zendeskSupportAPIService.addRecordToImporter(anyInt(), any())).thenReturn(taskReport);
 
         ConfigDiff configDiff = zendeskInputPlugin.transaction(src, new Control());
         String nextStartTime = configDiff.get(String.class, ZendeskConstants.Field.START_TIME);
+        String nextEndTime = configDiff.get(String.class, ZendeskConstants.Field.END_TIME);
         verify(pageBuilder, times(1)).finish();
         assertEquals("2019-05-05 03:22:56 +0000", nextStartTime);
+        assertEquals("2019-06-12 03:22:56 +0000", nextEndTime);
     }
 
     @Test
@@ -266,8 +271,14 @@ public class TestZendeskInputPlugin
         ConfigSource configSource = ZendeskTestHelper.getConfigSource("base_validator.yml");
         configSource.set("target", Target.USER_EVENTS.name().toLowerCase());
         assertValidation(configSource, "Profile Source is required for User Event Target");
+    }
 
-        configSource.set("profile_source", "");
+    @Test
+    public void validateTimeShouldThrowException()
+    {
+        ConfigSource configSource = ZendeskTestHelper.getConfigSource("base_validator.yml");
+        when(zendeskSupportAPIService.isSupportIncremental()).thenReturn(true);
+        configSource.set("target", Target.TICKETS.name().toLowerCase());
         configSource.set("start_time", OffsetDateTime.ofInstant(Instant.ofEpochSecond(Instant.now().getEpochSecond()), ZoneOffset.UTC).format(DateTimeFormatter.ISO_INSTANT));
         configSource.set("end_time", "2019-12-2 22-12-22");
         assertValidation(configSource, "End Time should follow these format " + ZendeskConstants.Misc.SUPPORT_DATE_TIME_FORMAT.toString());
@@ -277,6 +288,43 @@ public class TestZendeskInputPlugin
         assertValidation(configSource, "End Time should be later or equal than Start Time");
     }
 
+    @Test
+    public void isValidTimeRangeShouldThrowException()
+    {
+        ZendeskTestHelper.setPreviewMode(embulk, true);
+        String expectedMessage = "Invalid End time. End time is greater than current time";
+        ConfigSource configSource = ZendeskTestHelper.getConfigSource("base_validator.yml");
+        when(zendeskSupportAPIService.isSupportIncremental()).thenReturn(true);
+        configSource.set("start_time", OffsetDateTime.ofInstant(Instant.ofEpochSecond(Instant.now().getEpochSecond()), ZoneOffset.UTC).format(DateTimeFormatter.ISO_INSTANT));
+        configSource.set("end_time", OffsetDateTime.ofInstant(Instant.ofEpochSecond(Instant.now().getEpochSecond() + 1000), ZoneOffset.UTC).format(DateTimeFormatter.ISO_INSTANT));
+
+        assertValidation(configSource, expectedMessage);
+
+        try {
+            zendeskInputPlugin.transaction(configSource, new Control());
+            fail("Should not reach here");
+        }
+        catch (final Exception e) {
+            assertEquals(expectedMessage, e.getMessage());
+        }
+    }
+
+    @Test
+    public void runShouldKeepOldStartTimeAndEndTimeInConfigDiff()
+    {
+        ConfigSource configSource = ZendeskTestHelper.getConfigSource("base_validator.yml");
+        ZendeskTestHelper.setPreviewMode(embulk, false);
+        when(zendeskSupportAPIService.isSupportIncremental()).thenReturn(true);
+        configSource.set("start_time", OffsetDateTime.ofInstant(Instant.ofEpochSecond(Instant.now().getEpochSecond()), ZoneOffset.UTC).format(DateTimeFormatter.ISO_INSTANT));
+        configSource.set("end_time", OffsetDateTime.ofInstant(Instant.ofEpochSecond(Instant.now().getEpochSecond() + 1000), ZoneOffset.UTC).format(DateTimeFormatter.ISO_INSTANT));
+
+        ConfigDiff configDiff = zendeskInputPlugin.transaction(configSource, new Control());
+        assertEquals(ZendeskDateUtils.convertToDateTimeFormat(configSource.get(String.class, ZendeskConstants.Field.START_TIME), ZendeskConstants.Misc.RUBY_TIMESTAMP_FORMAT_INPUT),
+                configDiff.get(String.class, ZendeskConstants.Field.START_TIME));
+        assertEquals(ZendeskDateUtils.convertToDateTimeFormat(configSource.get(String.class, ZendeskConstants.Field.END_TIME), ZendeskConstants.Misc.RUBY_TIMESTAMP_FORMAT_INPUT),
+                configDiff.get(String.class, ZendeskConstants.Field.END_TIME));
+    }
+
     private void assertValidation(final ConfigSource configSource, final String message)
     {
         try {

data/src/test/java/org/embulk/input/zendesk/services/TestZendeskNormalService.java

@@ -153,7 +153,7 @@ public class TestZendeskNormalService
     }
 
     @Test
-    public void testTicketEventsAddRecordToImporterIncrementaAndAllRecordsShareTheSameTime()
+    public void testTicketEventsAddRecordToImporterIncrementalAndAllRecordsShareTheSameTime()
     {
         setupSupportAPIService("incremental.yml");
         ZendeskInputPlugin.PluginTask task = ZendeskTestHelper.getConfigSource("incremental.yml")
@@ -168,6 +168,78 @@ public class TestZendeskNormalService
         Assert.assertEquals(1550645444, taskReport.get(JsonNode.class, ZendeskConstants.Field.START_TIME).asLong());
     }
 
+    @Test
+    public void executeIncrementalContainEndTime()
+    {
+        ConfigSource src = ZendeskTestHelper.getConfigSource("incremental.yml");
+        // same updated_at time of last record
+        src.set("end_time", "2019-02-20T07:17:32Z");
+        ZendeskInputPlugin.PluginTask task = src.loadConfig(ZendeskInputPlugin.PluginTask.class);
+        setupZendeskSupportAPIService(task);
+        loadData("data/tickets.json");
+
+        TaskReport taskReport = zendeskSupportAPIService.addRecordToImporter(0, recordImporter);
+        verify(recordImporter, times(3)).addRecord(any());
+        Assert.assertFalse(taskReport.isEmpty());
+        // start_time = end_time + 1
+        Assert.assertEquals(1550647053, taskReport.get(JsonNode.class, ZendeskConstants.Field.START_TIME).asLong());
+    }
+
+    @Test
+    public void executeIncrementalContainEndTimeFilterOutLastRecord()
+    {
+        ConfigSource src = ZendeskTestHelper.getConfigSource("incremental.yml");
+        // earlier than updated_at time of last record
+        src.set("end_time", "2019-02-20T07:17:32Z");
+        ZendeskInputPlugin.PluginTask task = src.loadConfig(ZendeskInputPlugin.PluginTask.class);
+        setupZendeskSupportAPIService(task);
+        loadData("data/tickets.json");
+
+        TaskReport taskReport = zendeskSupportAPIService.addRecordToImporter(0, recordImporter);
+        verify(recordImporter, times(3)).addRecord(any());
+        Assert.assertFalse(taskReport.isEmpty());
+        //
+        Assert.assertEquals(1550647053, taskReport.get(JsonNode.class, ZendeskConstants.Field.START_TIME).asLong());
+    }
+
+    @Test
+    public void executeIncrementalContainEndTimeFilterOutLastRecordTicketEvents()
+    {
+        ConfigSource src = ZendeskTestHelper.getConfigSource("incremental.yml");
+        src.set("target", Target.TICKET_EVENTS.toString());
+        // earlier than updated_at time of last record
+        // 1550645520
+        src.set("end_time", "2019-02-20T06:52:00Z");
+        ZendeskInputPlugin.PluginTask task = src.loadConfig(ZendeskInputPlugin.PluginTask.class);
+        setupZendeskSupportAPIService(task);
+        loadData("data/ticket_events_updated_by_system_records.json");
+
+        TaskReport taskReport = zendeskSupportAPIService.addRecordToImporter(0, recordImporter);
+        verify(recordImporter, times(3)).addRecord(any());
+        Assert.assertFalse(taskReport.isEmpty());
+        // end_time + 1
+        Assert.assertEquals(1550645521, taskReport.get(JsonNode.class, ZendeskConstants.Field.START_TIME).asLong());
+    }
+
+    @Test
+    public void executeIncrementalContainEndTimeFilterOutLastRecordNPSScore()
+    {
+        ConfigSource src = ZendeskTestHelper.getConfigSource("nps.yml");
+        src.set("target", Target.SCORES.toString());
+        // earlier than updated_at time of last record
+        // 1550645520
+        src.set("end_time", "2019-02-20T06:52:00Z");
+        ZendeskInputPlugin.PluginTask task = src.loadConfig(ZendeskInputPlugin.PluginTask.class);
+        setupZendeskSupportAPIService(task);
+        loadData("data/scores.json");
+
+        TaskReport taskReport = zendeskSupportAPIService.addRecordToImporter(0, recordImporter);
+        verify(recordImporter, times(1)).addRecord(any());
+        Assert.assertFalse(taskReport.isEmpty());
+        // end_time + 1
+        Assert.assertEquals(1550645521, taskReport.get(JsonNode.class, ZendeskConstants.Field.START_TIME).asLong());
+    }
+
     private void setupSupportAPIService(String file)
     {
         ZendeskInputPlugin.PluginTask task = ZendeskTestHelper.getConfigSource(file)

data/src/test/java/org/embulk/input/zendesk/utils/TestZendeskDateUtils.java

@@ -79,7 +79,7 @@ public class TestZendeskDateUtils
         assertEquals(0, actualValue);
 
         actualValue = ZendeskDateUtils.getStartTime("2019-02-30 06:50:45 +0000");
-        assertEquals(ZendeskDateUtils.getStartTime("2019-02-28 06:50:45 +0000"), actualValue);
+        assertEquals(0, actualValue);
 
         actualValue = ZendeskDateUtils.getStartTime("2019-02-20 06:50:45 +0000");
         assertEquals(expectedValue, actualValue);

data/src/test/java/org/embulk/input/zendesk/utils/ZendeskTestHelper.java

@@ -76,4 +76,17 @@ public final class ZendeskTestHelper
             Assert.fail(e.getMessage());
         }
     }
+
+    public static void setPreviewMode(final ZendeskPluginTestRuntime runtime, final boolean isPreview)
+    {
+        // A small hack to make the plugin executed in preview mode so
+        try {
+            final Field previewField = ExecSession.class.getDeclaredField("preview");
+            previewField.setAccessible(true);
+            previewField.set(runtime.getExec(), isPreview);
+        }
+        catch (NoSuchFieldException | IllegalAccessException e) {
+            Assert.fail(e.getMessage());
+        }
+    }
 }

data/src/test/resources/config/nps.yml

@@ -13,6 +13,8 @@ incremental: true
 retry_initial_wait_sec: 1
 max_retry_wait_sec: 30
 retry_limit: 2
+start_time: '2019-01-20T07:14:50Z'
+end_time: '2019-01-20T07:14:53Z'
 columns:
   - {name: id, type: long}
   - {name: survey_id, type: string}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-zendesk
 version: !ruby/object:Gem::Version
-  version: 0.3.5
+  version: 0.3.6
 platform: ruby
 authors:
 - hieu.duong
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-06-04 00:00:00.000000000 Z
+date: 2019-07-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -138,7 +138,7 @@ files:
 - src/test/resources/data/util_page.json
 - classpath/commons-logging-1.2.jar
 - classpath/httpcore-4.4.10.jar
-- classpath/embulk-input-zendesk-0.3.5.jar
+- classpath/embulk-input-zendesk-0.3.6.jar
 - classpath/httpclient-4.5.6.jar
 - classpath/commons-codec-1.10.jar
 homepage: https://github.com/treasure-data/embulk-input-zendesk