embulk-input-zendesk 0.3.1 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ad43cfba07a05a5677083290abad4876dfaa652a
-  data.tar.gz: 90dbefa73c45d26e7a499e8c5b1f6618af4512f5
+  metadata.gz: d972c5116a66a8dab7e295a806f977bc1cbcf1f5
+  data.tar.gz: f51b015c3bfc0e62093b2bb83b78781a5fd22afc
 SHA512:
-  metadata.gz: 1bcbdc100b762d7bfa73cddc605d34b64040dcb218e37f31e03b0342dbf4594cbfe5f970d861997dbb85a6b734911fa291db526bbc50a6a933f424ae4924a6ac
-  data.tar.gz: ceef0158dc916745848727ccc3dd58bd8f63d38647475b3f4a0fee75aa95a690b690ba030d99a08fecdd3d14a0a6fa5fa0e2cf2a34ef109d45aba597dd48cdf5
+  metadata.gz: cfee244009630fcff30c4a3ced32729b451e9a88b3f889efc281fbdcf6f671e2f1aa80e094fe1af3cd40b8683f1cc1e0f210b847bc662e43d4737e7619d33af2
+  data.tar.gz: d23006d4502f46397ffd406813e55a08532cdf6f6cb6d86ab564240edea303f1b81082c6cd33c11d5870168acf8b5e882ec7168e1c8bdb95f3d58b4e7e535dc9

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.3.2 - 2019-04-10
+* [fixed] Fix generate config diff based on incremental config #54 [#54](https://github.com/treasure-data/embulk-input-zendesk/pull/54)
+
 ## 0.3.1 - 2019-04-09
 * [fixed] Fix checking 404 by status code when fetching related objects #52 [#52](https://github.com/treasure-data/embulk-input-zendesk/pull/52)
 

data/build.gradle

@@ -15,7 +15,7 @@ configurations {
     provided
 }
 
-version = "0.3.1"
+version = "0.3.2"
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8

data/src/main/java/org/embulk/input/zendesk/ZendeskInputPlugin.java

@@ -213,17 +213,14 @@ public class ZendeskInputPlugin implements InputPlugin
     {
         final ConfigDiff configDiff = Exec.newConfigDiff();
 
-        if (!taskReports.isEmpty()) {
-            if (ZendeskUtils.isSupportAPIIncremental(task.getTarget())) {
-                final TaskReport taskReport = taskReports.get(0);
-                if (taskReport.has(ZendeskConstants.Field.START_TIME)) {
-                    final OffsetDateTime offsetDateTime = OffsetDateTime.ofInstant(Instant.ofEpochSecond(
-                            taskReport.get(JsonNode.class, ZendeskConstants.Field.START_TIME).asLong()),
-                            ZoneOffset.UTC);
-
-                    configDiff.set(ZendeskConstants.Field.START_TIME,
+        if (!taskReports.isEmpty() && task.getIncremental()) {
+            final TaskReport taskReport = taskReports.get(0);
+            if (taskReport.has(ZendeskConstants.Field.START_TIME)) {
+                final OffsetDateTime offsetDateTime = OffsetDateTime.ofInstant(Instant.ofEpochSecond(
+                        taskReport.get(JsonNode.class, ZendeskConstants.Field.START_TIME).asLong()), ZoneOffset.UTC);
+
+                configDiff.set(ZendeskConstants.Field.START_TIME,
                             offsetDateTime.format(DateTimeFormatter.ofPattern(ZendeskConstants.Misc.RUBY_TIMESTAMP_FORMAT_INPUT)));
-                }
             }
         }
         return configDiff;
@@ -296,16 +293,19 @@ public class ZendeskInputPlugin implements InputPlugin
                     }
                 }
                 logger.info("Fetched '{}' records from start_time '{}'", recordCount, startTime);
-                if (result.has(ZendeskConstants.Field.END_TIME) && !result.get(ZendeskConstants.Field.END_TIME).isNull()
-                        && result.has(task.getTarget().getJsonName())) {
-                    // NOTE: start_time compared as "=>", not ">".
-                    // If we will use end_time for next start_time, we got the same record that is last fetched
-                    // end_time + 1 is workaround for that
-                    taskReport.set("start_time", result.get(ZendeskConstants.Field.END_TIME).asLong() + 1);
-                }
-                else {
-                    // Sometimes no record and no end_time fetched on the job, but we should generate start_time on config_diff.
-                    taskReport.set("start_time", Instant.now().getEpochSecond());
+
+                if (task.getIncremental()) {
+                    if (result.has(ZendeskConstants.Field.END_TIME) && !result.get(ZendeskConstants.Field.END_TIME).isNull()
+                            && result.has(task.getTarget().getJsonName())) {
+                        // NOTE: start_time compared as "=>", not ">".
+                        // If we will use end_time for next start_time, we got the same record that is last fetched
+                        // end_time + 1 is workaround for that
+                        taskReport.set(ZendeskConstants.Field.START_TIME, result.get(ZendeskConstants.Field.END_TIME).asLong() + 1);
+                    }
+                    else {
+                        // Sometimes no record and no end_time fetched on the job, but we should generate start_time on config_diff.
+                        taskReport.set(ZendeskConstants.Field.START_TIME, Instant.now().getEpochSecond());
+                    }
                 }
 
                 if (numberOfRecords < ZendeskConstants.Misc.MAXIMUM_RECORDS_INCREMENTAL) {

data/src/test/java/org/embulk/input/zendesk/TestZendeskInputPlugin.java

@@ -8,6 +8,7 @@ import org.embulk.config.ConfigSource;
 import org.embulk.config.TaskReport;
 import org.embulk.config.TaskSource;
 import org.embulk.input.zendesk.services.ZendeskSupportAPIService;
+import org.embulk.input.zendesk.utils.ZendeskConstants;
 import org.embulk.input.zendesk.utils.ZendeskPluginTestRuntime;
 import org.embulk.input.zendesk.utils.ZendeskTestHelper;
 
@@ -117,7 +118,7 @@ public class TestZendeskInputPlugin
         when(zendeskSupportAPIService.getData(anyString(), anyInt(), anyBoolean(), anyLong())).thenReturn(dataJson);
 
         ConfigDiff configDiff = zendeskInputPlugin.transaction(src, new Control());
-        String nextStartTime = configDiff.get(String.class, "start_time");
+        String nextStartTime = configDiff.get(String.class, ZendeskConstants.Field.START_TIME);
         verify(pageBuilder, times(4)).addRecord();
         verify(pageBuilder, times(1)).finish();
         assertEquals("2019-02-20 07:17:34 +0000", nextStartTime);
@@ -134,7 +135,7 @@ public class TestZendeskInputPlugin
                 .thenReturn(dataJsonNext);
 
         ConfigDiff configDiff = zendeskInputPlugin.transaction(src, new Control());
-        String nextStartTime = configDiff.get(String.class, "start_time");
+        String nextStartTime = configDiff.get(String.class, ZendeskConstants.Field.START_TIME);
         verify(pageBuilder, times(1)).addRecord();
         verify(pageBuilder, times(1)).finish();
         assertEquals("2019-02-20 07:17:34 +0000", nextStartTime);
@@ -150,7 +151,7 @@ public class TestZendeskInputPlugin
         when(zendeskSupportAPIService.getData(anyString(), anyInt(), anyBoolean(), anyLong())).thenReturn(dataJsonNext);
 
         ConfigDiff configDiff = zendeskInputPlugin.transaction(src, new Control());
-        String nextStartTime = configDiff.get(String.class, "start_time");
+        String nextStartTime = configDiff.get(String.class, ZendeskConstants.Field.START_TIME);
         verify(pageBuilder, times(5)).addRecord();
         verify(pageBuilder, times(1)).finish();
         assertEquals("2019-02-20 07:17:34 +0000", nextStartTime);
@@ -166,7 +167,7 @@ public class TestZendeskInputPlugin
         when(zendeskSupportAPIService.getData(anyString(), anyInt(), anyBoolean(), anyLong())).thenReturn(dataJson);
 
         ConfigDiff configDiff = zendeskInputPlugin.transaction(src, new Control());
-        String nextStartTime = configDiff.get(String.class, "start_time");
+        String nextStartTime = configDiff.get(String.class, ZendeskConstants.Field.START_TIME);
         verify(pageBuilder, times(3)).addRecord();
         verify(pageBuilder, times(1)).finish();
         assertEquals("2019-02-20 07:17:34 +0000", nextStartTime);
@@ -187,7 +188,7 @@ public class TestZendeskInputPlugin
         ConfigDiff configDiff = zendeskInputPlugin.transaction(src, new Control());
 
         verify(pageBuilder, times(1)).finish();
-        String nextStartTime = configDiff.get(String.class, "start_time");
+        String nextStartTime = configDiff.get(String.class, ZendeskConstants.Field.START_TIME);
         assertEquals("2019-02-20 07:17:34 +0000", nextStartTime);
     }
 
@@ -204,6 +205,42 @@ public class TestZendeskInputPlugin
         Assert.assertTrue(configDiff.isEmpty());
     }
 
+    @Test
+    public void testRunNotGenerateStartTimeWhenRunningInNonIncrementalModeAndTargetSupportIncremental()
+    {
+        final ConfigSource src = ZendeskTestHelper.getConfigSource("incremental.yml");
+        src.set("incremental", false);
+        loadData("data/tickets.json");
+
+        ConfigDiff configDiff = zendeskInputPlugin.transaction(src, new Control());
+
+        Assert.assertTrue(configDiff.isEmpty());
+    }
+
+    @Test
+    public void testRunNotGenerateStartTimeWhenRunningInIncrementalModeAndTargetDoesNotSupportIncremental()
+    {
+        final ConfigSource src = ZendeskTestHelper.getConfigSource("non-incremental.yml");
+        src.set("incremental", true);
+        loadData("data/ticket_fields.json");
+
+        ConfigDiff configDiff = zendeskInputPlugin.transaction(src, new Control());
+
+        Assert.assertTrue(configDiff.isEmpty());
+    }
+
+    @Test
+    public void testRunIncrementalGenerateStartTimeWhenRunningInIncrementalModeAndTargetSupportIncremental()
+    {
+        String expectedStartTime = "2019-02-20 07:17:34 +0000";
+        final ConfigSource src = ZendeskTestHelper.getConfigSource("incremental.yml");
+        loadData("data/tickets.json");
+
+        ConfigDiff configDiff = zendeskInputPlugin.transaction(src, new Control());
+        Assert.assertFalse(configDiff.isEmpty());
+        Assert.assertEquals(expectedStartTime, configDiff.get(String.class, ZendeskConstants.Field.START_TIME));
+    }
+
     private class Control implements InputPlugin.Control
     {
         @Override

data/src/test/java/org/embulk/input/zendesk/utils/TestZendeskValidatorUtils.java

@@ -114,7 +114,7 @@ public class TestZendeskValidatorUtils
         configSource.set("target", "ticket_fields");
         configSource.set("incremental", true);
         configSource.set("dedup", false);
-        configSource.set("start_time", "");
+        configSource.set(ZendeskConstants.Field.START_TIME, "");
 
         final ZendeskInputPlugin.PluginTask task = configSource.loadConfig(ZendeskInputPlugin.PluginTask.class);
         ZendeskValidatorUtils.validateInputTask(task, zendeskSupportAPIService);

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-zendesk
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
 platform: ruby
 authors:
 - hieu.duong
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-04-09 00:00:00.000000000 Z
+date: 2019-04-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -95,8 +95,8 @@ files:
 - src/test/resources/data/tickets_continue.json
 - src/test/resources/data/util.json
 - src/test/resources/data/util_page.json
+- classpath/embulk-input-zendesk-0.3.2.jar
 - classpath/commons-logging-1.2.jar
-- classpath/embulk-input-zendesk-0.3.1.jar
 - classpath/httpcore-4.4.10.jar
 - classpath/httpclient-4.5.6.jar
 - classpath/commons-codec-1.10.jar