embulk-input-zendesk 0.3.4 → 0.3.5

data/src/test/java/org/embulk/input/zendesk/services/TestZendeskUserEventService.java

@@ -0,0 +1,158 @@
+package org.embulk.input.zendesk.services;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.input.zendesk.RecordImporter;
+import org.embulk.input.zendesk.ZendeskInputPlugin.PluginTask;
+import org.embulk.input.zendesk.clients.ZendeskRestClient;
+import org.embulk.input.zendesk.utils.ZendeskTestHelper;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class TestZendeskUserEventService
+{
+    private RecordImporter recordImporter;
+
+    @Rule
+    @SuppressFBWarnings("URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD")
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+    private ZendeskRestClient zendeskRestClient;
+    private ZendeskUserEventService zendeskUserEventService;
+
+    @Before
+    public void prepare()
+    {
+        zendeskRestClient = mock(ZendeskRestClient.class);
+        recordImporter = mock(RecordImporter.class);
+    }
+
+    @Test
+    public void testGetData()
+    {
+        setup();
+
+        JsonNode jsonNode = zendeskUserEventService.getDataFromPath("https://abc.zendesk.com/api/sunshine/objects/records?type=user&per_page=1000", 0, true, 0);
+        assertFalse(jsonNode.isNull());
+        assertTrue(jsonNode.has("data"));
+        assertTrue(jsonNode.get("data").isArray());
+    }
+
+    @Test
+    public void testAddRecordToImporterInPreviewMode()
+    {
+        setup();
+        ZendeskTestHelper.setPreviewMode(runtime, true);
+        zendeskUserEventService.addRecordToImporter(0, recordImporter);
+        verify(recordImporter, times(1)).addRecord(any());
+    }
+
+    @Test
+    public void testUrlForUserEvent()
+    {
+        ZendeskTestHelper.setPreviewMode(runtime, false);
+        setup();
+        JsonNode dataJsonOrganization = ZendeskTestHelper.getJsonFromFile("data/simple_organization.json");
+        JsonNode dataJsonUser = ZendeskTestHelper.getJsonFromFile("data/simple_user.json");
+        JsonNode dataJsonUserEventWithLatterTime = ZendeskTestHelper.getJsonFromFile("data/user_event_contain_latter_create_at.json");
+
+        when(zendeskRestClient.doGet(any(), any(), anyBoolean()))
+                .thenReturn(dataJsonOrganization.toString())
+                .thenReturn(dataJsonUser.toString())
+                .thenReturn(dataJsonUserEventWithLatterTime.toString());
+
+        String expectedURIForOrganization = "https://abc.zendesk.com/api/v2/organizations?per_page=100&page=1";
+        String expectedURIForUser = "https://abc.zendesk.com/api/v2/organizations/360857467053/users.json?per_page=100&page=1";
+        String expectedURIForUserEvent = "https://abc.zendesk.com/api/sunshine/events?identifier=support%3Auser_id%3A1194092277&start_time=2019-01-20T07%3A14%3A50Z&end_time=2019-06-20T07%3A14%3A53Z";
+        List<String> expectedURI = Arrays.asList(expectedURIForOrganization, expectedURIForUser, expectedURIForUserEvent);
+
+        zendeskUserEventService.addRecordToImporter(0, recordImporter);
+        ArgumentCaptor<String> uri = ArgumentCaptor.forClass(String.class);
+        verify(zendeskRestClient, times(3)).doGet(uri.capture(), any(), anyBoolean());
+        assertEquals(expectedURI, uri.getAllValues());
+    }
+
+    @Test
+    public void testAddRecordToImporterInNonPreviewMode()
+    {
+        ZendeskTestHelper.setPreviewMode(runtime, false);
+        PluginTask task = setup();
+
+        JsonNode dataJsonOrganization = ZendeskTestHelper.getJsonFromFile("data/organization.json");
+        JsonNode dataJsonUser = ZendeskTestHelper.getJsonFromFile("data/simple_user.json");
+        JsonNode dataJsonUserEvent = ZendeskTestHelper.getJsonFromFile("data/user_event.json");
+
+        // 2 organizations but return the same user.
+        when(zendeskRestClient.doGet(eq("https://abc.zendesk.com/api/v2/organizations?per_page=100&page=1"), eq(task), eq(false)))
+                .thenReturn(dataJsonOrganization.toString());
+        when(zendeskRestClient.doGet(eq("https://abc.zendesk.com/api/v2/organizations/360857467053/users.json?per_page=100&page=1"), eq(task), eq(false)))
+                .thenReturn(dataJsonUser.toString());
+        when(zendeskRestClient.doGet(eq("https://abc.zendesk.com/api/v2/organizations/360857467055/users.json?per_page=100&page=1"), eq(task), eq(false)))
+                .thenReturn(dataJsonUser.toString());
+        when(zendeskRestClient.doGet(eq("https://abc.zendesk.com/api/sunshine/events?identifier=support%3Auser_id%3A1194092277&start_time=2019-01-20T07%3A14%3A50Z&end_time=2019-06-20T07%3A14%3A53Z"), eq(task), eq(false)))
+                .thenReturn(dataJsonUserEvent.toString());
+
+        zendeskUserEventService.addRecordToImporter(0, recordImporter);
+        // non dedup
+        verify(recordImporter, times(2)).addRecord(any());
+    }
+
+    @Test
+    public void testAddRecordToImporterWithDuplicateUser()
+    {
+        ZendeskTestHelper.setPreviewMode(runtime, false);
+        PluginTask task = ZendeskTestHelper.getConfigSource("user_events.yml")
+                .set("dedup", true)
+                .loadConfig(PluginTask.class);
+        setupZendeskSupportAPIService(task);
+
+        JsonNode dataJsonOrganization = ZendeskTestHelper.getJsonFromFile("data/organization.json");
+        JsonNode dataJsonUser = ZendeskTestHelper.getJsonFromFile("data/simple_user.json");
+        JsonNode dataJsonUserEvent = ZendeskTestHelper.getJsonFromFile("data/user_event.json");
+        // 2 organizations but return the same user.
+        when(zendeskRestClient.doGet(eq("https://abc.zendesk.com/api/v2/organizations?per_page=100&page=1"), eq(task), eq(false)))
+                .thenReturn(dataJsonOrganization.toString());
+        when(zendeskRestClient.doGet(eq("https://abc.zendesk.com/api/v2/organizations/360857467053/users.json?per_page=100&page=1"), eq(task), eq(false)))
+                .thenReturn(dataJsonUser.toString());
+        when(zendeskRestClient.doGet(eq("https://abc.zendesk.com/api/v2/organizations/360857467055/users.json?per_page=100&page=1"), eq(task), eq(false)))
+                .thenReturn(dataJsonUser.toString());
+        when(zendeskRestClient.doGet(eq("https://abc.zendesk.com/api/sunshine/events?identifier=support%3Auser_id%3A1194092277&start_time=2019-01-20T07%3A14%3A50Z&end_time=2019-06-20T07%3A14%3A53Z"), eq(task), eq(false)))
+                .thenReturn(dataJsonUserEvent.toString());
+
+        zendeskUserEventService.addRecordToImporter(0, recordImporter);
+
+        // expected to call fetchUserEvent only one time
+        verify(recordImporter, times(1)).addRecord(any());
+    }
+
+    private void setupZendeskSupportAPIService(PluginTask task)
+    {
+        zendeskUserEventService = spy(new ZendeskUserEventService(task));
+        when(zendeskUserEventService.getZendeskRestClient()).thenReturn(zendeskRestClient);
+    }
+
+    private PluginTask setup()
+    {
+        PluginTask task = ZendeskTestHelper.getConfigSource("user_events.yml")
+                .loadConfig(PluginTask.class);
+        setupZendeskSupportAPIService(task);
+        return task;
+    }
+}

data/src/test/java/org/embulk/input/zendesk/utils/TestZendeskDateUtils.java

@@ -3,12 +3,18 @@ package org.embulk.input.zendesk.utils;
 import org.embulk.spi.DataException;
 import org.junit.Assert;
 import org.junit.Test;
+
+import java.util.Optional;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertThrows;
+import static org.junit.Assert.assertTrue;
 
 public class TestZendeskDateUtils
 {
     @Test
-    public void isoToEpochSecondShouldReturnCorrectValue()
+    public void testIsoToEpochSecondShouldReturnCorrectValue()
     {
         long expectedValue = 1550645445;
 
@@ -29,11 +35,53 @@ public class TestZendeskDateUtils
     }
 
     @Test
-    public void isoToEpochSecondShouldThrowException()
+    public void testIsoToEpochSecondShouldThrowException()
     {
         assertThrows(DataException.class, () -> ZendeskDateUtils.isoToEpochSecond("2019-02asdasdasd-20T06:50:45Z"));
         assertThrows(DataException.class, () -> ZendeskDateUtils.isoToEpochSecond("2019-002-20T06:50:45Z"));
         assertThrows(DataException.class, () -> ZendeskDateUtils.isoToEpochSecond("2019-02-200T06:50:45.000Z"));
         assertThrows(DataException.class, () -> ZendeskDateUtils.isoToEpochSecond("2019-02-20T24:01:00Z"));
     }
+
+    @Test
+    public void testConvertToDateTimeFormat()
+    {
+        String expectedString = "2019-05-01T07:14:50Z";
+
+        String actualString = ZendeskDateUtils.convertToDateTimeFormat("2019-05-01 07:14:50 +0000", ZendeskConstants.Misc.ISO_INSTANT);
+        assertEquals(expectedString, actualString);
+
+        actualString = ZendeskDateUtils.convertToDateTimeFormat("2019-05-01 07:14:50+0000", ZendeskConstants.Misc.ISO_INSTANT);
+        assertEquals(expectedString, actualString);
+
+        actualString = ZendeskDateUtils.convertToDateTimeFormat("2019-05-01T07:14:50.000Z", ZendeskConstants.Misc.ISO_INSTANT);
+        assertEquals(expectedString, actualString);
+
+        actualString = ZendeskDateUtils.convertToDateTimeFormat("2019-05-01T07:14:50+00:00", ZendeskConstants.Misc.ISO_INSTANT);
+        assertEquals(expectedString, actualString);
+    }
+
+    @Test
+    public void testSupportedTimeFormat()
+    {
+        Optional optional = ZendeskDateUtils.supportedTimeFormat("2019-05-01 07:14:50+0000");
+        assertTrue(optional.isPresent());
+
+        optional = ZendeskDateUtils.supportedTimeFormat("2019-05-01T07:14:50");
+        assertFalse(optional.isPresent());
+    }
+
+    @Test
+    public void testGetStartTime()
+    {
+        long expectedValue = 1550645445;
+        long actualValue = ZendeskDateUtils.getStartTime("2019-02-20 06:512310:45 +0000");
+        assertEquals(0, actualValue);
+
+        actualValue = ZendeskDateUtils.getStartTime("2019-02-30 06:50:45 +0000");
+        assertEquals(ZendeskDateUtils.getStartTime("2019-02-28 06:50:45 +0000"), actualValue);
+
+        actualValue = ZendeskDateUtils.getStartTime("2019-02-20 06:50:45 +0000");
+        assertEquals(expectedValue, actualValue);
+    }
 }

data/src/test/java/org/embulk/input/zendesk/utils/TestZendeskUtil.java

@@ -1,35 +1,10 @@
 package org.embulk.input.zendesk.utils;
 
-import com.fasterxml.jackson.databind.JsonNode;
 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
 import org.embulk.EmbulkTestRuntime;
-import org.embulk.input.zendesk.ZendeskInputPlugin.PluginTask;
-import org.embulk.input.zendesk.models.Target;
-import org.embulk.input.zendesk.services.ZendeskSupportAPIService;
-import org.embulk.spi.Column;
-import org.embulk.spi.PageBuilder;
-import org.embulk.spi.Schema;
-import org.embulk.spi.json.JsonParser;
-import org.embulk.spi.time.Timestamp;
-import org.embulk.spi.time.TimestampParser;
-
-import org.junit.Assert;
-import org.junit.BeforeClass;
 import org.junit.Rule;
 import org.junit.Test;
-import org.mockito.Mockito;
-import org.msgpack.value.Value;
-
 import static org.junit.Assert.assertEquals;
-import static org.mockito.ArgumentMatchers.anyBoolean;
-import static org.mockito.ArgumentMatchers.anyInt;
-import static org.mockito.ArgumentMatchers.anyLong;
-import static org.mockito.ArgumentMatchers.anyString;
-
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
 
 public class TestZendeskUtil
 {
@@ -37,119 +12,6 @@ public class TestZendeskUtil
     @SuppressFBWarnings("URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD")
     public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
 
-    private static Schema schema;
-    private static Column booleanColumn;
-    private static Column longColumn;
-    private static Column doubleColumn;
-    private static Column stringColumn;
-    private static Column dateColumn;
-    private static Column jsonColumn;
-
-    private static ZendeskSupportAPIService zendeskSupportAPIService;
-
-    @BeforeClass
-    public static void setUp()
-    {
-        zendeskSupportAPIService = mock(ZendeskSupportAPIService.class);
-        PluginTask pluginTask = ZendeskTestHelper.getConfigSource("util.yml").loadConfig(PluginTask.class);
-        schema = pluginTask.getColumns().toSchema();
-        booleanColumn = schema.getColumn(0);
-        longColumn = schema.getColumn(1);
-        doubleColumn = schema.getColumn(2);
-        stringColumn = schema.getColumn(3);
-        dateColumn = schema.getColumn(4);
-        jsonColumn = schema.getColumn(5);
-    }
-
-    @Test
-    public void testIsSupportIncrementalShouldReturnTrue()
-    {
-        boolean result = ZendeskUtils.isSupportAPIIncremental(Target.TICKETS);
-        Assert.assertTrue(result);
-
-        result = ZendeskUtils.isSupportAPIIncremental(Target.USERS);
-        Assert.assertTrue(result);
-
-        result = ZendeskUtils.isSupportAPIIncremental(Target.ORGANIZATIONS);
-        Assert.assertTrue(result);
-
-        result = ZendeskUtils.isSupportAPIIncremental(Target.TICKET_EVENTS);
-        Assert.assertTrue(result);
-
-        result = ZendeskUtils.isSupportAPIIncremental(Target.TICKET_METRICS);
-        Assert.assertTrue(result);
-    }
-
-    @Test
-    public void testNumberToSplitWithHintingInTaskWithNonIncrementalTarget()
-    {
-        JsonNode dataJson = ZendeskTestHelper.getJsonFromFile("data/util_page.json");
-        when(zendeskSupportAPIService.getData(anyString(), anyInt(), anyBoolean(), anyLong())).thenReturn(dataJson);
-        int number = ZendeskUtils.numberToSplitWithHintingInTask(2101);
-        assertEquals(22, number);
-
-        number = ZendeskUtils.numberToSplitWithHintingInTask(2100);
-        assertEquals(21, number);
-
-        number = ZendeskUtils.numberToSplitWithHintingInTask(2099);
-        assertEquals(21, number);
-    }
-
-    @Test
-    public void testAddRecordAllRight()
-    {
-        String name = "allRight";
-        JsonNode dataJson = ZendeskTestHelper.getJsonFromFile("data/util.json").get(name);
-        PageBuilder mock = Mockito.mock(PageBuilder.class);
-
-        Boolean boolValue = Boolean.TRUE;
-        long longValue = 1;
-        double doubleValue = 1;
-        String stringValue = "string";
-        Timestamp dateValue = TimestampParser.of("%Y-%m-%dT%H:%M:%S%z", "UTC").parse("2019-01-01T00:00:00Z");
-        Value jsonValue = new JsonParser().parse("{}");
-
-        ZendeskUtils.addRecord(dataJson, schema, mock);
-
-        verify(mock, times(1)).setBoolean(booleanColumn, boolValue);
-        verify(mock, times(1)).setLong(longColumn, longValue);
-        verify(mock, times(1)).setDouble(doubleColumn, doubleValue);
-        verify(mock, times(1)).setString(stringColumn, stringValue);
-        verify(mock, times(1)).setTimestamp(dateColumn, dateValue);
-        verify(mock, times(1)).setJson(jsonColumn, jsonValue);
-    }
-
-    @Test
-    public void testAddRecordAllWrong()
-    {
-        String name = "allWrong";
-        JsonNode dataJson = ZendeskTestHelper.getJsonFromFile("data/util.json").get(name);
-        PageBuilder mock = Mockito.mock(PageBuilder.class);
-
-        Value jsonValue = new JsonParser().parse("{}");
-
-        ZendeskUtils.addRecord(dataJson, schema, mock);
-
-        verify(mock, times(1)).setNull(booleanColumn);
-        verify(mock, times(1)).setNull(longColumn);
-        verify(mock, times(1)).setNull(doubleColumn);
-        verify(mock, times(1)).setNull(stringColumn);
-        verify(mock, times(1)).setNull(dateColumn);
-        verify(mock, times(1)).setJson(jsonColumn, jsonValue);
-    }
-
-    @Test
-    public void testAddRecordAllMissing()
-    {
-        String name = "allMissing";
-        JsonNode dataJson = ZendeskTestHelper.getJsonFromFile("data/util.json").get(name);
-        PageBuilder mock = Mockito.mock(PageBuilder.class);
-
-        ZendeskUtils.addRecord(dataJson, schema, mock);
-
-        verify(mock, times(6)).setNull(Mockito.any(Column.class));
-    }
-
     @Test
     public void testConvertBase64()
     {

data/src/test/java/org/embulk/input/zendesk/utils/ZendeskTestHelper.java

@@ -4,14 +4,17 @@ import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
 
+import org.embulk.EmbulkTestRuntime;
 import org.embulk.config.ConfigLoader;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.ModelManager;
+import org.embulk.spi.ExecSession;
 import org.embulk.test.EmbulkTests;
 import org.junit.Assert;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.lang.reflect.Field;
 
 import static junit.framework.TestCase.fail;
 
@@ -60,4 +63,17 @@ public final class ZendeskTestHelper
         }
         return configSource;
     }
+
+    public static void setPreviewMode(final EmbulkTestRuntime runtime, final boolean isPreview)
+    {
+        // A small hack to make the plugin executed in preview mode so
+        try {
+            final Field previewField = ExecSession.class.getDeclaredField("preview");
+            previewField.setAccessible(true);
+            previewField.set(runtime.getExec(), isPreview);
+        }
+        catch (NoSuchFieldException | IllegalAccessException e) {
+            Assert.fail(e.getMessage());
+        }
+    }
 }

data/src/test/resources/config/nps.yml

@@ -0,0 +1,29 @@
+type: zendesk
+login_url: https://abc.zendesk.com
+auth_method: oauth
+username: dummy
+password: dummy
+token: dummy
+access_token: dummy
+target: scores
+app_marketplace_integration_name: abc
+app_marketplace_app_id: abc
+app_marketplace_org_id: abc
+incremental: true
+retry_initial_wait_sec: 1
+max_retry_wait_sec: 30
+retry_limit: 2
+columns:
+  - {name: id, type: long}
+  - {name: survey_id, type: string}
+  - {name: delivery_id, type: string}
+  - {name: recipient_id, type: string}
+  - {name: user_id, type: string}
+  - {name: rating, type: long}
+  - {name: comment, type: string}
+  - {name: delivered_at, type: timestamp, format: '%Y-%m-%dT%H:%M:%S%z'}
+  - {name: rated_at, type: timestamp, format: '%Y-%m-%dT%H:%M:%S%z'}
+  - {name: user_email, type: string}
+  - {name: user_name, type: string}
+  - {name: survey_name, type: string}
+  - {name: account, type: json}