embulk-input-zendesk 0.2.14 → 0.3.0

data/src/main/java/org/embulk/input/zendesk/utils/ZendeskConstants.java

@@ -0,0 +1,61 @@
+package org.embulk.input.zendesk.utils;
+
+public class ZendeskConstants
+{
+    private ZendeskConstants()
+    {
+    }
+
+    public static class Header
+    {
+        public static final String APPLICATION_JSON = "application/json";
+
+        public static final String ZENDESK_MARKETPLACE_NAME = "X-Zendesk-Marketplace-Name";
+        public static final String ZENDESK_MARKETPLACE_ORGANIZATION_ID = "X-Zendesk-Marketplace-Organization-Id";
+        public static final String ZENDESK_MARKETPLACE_APP_ID = "X-Zendesk-Marketplace-App-Id";
+    }
+
+    public static class Field
+    {
+        public static final String START_TIME = "start_time";
+        public static final String END_TIME = "end_time";
+        public static final String COUNT = "count";
+        public static final String GENERATED_TIMESTAMP = "generated_timestamp";
+        public static final String UPDATED_AT = "updated_at";
+        public static final String ID = "id";
+    }
+
+    public static class Url
+    {
+        public static final String API = "api/v2";
+        public static final String API_INCREMENTAL = API + "/incremental";
+    }
+
+    public static class Misc
+    {
+        public static final String RUBY_TIMESTAMP_FORMAT = "%Y-%m-%dT%H:%M:%S%z";
+        public static final String RUBY_TIMESTAMP_FORMAT_INPUT = "yyyy-MM-dd HH:mm:ss Z";
+        public static final String JAVA_TIMESTAMP_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'";
+        public static final String ISO_TIMESTAMP_FORMAT = "yyyy-MM-dd'T'HH:mm:ssXXX";
+        public static final String ISO_INSTANT = "yyyy-MM-dd'T'HH:mm:ss'Z'";
+        public static final String TOO_RECENT_START_TIME = "Too recent start_time.";
+        public static final String INVALID_END_POINT_RESPONSE = "\"error\":\"InvalidEndpoint\"";
+        public static final int RECORDS_SIZE_PER_PAGE = 100;
+        public static final int MAXIMUM_RECORDS_INCREMENTAL = 1000;
+        public static final int TOO_MANY_REQUEST = 429;
+
+        // 1 MB
+        public static final int GUESS_BUFFER_SIZE = 1024 * 1024;
+    }
+
+    public static class Regex
+    {
+        public static final String ID = "_id$";
+        public static final String HOST = "^(https:\\/\\/)?(www.)?([a-zA-Z0-9]+).zendesk.com/$";
+    }
+
+    public static class HttpStatus
+    {
+        public static final int TOO_MANY_REQUEST = 429;
+    }
+}

data/src/main/java/org/embulk/input/zendesk/utils/ZendeskDateUtils.java

@@ -0,0 +1,51 @@
+package org.embulk.input.zendesk.utils;
+
+import com.google.common.base.Joiner;
+import org.embulk.spi.DataException;
+
+import java.time.LocalDateTime;
+import java.time.OffsetDateTime;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeParseException;
+import java.util.Arrays;
+
+import java.util.List;
+import java.util.Optional;
+
+public class ZendeskDateUtils
+{
+    private ZendeskDateUtils()
+    {
+    }
+
+    private static final List<String> supportedFormats =  Arrays.asList(ZendeskConstants.Misc.ISO_INSTANT, ZendeskConstants.Misc.RUBY_TIMESTAMP_FORMAT_INPUT,
+                                                                        ZendeskConstants.Misc.JAVA_TIMESTAMP_FORMAT, ZendeskConstants.Misc.ISO_TIMESTAMP_FORMAT);
+
+    public static long isoToEpochSecond(final String time)
+    {
+        Optional<String> pattern = supportedTimeFormat(time, supportedFormats);
+        if (pattern.isPresent()) {
+            final DateTimeFormatter formatter = DateTimeFormatter.ofPattern(pattern.get()).withZone(ZoneOffset.UTC);
+            final OffsetDateTime offsetDateTime = LocalDateTime.parse(time, formatter).atOffset(ZoneOffset.UTC);
+            return offsetDateTime.toInstant().getEpochSecond();
+        }
+
+        throw new DataException("Fail to parse value '" + time + "' follow formats " + "[ " + Joiner.on(",").join(supportedFormats) + "]");
+    }
+
+    private static Optional<String> supportedTimeFormat(final String value, final List<String> supportedFormats)
+    {
+        for (final String fmt : supportedFormats) {
+            final DateTimeFormatter formatter = DateTimeFormatter.ofPattern(fmt);
+            try {
+                formatter.parse(value);
+                return Optional.of(fmt);
+            }
+            catch (final DateTimeParseException e) {
+                // Do nothing
+            }
+        }
+        return Optional.empty();
+    }
+}

data/src/main/java/org/embulk/input/zendesk/utils/ZendeskUtils.java

@@ -0,0 +1,150 @@
+package org.embulk.input.zendesk.utils;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.base.Charsets;
+import org.embulk.input.zendesk.models.Target;
+
+import org.embulk.spi.Column;
+import org.embulk.spi.ColumnVisitor;
+
+import org.embulk.spi.Exec;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.Schema;
+import org.embulk.spi.json.JsonParser;
+import org.embulk.spi.time.Timestamp;
+import org.slf4j.Logger;
+
+import java.util.Base64;
+import java.util.function.Function;
+
+public class ZendeskUtils
+{
+    private static final Logger logger = Exec.getLogger(ZendeskUtils.class);
+
+    private ZendeskUtils()
+    {}
+
+    public static boolean isSupportAPIIncremental(final Target target)
+    {
+        return !Target.TICKET_FORMS.equals(target)
+                && !Target.TICKET_FIELDS.equals(target);
+    }
+
+    public static String convertBase64(final String text)
+    {
+        return Base64.getEncoder().encodeToString(text.getBytes(Charsets.UTF_8));
+    }
+
+    public static int numberToSplitWithHintingInTask(int count)
+    {
+        return (int) Math.ceil((double) count / ZendeskConstants.Misc.RECORDS_SIZE_PER_PAGE);
+    }
+
+    public static synchronized void addRecord(JsonNode record, Schema schema, PageBuilder pageBuilder)
+    {
+        schema.visitColumns(new ColumnVisitor() {
+            @Override
+            public void jsonColumn(Column column)
+            {
+                JsonNode data = record.get(column.getName());
+
+                setColumn(column, data, (value) -> {
+                    pageBuilder.setJson(column, new JsonParser().parse(value.toString()));
+                    return null;
+                });
+            }
+
+            @Override
+            public void stringColumn(Column column)
+            {
+                JsonNode data = record.get(column.getName());
+
+                setColumn(column, data, (value) -> {
+                    pageBuilder.setString(column, value.asText());
+                    return null;
+                });
+            }
+
+            @Override
+            public void timestampColumn(Column column)
+            {
+                JsonNode data = record.get(column.getName());
+                setColumn(column, data, (value) -> {
+                    Timestamp timestamp = getTimestampValue(value.asText());
+                    if (timestamp == null) {
+                        pageBuilder.setNull(column);
+                    }
+                    else {
+                        pageBuilder.setTimestamp(column, timestamp);
+                    }
+                    return null;
+                });
+            }
+
+            @Override
+            public void booleanColumn(Column column)
+            {
+                JsonNode data = record.get(column.getName());
+
+                setColumn(column, data, (value) -> {
+                    pageBuilder.setBoolean(column, value.asBoolean());
+                    return null;
+                });
+            }
+
+            @Override
+            public void longColumn(Column column)
+            {
+                JsonNode data = record.get(column.getName());
+
+                setColumn(column, data, (value) -> {
+                    pageBuilder.setLong(column, value.asLong());
+                    return null;
+                });
+            }
+
+            @Override
+            public void doubleColumn(Column column)
+            {
+                JsonNode data = record.get(column.getName());
+
+                setColumn(column, data, (value) -> {
+                    pageBuilder.setDouble(column, value.asDouble());
+                    return null;
+                });
+            }
+
+            private void setColumn(Column column, JsonNode data, Function<JsonNode, Void> setter)
+            {
+                if (isNull(data)) {
+                    pageBuilder.setNull(column);
+                    return;
+                }
+                setter.apply(data);
+            }
+        });
+        pageBuilder.addRecord();
+    }
+
+    private static boolean isNull(JsonNode jsonNode)
+    {
+        return jsonNode == null || jsonNode.isNull();
+    }
+
+    /*
+     * For getting the timestamp value of the node
+     * Sometime if the parser could not parse the value then return null
+     * */
+    private static Timestamp getTimestampValue(String value)
+    {
+        Timestamp result = null;
+        try {
+            long timeStamp = ZendeskDateUtils.isoToEpochSecond(value);
+            result = Timestamp.ofEpochSecond(timeStamp);
+        }
+        catch (Exception e) {
+            logger.warn("Error when parse time stamp data " + value);
+        }
+        return result;
+    }
+}

data/src/main/java/org/embulk/input/zendesk/utils/ZendeskValidatorUtils.java

@@ -0,0 +1,92 @@
+package org.embulk.input.zendesk.utils;
+
+import org.embulk.config.ConfigException;
+import org.embulk.input.zendesk.ZendeskInputPlugin;
+import org.embulk.input.zendesk.services.ZendeskSupportAPIService;
+import org.embulk.spi.Exec;
+import org.slf4j.Logger;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class ZendeskValidatorUtils
+{
+    private ZendeskValidatorUtils(){}
+
+    private static final Logger logger = Exec.getLogger(ZendeskValidatorUtils.class);
+
+    public static void validateInputTask(final ZendeskInputPlugin.PluginTask task, final ZendeskSupportAPIService zendeskSupportAPIService)
+    {
+        validateHost(task.getLoginUrl());
+        validateAppMarketPlace(task.getAppMarketPlaceIntegrationName().isPresent(),
+                task.getAppMarketPlaceAppId().isPresent(),
+                task.getAppMarketPlaceOrgId().isPresent());
+        validateCredentials(task);
+        validateIncremental(task);
+    }
+
+    private static void validateHost(final String loginUrl)
+    {
+        final Matcher matcher = Pattern.compile(ZendeskConstants.Regex.HOST).matcher(loginUrl);
+        if (!matcher.matches()) {
+            throw new ConfigException(String.format("Login URL, '%s', is unmatched expectation. " +
+                    "It should be followed this format: https://abc.zendesk.com/", loginUrl));
+        }
+    }
+
+    private static void validateCredentials(final ZendeskInputPlugin.PluginTask task)
+    {
+        switch (task.getAuthenticationMethod()) {
+            case OAUTH:
+                if (!task.getAccessToken().isPresent()) {
+                    throw new ConfigException(String.format("access_token is required for authentication method '%s'",
+                            task.getAuthenticationMethod().name().toLowerCase()));
+                }
+                break;
+            case TOKEN:
+                if (!task.getUsername().isPresent() || !task.getToken().isPresent()) {
+                    throw new ConfigException(String.format("username and token are required for authentication method '%s'",
+                            task.getAuthenticationMethod().name().toLowerCase()));
+                }
+                break;
+            case BASIC:
+                if (!task.getUsername().isPresent() || !task.getPassword().isPresent()) {
+                    throw new ConfigException(String.format("username and password are required for authentication method '%s'",
+                            task.getAuthenticationMethod().name().toLowerCase()));
+                }
+                break;
+            default:
+                throw new ConfigException("Unknown authentication method");
+        }
+    }
+
+    private static void validateAppMarketPlace(final boolean isAppMarketIntegrationNamePresent,
+                                        final boolean isAppMarketAppIdPresent,
+                                        final boolean isAppMarketOrgIdPresent)
+    {
+        final boolean isAllAvailable =
+                isAppMarketIntegrationNamePresent && isAppMarketAppIdPresent && isAppMarketOrgIdPresent;
+        final boolean isAllUnAvailable =
+                !isAppMarketIntegrationNamePresent && !isAppMarketAppIdPresent && !isAppMarketOrgIdPresent;
+        // All or nothing needed
+        if (!(isAllAvailable || isAllUnAvailable)) {
+            throw new ConfigException("All of app_marketplace_integration_name, app_marketplace_org_id, " +
+                    "app_marketplace_app_id " +
+                    "are required to fill out for Apps Marketplace API header");
+        }
+    }
+
+    private static void validateIncremental(final ZendeskInputPlugin.PluginTask task)
+    {
+        if (task.getIncremental()) {
+            if (!task.getDedup()) {
+                logger.warn("You've selected to skip de-duplicating records, result may contain duplicated data");
+            }
+
+            if (!ZendeskUtils.isSupportAPIIncremental(task.getTarget()) && task.getStartTime().isPresent()) {
+                logger.warn(String.format("Target: '%s' doesn't support incremental export API. Will be ignored start_time option",
+                        task.getTarget()));
+            }
+        }
+    }
+}

data/src/test/java/org/embulk/input/zendesk/TestZendeskInputPlugin.java

@@ -0,0 +1,232 @@
+package org.embulk.input.zendesk;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskReport;
+import org.embulk.config.TaskSource;
+import org.embulk.input.zendesk.services.ZendeskSupportAPIService;
+import org.embulk.input.zendesk.utils.ZendeskPluginTestRuntime;
+import org.embulk.input.zendesk.utils.ZendeskTestHelper;
+
+import org.embulk.spi.InputPlugin;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.Schema;
+import org.embulk.spi.TestPageBuilderReader;
+import org.junit.Assert;
+
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import static org.junit.Assert.assertEquals;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyString;
+
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+public class TestZendeskInputPlugin
+{
+    @Rule
+    @SuppressFBWarnings("URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD")
+    public ZendeskPluginTestRuntime embulk = new ZendeskPluginTestRuntime();
+
+    private ZendeskSupportAPIService zendeskSupportAPIService = mock(ZendeskSupportAPIService.class);
+
+    private ZendeskInputPlugin zendeskInputPlugin;
+
+    private TestPageBuilderReader.MockPageOutput output = new TestPageBuilderReader.MockPageOutput();
+
+    private PageBuilder pageBuilder = Mockito.mock(PageBuilder.class);
+
+    @Before
+    public void prepare()
+    {
+        zendeskInputPlugin = spy(new ZendeskInputPlugin());
+        when(zendeskInputPlugin.getZendeskSupportAPIService(any(ZendeskInputPlugin.PluginTask.class))).thenReturn(zendeskSupportAPIService);
+        doReturn(pageBuilder).when(zendeskInputPlugin).getPageBuilder(any(Schema.class), any(PageOutput.class));
+    }
+
+    @Test
+    public void testGuessGenerateColumnsForIncrementalTarget()
+    {
+        final ConfigSource src = ZendeskTestHelper.getConfigSource("base.yml");
+        src.set("target", "tickets");
+        setupTestGuessGenerateColumn(src, "data/tickets.json", "data/expected/ticket_column.json");
+    }
+
+    @Test
+    public void testGuessGenerateColumnsForIncrementalTargetIncludeRelatedObject()
+    {
+        final ConfigSource src = ZendeskTestHelper.getConfigSource("base.yml");
+        src.set("target", "tickets");
+        src.set("includes", Collections.singletonList("organizations"));
+        setupTestGuessGenerateColumn(src, "data/tickets.json", "data/expected/ticket_column_with_related_objects.json");
+    }
+
+    @Test
+    public void testGuessGenerateColumnsForTicketMetrics()
+    {
+        final ConfigSource src = ZendeskTestHelper.getConfigSource("base.yml");
+        src.set("target", "ticket_metrics");
+        setupTestGuessGenerateColumn(src, "data/ticket_metrics.json", "data/expected/ticket_metrics_column.json");
+    }
+
+    @Test
+    public void testGuessGenerateColumnsForNonIncrementalTarget()
+    {
+        final ConfigSource src = ZendeskTestHelper.getConfigSource("base.yml");
+        src.set("target", "ticket_fields");
+        setupTestGuessGenerateColumn(src, "data/ticket_fields.json", "data/expected/ticket_fields_column.json");
+    }
+
+    @Test(expected = ConfigException.class)
+    public void testGuessFail()
+    {
+        final ConfigSource src = ZendeskTestHelper.getConfigSource("base.yml");
+        src.set("target", "tickets");
+        loadData("data/error_data.json");
+
+        zendeskInputPlugin.guess(src);
+    }
+
+    @Test
+    public void testRunIncrementalDedup()
+    {
+        final ConfigSource src = ZendeskTestHelper.getConfigSource("incremental.yml");
+        JsonNode dataJson = ZendeskTestHelper.getJsonFromFile("data/tickets.json");
+
+        when(zendeskSupportAPIService.getData(anyString(), anyInt(), anyBoolean(), anyLong())).thenReturn(dataJson);
+
+        ConfigDiff configDiff = zendeskInputPlugin.transaction(src, new Control());
+        String nextStartTime = configDiff.get(String.class, "start_time");
+        verify(pageBuilder, times(4)).addRecord();
+        verify(pageBuilder, times(1)).finish();
+        assertEquals("2019-02-20 07:17:34 +0000", nextStartTime);
+    }
+
+    @Test
+    public void testRunIncrementalWithNextPage()
+    {
+        final ConfigSource src = ZendeskTestHelper.getConfigSource("incremental.yml");
+        JsonNode dataJson = ZendeskTestHelper.getJsonFromFile("data/tickets_continue.json");
+        JsonNode dataJsonNext = ZendeskTestHelper.getJsonFromFile("data/tickets.json");
+        when(zendeskSupportAPIService.getData(anyString(), anyInt(), anyBoolean(), anyLong()))
+                .thenReturn(dataJson)
+                .thenReturn(dataJsonNext);
+
+        ConfigDiff configDiff = zendeskInputPlugin.transaction(src, new Control());
+        String nextStartTime = configDiff.get(String.class, "start_time");
+        verify(pageBuilder, times(1)).addRecord();
+        verify(pageBuilder, times(1)).finish();
+        assertEquals("2019-02-20 07:17:34 +0000", nextStartTime);
+    }
+
+    @Test
+    public void testRunIncrementalNonDedup()
+    {
+        final ConfigSource src = ZendeskTestHelper.getConfigSource("incremental.yml");
+        src.set("dedup", false);
+
+        JsonNode dataJsonNext = ZendeskTestHelper.getJsonFromFile("data/tickets.json");
+        when(zendeskSupportAPIService.getData(anyString(), anyInt(), anyBoolean(), anyLong())).thenReturn(dataJsonNext);
+
+        ConfigDiff configDiff = zendeskInputPlugin.transaction(src, new Control());
+        String nextStartTime = configDiff.get(String.class, "start_time");
+        verify(pageBuilder, times(5)).addRecord();
+        verify(pageBuilder, times(1)).finish();
+        assertEquals("2019-02-20 07:17:34 +0000", nextStartTime);
+    }
+
+    @Test
+    public void testRunIncrementalForTicketMetrics()
+    {
+        final ConfigSource src = ZendeskTestHelper.getConfigSource("incremental.yml");
+        src.set("target", "ticket_metrics");
+        JsonNode dataJson = ZendeskTestHelper.getJsonFromFile("data/ticket_metrics.json");
+
+        when(zendeskSupportAPIService.getData(anyString(), anyInt(), anyBoolean(), anyLong())).thenReturn(dataJson);
+
+        ConfigDiff configDiff = zendeskInputPlugin.transaction(src, new Control());
+        String nextStartTime = configDiff.get(String.class, "start_time");
+        verify(pageBuilder, times(3)).addRecord();
+        verify(pageBuilder, times(1)).finish();
+        assertEquals("2019-02-20 07:17:34 +0000", nextStartTime);
+    }
+
+    @Test
+    public void testRunIncrementalWithRelatedObject()
+    {
+        final ConfigSource src = ZendeskTestHelper.getConfigSource("incremental.yml");
+        src.set("includes", Collections.singletonList("organizations"));
+
+        JsonNode dataJson = ZendeskTestHelper.getJsonFromFile("data/tickets.json");
+        JsonNode dataJsonObject = ZendeskTestHelper.getJsonFromFile("data/ticket_with_related_objects.json");
+        when(zendeskSupportAPIService.getData(anyString(), anyInt(), anyBoolean(), anyLong()))
+                .thenReturn(dataJson)
+                .thenReturn(dataJsonObject);
+
+        ConfigDiff configDiff = zendeskInputPlugin.transaction(src, new Control());
+
+        verify(pageBuilder, times(1)).finish();
+        String nextStartTime = configDiff.get(String.class, "start_time");
+        assertEquals("2019-02-20 07:17:34 +0000", nextStartTime);
+    }
+
+    @Test
+    public void testRunNonIncremental()
+    {
+        final ConfigSource src = ZendeskTestHelper.getConfigSource("non-incremental.yml");
+        loadData("data/ticket_fields.json");
+
+        ConfigDiff configDiff = zendeskInputPlugin.transaction(src, new Control());
+        // running in 2 pages
+        verify(pageBuilder, times(7 * 2)).addRecord();
+        verify(pageBuilder, times(2)).finish();
+        Assert.assertTrue(configDiff.isEmpty());
+    }
+
+    private class Control implements InputPlugin.Control
+    {
+        @Override
+        public List<TaskReport> run(final TaskSource taskSource, final Schema schema, final int taskCount)
+        {
+            List<TaskReport> reports = IntStream.range(0, taskCount)
+                    .mapToObj(i -> zendeskInputPlugin.run(taskSource, schema, i, output))
+                    .collect(Collectors.toList());
+            return reports;
+        }
+    }
+
+    private void loadData(String fileName)
+    {
+        JsonNode dataJson = ZendeskTestHelper.getJsonFromFile(fileName);
+        when(zendeskSupportAPIService.getData(anyString(), anyInt(), anyBoolean(), anyLong())).thenReturn(dataJson);
+    }
+
+    private void setupTestGuessGenerateColumn(ConfigSource src, String fileName, String expectedSource)
+    {
+        loadData(fileName);
+        ConfigDiff configDiff = zendeskInputPlugin.guess(src);
+        JsonNode columns = configDiff.get(JsonNode.class, "columns");
+        assertEquals(ZendeskTestHelper.getJsonFromFile(expectedSource), columns);
+    }
+}