embulk-input-zendesk 0.2.14 → 0.3.0

data/src/test/java/org/embulk/input/zendesk/utils/TestZendeskDateUtils.java

@@ -0,0 +1,36 @@
+package org.embulk.input.zendesk.utils;
+
+import org.embulk.spi.DataException;
+import org.junit.Assert;
+import org.junit.Test;
+import static org.junit.Assert.assertThrows;
+
+public class TestZendeskDateUtils
+{
+    @Test
+    public void isoToEpochSecondShouldReturnCorrectValue()
+    {
+        long expectedValue = 1550645445;
+
+        long value = ZendeskDateUtils.isoToEpochSecond("2019-02-20T06:50:45Z");
+        Assert.assertEquals(expectedValue, value);
+
+        value = ZendeskDateUtils.isoToEpochSecond("2019-02-20 06:50:45 +0000");
+        Assert.assertEquals(expectedValue, value);
+
+        value = ZendeskDateUtils.isoToEpochSecond("2019-02-20T06:50:45.000Z");
+        Assert.assertEquals(expectedValue, value);
+
+        value = ZendeskDateUtils.isoToEpochSecond("2019-02-20T06:50:45+00:00");
+        Assert.assertEquals(expectedValue, value);
+    }
+
+    @Test
+    public void isoToEpochSecondShouldThrowException()
+    {
+        assertThrows(DataException.class, () -> ZendeskDateUtils.isoToEpochSecond("2019-02asdasdasd-20T06:50:45Z"));
+        assertThrows(DataException.class, () -> ZendeskDateUtils.isoToEpochSecond("2019-002-20T06:50:45Z"));
+        assertThrows(DataException.class, () -> ZendeskDateUtils.isoToEpochSecond("2019-02-200T06:50:45.000Z"));
+        assertThrows(DataException.class, () -> ZendeskDateUtils.isoToEpochSecond("2019-02-20T24:01:00Z"));
+    }
+}

data/src/test/java/org/embulk/input/zendesk/utils/TestZendeskUtil.java

@@ -0,0 +1,160 @@
+package org.embulk.input.zendesk.utils;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.input.zendesk.ZendeskInputPlugin.PluginTask;
+import org.embulk.input.zendesk.models.Target;
+import org.embulk.input.zendesk.services.ZendeskSupportAPIService;
+import org.embulk.spi.Column;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.Schema;
+import org.embulk.spi.json.JsonParser;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.time.TimestampParser;
+
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.mockito.Mockito;
+import org.msgpack.value.Value;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyString;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class TestZendeskUtil
+{
+    @Rule
+    @SuppressFBWarnings("URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD")
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    private static Schema schema;
+    private static Column booleanColumn;
+    private static Column longColumn;
+    private static Column doubleColumn;
+    private static Column stringColumn;
+    private static Column dateColumn;
+    private static Column jsonColumn;
+
+    private static ZendeskSupportAPIService zendeskSupportAPIService;
+
+    @BeforeClass
+    public static void setUp()
+    {
+        zendeskSupportAPIService = mock(ZendeskSupportAPIService.class);
+        PluginTask pluginTask = ZendeskTestHelper.getConfigSource("util.yml").loadConfig(PluginTask.class);
+        schema = pluginTask.getColumns().toSchema();
+        booleanColumn = schema.getColumn(0);
+        longColumn = schema.getColumn(1);
+        doubleColumn = schema.getColumn(2);
+        stringColumn = schema.getColumn(3);
+        dateColumn = schema.getColumn(4);
+        jsonColumn = schema.getColumn(5);
+    }
+
+    @Test
+    public void testIsSupportIncrementalShouldReturnTrue()
+    {
+        boolean result = ZendeskUtils.isSupportAPIIncremental(Target.TICKETS);
+        Assert.assertTrue(result);
+
+        result = ZendeskUtils.isSupportAPIIncremental(Target.USERS);
+        Assert.assertTrue(result);
+
+        result = ZendeskUtils.isSupportAPIIncremental(Target.ORGANIZATIONS);
+        Assert.assertTrue(result);
+
+        result = ZendeskUtils.isSupportAPIIncremental(Target.TICKET_EVENTS);
+        Assert.assertTrue(result);
+
+        result = ZendeskUtils.isSupportAPIIncremental(Target.TICKET_METRICS);
+        Assert.assertTrue(result);
+    }
+
+    @Test
+    public void testNumberToSplitWithHintingInTaskWithNonIncrementalTarget()
+    {
+        JsonNode dataJson = ZendeskTestHelper.getJsonFromFile("data/util_page.json");
+        when(zendeskSupportAPIService.getData(anyString(), anyInt(), anyBoolean(), anyLong())).thenReturn(dataJson);
+        int number = ZendeskUtils.numberToSplitWithHintingInTask(2101);
+        assertEquals(22, number);
+
+        number = ZendeskUtils.numberToSplitWithHintingInTask(2100);
+        assertEquals(21, number);
+
+        number = ZendeskUtils.numberToSplitWithHintingInTask(2099);
+        assertEquals(21, number);
+    }
+
+    @Test
+    public void testAddRecordAllRight()
+    {
+        String name = "allRight";
+        JsonNode dataJson = ZendeskTestHelper.getJsonFromFile("data/util.json").get(name);
+        PageBuilder mock = Mockito.mock(PageBuilder.class);
+
+        Boolean boolValue = Boolean.TRUE;
+        long longValue = 1;
+        double doubleValue = 1;
+        String stringValue = "string";
+        Timestamp dateValue = TimestampParser.of("%Y-%m-%dT%H:%M:%S%z", "UTC").parse("2019-01-01T00:00:00Z");
+        Value jsonValue = new JsonParser().parse("{}");
+
+        ZendeskUtils.addRecord(dataJson, schema, mock);
+
+        verify(mock, times(1)).setBoolean(booleanColumn, boolValue);
+        verify(mock, times(1)).setLong(longColumn, longValue);
+        verify(mock, times(1)).setDouble(doubleColumn, doubleValue);
+        verify(mock, times(1)).setString(stringColumn, stringValue);
+        verify(mock, times(1)).setTimestamp(dateColumn, dateValue);
+        verify(mock, times(1)).setJson(jsonColumn, jsonValue);
+    }
+
+    @Test
+    public void testAddRecordAllWrong()
+    {
+        String name = "allWrong";
+        JsonNode dataJson = ZendeskTestHelper.getJsonFromFile("data/util.json").get(name);
+        PageBuilder mock = Mockito.mock(PageBuilder.class);
+
+        Value jsonValue = new JsonParser().parse("{}");
+
+        ZendeskUtils.addRecord(dataJson, schema, mock);
+
+        verify(mock, times(1)).setNull(booleanColumn);
+        verify(mock, times(1)).setNull(longColumn);
+        verify(mock, times(1)).setNull(doubleColumn);
+        verify(mock, times(1)).setNull(stringColumn);
+        verify(mock, times(1)).setNull(dateColumn);
+        verify(mock, times(1)).setJson(jsonColumn, jsonValue);
+    }
+
+    @Test
+    public void testAddRecordAllMissing()
+    {
+        String name = "allMissing";
+        JsonNode dataJson = ZendeskTestHelper.getJsonFromFile("data/util.json").get(name);
+        PageBuilder mock = Mockito.mock(PageBuilder.class);
+
+        ZendeskUtils.addRecord(dataJson, schema, mock);
+
+        verify(mock, times(6)).setNull(Mockito.any(Column.class));
+    }
+
+    @Test
+    public void testConvertBase64()
+    {
+        String expectedResult = "YWhrc2RqZmhramFzZGhma2phaGRma2phaGRramZoYWtqZGY=";
+        String encode = ZendeskUtils.convertBase64("ahksdjfhkjasdhfkjahdfkjahdkjfhakjdf");
+        assertEquals(expectedResult, encode);
+    }
+}

data/src/test/java/org/embulk/input/zendesk/utils/TestZendeskValidatorUtils.java

@@ -0,0 +1,138 @@
+package org.embulk.input.zendesk.utils;
+
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+import org.embulk.config.ConfigSource;
+
+import org.embulk.input.zendesk.ZendeskInputPlugin;
+import org.embulk.input.zendesk.services.ZendeskSupportAPIService;
+
+import org.embulk.spi.InputPlugin;
+import org.embulk.test.TestingEmbulk;
+
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.rules.RuleChain;
+import org.junit.rules.TestRule;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import static org.mockito.Mockito.mock;
+
+import java.util.Collections;
+
+public class TestZendeskValidatorUtils
+{
+    private final ExpectedException thrown = ExpectedException.none();
+    private final TestingEmbulk embulk = TestingEmbulk.builder()
+            .registerPlugin(InputPlugin.class, "zendesk", ZendeskInputPlugin.class)
+            .build();
+    public ZendeskPluginTestRuntime runtime = new ZendeskPluginTestRuntime();
+    ConfigSource configSource;
+
+    @Rule
+    @SuppressFBWarnings("URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD")
+    public TestRule chain = RuleChain.outerRule(embulk).around(runtime).around(thrown);
+
+    private ZendeskSupportAPIService zendeskSupportAPIService = mock(ZendeskSupportAPIService.class);
+
+    @Before
+    public void prepare()
+    {
+        configSource = getBaseConfigSource();
+    }
+
+    @Test
+    public void validateHostShouldThrowException()
+    {
+        configSource.set("login_url", "dummyhost");
+        final ZendeskInputPlugin.PluginTask task = configSource.loadConfig(ZendeskInputPlugin.PluginTask.class);
+        assertValidation(task, "Login URL, 'dummyhost', is unmatched expectation. It should be followed this format: https://abc.zendesk.com/");
+    }
+
+    @Test
+    public void validateCredentialOauthShouldThrowException()
+    {
+        configSource.set("auth_method", "oauth");
+        configSource.remove("access_token");
+        final ZendeskInputPlugin.PluginTask task = configSource.loadConfig(ZendeskInputPlugin.PluginTask.class);
+        assertValidation(task, "access_token is required for authentication method 'oauth'");
+    }
+
+    @Test
+    public void validateCredentialBasicShouldThrowException()
+    {
+        configSource.set("auth_method", "basic");
+        configSource.remove("username");
+        ZendeskInputPlugin.PluginTask task = configSource.loadConfig(ZendeskInputPlugin.PluginTask.class);
+        assertValidation(task, "username and password are required for authentication method 'basic'");
+
+        configSource.set("username", "");
+        configSource.remove("password");
+        task = configSource.loadConfig(ZendeskInputPlugin.PluginTask.class);
+        assertValidation(task, "username and password are required for authentication method 'basic'");
+    }
+
+    @Test
+    public void validateCredentialTokenShouldThrowException()
+    {
+        configSource.set("auth_method", "token");
+        configSource.remove("token");
+        ZendeskInputPlugin.PluginTask task = configSource.loadConfig(ZendeskInputPlugin.PluginTask.class);
+        assertValidation(task, "username and token are required for authentication method 'token'");
+
+        configSource.set("token", "");
+        configSource.remove("username");
+        task = configSource.loadConfig(ZendeskInputPlugin.PluginTask.class);
+        assertValidation(task, "username and token are required for authentication method 'token'");
+    }
+
+    @Test
+    public void validateAppMarketPlaceShouldThrowException()
+    {
+        configSource.remove("app_marketplace_integration_name");
+        ZendeskInputPlugin.PluginTask task = configSource.loadConfig(ZendeskInputPlugin.PluginTask.class);
+        assertValidation(task, "All of app_marketplace_integration_name, app_marketplace_org_id, " +
+                "app_marketplace_app_id " +
+                "are required to fill out for Apps Marketplace API header");
+    }
+
+    @Test
+    public void validateInclude()
+    {
+        configSource.set("target", "ticket_fields");
+        configSource.set("includes", Collections.singletonList("organizations"));
+        final ZendeskInputPlugin.PluginTask task = configSource.loadConfig(ZendeskInputPlugin.PluginTask.class);
+        ZendeskValidatorUtils.validateInputTask(task, zendeskSupportAPIService);
+    }
+
+    @Test
+    public void validateIncremental()
+    {
+        configSource.set("target", "ticket_fields");
+        configSource.set("incremental", true);
+        configSource.set("dedup", false);
+        configSource.set("start_time", "");
+
+        final ZendeskInputPlugin.PluginTask task = configSource.loadConfig(ZendeskInputPlugin.PluginTask.class);
+        ZendeskValidatorUtils.validateInputTask(task, zendeskSupportAPIService);
+    }
+
+    private static ConfigSource getBaseConfigSource()
+    {
+        return ZendeskTestHelper.getConfigSource("base_validator.yml");
+    }
+
+    private void assertValidation(final ZendeskInputPlugin.PluginTask task, final String message)
+    {
+        try {
+            ZendeskValidatorUtils.validateInputTask(task, zendeskSupportAPIService);
+            fail("Should not reach here");
+        }
+        catch (final Exception e) {
+            assertEquals(message, e.getMessage());
+        }
+    }
+}

data/src/test/java/org/embulk/input/zendesk/utils/ZendeskPluginTestRuntime.java

@@ -0,0 +1,133 @@
+package org.embulk.input.zendesk.utils;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.inject.Binder;
+import com.google.inject.Injector;
+import com.google.inject.Module;
+import org.embulk.GuiceBinder;
+import org.embulk.RandomManager;
+import org.embulk.TestPluginSourceModule;
+import org.embulk.TestUtilityModule;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.DataSourceImpl;
+import org.embulk.config.ModelManager;
+import org.embulk.exec.ExecModule;
+import org.embulk.exec.ExtensionServiceLoaderModule;
+import org.embulk.exec.SystemConfigModule;
+import org.embulk.jruby.JRubyScriptingModule;
+import org.embulk.plugin.BuiltinPluginSourceModule;
+import org.embulk.plugin.PluginClassLoaderFactory;
+import org.embulk.plugin.PluginClassLoaderModule;
+import org.embulk.spi.BufferAllocator;
+import org.embulk.spi.Exec;
+import org.embulk.spi.ExecAction;
+import org.embulk.spi.ExecSession;
+import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
+
+import java.util.Random;
+
+/**
+ * This is a clone from {@link org.embulk.EmbulkTestRuntime}, since there is no easy way to extend it.
+ * The only modification is on the provided systemConfig, enable tests to run `embulk/guess/zendesk.rb`
+ */
+public class ZendeskPluginTestRuntime extends GuiceBinder
+{
+    public static class TestRuntimeModule implements Module
+    {
+        @Override
+        public void configure(Binder binder)
+        {
+            ConfigSource systemConfig = getSystemConfig();
+            new SystemConfigModule(systemConfig).configure(binder);
+            new ExecModule().configure(binder);
+            new ExtensionServiceLoaderModule(systemConfig).configure(binder);
+            new BuiltinPluginSourceModule().configure(binder);
+            new JRubyScriptingModule(systemConfig).configure(binder);
+            new PluginClassLoaderModule(systemConfig).configure(binder);
+            new TestUtilityModule().configure(binder);
+            new TestPluginSourceModule().configure(binder);
+        }
+    }
+
+    private ExecSession exec;
+
+    public ZendeskPluginTestRuntime()
+    {
+        super(new TestRuntimeModule());
+        Injector injector = getInjector();
+        ConfigSource execConfig = new DataSourceImpl(injector.getInstance(ModelManager.class));
+        this.exec = ExecSession.builder(injector).fromExecConfig(execConfig).build();
+    }
+
+    public ExecSession getExec()
+    {
+        return exec;
+    }
+
+    public BufferAllocator getBufferAllocator()
+    {
+        return getInstance(BufferAllocator.class);
+    }
+
+    public ModelManager getModelManager()
+    {
+        return getInstance(ModelManager.class);
+    }
+
+    public Random getRandom()
+    {
+        return getInstance(RandomManager.class).getRandom();
+    }
+
+    public PluginClassLoaderFactory getPluginClassLoaderFactory()
+    {
+        return getInstance(PluginClassLoaderFactory.class);
+    }
+
+    @Override
+    public Statement apply(Statement base, Description description)
+    {
+        final Statement superStatement = ZendeskPluginTestRuntime.super.apply(base, description);
+        return new Statement() {
+            public void evaluate() throws Throwable
+            {
+                try {
+                    Exec.doWith(exec, (ExecAction<Void>) () -> {
+                        try {
+                            superStatement.evaluate();
+                        }
+                        catch (Throwable ex) {
+                            throw new RuntimeExecutionException(ex);
+                        }
+                        return null;
+                    });
+                }
+                catch (RuntimeException ex) {
+                    throw ex.getCause();
+                }
+                finally {
+                    exec.cleanup();
+                }
+            }
+        };
+    }
+
+    private static class RuntimeExecutionException extends RuntimeException
+    {
+        public RuntimeExecutionException(Throwable cause)
+        {
+            super(cause);
+        }
+    }
+
+    private static ConfigSource getSystemConfig()
+    {
+        ObjectNode configNode = JsonNodeFactory.instance.objectNode();
+        configNode.set("jruby_load_path", JsonNodeFactory.instance.arrayNode().add("lib"));
+
+        return new DataSourceImpl(new ModelManager(null, new ObjectMapper()), configNode);
+    }
+}