embulk-input-zendesk 0.3.4 → 0.3.5

data/src/main/java/org/embulk/input/zendesk/stream/paginator/sunshine/UserEventSpliterator.java

@@ -0,0 +1,35 @@
+package org.embulk.input.zendesk.stream.paginator.sunshine;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.embulk.input.zendesk.ZendeskInputPlugin;
+import org.embulk.input.zendesk.clients.ZendeskRestClient;
+import org.embulk.input.zendesk.utils.ZendeskUtils;
+
+import java.util.Iterator;
+import java.util.function.Consumer;
+
+public class UserEventSpliterator extends SunshineSpliterator
+{
+    private String userID;
+
+    public UserEventSpliterator(final String userID, final String path, final ZendeskRestClient zendeskRestClient, final ZendeskInputPlugin.PluginTask task, final boolean isPreview)
+    {
+        super(path, zendeskRestClient, task, isPreview);
+        this.userID = userID;
+    }
+
+    @Override
+    protected void handleRunIterator(final Iterator<JsonNode> iterator, final Consumer<? super JsonNode> action)
+    {
+        iterator.forEachRemaining(
+                item -> {
+                    if (!ZendeskUtils.isNull(item)) {
+                        JsonNode temp = iterator.next();
+                        // Because in the returned json doesn't have user_id, so we try to add to it
+                        ((ObjectNode) temp).put("user_id", this.userID);
+                        action.accept(temp);
+                    }
+                });
+    }
+}

data/src/main/java/org/embulk/input/zendesk/stream/paginator/support/OrganizationSpliterator.java

@@ -0,0 +1,13 @@
+package org.embulk.input.zendesk.stream.paginator.support;
+
+import org.embulk.input.zendesk.ZendeskInputPlugin;
+import org.embulk.input.zendesk.clients.ZendeskRestClient;
+import org.embulk.input.zendesk.models.Target;
+
+public class OrganizationSpliterator extends SupportSpliterator
+{
+    public OrganizationSpliterator(final String path, final ZendeskRestClient zendeskRestClient, final ZendeskInputPlugin.PluginTask task)
+    {
+        super(Target.ORGANIZATIONS, path, zendeskRestClient, task, false);
+    }
+}

data/src/main/java/org/embulk/input/zendesk/stream/paginator/support/SupportSpliterator.java

@@ -0,0 +1,44 @@
+package org.embulk.input.zendesk.stream.paginator.support;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import org.embulk.input.zendesk.ZendeskInputPlugin;
+import org.embulk.input.zendesk.clients.ZendeskRestClient;
+import org.embulk.input.zendesk.models.Target;
+import org.embulk.input.zendesk.stream.PagingSpliterator;
+import org.embulk.input.zendesk.utils.ZendeskUtils;
+
+import java.util.Iterator;
+import java.util.function.Consumer;
+
+public class SupportSpliterator extends PagingSpliterator<JsonNode>
+{
+    Target target;
+
+    public SupportSpliterator(final Target target, final String path, final ZendeskRestClient zendeskRestClient, final ZendeskInputPlugin.PluginTask task, final boolean isPreview)
+    {
+        super(path, zendeskRestClient, task, isPreview);
+        this.target = target;
+    }
+
+    @Override
+    public boolean tryAdvance(final Consumer<? super JsonNode> action)
+    {
+        final String result = zendeskRestClient.doGet(path, task, isPreview);
+        if (result != null && !result.isEmpty()) {
+            final JsonNode jsonNode = ZendeskUtils.parseJsonObject(result);
+            final Iterator<JsonNode> iterator = ZendeskUtils.getListRecords(jsonNode, target.getJsonName());
+            iterator.forEachRemaining(
+                    item -> {
+                        if (!ZendeskUtils.isNull(item)) {
+                            action.accept(item);
+                        }
+                    });
+            if (jsonNode.get("next_page") != null && !jsonNode.get("next_page").isNull()) {
+                path = jsonNode.get("next_page").asText();
+                return true;
+            }
+        }
+
+        return false;
+    }
+}

data/src/main/java/org/embulk/input/zendesk/stream/paginator/support/UserSpliterator.java

@@ -0,0 +1,13 @@
+package org.embulk.input.zendesk.stream.paginator.support;
+
+import org.embulk.input.zendesk.ZendeskInputPlugin;
+import org.embulk.input.zendesk.clients.ZendeskRestClient;
+import org.embulk.input.zendesk.models.Target;
+
+public class UserSpliterator extends SupportSpliterator
+{
+    public UserSpliterator(final String path, final ZendeskRestClient zendeskRestClient, final ZendeskInputPlugin.PluginTask task, final boolean isPreview)
+    {
+        super(Target.USERS, path, zendeskRestClient, task, isPreview);
+    }
+}

data/src/main/java/org/embulk/input/zendesk/utils/ZendeskConstants.java

@@ -1,5 +1,10 @@
 package org.embulk.input.zendesk.utils;
 
+import com.google.common.collect.ImmutableList;
+
+import java.util.Arrays;
+import java.util.List;
+
 public class ZendeskConstants
 {
     private ZendeskConstants()
@@ -27,8 +32,12 @@ public class ZendeskConstants
 
     public static class Url
     {
-        public static final String API = "api/v2";
+        public static final String API = "/api/v2";
         public static final String API_INCREMENTAL = API + "/incremental";
+        public static final String API_NPS_INCREMENTAL = API + "/nps/incremental";
+        public static final String API_OBJECT_RECORD = "api/sunshine/objects/records";
+        public static final String API_RELATIONSHIP_RECORD = "api/sunshine/relationships/records";
+        public static final String API_USER_EVENT = "api/sunshine/events";
     }
 
     public static class Misc
@@ -45,6 +54,9 @@ public class ZendeskConstants
 
         // 1 MB
         public static final int GUESS_BUFFER_SIZE = 1024 * 1024;
+        public static final List<String> SUPPORT_DATE_TIME_FORMAT = ImmutableList.copyOf(Arrays.asList(ZendeskConstants.Misc.ISO_INSTANT, ZendeskConstants.Misc.RUBY_TIMESTAMP_FORMAT_INPUT,
+                ZendeskConstants.Misc.JAVA_TIMESTAMP_FORMAT, ZendeskConstants.Misc.ISO_TIMESTAMP_FORMAT,
+                ZendeskConstants.Misc.RUBY_TIMESTAMP_FORMAT_INPUT_NO_SPACE));
     }
 
     public static class Regex

data/src/main/java/org/embulk/input/zendesk/utils/ZendeskDateUtils.java

@@ -1,16 +1,14 @@
 package org.embulk.input.zendesk.utils;
 
-import com.google.common.base.Joiner;
 import org.embulk.spi.DataException;
 
+import java.time.Instant;
 import java.time.LocalDateTime;
 import java.time.OffsetDateTime;
 import java.time.ZoneOffset;
 import java.time.format.DateTimeFormatter;
 import java.time.format.DateTimeParseException;
-import java.util.Arrays;
 
-import java.util.List;
 import java.util.Optional;
 
 public class ZendeskDateUtils
@@ -19,25 +17,21 @@ public class ZendeskDateUtils
     {
     }
 
-    private static final List<String> supportedFormats =  Arrays.asList(ZendeskConstants.Misc.ISO_INSTANT, ZendeskConstants.Misc.RUBY_TIMESTAMP_FORMAT_INPUT,
-                                                                        ZendeskConstants.Misc.JAVA_TIMESTAMP_FORMAT, ZendeskConstants.Misc.ISO_TIMESTAMP_FORMAT,
-                                                                        ZendeskConstants.Misc.RUBY_TIMESTAMP_FORMAT_INPUT_NO_SPACE);
-
     public static long isoToEpochSecond(final String time)
     {
-        Optional<String> pattern = supportedTimeFormat(time, supportedFormats);
+        final Optional<String> pattern = supportedTimeFormat(time);
         if (pattern.isPresent()) {
             final DateTimeFormatter formatter = DateTimeFormatter.ofPattern(pattern.get()).withZone(ZoneOffset.UTC);
             final OffsetDateTime offsetDateTime = LocalDateTime.parse(time, formatter).atOffset(ZoneOffset.UTC);
             return offsetDateTime.toInstant().getEpochSecond();
         }
 
-        throw new DataException("Fail to parse value '" + time + "' follow formats " + "[ " + Joiner.on(",").join(supportedFormats) + "]");
+        throw new DataException("Fail to parse value '" + time + "' follow formats " + ZendeskConstants.Misc.SUPPORT_DATE_TIME_FORMAT.toString());
     }
 
-    private static Optional<String> supportedTimeFormat(final String value, final List<String> supportedFormats)
+    public static Optional<String> supportedTimeFormat(final String value)
     {
-        for (final String fmt : supportedFormats) {
+        for (final String fmt : ZendeskConstants.Misc.SUPPORT_DATE_TIME_FORMAT) {
             final DateTimeFormatter formatter = DateTimeFormatter.ofPattern(fmt);
             try {
                 formatter.parse(value);
@@ -49,4 +43,21 @@ public class ZendeskDateUtils
         }
         return Optional.empty();
     }
+
+    public static String convertToDateTimeFormat(String datetime, String dateTimeFormat)
+    {
+        return OffsetDateTime.ofInstant(Instant.ofEpochSecond(ZendeskDateUtils.isoToEpochSecond(datetime)), ZoneOffset.UTC)
+                .format(DateTimeFormatter.ofPattern(dateTimeFormat));
+    }
+
+    // start_time should be start from 0
+    public static long getStartTime(final  String time)
+    {
+        try {
+            return isoToEpochSecond(time);
+        }
+        catch (DataException ex) {
+            return 0;
+        }
+    }
 }

data/src/main/java/org/embulk/input/zendesk/utils/ZendeskUtils.java

@@ -1,150 +1,88 @@
 package org.embulk.input.zendesk.utils;
 
+import com.fasterxml.jackson.databind.DeserializationFeature;
 import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.common.base.Charsets;
-import org.embulk.input.zendesk.models.Target;
-
-import org.embulk.spi.Column;
-import org.embulk.spi.ColumnVisitor;
-
-import org.embulk.spi.Exec;
-import org.embulk.spi.PageBuilder;
-import org.embulk.spi.Schema;
-import org.embulk.spi.json.JsonParser;
-import org.embulk.spi.time.Timestamp;
-import org.slf4j.Logger;
-
+import com.google.common.base.Throwables;
+import org.apache.http.client.utils.URIBuilder;
+import org.embulk.config.ConfigException;
+import org.embulk.spi.DataException;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.util.Base64;
-import java.util.function.Function;
+import java.util.Iterator;
 
 public class ZendeskUtils
 {
-    private static final Logger logger = Exec.getLogger(ZendeskUtils.class);
+    private static final ObjectMapper mapper = new ObjectMapper();
+
+    static {
+        mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
+        mapper.configure(com.fasterxml.jackson.core.JsonParser.Feature.ALLOW_UNQUOTED_CONTROL_CHARS, false);
+    }
 
     private ZendeskUtils()
     {}
 
-    public static boolean isSupportAPIIncremental(final Target target)
-    {
-        return !Target.TICKET_FORMS.equals(target)
-                && !Target.TICKET_FIELDS.equals(target);
-    }
-
     public static String convertBase64(final String text)
     {
         return Base64.getEncoder().encodeToString(text.getBytes(Charsets.UTF_8));
     }
 
-    public static int numberToSplitWithHintingInTask(int count)
+    public static int numberToSplitWithHintingInTask(final int count)
     {
         return (int) Math.ceil((double) count / ZendeskConstants.Misc.RECORDS_SIZE_PER_PAGE);
     }
 
-    public static synchronized void addRecord(JsonNode record, Schema schema, PageBuilder pageBuilder)
+    public static ObjectNode parseJsonObject(final String jsonText)
     {
-        schema.visitColumns(new ColumnVisitor() {
-            @Override
-            public void jsonColumn(Column column)
-            {
-                JsonNode data = record.get(column.getName());
-
-                setColumn(column, data, (value) -> {
-                    pageBuilder.setJson(column, new JsonParser().parse(value.toString()));
-                    return null;
-                });
-            }
-
-            @Override
-            public void stringColumn(Column column)
-            {
-                JsonNode data = record.get(column.getName());
-
-                setColumn(column, data, (value) -> {
-                    pageBuilder.setString(column, value.asText());
-                    return null;
-                });
-            }
-
-            @Override
-            public void timestampColumn(Column column)
-            {
-                JsonNode data = record.get(column.getName());
-                setColumn(column, data, (value) -> {
-                    Timestamp timestamp = getTimestampValue(value.asText());
-                    if (timestamp == null) {
-                        pageBuilder.setNull(column);
-                    }
-                    else {
-                        pageBuilder.setTimestamp(column, timestamp);
-                    }
-                    return null;
-                });
-            }
-
-            @Override
-            public void booleanColumn(Column column)
-            {
-                JsonNode data = record.get(column.getName());
-
-                setColumn(column, data, (value) -> {
-                    pageBuilder.setBoolean(column, value.asBoolean());
-                    return null;
-                });
-            }
-
-            @Override
-            public void longColumn(Column column)
-            {
-                JsonNode data = record.get(column.getName());
-
-                setColumn(column, data, (value) -> {
-                    pageBuilder.setLong(column, value.asLong());
-                    return null;
-                });
-            }
-
-            @Override
-            public void doubleColumn(Column column)
-            {
-                JsonNode data = record.get(column.getName());
+        final JsonNode node = ZendeskUtils.parseJsonNode(jsonText);
+        if (node.isObject()) {
+            return (ObjectNode) node;
+        }
 
-                setColumn(column, data, (value) -> {
-                    pageBuilder.setDouble(column, value.asDouble());
-                    return null;
-                });
-            }
+        throw new DataException("Expected object node to parse but doesn't get");
+    }
 
-            private void setColumn(Column column, JsonNode data, Function<JsonNode, Void> setter)
-            {
-                if (isNull(data)) {
-                    pageBuilder.setNull(column);
-                    return;
-                }
-                setter.apply(data);
-            }
-        });
-        pageBuilder.addRecord();
+    public static Iterator<JsonNode> getListRecords(final JsonNode result, final String targetJsonName)
+    {
+        if (!result.has(targetJsonName) || !result.get(targetJsonName).isArray()) {
+            throw new DataException(String.format("Missing '%s' from Zendesk API response", targetJsonName));
+        }
+        return result.get(targetJsonName).elements();
     }
 
-    private static boolean isNull(JsonNode jsonNode)
+    public static boolean isNull(final JsonNode jsonNode)
     {
         return jsonNode == null || jsonNode.isNull();
     }
 
-    /*
-     * For getting the timestamp value of the node
-     * Sometime if the parser could not parse the value then return null
-     * */
-    private static Timestamp getTimestampValue(String value)
+    public static URIBuilder getURIBuilder(final String urlString)
+    {
+        final URI uri;
+        try {
+            uri = new URI(urlString);
+        }
+        catch (final URISyntaxException e) {
+            throw new ConfigException("URL is invalid format " + urlString);
+        }
+
+        return new URIBuilder()
+                .setScheme(uri.getScheme())
+                .setHost(uri.getHost());
+    }
+
+    private static JsonNode parseJsonNode(final String jsonText)
     {
-        Timestamp result = null;
         try {
-            long timeStamp = ZendeskDateUtils.isoToEpochSecond(value);
-            result = Timestamp.ofEpochSecond(timeStamp);
+            return ZendeskUtils.mapper.readTree(jsonText);
         }
-        catch (Exception e) {
-            logger.warn("Error when parse time stamp data " + value);
+        catch (final IOException e) {
+            throw Throwables.propagate(e);
         }
-        return result;
     }
 }

data/src/test/java/org/embulk/input/zendesk/TestRecordImporter.java

@@ -0,0 +1,114 @@
+package org.embulk.input.zendesk;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.input.zendesk.utils.ZendeskTestHelper;
+import org.embulk.spi.Column;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.Schema;
+import org.embulk.spi.json.JsonParser;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.time.TimestampParser;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.mockito.Mockito;
+import org.msgpack.value.Value;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
+public class TestRecordImporter
+{
+    @Rule
+    @SuppressFBWarnings("URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD")
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    public PageBuilder pageBuilder = mock(PageBuilder.class);
+
+    private static Schema schema;
+    private static Column booleanColumn;
+    private static Column longColumn;
+    private static Column doubleColumn;
+    private static Column stringColumn;
+    private static Column dateColumn;
+    private static Column jsonColumn;
+    private static ZendeskInputPlugin.PluginTask pluginTask;
+
+    private RecordImporter recordImporter;
+
+    @BeforeClass
+    public static void setUp()
+    {
+        pluginTask = ZendeskTestHelper.getConfigSource("util.yml").loadConfig(ZendeskInputPlugin.PluginTask.class);
+        schema = pluginTask.getColumns().toSchema();
+        booleanColumn = schema.getColumn(0);
+        longColumn = schema.getColumn(1);
+        doubleColumn = schema.getColumn(2);
+        stringColumn = schema.getColumn(3);
+        dateColumn = schema.getColumn(4);
+        jsonColumn = schema.getColumn(5);
+    }
+
+    @Before
+    public void prepare()
+    {
+        recordImporter = spy(new RecordImporter(schema, pageBuilder));
+    }
+
+    @Test
+    public void testAddRecordAllRight()
+    {
+        String name = "allRight";
+        JsonNode dataJson = ZendeskTestHelper.getJsonFromFile("data/util.json").get(name);
+
+        Boolean boolValue = Boolean.TRUE;
+        long longValue = 1;
+        double doubleValue = 1;
+        String stringValue = "string";
+        Timestamp dateValue = TimestampParser.of("%Y-%m-%dT%H:%M:%S%z", "UTC").parse("2019-01-01T00:00:00Z");
+        Value jsonValue = new JsonParser().parse("{}");
+
+        recordImporter.addRecord(dataJson);
+
+        verify(pageBuilder, times(1)).setBoolean(booleanColumn, boolValue);
+        verify(pageBuilder, times(1)).setLong(longColumn, longValue);
+        verify(pageBuilder, times(1)).setDouble(doubleColumn, doubleValue);
+        verify(pageBuilder, times(1)).setString(stringColumn, stringValue);
+        verify(pageBuilder, times(1)).setTimestamp(dateColumn, dateValue);
+        verify(pageBuilder, times(1)).setJson(jsonColumn, jsonValue);
+    }
+
+    @Test
+    public void testAddRecordAllWrong()
+    {
+        String name = "allWrong";
+        JsonNode dataJson = ZendeskTestHelper.getJsonFromFile("data/util.json").get(name);
+
+        Value jsonValue = new JsonParser().parse("{}");
+
+        recordImporter.addRecord(dataJson);
+
+        verify(pageBuilder, times(1)).setNull(booleanColumn);
+        verify(pageBuilder, times(1)).setNull(longColumn);
+        verify(pageBuilder, times(1)).setNull(doubleColumn);
+        verify(pageBuilder, times(1)).setNull(stringColumn);
+        verify(pageBuilder, times(1)).setNull(dateColumn);
+        verify(pageBuilder, times(1)).setJson(jsonColumn, jsonValue);
+    }
+
+    @Test
+    public void testAddRecordAllMissing()
+    {
+        String name = "allMissing";
+        JsonNode dataJson = ZendeskTestHelper.getJsonFromFile("data/util.json").get(name);
+
+        recordImporter.addRecord(dataJson);
+
+        verify(pageBuilder, times(6)).setNull(Mockito.any());
+    }
+}