embulk-input-zendesk-all 0.3.9 → 0.4.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 66b582f9b7eb1e5b3269b334e7c8dd46fb8bcd01
-  data.tar.gz: 1fd7ead56fc82aa40b9d9c9786491b9de5d20fe3
+  metadata.gz: 0bfa12fca52df30c6d53c787b0c1fbcc3607d83e
+  data.tar.gz: 9366de09a25cc4a53db47131de1381edd136bcda
 SHA512:
-  metadata.gz: 7f8b5f051215b0cc328a96b1a7d5c6729102283821f0fbc5a9e9cf0f8a58d45e32a2d9da240a97924a6bcc0b7ccef16c35cd36936b934a96cba46f365845d25e
-  data.tar.gz: c06cf6a71c462fe6e598d2977745ec129dc5c32130c0fa63d05015b2f08c8942ed1803edb391338f760ab2652a0a3c485596672d5c842a24a2b872e546055dd8
+  metadata.gz: ac55e4011b761639f55d3fcbd1fd2f9f3d54556114d164cbd4bf13eb51fa5ad5dd3d0ed2d8840303274c6e6348dc835a6026fac4d6ed0d5db4d1f321cfec6bf8
+  data.tar.gz: 935c10be956e9ef20c5cbf761f3f90e800d37f0bb8e844813926b03aab90fd9bedb8ddcf3b8f71963ecef2b7342ee8a0df4d2de590eb6a36fa7fdabca09379a8

data/README.md

@@ -15,6 +15,8 @@ Required Embulk version >= 0.9.6.
 * DOES support JSON type columns
 * Supports satisfaction_ratings API
 * Supports ticket_metric_events API
+* Supports SLA Policies API
+* Supports Groups API
 
 
 * **Plugin type**: input

data/build.gradle

@@ -15,7 +15,7 @@ configurations {
     provided
 }
 
-version = "0.3.9"
+version = "0.4.5"
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8

data/gradlew

@@ -168,5 +168,5 @@ eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$A
 if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
   cd "$(dirname "$0")"
 fi
-
+echo "$JAVACMD" "$@"
 exec "$JAVACMD" "$@"

data/src/main/java/org/embulk/input/zendesk/ZendeskInputPlugin.java

@@ -374,6 +374,8 @@ public class ZendeskInputPlugin implements InputPlugin
                 return new ZendeskCustomObjectService(task);
             case USER_EVENTS:
                 return new ZendeskUserEventService(task);
+            case GROUPS:
+                return new ZendeskSupportAPIService(task);
             default:
                 throw new ConfigException("Unsupported " + task.getTarget() + ", supported values: '" + Arrays.toString(Target.values()) + "'");
         }

data/src/main/java/org/embulk/input/zendesk/clients/ZendeskRestClient.java

@@ -235,6 +235,8 @@ public class ZendeskRestClient
         final HttpGet request = new HttpGet(url);
         final ImmutableMap<String, String> headers = buildAuthHeader(task);
         headers.forEach(request::setHeader);
+
+//        headers.values().forEach(logger::info);
         return request;
     }
 

data/src/main/java/org/embulk/input/zendesk/models/Target.java

@@ -14,7 +14,7 @@ public enum Target
     TICKET_METRICS("metric_sets"), TICKET_FIELDS("ticket_fields"), TICKET_FORMS("ticket_forms"),
     TICKET_METRIC_EVENTS("ticket_metric_events"), SATISFACTION_RATINGS("satisfaction_ratings"),
     SLA_POLICIES("sla_policies"), SCORES("responses"), OBJECT_RECORDS("data"),
-    RECIPIENTS("recipients"),  RELATIONSHIP_RECORDS("data"), USER_EVENTS("data");
+    RECIPIENTS("recipients"), RELATIONSHIP_RECORDS("data"), USER_EVENTS("data"), GROUPS("groups");
 
     String jsonName;
 

data/src/main/java/org/embulk/input/zendesk/services/ZendeskNPSService.java

@@ -15,6 +15,11 @@ public class ZendeskNPSService extends ZendeskNormalServices
     {
         return true;
     }
+    @Override
+    protected String buildURI(final int page, final long startTime, final long endTime)
+    {
+        return buildURI(page, startTime);
+    }
 
     @Override
     protected String buildURI(final int page, final long startTime)

data/src/main/java/org/embulk/input/zendesk/services/ZendeskNormalServices.java

@@ -59,11 +59,15 @@ public abstract class ZendeskNormalServices implements ZendeskService
 
         return taskReport;
     }
-
     public JsonNode getDataFromPath(String path, final int page, final boolean isPreview, final long startTime)
+    {
+        return getDataFromPath(path, page,isPreview, startTime, 0);
+    }
+
+    public JsonNode getDataFromPath(String path, final int page, final boolean isPreview, final long startTime, final long endTime)
     {
         if (path.isEmpty()) {
-            path = buildURI(page, startTime);
+            path = buildURI(page, startTime, endTime);
         }
 
         final String response = getZendeskRestClient().doGet(path, task, isPreview);
@@ -71,6 +75,8 @@ public abstract class ZendeskNormalServices implements ZendeskService
     }
 
     protected abstract String buildURI(int page, long startTime);
+    protected abstract String buildURI(int page, long startTime, long endTime);
+
 
     @VisibleForTesting
     protected ZendeskRestClient getZendeskRestClient()
@@ -127,14 +133,16 @@ public abstract class ZendeskNormalServices implements ZendeskService
                 int numberOfRecords = 0;
                 if (result.has(ZendeskConstants.Field.COUNT)) {
                     numberOfRecords = result.get(ZendeskConstants.Field.COUNT).asInt();
+                    logger.info("Number of Records {}", numberOfRecords);
                 }
 
                 while (iterator.hasNext()) {
                     final JsonNode recordJsonNode = iterator.next();
 
-                    if (isUpdatedBySystem(recordJsonNode, startTime)) {
-                        continue;
-                    }
+//                    if (isUpdatedBySystem(recordJsonNode, startTime)) {
+//                        logger.info("This record is update by system? {} IGNORED!! ", recordJsonNode.get("id"));
+//                        continue;
+//                    }
 
                     // Contain some records  that later than end_time. Checked and don't add.
                     // Because the api already sorted by updated_at or timestamp for ticket_events, we just need to break no need to check further.
@@ -142,6 +150,7 @@ public abstract class ZendeskNormalServices implements ZendeskService
                         long checkedTime = 0;
                         if (recordJsonNode.has(ZendeskConstants.Field.UPDATED_AT) && !recordJsonNode.get(ZendeskConstants.Field.UPDATED_AT).isNull()) {
                             checkedTime = ZendeskDateUtils.isoToEpochSecond(recordJsonNode.get(ZendeskConstants.Field.UPDATED_AT).textValue());
+//                            logger.info("Checked time {}", checkedTime);
                         }
 
                         // ticket events is updated by system not user's action so it only has timestamp field
@@ -155,11 +164,13 @@ public abstract class ZendeskNormalServices implements ZendeskService
                         }
 
                         if (checkedTime > endTime) {
+                            logger.info("Lets break since checked time > endTime {}", endTime);
                             break;
                         }
                     }
 
                     if (task.getDedup()) {
+//                        logger.info("Do we dedup??");
                         final String recordID = recordJsonNode.get(ZendeskConstants.Field.ID).asText();
 
                         // add success -> no duplicate
@@ -170,6 +181,7 @@ public abstract class ZendeskNormalServices implements ZendeskService
 
                     pool.submit(() -> fetchSubResourceAndAddToImporter(recordJsonNode, task, recordImporter));
                     recordCount++;
+//                    logger.info("Record count {}", recordCount);
                     if (Exec.isPreview()) {
                         return;
                     }
@@ -184,6 +196,8 @@ public abstract class ZendeskNormalServices implements ZendeskService
                         : apiEndTime;
 
                 if (numberOfRecords < ZendeskConstants.Misc.MAXIMUM_RECORDS_INCREMENTAL || startTime > endTime) {
+                    logger.info("Break because number of Records < MAXIMUM RECORDS {}", numberOfRecords);
+                    logger.info("or start time {}> end Time {}", startTime, endTime);
                     break;
                 }
             }
@@ -239,6 +253,7 @@ public abstract class ZendeskNormalServices implements ZendeskService
 
     private void fetchSubResourceAndAddToImporter(final JsonNode jsonNode, final ZendeskInputPlugin.PluginTask task, final RecordImporter recordImporter)
     {
+//        logger.info("Fetch Sub Resource {}", jsonNode.get("id"));
         task.getIncludes().forEach(include -> {
             final String relatedObjectName = include.trim();
 
@@ -254,13 +269,14 @@ public abstract class ZendeskNormalServices implements ZendeskService
                 }
             }
             catch (final ConfigException e) {
+                logger.info("Some exception for id {}", jsonNode.get("id"));
                 // Sometimes we get 404 when having invalid endpoint, so ignore when we get 404 InvalidEndpoint
                 if (!(e.getCause() instanceof ZendeskException && ((ZendeskException) e.getCause()).getStatusCode() == HttpStatus.SC_NOT_FOUND)) {
                     throw e;
                 }
             }
         });
-
+//        logger.info("Add record {}", jsonNode.get("id"));
         recordImporter.addRecord(jsonNode);
     }
 
@@ -287,6 +303,7 @@ public abstract class ZendeskNormalServices implements ZendeskService
     private void importDataForNonIncremental(final ZendeskInputPlugin.PluginTask task,  final int taskIndex, RecordImporter recordImporter, final boolean getStartDate)
     {
         long startTime = 0;
+        long endTime = 0;
 
         if (Target.SATISFACTION_RATINGS.equals(task.getTarget())){
             if (getStartDate && task.getStartTime().isPresent()) {
@@ -294,10 +311,16 @@ public abstract class ZendeskNormalServices implements ZendeskService
             }
             logger.info("Start time = {}", startTime);
 
+            if (task.getEndTime().isPresent()) {
+                endTime = ZendeskDateUtils.isoToEpochSecond(task.getEndTime().get());
+            }
+
+            logger.info("End time = {}", endTime);
+
             int i = 1;
             final Set<String> knownIds = ConcurrentHashMap.newKeySet();
             while (true) {
-                final JsonNode result = getDataFromPath("", taskIndex +i, false, startTime);
+                final JsonNode result = getDataFromPath("", taskIndex +i, false, startTime, endTime);
                 final Iterator<JsonNode> iterator = ZendeskUtils.getListRecords(result, task.getTarget().getJsonName());
                 while (iterator.hasNext()) {
                     final JsonNode recordJsonNode = iterator.next();

data/src/main/java/org/embulk/input/zendesk/services/ZendeskSupportAPIService.java

@@ -18,11 +18,17 @@ public class ZendeskSupportAPIService extends ZendeskNormalServices
         return !(task.getTarget().equals(Target.TICKET_FORMS)
                 || task.getTarget().equals(Target.SLA_POLICIES)
                 || task.getTarget().equals(Target.TICKET_FIELDS)
-                || task.getTarget().equals(Target.SATISFACTION_RATINGS));
+                || task.getTarget().equals(Target.SATISFACTION_RATINGS)
+                || task.getTarget().equals(Target.GROUPS));
     }
 
     @Override
-    protected String buildURI(final int page, long startTime)
+    protected String buildURI(final int page, long startTime){
+        return buildURI(page, startTime);
+    }
+
+    @Override
+    protected String buildURI(final int page, long startTime, long endTime)
     {
         final URIBuilder uriBuilder = ZendeskUtils.getURIBuilder(task.getLoginUrl()).setPath(buildPath());
 
@@ -34,11 +40,18 @@ public class ZendeskSupportAPIService extends ZendeskNormalServices
         }
         else {
             if (Target.SATISFACTION_RATINGS.equals(task.getTarget())){
-                uriBuilder.setParameter(ZendeskConstants.Field.START_TIME, String.valueOf(startTime))
-                        .setParameter("sort_by", "id")
-                        .setParameter("per_page", String.valueOf(100))
-                        .setParameter("page", String.valueOf(page));
-
+                if (endTime>0){
+                    uriBuilder.setParameter(ZendeskConstants.Field.START_TIME, String.valueOf(startTime))
+                            .setParameter(ZendeskConstants.Field.END_TIME, String.valueOf(endTime))
+                            .setParameter("sort_by", "id")
+                            .setParameter("per_page", String.valueOf(100))
+                            .setParameter("page", String.valueOf(page));
+                }else{
+                    uriBuilder.setParameter(ZendeskConstants.Field.START_TIME, String.valueOf(startTime))
+                            .setParameter("sort_by", "id")
+                            .setParameter("per_page", String.valueOf(100))
+                            .setParameter("page", String.valueOf(page));
+                }
             }
             else{
                 uriBuilder.setParameter("sort_by", "id")

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-zendesk-all
 version: !ruby/object:Gem::Version
-  version: 0.3.9
+  version: 0.4.5
 platform: ruby
 authors:
 - hieu.duong
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-10-31 00:00:00.000000000 Z
+date: 2021-05-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -138,9 +138,9 @@ files:
 - src/test/resources/data/util_page.json
 - classpath/commons-logging-1.2.jar
 - classpath/httpcore-4.4.10.jar
+- classpath/embulk-input-zendesk-all-0.4.5.jar
 - classpath/httpclient-4.5.6.jar
 - classpath/commons-codec-1.10.jar
-- classpath/embulk-input-zendesk-all-0.3.9.jar
 homepage: https://github.com/treasure-data/embulk-input-zendesk
 licenses:
 - MIT