embulk-input-zendesk-all 0.3.9 → 0.4.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +2 -0
- data/build.gradle +1 -1
- data/gradlew +1 -1
- data/src/main/java/org/embulk/input/zendesk/ZendeskInputPlugin.java +2 -0
- data/src/main/java/org/embulk/input/zendesk/clients/ZendeskRestClient.java +2 -0
- data/src/main/java/org/embulk/input/zendesk/models/Target.java +1 -1
- data/src/main/java/org/embulk/input/zendesk/services/ZendeskNPSService.java +5 -0
- data/src/main/java/org/embulk/input/zendesk/services/ZendeskNormalServices.java +30 -7
- data/src/main/java/org/embulk/input/zendesk/services/ZendeskSupportAPIService.java +20 -7
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0bfa12fca52df30c6d53c787b0c1fbcc3607d83e
|
4
|
+
data.tar.gz: 9366de09a25cc4a53db47131de1381edd136bcda
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ac55e4011b761639f55d3fcbd1fd2f9f3d54556114d164cbd4bf13eb51fa5ad5dd3d0ed2d8840303274c6e6348dc835a6026fac4d6ed0d5db4d1f321cfec6bf8
|
7
|
+
data.tar.gz: 935c10be956e9ef20c5cbf761f3f90e800d37f0bb8e844813926b03aab90fd9bedb8ddcf3b8f71963ecef2b7342ee8a0df4d2de590eb6a36fa7fdabca09379a8
|
data/README.md
CHANGED
data/build.gradle
CHANGED
data/gradlew
CHANGED
@@ -374,6 +374,8 @@ public class ZendeskInputPlugin implements InputPlugin
|
|
374
374
|
return new ZendeskCustomObjectService(task);
|
375
375
|
case USER_EVENTS:
|
376
376
|
return new ZendeskUserEventService(task);
|
377
|
+
case GROUPS:
|
378
|
+
return new ZendeskSupportAPIService(task);
|
377
379
|
default:
|
378
380
|
throw new ConfigException("Unsupported " + task.getTarget() + ", supported values: '" + Arrays.toString(Target.values()) + "'");
|
379
381
|
}
|
@@ -235,6 +235,8 @@ public class ZendeskRestClient
|
|
235
235
|
final HttpGet request = new HttpGet(url);
|
236
236
|
final ImmutableMap<String, String> headers = buildAuthHeader(task);
|
237
237
|
headers.forEach(request::setHeader);
|
238
|
+
|
239
|
+
// headers.values().forEach(logger::info);
|
238
240
|
return request;
|
239
241
|
}
|
240
242
|
|
@@ -14,7 +14,7 @@ public enum Target
|
|
14
14
|
TICKET_METRICS("metric_sets"), TICKET_FIELDS("ticket_fields"), TICKET_FORMS("ticket_forms"),
|
15
15
|
TICKET_METRIC_EVENTS("ticket_metric_events"), SATISFACTION_RATINGS("satisfaction_ratings"),
|
16
16
|
SLA_POLICIES("sla_policies"), SCORES("responses"), OBJECT_RECORDS("data"),
|
17
|
-
RECIPIENTS("recipients"),
|
17
|
+
RECIPIENTS("recipients"), RELATIONSHIP_RECORDS("data"), USER_EVENTS("data"), GROUPS("groups");
|
18
18
|
|
19
19
|
String jsonName;
|
20
20
|
|
@@ -15,6 +15,11 @@ public class ZendeskNPSService extends ZendeskNormalServices
|
|
15
15
|
{
|
16
16
|
return true;
|
17
17
|
}
|
18
|
+
@Override
|
19
|
+
protected String buildURI(final int page, final long startTime, final long endTime)
|
20
|
+
{
|
21
|
+
return buildURI(page, startTime);
|
22
|
+
}
|
18
23
|
|
19
24
|
@Override
|
20
25
|
protected String buildURI(final int page, final long startTime)
|
@@ -59,11 +59,15 @@ public abstract class ZendeskNormalServices implements ZendeskService
|
|
59
59
|
|
60
60
|
return taskReport;
|
61
61
|
}
|
62
|
-
|
63
62
|
public JsonNode getDataFromPath(String path, final int page, final boolean isPreview, final long startTime)
|
63
|
+
{
|
64
|
+
return getDataFromPath(path, page,isPreview, startTime, 0);
|
65
|
+
}
|
66
|
+
|
67
|
+
public JsonNode getDataFromPath(String path, final int page, final boolean isPreview, final long startTime, final long endTime)
|
64
68
|
{
|
65
69
|
if (path.isEmpty()) {
|
66
|
-
path = buildURI(page, startTime);
|
70
|
+
path = buildURI(page, startTime, endTime);
|
67
71
|
}
|
68
72
|
|
69
73
|
final String response = getZendeskRestClient().doGet(path, task, isPreview);
|
@@ -71,6 +75,8 @@ public abstract class ZendeskNormalServices implements ZendeskService
|
|
71
75
|
}
|
72
76
|
|
73
77
|
protected abstract String buildURI(int page, long startTime);
|
78
|
+
protected abstract String buildURI(int page, long startTime, long endTime);
|
79
|
+
|
74
80
|
|
75
81
|
@VisibleForTesting
|
76
82
|
protected ZendeskRestClient getZendeskRestClient()
|
@@ -127,14 +133,16 @@ public abstract class ZendeskNormalServices implements ZendeskService
|
|
127
133
|
int numberOfRecords = 0;
|
128
134
|
if (result.has(ZendeskConstants.Field.COUNT)) {
|
129
135
|
numberOfRecords = result.get(ZendeskConstants.Field.COUNT).asInt();
|
136
|
+
logger.info("Number of Records {}", numberOfRecords);
|
130
137
|
}
|
131
138
|
|
132
139
|
while (iterator.hasNext()) {
|
133
140
|
final JsonNode recordJsonNode = iterator.next();
|
134
141
|
|
135
|
-
if (isUpdatedBySystem(recordJsonNode, startTime)) {
|
136
|
-
|
137
|
-
|
142
|
+
// if (isUpdatedBySystem(recordJsonNode, startTime)) {
|
143
|
+
// logger.info("This record is update by system? {} IGNORED!! ", recordJsonNode.get("id"));
|
144
|
+
// continue;
|
145
|
+
// }
|
138
146
|
|
139
147
|
// Contain some records that later than end_time. Checked and don't add.
|
140
148
|
// Because the api already sorted by updated_at or timestamp for ticket_events, we just need to break no need to check further.
|
@@ -142,6 +150,7 @@ public abstract class ZendeskNormalServices implements ZendeskService
|
|
142
150
|
long checkedTime = 0;
|
143
151
|
if (recordJsonNode.has(ZendeskConstants.Field.UPDATED_AT) && !recordJsonNode.get(ZendeskConstants.Field.UPDATED_AT).isNull()) {
|
144
152
|
checkedTime = ZendeskDateUtils.isoToEpochSecond(recordJsonNode.get(ZendeskConstants.Field.UPDATED_AT).textValue());
|
153
|
+
// logger.info("Checked time {}", checkedTime);
|
145
154
|
}
|
146
155
|
|
147
156
|
// ticket events is updated by system not user's action so it only has timestamp field
|
@@ -155,11 +164,13 @@ public abstract class ZendeskNormalServices implements ZendeskService
|
|
155
164
|
}
|
156
165
|
|
157
166
|
if (checkedTime > endTime) {
|
167
|
+
logger.info("Lets break since checked time > endTime {}", endTime);
|
158
168
|
break;
|
159
169
|
}
|
160
170
|
}
|
161
171
|
|
162
172
|
if (task.getDedup()) {
|
173
|
+
// logger.info("Do we dedup??");
|
163
174
|
final String recordID = recordJsonNode.get(ZendeskConstants.Field.ID).asText();
|
164
175
|
|
165
176
|
// add success -> no duplicate
|
@@ -170,6 +181,7 @@ public abstract class ZendeskNormalServices implements ZendeskService
|
|
170
181
|
|
171
182
|
pool.submit(() -> fetchSubResourceAndAddToImporter(recordJsonNode, task, recordImporter));
|
172
183
|
recordCount++;
|
184
|
+
// logger.info("Record count {}", recordCount);
|
173
185
|
if (Exec.isPreview()) {
|
174
186
|
return;
|
175
187
|
}
|
@@ -184,6 +196,8 @@ public abstract class ZendeskNormalServices implements ZendeskService
|
|
184
196
|
: apiEndTime;
|
185
197
|
|
186
198
|
if (numberOfRecords < ZendeskConstants.Misc.MAXIMUM_RECORDS_INCREMENTAL || startTime > endTime) {
|
199
|
+
logger.info("Break because number of Records < MAXIMUM RECORDS {}", numberOfRecords);
|
200
|
+
logger.info("or start time {}> end Time {}", startTime, endTime);
|
187
201
|
break;
|
188
202
|
}
|
189
203
|
}
|
@@ -239,6 +253,7 @@ public abstract class ZendeskNormalServices implements ZendeskService
|
|
239
253
|
|
240
254
|
private void fetchSubResourceAndAddToImporter(final JsonNode jsonNode, final ZendeskInputPlugin.PluginTask task, final RecordImporter recordImporter)
|
241
255
|
{
|
256
|
+
// logger.info("Fetch Sub Resource {}", jsonNode.get("id"));
|
242
257
|
task.getIncludes().forEach(include -> {
|
243
258
|
final String relatedObjectName = include.trim();
|
244
259
|
|
@@ -254,13 +269,14 @@ public abstract class ZendeskNormalServices implements ZendeskService
|
|
254
269
|
}
|
255
270
|
}
|
256
271
|
catch (final ConfigException e) {
|
272
|
+
logger.info("Some exception for id {}", jsonNode.get("id"));
|
257
273
|
// Sometimes we get 404 when having invalid endpoint, so ignore when we get 404 InvalidEndpoint
|
258
274
|
if (!(e.getCause() instanceof ZendeskException && ((ZendeskException) e.getCause()).getStatusCode() == HttpStatus.SC_NOT_FOUND)) {
|
259
275
|
throw e;
|
260
276
|
}
|
261
277
|
}
|
262
278
|
});
|
263
|
-
|
279
|
+
// logger.info("Add record {}", jsonNode.get("id"));
|
264
280
|
recordImporter.addRecord(jsonNode);
|
265
281
|
}
|
266
282
|
|
@@ -287,6 +303,7 @@ public abstract class ZendeskNormalServices implements ZendeskService
|
|
287
303
|
private void importDataForNonIncremental(final ZendeskInputPlugin.PluginTask task, final int taskIndex, RecordImporter recordImporter, final boolean getStartDate)
|
288
304
|
{
|
289
305
|
long startTime = 0;
|
306
|
+
long endTime = 0;
|
290
307
|
|
291
308
|
if (Target.SATISFACTION_RATINGS.equals(task.getTarget())){
|
292
309
|
if (getStartDate && task.getStartTime().isPresent()) {
|
@@ -294,10 +311,16 @@ public abstract class ZendeskNormalServices implements ZendeskService
|
|
294
311
|
}
|
295
312
|
logger.info("Start time = {}", startTime);
|
296
313
|
|
314
|
+
if (task.getEndTime().isPresent()) {
|
315
|
+
endTime = ZendeskDateUtils.isoToEpochSecond(task.getEndTime().get());
|
316
|
+
}
|
317
|
+
|
318
|
+
logger.info("End time = {}", endTime);
|
319
|
+
|
297
320
|
int i = 1;
|
298
321
|
final Set<String> knownIds = ConcurrentHashMap.newKeySet();
|
299
322
|
while (true) {
|
300
|
-
final JsonNode result = getDataFromPath("", taskIndex +i, false, startTime);
|
323
|
+
final JsonNode result = getDataFromPath("", taskIndex +i, false, startTime, endTime);
|
301
324
|
final Iterator<JsonNode> iterator = ZendeskUtils.getListRecords(result, task.getTarget().getJsonName());
|
302
325
|
while (iterator.hasNext()) {
|
303
326
|
final JsonNode recordJsonNode = iterator.next();
|
@@ -18,11 +18,17 @@ public class ZendeskSupportAPIService extends ZendeskNormalServices
|
|
18
18
|
return !(task.getTarget().equals(Target.TICKET_FORMS)
|
19
19
|
|| task.getTarget().equals(Target.SLA_POLICIES)
|
20
20
|
|| task.getTarget().equals(Target.TICKET_FIELDS)
|
21
|
-
|| task.getTarget().equals(Target.SATISFACTION_RATINGS)
|
21
|
+
|| task.getTarget().equals(Target.SATISFACTION_RATINGS)
|
22
|
+
|| task.getTarget().equals(Target.GROUPS));
|
22
23
|
}
|
23
24
|
|
24
25
|
@Override
|
25
|
-
protected String buildURI(final int page, long startTime)
|
26
|
+
protected String buildURI(final int page, long startTime){
|
27
|
+
return buildURI(page, startTime);
|
28
|
+
}
|
29
|
+
|
30
|
+
@Override
|
31
|
+
protected String buildURI(final int page, long startTime, long endTime)
|
26
32
|
{
|
27
33
|
final URIBuilder uriBuilder = ZendeskUtils.getURIBuilder(task.getLoginUrl()).setPath(buildPath());
|
28
34
|
|
@@ -34,11 +40,18 @@ public class ZendeskSupportAPIService extends ZendeskNormalServices
|
|
34
40
|
}
|
35
41
|
else {
|
36
42
|
if (Target.SATISFACTION_RATINGS.equals(task.getTarget())){
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
43
|
+
if (endTime>0){
|
44
|
+
uriBuilder.setParameter(ZendeskConstants.Field.START_TIME, String.valueOf(startTime))
|
45
|
+
.setParameter(ZendeskConstants.Field.END_TIME, String.valueOf(endTime))
|
46
|
+
.setParameter("sort_by", "id")
|
47
|
+
.setParameter("per_page", String.valueOf(100))
|
48
|
+
.setParameter("page", String.valueOf(page));
|
49
|
+
}else{
|
50
|
+
uriBuilder.setParameter(ZendeskConstants.Field.START_TIME, String.valueOf(startTime))
|
51
|
+
.setParameter("sort_by", "id")
|
52
|
+
.setParameter("per_page", String.valueOf(100))
|
53
|
+
.setParameter("page", String.valueOf(page));
|
54
|
+
}
|
42
55
|
}
|
43
56
|
else{
|
44
57
|
uriBuilder.setParameter("sort_by", "id")
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-zendesk-all
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- hieu.duong
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-05-05 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -138,9 +138,9 @@ files:
|
|
138
138
|
- src/test/resources/data/util_page.json
|
139
139
|
- classpath/commons-logging-1.2.jar
|
140
140
|
- classpath/httpcore-4.4.10.jar
|
141
|
+
- classpath/embulk-input-zendesk-all-0.4.5.jar
|
141
142
|
- classpath/httpclient-4.5.6.jar
|
142
143
|
- classpath/commons-codec-1.10.jar
|
143
|
-
- classpath/embulk-input-zendesk-all-0.3.9.jar
|
144
144
|
homepage: https://github.com/treasure-data/embulk-input-zendesk
|
145
145
|
licenses:
|
146
146
|
- MIT
|