embulk-input-zendesk-all 0.3.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +12 -0
- data/.travis.yml +5 -0
- data/CHANGELOG.md +126 -0
- data/LICENSE.txt +21 -0
- data/README.md +91 -0
- data/build.gradle +123 -0
- data/config/checkstyle/checkstyle.xml +128 -0
- data/config/checkstyle/default.xml +108 -0
- data/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +5 -0
- data/gradlew +172 -0
- data/gradlew.bat +84 -0
- data/lib/embulk/guess/zendesk.rb +21 -0
- data/lib/embulk/input/zendesk.rb +3 -0
- data/src/main/java/org/embulk/input/zendesk/RecordImporter.java +134 -0
- data/src/main/java/org/embulk/input/zendesk/ZendeskInputPlugin.java +513 -0
- data/src/main/java/org/embulk/input/zendesk/clients/ZendeskRestClient.java +291 -0
- data/src/main/java/org/embulk/input/zendesk/models/AuthenticationMethod.java +23 -0
- data/src/main/java/org/embulk/input/zendesk/models/Target.java +47 -0
- data/src/main/java/org/embulk/input/zendesk/models/ZendeskException.java +25 -0
- data/src/main/java/org/embulk/input/zendesk/services/ZendeskCustomObjectService.java +110 -0
- data/src/main/java/org/embulk/input/zendesk/services/ZendeskNPSService.java +30 -0
- data/src/main/java/org/embulk/input/zendesk/services/ZendeskNormalServices.java +347 -0
- data/src/main/java/org/embulk/input/zendesk/services/ZendeskService.java +14 -0
- data/src/main/java/org/embulk/input/zendesk/services/ZendeskSupportAPIService.java +63 -0
- data/src/main/java/org/embulk/input/zendesk/services/ZendeskUserEventService.java +158 -0
- data/src/main/java/org/embulk/input/zendesk/stream/PagingSpliterator.java +40 -0
- data/src/main/java/org/embulk/input/zendesk/stream/paginator/sunshine/CustomObjectSpliterator.java +42 -0
- data/src/main/java/org/embulk/input/zendesk/stream/paginator/sunshine/SunshineSpliterator.java +66 -0
- data/src/main/java/org/embulk/input/zendesk/stream/paginator/sunshine/UserEventSpliterator.java +35 -0
- data/src/main/java/org/embulk/input/zendesk/stream/paginator/support/OrganizationSpliterator.java +13 -0
- data/src/main/java/org/embulk/input/zendesk/stream/paginator/support/SupportSpliterator.java +44 -0
- data/src/main/java/org/embulk/input/zendesk/stream/paginator/support/UserSpliterator.java +13 -0
- data/src/main/java/org/embulk/input/zendesk/utils/ZendeskConstants.java +72 -0
- data/src/main/java/org/embulk/input/zendesk/utils/ZendeskDateUtils.java +68 -0
- data/src/main/java/org/embulk/input/zendesk/utils/ZendeskUtils.java +92 -0
- data/src/test/java/org/embulk/input/zendesk/TestRecordImporter.java +114 -0
- data/src/test/java/org/embulk/input/zendesk/TestZendeskInputPlugin.java +402 -0
- data/src/test/java/org/embulk/input/zendesk/clients/TestZendeskRestClient.java +337 -0
- data/src/test/java/org/embulk/input/zendesk/services/TestZendeskCustomObjectService.java +161 -0
- data/src/test/java/org/embulk/input/zendesk/services/TestZendeskNPSService.java +56 -0
- data/src/test/java/org/embulk/input/zendesk/services/TestZendeskNormalService.java +261 -0
- data/src/test/java/org/embulk/input/zendesk/services/TestZendeskSupportAPIService.java +130 -0
- data/src/test/java/org/embulk/input/zendesk/services/TestZendeskUserEventService.java +158 -0
- data/src/test/java/org/embulk/input/zendesk/utils/TestZendeskDateUtils.java +87 -0
- data/src/test/java/org/embulk/input/zendesk/utils/TestZendeskUtil.java +22 -0
- data/src/test/java/org/embulk/input/zendesk/utils/ZendeskPluginTestRuntime.java +133 -0
- data/src/test/java/org/embulk/input/zendesk/utils/ZendeskTestHelper.java +92 -0
- data/src/test/resources/config/base.yml +14 -0
- data/src/test/resources/config/base_validator.yml +48 -0
- data/src/test/resources/config/incremental.yml +54 -0
- data/src/test/resources/config/non-incremental.yml +39 -0
- data/src/test/resources/config/nps.yml +31 -0
- data/src/test/resources/config/object_records.yml +24 -0
- data/src/test/resources/config/relationship_records.yml +23 -0
- data/src/test/resources/config/user_events.yml +29 -0
- data/src/test/resources/config/util.yml +18 -0
- data/src/test/resources/data/client.json +293 -0
- data/src/test/resources/data/duplicate_user.json +0 -0
- data/src/test/resources/data/empty_result.json +7 -0
- data/src/test/resources/data/error_data.json +187 -0
- data/src/test/resources/data/expected/ticket_column.json +148 -0
- data/src/test/resources/data/expected/ticket_column_with_related_objects.json +152 -0
- data/src/test/resources/data/expected/ticket_fields_column.json +92 -0
- data/src/test/resources/data/expected/ticket_metrics_column.json +98 -0
- data/src/test/resources/data/expected/user_events_column.json +40 -0
- data/src/test/resources/data/object_records.json +30 -0
- data/src/test/resources/data/organization.json +39 -0
- data/src/test/resources/data/relationship_records.json +57 -0
- data/src/test/resources/data/scores.json +21 -0
- data/src/test/resources/data/scores_share_same_time_with_next_page.json +35 -0
- data/src/test/resources/data/scores_share_same_time_without_next_page.json +35 -0
- data/src/test/resources/data/simple_organization.json +23 -0
- data/src/test/resources/data/simple_user.json +50 -0
- data/src/test/resources/data/simple_user_event.json +19 -0
- data/src/test/resources/data/ticket_events_share_same_time_with_next_page.json +279 -0
- data/src/test/resources/data/ticket_events_share_same_time_without_next_page.json +279 -0
- data/src/test/resources/data/ticket_events_updated_by_system_records.json +279 -0
- data/src/test/resources/data/ticket_fields.json +225 -0
- data/src/test/resources/data/ticket_metrics.json +397 -0
- data/src/test/resources/data/ticket_share_same_time_with_next_page.json +232 -0
- data/src/test/resources/data/ticket_share_same_time_without_next_page.json +232 -0
- data/src/test/resources/data/ticket_with_related_objects.json +67 -0
- data/src/test/resources/data/ticket_with_updated_by_system_records.json +187 -0
- data/src/test/resources/data/tickets.json +232 -0
- data/src/test/resources/data/tickets_continue.json +52 -0
- data/src/test/resources/data/user_event.json +19 -0
- data/src/test/resources/data/user_event_contain_latter_create_at.json +19 -0
- data/src/test/resources/data/user_event_multiple.json +33 -0
- data/src/test/resources/data/util.json +19 -0
- data/src/test/resources/data/util_page.json +227 -0
- metadata +168 -0
@@ -0,0 +1,30 @@
|
|
1
|
+
package org.embulk.input.zendesk.services;
|
2
|
+
|
3
|
+
import org.embulk.input.zendesk.ZendeskInputPlugin;
|
4
|
+
import org.embulk.input.zendesk.utils.ZendeskConstants;
|
5
|
+
import org.embulk.input.zendesk.utils.ZendeskUtils;
|
6
|
+
|
7
|
+
public class ZendeskNPSService extends ZendeskNormalServices
|
8
|
+
{
|
9
|
+
public ZendeskNPSService(final ZendeskInputPlugin.PluginTask task)
|
10
|
+
{
|
11
|
+
super(task);
|
12
|
+
}
|
13
|
+
|
14
|
+
public boolean isSupportIncremental()
|
15
|
+
{
|
16
|
+
return true;
|
17
|
+
}
|
18
|
+
|
19
|
+
@Override
|
20
|
+
protected String buildURI(final int page, final long startTime)
|
21
|
+
{
|
22
|
+
return ZendeskUtils.getURIBuilder(task.getLoginUrl())
|
23
|
+
.setPath(ZendeskConstants.Url.API_NPS_INCREMENTAL
|
24
|
+
+ "/"
|
25
|
+
+ task.getTarget().getJsonName()
|
26
|
+
+ ".json")
|
27
|
+
.setParameter(ZendeskConstants.Field.START_TIME, String.valueOf(startTime))
|
28
|
+
.toString();
|
29
|
+
}
|
30
|
+
}
|
@@ -0,0 +1,347 @@
|
|
1
|
+
package org.embulk.input.zendesk.services;
|
2
|
+
|
3
|
+
import com.fasterxml.jackson.databind.JsonNode;
|
4
|
+
import com.fasterxml.jackson.databind.node.ObjectNode;
|
5
|
+
import com.google.common.annotations.VisibleForTesting;
|
6
|
+
import com.google.common.base.Throwables;
|
7
|
+
import org.apache.http.HttpStatus;
|
8
|
+
import org.apache.http.client.utils.URIBuilder;
|
9
|
+
import org.embulk.config.ConfigException;
|
10
|
+
import org.embulk.config.TaskReport;
|
11
|
+
import org.embulk.input.zendesk.RecordImporter;
|
12
|
+
import org.embulk.input.zendesk.ZendeskInputPlugin;
|
13
|
+
import org.embulk.input.zendesk.clients.ZendeskRestClient;
|
14
|
+
import org.embulk.input.zendesk.models.Target;
|
15
|
+
import org.embulk.input.zendesk.models.ZendeskException;
|
16
|
+
import org.embulk.input.zendesk.utils.ZendeskConstants;
|
17
|
+
import org.embulk.input.zendesk.utils.ZendeskDateUtils;
|
18
|
+
import org.embulk.input.zendesk.utils.ZendeskUtils;
|
19
|
+
import org.embulk.spi.Exec;
|
20
|
+
import org.slf4j.Logger;
|
21
|
+
|
22
|
+
import java.time.Instant;
|
23
|
+
|
24
|
+
import java.util.Iterator;
|
25
|
+
import java.util.Set;
|
26
|
+
import java.util.concurrent.ConcurrentHashMap;
|
27
|
+
import java.util.concurrent.LinkedBlockingQueue;
|
28
|
+
import java.util.concurrent.ThreadPoolExecutor;
|
29
|
+
import java.util.concurrent.TimeUnit;
|
30
|
+
|
31
|
+
public abstract class ZendeskNormalServices implements ZendeskService
|
32
|
+
{
|
33
|
+
private static final Logger logger = Exec.getLogger(ZendeskNormalServices.class);
|
34
|
+
|
35
|
+
protected ZendeskInputPlugin.PluginTask task;
|
36
|
+
|
37
|
+
private ZendeskRestClient zendeskRestClient;
|
38
|
+
|
39
|
+
protected ZendeskNormalServices(final ZendeskInputPlugin.PluginTask task)
|
40
|
+
{
|
41
|
+
this.task = task;
|
42
|
+
}
|
43
|
+
|
44
|
+
public TaskReport addRecordToImporter(final int taskIndex, final RecordImporter recordImporter)
|
45
|
+
{
|
46
|
+
TaskReport taskReport = Exec.newTaskReport();
|
47
|
+
|
48
|
+
if (isSupportIncremental()) {
|
49
|
+
importDataForIncremental(task, recordImporter, taskReport);
|
50
|
+
}
|
51
|
+
else {
|
52
|
+
if(task.getTarget().equals(Target.SATISFACTION_RATINGS)){
|
53
|
+
importDataForNonIncremental(task, taskIndex, recordImporter, true);
|
54
|
+
}
|
55
|
+
else{
|
56
|
+
importDataForNonIncremental(task, taskIndex, recordImporter, false);
|
57
|
+
}
|
58
|
+
}
|
59
|
+
|
60
|
+
return taskReport;
|
61
|
+
}
|
62
|
+
|
63
|
+
public JsonNode getDataFromPath(String path, final int page, final boolean isPreview, final long startTime)
|
64
|
+
{
|
65
|
+
if (path.isEmpty()) {
|
66
|
+
path = buildURI(page, startTime);
|
67
|
+
}
|
68
|
+
|
69
|
+
final String response = getZendeskRestClient().doGet(path, task, isPreview);
|
70
|
+
return ZendeskUtils.parseJsonObject(response);
|
71
|
+
}
|
72
|
+
|
73
|
+
protected abstract String buildURI(int page, long startTime);
|
74
|
+
|
75
|
+
@VisibleForTesting
|
76
|
+
protected ZendeskRestClient getZendeskRestClient()
|
77
|
+
{
|
78
|
+
if (zendeskRestClient == null) {
|
79
|
+
zendeskRestClient = new ZendeskRestClient();
|
80
|
+
}
|
81
|
+
return zendeskRestClient;
|
82
|
+
}
|
83
|
+
|
84
|
+
private void importDataForIncremental(final ZendeskInputPlugin.PluginTask task, final RecordImporter recordImporter, final TaskReport taskReport)
|
85
|
+
{
|
86
|
+
long initStartTime = 0;
|
87
|
+
long startTime = 0;
|
88
|
+
long endTime = Long.MAX_VALUE;
|
89
|
+
|
90
|
+
if (task.getStartTime().isPresent()) {
|
91
|
+
startTime = ZendeskDateUtils.getStartTime(task.getStartTime().get());
|
92
|
+
initStartTime = startTime;
|
93
|
+
}
|
94
|
+
|
95
|
+
if (task.getEndTime().isPresent()) {
|
96
|
+
endTime = ZendeskDateUtils.isoToEpochSecond(task.getEndTime().get());
|
97
|
+
}
|
98
|
+
|
99
|
+
// For incremental target, we will run in one task but split in multiple threads inside for data deduplication.
|
100
|
+
// Run with incremental will contain duplicated data.
|
101
|
+
ThreadPoolExecutor pool = null;
|
102
|
+
try {
|
103
|
+
final Set<String> knownIds = ConcurrentHashMap.newKeySet();
|
104
|
+
pool = new ThreadPoolExecutor(
|
105
|
+
10, 100, 60, TimeUnit.SECONDS, new LinkedBlockingQueue<>()
|
106
|
+
);
|
107
|
+
|
108
|
+
long apiEndTime = 0;
|
109
|
+
int afterStartTimeIndex = 11;
|
110
|
+
while (true) {
|
111
|
+
int recordCount = 0;
|
112
|
+
|
113
|
+
// Page argument isn't used in incremental API so we just set it to 0
|
114
|
+
final JsonNode result = getDataFromPath("", 0, false, startTime);
|
115
|
+
final Iterator<JsonNode> iterator = ZendeskUtils.getListRecords(result, task.getTarget().getJsonName());
|
116
|
+
|
117
|
+
if (result.has(ZendeskConstants.Field.END_TIME)){
|
118
|
+
apiEndTime = result.get(ZendeskConstants.Field.END_TIME).asLong();
|
119
|
+
}
|
120
|
+
else{
|
121
|
+
if(result.has(ZendeskConstants.Field.NEXT_PAGE)){
|
122
|
+
String next_page = result.get(ZendeskConstants.Field.NEXT_PAGE).textValue();
|
123
|
+
apiEndTime = Long.parseLong(next_page.substring(next_page.indexOf(ZendeskConstants.Field.START_TIME)+afterStartTimeIndex));
|
124
|
+
}
|
125
|
+
}
|
126
|
+
logger.info("Api END Time = '{}'", apiEndTime);
|
127
|
+
int numberOfRecords = 0;
|
128
|
+
if (result.has(ZendeskConstants.Field.COUNT)) {
|
129
|
+
numberOfRecords = result.get(ZendeskConstants.Field.COUNT).asInt();
|
130
|
+
}
|
131
|
+
|
132
|
+
while (iterator.hasNext()) {
|
133
|
+
final JsonNode recordJsonNode = iterator.next();
|
134
|
+
|
135
|
+
if (isUpdatedBySystem(recordJsonNode, startTime)) {
|
136
|
+
continue;
|
137
|
+
}
|
138
|
+
|
139
|
+
// Contain some records that later than end_time. Checked and don't add.
|
140
|
+
// Because the api already sorted by updated_at or timestamp for ticket_events, we just need to break no need to check further.
|
141
|
+
if (apiEndTime > endTime) {
|
142
|
+
long checkedTime = 0;
|
143
|
+
if (recordJsonNode.has(ZendeskConstants.Field.UPDATED_AT) && !recordJsonNode.get(ZendeskConstants.Field.UPDATED_AT).isNull()) {
|
144
|
+
checkedTime = ZendeskDateUtils.isoToEpochSecond(recordJsonNode.get(ZendeskConstants.Field.UPDATED_AT).textValue());
|
145
|
+
}
|
146
|
+
|
147
|
+
// ticket events is updated by system not user's action so it only has timestamp field
|
148
|
+
if (task.getTarget().equals(Target.TICKET_EVENTS) && recordJsonNode.has("timestamp") && !recordJsonNode.get("timestamp").isNull()) {
|
149
|
+
checkedTime = recordJsonNode.get("timestamp").asLong();
|
150
|
+
}
|
151
|
+
|
152
|
+
// scores (or response) is only store rated_at time
|
153
|
+
if (task.getTarget().equals(Target.SCORES) && recordJsonNode.has("rated_at") && !recordJsonNode.get("rated_at").isNull()) {
|
154
|
+
checkedTime = ZendeskDateUtils.isoToEpochSecond(recordJsonNode.get("rated_at").textValue());
|
155
|
+
}
|
156
|
+
|
157
|
+
if (checkedTime > endTime) {
|
158
|
+
break;
|
159
|
+
}
|
160
|
+
}
|
161
|
+
|
162
|
+
if (task.getDedup()) {
|
163
|
+
final String recordID = recordJsonNode.get(ZendeskConstants.Field.ID).asText();
|
164
|
+
|
165
|
+
// add success -> no duplicate
|
166
|
+
if (!knownIds.add(recordID)) {
|
167
|
+
continue;
|
168
|
+
}
|
169
|
+
}
|
170
|
+
|
171
|
+
pool.submit(() -> fetchSubResourceAndAddToImporter(recordJsonNode, task, recordImporter));
|
172
|
+
recordCount++;
|
173
|
+
if (Exec.isPreview()) {
|
174
|
+
return;
|
175
|
+
}
|
176
|
+
}
|
177
|
+
|
178
|
+
logger.info("Fetched '{}' records from start_time '{}'", recordCount, startTime);
|
179
|
+
|
180
|
+
// https://developer.zendesk.com/rest_api/docs/support/incremental_export#pagination
|
181
|
+
// When there are more than 1000 records share the same time stamp, the count > 1000
|
182
|
+
startTime = startTime == apiEndTime
|
183
|
+
? apiEndTime + 1
|
184
|
+
: apiEndTime;
|
185
|
+
|
186
|
+
if (numberOfRecords < ZendeskConstants.Misc.MAXIMUM_RECORDS_INCREMENTAL || startTime > endTime) {
|
187
|
+
break;
|
188
|
+
}
|
189
|
+
}
|
190
|
+
|
191
|
+
if (!Exec.isPreview()) {
|
192
|
+
storeStartTimeForConfigDiff(taskReport, initStartTime, startTime);
|
193
|
+
}
|
194
|
+
}
|
195
|
+
finally {
|
196
|
+
if (pool != null) {
|
197
|
+
pool.shutdown();
|
198
|
+
try {
|
199
|
+
pool.awaitTermination(Long.MAX_VALUE, TimeUnit.MILLISECONDS);
|
200
|
+
}
|
201
|
+
catch (final InterruptedException e) {
|
202
|
+
logger.warn("Error when wait pool to finish");
|
203
|
+
throw Throwables.propagate(e);
|
204
|
+
}
|
205
|
+
}
|
206
|
+
}
|
207
|
+
}
|
208
|
+
|
209
|
+
private void storeStartTimeForConfigDiff(final TaskReport taskReport, final long initStartTime, final long resultEndTime)
|
210
|
+
{
|
211
|
+
if (task.getIncremental()) {
|
212
|
+
long nextStartTime;
|
213
|
+
long now = Instant.now().getEpochSecond();
|
214
|
+
// no record to add
|
215
|
+
if (resultEndTime == 0) {
|
216
|
+
nextStartTime = now;
|
217
|
+
}
|
218
|
+
else {
|
219
|
+
if (task.getEndTime().isPresent()) {
|
220
|
+
long endTime = ZendeskDateUtils.isoToEpochSecond(task.getEndTime().get());
|
221
|
+
nextStartTime = endTime + 1;
|
222
|
+
}
|
223
|
+
else {
|
224
|
+
// NOTE: start_time compared as "=>", not ">".
|
225
|
+
// If we will use end_time for next start_time, we got the same records that are fetched
|
226
|
+
// end_time + 1 is workaround for that
|
227
|
+
nextStartTime = resultEndTime + 1;
|
228
|
+
}
|
229
|
+
}
|
230
|
+
|
231
|
+
if (task.getEndTime().isPresent()) {
|
232
|
+
long endTime = ZendeskDateUtils.isoToEpochSecond(task.getEndTime().get());
|
233
|
+
taskReport.set(ZendeskConstants.Field.END_TIME, nextStartTime + endTime - initStartTime);
|
234
|
+
}
|
235
|
+
|
236
|
+
taskReport.set(ZendeskConstants.Field.START_TIME, nextStartTime);
|
237
|
+
}
|
238
|
+
}
|
239
|
+
|
240
|
+
private void fetchSubResourceAndAddToImporter(final JsonNode jsonNode, final ZendeskInputPlugin.PluginTask task, final RecordImporter recordImporter)
|
241
|
+
{
|
242
|
+
task.getIncludes().forEach(include -> {
|
243
|
+
final String relatedObjectName = include.trim();
|
244
|
+
|
245
|
+
final URIBuilder uriBuilder = ZendeskUtils.getURIBuilder(task.getLoginUrl())
|
246
|
+
.setPath(ZendeskConstants.Url.API
|
247
|
+
+ "/" + task.getTarget().toString()
|
248
|
+
+ "/" + jsonNode.get(ZendeskConstants.Field.ID).asText()
|
249
|
+
+ "/" + relatedObjectName + ".json");
|
250
|
+
try {
|
251
|
+
final JsonNode result = getDataFromPath(uriBuilder.toString(), 0, false, 0);
|
252
|
+
if (result != null && result.has(relatedObjectName)) {
|
253
|
+
((ObjectNode) jsonNode).set(include, result.get(relatedObjectName));
|
254
|
+
}
|
255
|
+
}
|
256
|
+
catch (final ConfigException e) {
|
257
|
+
// Sometimes we get 404 when having invalid endpoint, so ignore when we get 404 InvalidEndpoint
|
258
|
+
if (!(e.getCause() instanceof ZendeskException && ((ZendeskException) e.getCause()).getStatusCode() == HttpStatus.SC_NOT_FOUND)) {
|
259
|
+
throw e;
|
260
|
+
}
|
261
|
+
}
|
262
|
+
});
|
263
|
+
|
264
|
+
recordImporter.addRecord(jsonNode);
|
265
|
+
}
|
266
|
+
|
267
|
+
private boolean isUpdatedBySystem(final JsonNode recordJsonNode, final long startTime)
|
268
|
+
{
|
269
|
+
/*
|
270
|
+
* https://developer.zendesk.com/rest_api/docs/core/incremental_export#excluding-system-updates
|
271
|
+
* "generated_timestamp" will be updated when Zendesk internal changing
|
272
|
+
* "updated_at" will be updated when ticket data was changed
|
273
|
+
* start_time for query parameter will be processed on Zendesk with generated_timestamp,
|
274
|
+
* but it was calculated by record' updated_at time.
|
275
|
+
* So the doesn't changed record from previous import would be appear by Zendesk internal changes.
|
276
|
+
*/
|
277
|
+
if (recordJsonNode.has(ZendeskConstants.Field.GENERATED_TIMESTAMP) && recordJsonNode.has(ZendeskConstants.Field.UPDATED_AT)) {
|
278
|
+
final String recordUpdatedAtTime = recordJsonNode.get(ZendeskConstants.Field.UPDATED_AT).asText();
|
279
|
+
final long recordUpdatedAtToEpochSecond = ZendeskDateUtils.isoToEpochSecond(recordUpdatedAtTime);
|
280
|
+
|
281
|
+
return recordUpdatedAtToEpochSecond <= startTime;
|
282
|
+
}
|
283
|
+
|
284
|
+
return false;
|
285
|
+
}
|
286
|
+
|
287
|
+
private void importDataForNonIncremental(final ZendeskInputPlugin.PluginTask task, final int taskIndex, RecordImporter recordImporter, final boolean getStartDate)
|
288
|
+
{
|
289
|
+
long startTime = 0;
|
290
|
+
|
291
|
+
if (Target.SATISFACTION_RATINGS.equals(task.getTarget())){
|
292
|
+
if (getStartDate && task.getStartTime().isPresent()) {
|
293
|
+
startTime = ZendeskDateUtils.getStartTime(task.getStartTime().get());
|
294
|
+
}
|
295
|
+
logger.info("Start time = {}", startTime);
|
296
|
+
|
297
|
+
int i = 1;
|
298
|
+
final Set<String> knownIds = ConcurrentHashMap.newKeySet();
|
299
|
+
while (true) {
|
300
|
+
final JsonNode result = getDataFromPath("", taskIndex +i, false, startTime);
|
301
|
+
final Iterator<JsonNode> iterator = ZendeskUtils.getListRecords(result, task.getTarget().getJsonName());
|
302
|
+
while (iterator.hasNext()) {
|
303
|
+
final JsonNode recordJsonNode = iterator.next();
|
304
|
+
|
305
|
+
if (task.getDedup()) {
|
306
|
+
final String recordID = recordJsonNode.get(ZendeskConstants.Field.ID).asText();
|
307
|
+
|
308
|
+
// add success -> no duplicate
|
309
|
+
if (!knownIds.add(recordID)) {
|
310
|
+
continue;
|
311
|
+
}
|
312
|
+
}
|
313
|
+
fetchSubResourceAndAddToImporter(recordJsonNode, task, recordImporter);
|
314
|
+
|
315
|
+
if (Exec.isPreview()) {
|
316
|
+
break;
|
317
|
+
}
|
318
|
+
}
|
319
|
+
|
320
|
+
i++;
|
321
|
+
if(result.has(ZendeskConstants.Field.NEXT_PAGE)){
|
322
|
+
String next_page = result.get(ZendeskConstants.Field.NEXT_PAGE).textValue();
|
323
|
+
if (next_page == null){
|
324
|
+
logger.info("No next page exists. Exiting...");
|
325
|
+
return;
|
326
|
+
}
|
327
|
+
logger.info("Next page = {}", next_page);
|
328
|
+
}
|
329
|
+
}
|
330
|
+
|
331
|
+
}
|
332
|
+
else{
|
333
|
+
|
334
|
+
// Page start from 1 => page = taskIndex + 1
|
335
|
+
final JsonNode result = getDataFromPath("", taskIndex + 1 , false, 0);
|
336
|
+
final Iterator<JsonNode> iterator = ZendeskUtils.getListRecords(result, task.getTarget().getJsonName());
|
337
|
+
while (iterator.hasNext()) {
|
338
|
+
fetchSubResourceAndAddToImporter(iterator.next(), task, recordImporter);
|
339
|
+
|
340
|
+
if (Exec.isPreview()) {
|
341
|
+
break;
|
342
|
+
}
|
343
|
+
}
|
344
|
+
}
|
345
|
+
|
346
|
+
}
|
347
|
+
}
|
@@ -0,0 +1,14 @@
|
|
1
|
+
package org.embulk.input.zendesk.services;
|
2
|
+
|
3
|
+
import com.fasterxml.jackson.databind.JsonNode;
|
4
|
+
import org.embulk.config.TaskReport;
|
5
|
+
import org.embulk.input.zendesk.RecordImporter;
|
6
|
+
|
7
|
+
public interface ZendeskService
|
8
|
+
{
|
9
|
+
boolean isSupportIncremental();
|
10
|
+
|
11
|
+
TaskReport addRecordToImporter(int taskIndex, RecordImporter recordImporter);
|
12
|
+
|
13
|
+
JsonNode getDataFromPath(String path, int page, boolean isPreview, long startTime);
|
14
|
+
}
|
@@ -0,0 +1,63 @@
|
|
1
|
+
package org.embulk.input.zendesk.services;
|
2
|
+
|
3
|
+
import org.apache.http.client.utils.URIBuilder;
|
4
|
+
import org.embulk.input.zendesk.ZendeskInputPlugin.PluginTask;
|
5
|
+
import org.embulk.input.zendesk.models.Target;
|
6
|
+
import org.embulk.input.zendesk.utils.ZendeskConstants;
|
7
|
+
import org.embulk.input.zendesk.utils.ZendeskUtils;
|
8
|
+
|
9
|
+
public class ZendeskSupportAPIService extends ZendeskNormalServices
|
10
|
+
{
|
11
|
+
public ZendeskSupportAPIService(final PluginTask task)
|
12
|
+
{
|
13
|
+
super(task);
|
14
|
+
}
|
15
|
+
|
16
|
+
public boolean isSupportIncremental()
|
17
|
+
{
|
18
|
+
return !(task.getTarget().equals(Target.TICKET_FORMS)
|
19
|
+
|| task.getTarget().equals(Target.TICKET_FIELDS)
|
20
|
+
|| task.getTarget().equals(Target.SATISFACTION_RATINGS));
|
21
|
+
}
|
22
|
+
|
23
|
+
@Override
|
24
|
+
protected String buildURI(final int page, long startTime)
|
25
|
+
{
|
26
|
+
final URIBuilder uriBuilder = ZendeskUtils.getURIBuilder(task.getLoginUrl()).setPath(buildPath());
|
27
|
+
|
28
|
+
if (isSupportIncremental()) {
|
29
|
+
uriBuilder.setParameter(ZendeskConstants.Field.START_TIME, String.valueOf(startTime));
|
30
|
+
if (Target.TICKET_METRICS.equals(task.getTarget())) {
|
31
|
+
uriBuilder.setParameter("include", "metric_sets");
|
32
|
+
}
|
33
|
+
}
|
34
|
+
else {
|
35
|
+
if (Target.SATISFACTION_RATINGS.equals(task.getTarget())){
|
36
|
+
uriBuilder.setParameter(ZendeskConstants.Field.START_TIME, String.valueOf(startTime))
|
37
|
+
.setParameter("sort_by", "id")
|
38
|
+
.setParameter("per_page", String.valueOf(100))
|
39
|
+
.setParameter("page", String.valueOf(page));
|
40
|
+
|
41
|
+
}
|
42
|
+
else{
|
43
|
+
uriBuilder.setParameter("sort_by", "id")
|
44
|
+
.setParameter("per_page", String.valueOf(100))
|
45
|
+
.setParameter("page", String.valueOf(page));
|
46
|
+
}
|
47
|
+
}
|
48
|
+
|
49
|
+
return uriBuilder.toString();
|
50
|
+
}
|
51
|
+
|
52
|
+
private String buildPath()
|
53
|
+
{
|
54
|
+
return (isSupportIncremental() && !(Target.SATISFACTION_RATINGS.equals(task.getTarget()))
|
55
|
+
? ZendeskConstants.Url.API_INCREMENTAL
|
56
|
+
: ZendeskConstants.Url.API) +
|
57
|
+
"/" +
|
58
|
+
(Target.TICKET_METRICS.equals(task.getTarget())
|
59
|
+
? Target.TICKETS.toString()
|
60
|
+
: task.getTarget().toString())
|
61
|
+
+ ".json";
|
62
|
+
}
|
63
|
+
}
|