embulk-input-zendesk 0.3.5 → 0.3.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/build.gradle +1 -1
- data/src/main/java/org/embulk/input/zendesk/ZendeskInputPlugin.java +53 -0
- data/src/main/java/org/embulk/input/zendesk/clients/ZendeskRestClient.java +20 -0
- data/src/main/java/org/embulk/input/zendesk/services/ZendeskNormalServices.java +57 -10
- data/src/main/java/org/embulk/input/zendesk/utils/ZendeskConstants.java +5 -5
- data/src/main/java/org/embulk/input/zendesk/utils/ZendeskDateUtils.java +10 -5
- data/src/test/java/org/embulk/input/zendesk/TestZendeskInputPlugin.java +51 -3
- data/src/test/java/org/embulk/input/zendesk/services/TestZendeskNormalService.java +73 -1
- data/src/test/java/org/embulk/input/zendesk/utils/TestZendeskDateUtils.java +1 -1
- data/src/test/java/org/embulk/input/zendesk/utils/ZendeskTestHelper.java +13 -0
- data/src/test/resources/config/nps.yml +2 -0
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 59c7819359a24e229366275644b6d543809d7c08
|
4
|
+
data.tar.gz: 818fe70947171c2c96d7951f54316f8b13a357e9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 477172f0826c449a67af48191f29796887c9c8a8b9d8e9550cbfd613b194c406f8047c80e2ab31d0cdcff70410e3f8b4feda7e94316ffbaad3db050033b90b9a
|
7
|
+
data.tar.gz: 7a24e2b1bbe4ad103bf5e4c86055e1c069b9f0ca38898b2a1bbbcafc2afa9dad6ccb25bc0d5e45136adf7f8a669c16df3b293fb5a65f32d7799b1f42989d4f9e
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,7 @@
|
|
1
|
+
## 0.3.6 - 2019-07-02
|
2
|
+
* [enhancement] Improve error message #61 [#61](https://github.com/treasure-data/embulk-input-zendesk/pull/61)
|
3
|
+
* [enhancement] Support `end_time` field for incremental #60 [#60](https://github.com/treasure-data/embulk-input-zendesk/pull/60)
|
4
|
+
|
1
5
|
## 0.3.5 - 2019-06-03
|
2
6
|
* [enhancement] Add new targets #58 [#58](https://github.com/treasure-data/embulk-input-zendesk/pull/58)
|
3
7
|
|
data/build.gradle
CHANGED
@@ -167,6 +167,7 @@ public class ZendeskInputPlugin implements InputPlugin
|
|
167
167
|
{
|
168
168
|
final PluginTask task = config.loadConfig(PluginTask.class);
|
169
169
|
validateInputTask(task);
|
170
|
+
|
170
171
|
final Schema schema = task.getColumns().toSchema();
|
171
172
|
int taskCount = 1;
|
172
173
|
|
@@ -199,6 +200,21 @@ public class ZendeskInputPlugin implements InputPlugin
|
|
199
200
|
public TaskReport run(final TaskSource taskSource, final Schema schema, final int taskIndex, final PageOutput output)
|
200
201
|
{
|
201
202
|
final PluginTask task = taskSource.loadTask(PluginTask.class);
|
203
|
+
|
204
|
+
if (getZendeskService(task).isSupportIncremental() && !isValidTimeRange(task)) {
|
205
|
+
if (Exec.isPreview()) {
|
206
|
+
throw new ConfigException("Invalid End time. End time is greater than current time");
|
207
|
+
}
|
208
|
+
|
209
|
+
logger.warn("The end time, '" + task.getEndTime().get() + "', is greater than the current time. No records will be imported");
|
210
|
+
|
211
|
+
// we just need to store config_diff when incremental_mode is enable
|
212
|
+
if (task.getIncremental()) {
|
213
|
+
return buildTaskReportKeepOldStartAndEndTime(task);
|
214
|
+
}
|
215
|
+
return Exec.newTaskReport();
|
216
|
+
}
|
217
|
+
|
202
218
|
try (final PageBuilder pageBuilder = getPageBuilder(schema, output)) {
|
203
219
|
final TaskReport taskReport = getZendeskService(task).addRecordToImporter(taskIndex, getRecordImporter(schema, pageBuilder));
|
204
220
|
pageBuilder.finish();
|
@@ -212,6 +228,9 @@ public class ZendeskInputPlugin implements InputPlugin
|
|
212
228
|
config.set("columns", new ObjectMapper().createArrayNode());
|
213
229
|
final PluginTask task = config.loadConfig(PluginTask.class);
|
214
230
|
validateInputTask(task);
|
231
|
+
if (!isValidTimeRange(task)) {
|
232
|
+
throw new ConfigException("Invalid End time. End time is greater than current time");
|
233
|
+
}
|
215
234
|
return Exec.newConfigDiff().set("columns", buildColumns(task));
|
216
235
|
}
|
217
236
|
|
@@ -234,6 +253,14 @@ public class ZendeskInputPlugin implements InputPlugin
|
|
234
253
|
configDiff.set(ZendeskConstants.Field.START_TIME,
|
235
254
|
offsetDateTime.format(DateTimeFormatter.ofPattern(ZendeskConstants.Misc.RUBY_TIMESTAMP_FORMAT_INPUT)));
|
236
255
|
}
|
256
|
+
|
257
|
+
if (taskReport.has(ZendeskConstants.Field.END_TIME)) {
|
258
|
+
final OffsetDateTime offsetDateTime = OffsetDateTime.ofInstant(Instant.ofEpochSecond(
|
259
|
+
taskReport.get(JsonNode.class, ZendeskConstants.Field.END_TIME).asLong()), ZoneOffset.UTC);
|
260
|
+
|
261
|
+
configDiff.set(ZendeskConstants.Field.END_TIME,
|
262
|
+
offsetDateTime.format(DateTimeFormatter.ofPattern(ZendeskConstants.Misc.RUBY_TIMESTAMP_FORMAT_INPUT)));
|
263
|
+
}
|
237
264
|
}
|
238
265
|
return configDiff;
|
239
266
|
}
|
@@ -364,6 +391,7 @@ public class ZendeskInputPlugin implements InputPlugin
|
|
364
391
|
validateIncremental(task);
|
365
392
|
validateCustomObject(task);
|
366
393
|
validateUserEvent(task);
|
394
|
+
validateTime(task);
|
367
395
|
}
|
368
396
|
|
369
397
|
private void validateCredentials(PluginTask task)
|
@@ -439,7 +467,12 @@ public class ZendeskInputPlugin implements InputPlugin
|
|
439
467
|
if (!task.getProfileSource().isPresent()) {
|
440
468
|
throw new ConfigException("Profile Source is required for User Event Target");
|
441
469
|
}
|
470
|
+
}
|
471
|
+
}
|
442
472
|
|
473
|
+
private void validateTime(PluginTask task)
|
474
|
+
{
|
475
|
+
if (getZendeskService(task).isSupportIncremental()) {
|
443
476
|
// Can't set end_time to 0, so it should be valid
|
444
477
|
task.getEndTime().ifPresent(time -> {
|
445
478
|
if (!ZendeskDateUtils.supportedTimeFormat(task.getEndTime().get()).isPresent()) {
|
@@ -453,4 +486,24 @@ public class ZendeskInputPlugin implements InputPlugin
|
|
453
486
|
}
|
454
487
|
}
|
455
488
|
}
|
489
|
+
|
490
|
+
private boolean isValidTimeRange(PluginTask task)
|
491
|
+
{
|
492
|
+
return !task.getEndTime().isPresent() || ZendeskDateUtils.isoToEpochSecond(task.getEndTime().get()) <= Instant.now().getEpochSecond();
|
493
|
+
}
|
494
|
+
|
495
|
+
private TaskReport buildTaskReportKeepOldStartAndEndTime(PluginTask task)
|
496
|
+
{
|
497
|
+
final TaskReport taskReport = Exec.newTaskReport();
|
498
|
+
|
499
|
+
if (task.getStartTime().isPresent()) {
|
500
|
+
taskReport.set(ZendeskConstants.Field.START_TIME, ZendeskDateUtils.isoToEpochSecond(task.getStartTime().get()));
|
501
|
+
}
|
502
|
+
|
503
|
+
if (task.getEndTime().isPresent()) {
|
504
|
+
taskReport.set(ZendeskConstants.Field.END_TIME, ZendeskDateUtils.isoToEpochSecond(task.getEndTime().get()));
|
505
|
+
}
|
506
|
+
|
507
|
+
return taskReport;
|
508
|
+
}
|
456
509
|
}
|
@@ -1,6 +1,7 @@
|
|
1
1
|
package org.embulk.input.zendesk.clients;
|
2
2
|
|
3
3
|
import com.fasterxml.jackson.databind.DeserializationFeature;
|
4
|
+
import com.fasterxml.jackson.databind.JsonNode;
|
4
5
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
5
6
|
import com.google.common.annotations.VisibleForTesting;
|
6
7
|
import com.google.common.collect.ImmutableMap;
|
@@ -170,6 +171,17 @@ public class ZendeskRestClient
|
|
170
171
|
private boolean isResponseStatusToRetry(final int status, final String message, final int retryAfter, final boolean isPreview)
|
171
172
|
{
|
172
173
|
if (status == HttpStatus.SC_NOT_FOUND) {
|
174
|
+
ObjectMapper objectMapper = new ObjectMapper();
|
175
|
+
try {
|
176
|
+
JsonNode jsonNode = objectMapper.readTree(message);
|
177
|
+
if (jsonNode.has("error") && jsonNode.get("error").has("title") && jsonNode.get("error").get("title").asText().startsWith("No help desk at ")) {
|
178
|
+
throw new ConfigException("This address is not registered in Zendesk. Please check the login_url again");
|
179
|
+
}
|
180
|
+
}
|
181
|
+
catch (IOException e) {
|
182
|
+
// In case we can't parse the message, error should not be show here
|
183
|
+
}
|
184
|
+
|
173
185
|
// 404 would be returned e.g. ticket comments are empty (on fetchRelatedObjects method)
|
174
186
|
return false;
|
175
187
|
}
|
@@ -203,6 +215,14 @@ public class ZendeskRestClient
|
|
203
215
|
|
204
216
|
// Won't retry for 4xx range errors except above. Almost they should be ConfigError e.g. 403 Forbidden
|
205
217
|
if (status / 100 == 4) {
|
218
|
+
if (status == HttpStatus.SC_UNAUTHORIZED) {
|
219
|
+
throw new ConfigException("Cannot authenticate due to invalid login credentials");
|
220
|
+
}
|
221
|
+
|
222
|
+
if (status == HttpStatus.SC_FORBIDDEN) {
|
223
|
+
throw new ConfigException("You do not have access to this resource. Please contact the account owner of this help desk for further help.");
|
224
|
+
}
|
225
|
+
|
206
226
|
throw new ConfigException("Status '" + status + "', message '" + message + "'");
|
207
227
|
}
|
208
228
|
|
@@ -11,6 +11,7 @@ import org.embulk.config.TaskReport;
|
|
11
11
|
import org.embulk.input.zendesk.RecordImporter;
|
12
12
|
import org.embulk.input.zendesk.ZendeskInputPlugin;
|
13
13
|
import org.embulk.input.zendesk.clients.ZendeskRestClient;
|
14
|
+
import org.embulk.input.zendesk.models.Target;
|
14
15
|
import org.embulk.input.zendesk.models.ZendeskException;
|
15
16
|
import org.embulk.input.zendesk.utils.ZendeskConstants;
|
16
17
|
import org.embulk.input.zendesk.utils.ZendeskDateUtils;
|
@@ -77,10 +78,17 @@ public abstract class ZendeskNormalServices implements ZendeskService
|
|
77
78
|
|
78
79
|
private void importDataForIncremental(final ZendeskInputPlugin.PluginTask task, final RecordImporter recordImporter, final TaskReport taskReport)
|
79
80
|
{
|
81
|
+
long initStartTime = 0;
|
80
82
|
long startTime = 0;
|
83
|
+
long endTime = Long.MAX_VALUE;
|
81
84
|
|
82
85
|
if (task.getStartTime().isPresent()) {
|
83
86
|
startTime = ZendeskDateUtils.getStartTime(task.getStartTime().get());
|
87
|
+
initStartTime = startTime;
|
88
|
+
}
|
89
|
+
|
90
|
+
if (task.getEndTime().isPresent()) {
|
91
|
+
endTime = ZendeskDateUtils.isoToEpochSecond(task.getEndTime().get());
|
84
92
|
}
|
85
93
|
|
86
94
|
// For incremental target, we will run in one task but split in multiple threads inside for data deduplication.
|
@@ -92,12 +100,14 @@ public abstract class ZendeskNormalServices implements ZendeskService
|
|
92
100
|
10, 100, 60, TimeUnit.SECONDS, new LinkedBlockingQueue<>()
|
93
101
|
);
|
94
102
|
|
103
|
+
long apiEndTime = 0;
|
95
104
|
while (true) {
|
96
105
|
int recordCount = 0;
|
97
106
|
|
98
107
|
// Page argument isn't used in incremental API so we just set it to 0
|
99
108
|
final JsonNode result = getDataFromPath("", 0, false, startTime);
|
100
109
|
final Iterator<JsonNode> iterator = ZendeskUtils.getListRecords(result, task.getTarget().getJsonName());
|
110
|
+
apiEndTime = result.get(ZendeskConstants.Field.END_TIME).asLong();
|
101
111
|
|
102
112
|
int numberOfRecords = 0;
|
103
113
|
if (result.has(ZendeskConstants.Field.COUNT)) {
|
@@ -111,6 +121,29 @@ public abstract class ZendeskNormalServices implements ZendeskService
|
|
111
121
|
continue;
|
112
122
|
}
|
113
123
|
|
124
|
+
// Contain some records that later than end_time. Checked and don't add.
|
125
|
+
// Because the api already sorted by updated_at or timestamp for ticket_events, we just need to break no need to check further.
|
126
|
+
if (apiEndTime > endTime) {
|
127
|
+
long checkedTime = 0;
|
128
|
+
if (recordJsonNode.has(ZendeskConstants.Field.UPDATED_AT) && !recordJsonNode.get(ZendeskConstants.Field.UPDATED_AT).isNull()) {
|
129
|
+
checkedTime = ZendeskDateUtils.isoToEpochSecond(recordJsonNode.get(ZendeskConstants.Field.UPDATED_AT).textValue());
|
130
|
+
}
|
131
|
+
|
132
|
+
// ticket events is updated by system not user's action so it only has timestamp field
|
133
|
+
if (task.getTarget().equals(Target.TICKET_EVENTS) && recordJsonNode.has("timestamp") && !recordJsonNode.get("timestamp").isNull()) {
|
134
|
+
checkedTime = recordJsonNode.get("timestamp").asLong();
|
135
|
+
}
|
136
|
+
|
137
|
+
// scores (or response) is only store rated_at time
|
138
|
+
if (task.getTarget().equals(Target.SCORES) && recordJsonNode.has("rated_at") && !recordJsonNode.get("rated_at").isNull()) {
|
139
|
+
checkedTime = ZendeskDateUtils.isoToEpochSecond(recordJsonNode.get("rated_at").textValue());
|
140
|
+
}
|
141
|
+
|
142
|
+
if (checkedTime > endTime) {
|
143
|
+
break;
|
144
|
+
}
|
145
|
+
}
|
146
|
+
|
114
147
|
if (task.getDedup()) {
|
115
148
|
final String recordID = recordJsonNode.get(ZendeskConstants.Field.ID).asText();
|
116
149
|
|
@@ -131,18 +164,17 @@ public abstract class ZendeskNormalServices implements ZendeskService
|
|
131
164
|
|
132
165
|
// https://developer.zendesk.com/rest_api/docs/support/incremental_export#pagination
|
133
166
|
// When there are more than 1000 records share the same time stamp, the count > 1000
|
134
|
-
long apiEndTime = result.get(ZendeskConstants.Field.END_TIME).asLong();
|
135
167
|
startTime = startTime == apiEndTime
|
136
168
|
? apiEndTime + 1
|
137
169
|
: apiEndTime;
|
138
170
|
|
139
|
-
if (numberOfRecords < ZendeskConstants.Misc.MAXIMUM_RECORDS_INCREMENTAL) {
|
171
|
+
if (numberOfRecords < ZendeskConstants.Misc.MAXIMUM_RECORDS_INCREMENTAL || startTime > endTime) {
|
140
172
|
break;
|
141
173
|
}
|
142
174
|
}
|
143
175
|
|
144
176
|
if (!Exec.isPreview()) {
|
145
|
-
storeStartTimeForConfigDiff(taskReport, startTime);
|
177
|
+
storeStartTimeForConfigDiff(taskReport, initStartTime, startTime);
|
146
178
|
}
|
147
179
|
}
|
148
180
|
finally {
|
@@ -159,19 +191,34 @@ public abstract class ZendeskNormalServices implements ZendeskService
|
|
159
191
|
}
|
160
192
|
}
|
161
193
|
|
162
|
-
private void storeStartTimeForConfigDiff(final TaskReport taskReport, final long resultEndTime)
|
194
|
+
private void storeStartTimeForConfigDiff(final TaskReport taskReport, final long initStartTime, final long resultEndTime)
|
163
195
|
{
|
164
196
|
if (task.getIncremental()) {
|
165
|
-
|
197
|
+
long nextStartTime;
|
198
|
+
long now = Instant.now().getEpochSecond();
|
199
|
+
// no record to add
|
166
200
|
if (resultEndTime == 0) {
|
167
|
-
|
201
|
+
nextStartTime = now;
|
168
202
|
}
|
169
203
|
else {
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
204
|
+
if (task.getEndTime().isPresent()) {
|
205
|
+
long endTime = ZendeskDateUtils.isoToEpochSecond(task.getEndTime().get());
|
206
|
+
nextStartTime = endTime + 1;
|
207
|
+
}
|
208
|
+
else {
|
209
|
+
// NOTE: start_time compared as "=>", not ">".
|
210
|
+
// If we will use end_time for next start_time, we got the same records that are fetched
|
211
|
+
// end_time + 1 is workaround for that
|
212
|
+
nextStartTime = resultEndTime + 1;
|
213
|
+
}
|
214
|
+
}
|
215
|
+
|
216
|
+
if (task.getEndTime().isPresent()) {
|
217
|
+
long endTime = ZendeskDateUtils.isoToEpochSecond(task.getEndTime().get());
|
218
|
+
taskReport.set(ZendeskConstants.Field.END_TIME, nextStartTime + endTime - initStartTime);
|
174
219
|
}
|
220
|
+
|
221
|
+
taskReport.set(ZendeskConstants.Field.START_TIME, nextStartTime);
|
175
222
|
}
|
176
223
|
}
|
177
224
|
|
@@ -43,11 +43,11 @@ public class ZendeskConstants
|
|
43
43
|
public static class Misc
|
44
44
|
{
|
45
45
|
public static final String RUBY_TIMESTAMP_FORMAT = "%Y-%m-%dT%H:%M:%S%z";
|
46
|
-
public static final String RUBY_TIMESTAMP_FORMAT_INPUT = "
|
47
|
-
public static final String JAVA_TIMESTAMP_FORMAT = "
|
48
|
-
public static final String ISO_TIMESTAMP_FORMAT = "
|
49
|
-
public static final String ISO_INSTANT = "
|
50
|
-
public static final String RUBY_TIMESTAMP_FORMAT_INPUT_NO_SPACE = "
|
46
|
+
public static final String RUBY_TIMESTAMP_FORMAT_INPUT = "uuuu-MM-dd HH:mm:ss Z";
|
47
|
+
public static final String JAVA_TIMESTAMP_FORMAT = "uuuu-MM-dd'T'HH:mm:ss.SSS'Z'";
|
48
|
+
public static final String ISO_TIMESTAMP_FORMAT = "uuuu-MM-dd'T'HH:mm:ssXXX";
|
49
|
+
public static final String ISO_INSTANT = "uuuu-MM-dd'T'HH:mm:ss'Z'";
|
50
|
+
public static final String RUBY_TIMESTAMP_FORMAT_INPUT_NO_SPACE = "uuuu-MM-dd HH:mm:ssZ";
|
51
51
|
public static final String TOO_RECENT_START_TIME = "Too recent start_time.";
|
52
52
|
public static final int RECORDS_SIZE_PER_PAGE = 100;
|
53
53
|
public static final int MAXIMUM_RECORDS_INCREMENTAL = 1000;
|
@@ -9,6 +9,7 @@ import java.time.ZoneOffset;
|
|
9
9
|
import java.time.format.DateTimeFormatter;
|
10
10
|
import java.time.format.DateTimeParseException;
|
11
11
|
|
12
|
+
import java.time.format.ResolverStyle;
|
12
13
|
import java.util.Optional;
|
13
14
|
|
14
15
|
public class ZendeskDateUtils
|
@@ -21,9 +22,14 @@ public class ZendeskDateUtils
|
|
21
22
|
{
|
22
23
|
final Optional<String> pattern = supportedTimeFormat(time);
|
23
24
|
if (pattern.isPresent()) {
|
24
|
-
final DateTimeFormatter formatter = DateTimeFormatter.ofPattern(pattern.get()).withZone(ZoneOffset.UTC);
|
25
|
-
|
26
|
-
|
25
|
+
final DateTimeFormatter formatter = DateTimeFormatter.ofPattern(pattern.get()).withZone(ZoneOffset.UTC).withResolverStyle(ResolverStyle.STRICT);
|
26
|
+
try {
|
27
|
+
final OffsetDateTime offsetDateTime = LocalDateTime.parse(time, formatter).atOffset(ZoneOffset.UTC);
|
28
|
+
return offsetDateTime.toInstant().getEpochSecond();
|
29
|
+
}
|
30
|
+
catch (DateTimeParseException e) {
|
31
|
+
throw new DataException(e.getMessage());
|
32
|
+
}
|
27
33
|
}
|
28
34
|
|
29
35
|
throw new DataException("Fail to parse value '" + time + "' follow formats " + ZendeskConstants.Misc.SUPPORT_DATE_TIME_FORMAT.toString());
|
@@ -46,8 +52,7 @@ public class ZendeskDateUtils
|
|
46
52
|
|
47
53
|
public static String convertToDateTimeFormat(String datetime, String dateTimeFormat)
|
48
54
|
{
|
49
|
-
return
|
50
|
-
.format(DateTimeFormatter.ofPattern(dateTimeFormat));
|
55
|
+
return Instant.ofEpochSecond(ZendeskDateUtils.isoToEpochSecond(datetime)).atOffset(ZoneOffset.UTC).format(DateTimeFormatter.ofPattern(dateTimeFormat));
|
51
56
|
}
|
52
57
|
|
53
58
|
// start_time should be start from 0
|
@@ -14,6 +14,7 @@ import org.embulk.input.zendesk.services.ZendeskService;
|
|
14
14
|
import org.embulk.input.zendesk.services.ZendeskSupportAPIService;
|
15
15
|
import org.embulk.input.zendesk.services.ZendeskUserEventService;
|
16
16
|
import org.embulk.input.zendesk.utils.ZendeskConstants;
|
17
|
+
import org.embulk.input.zendesk.utils.ZendeskDateUtils;
|
17
18
|
import org.embulk.input.zendesk.utils.ZendeskPluginTestRuntime;
|
18
19
|
import org.embulk.input.zendesk.utils.ZendeskTestHelper;
|
19
20
|
|
@@ -143,19 +144,23 @@ public class TestZendeskInputPlugin
|
|
143
144
|
}
|
144
145
|
|
145
146
|
@Test
|
146
|
-
public void
|
147
|
+
public void testRunIncrementalStoreStartTimeAndEndTime()
|
147
148
|
{
|
148
|
-
final ConfigSource src = ZendeskTestHelper.getConfigSource("incremental.yml")
|
149
|
+
final ConfigSource src = ZendeskTestHelper.getConfigSource("incremental.yml")
|
150
|
+
.set("end_time", "2019-04-12 06:51:50 +0000");
|
149
151
|
TaskReport taskReport = Exec.newTaskReport();
|
150
152
|
taskReport.set(ZendeskConstants.Field.START_TIME, 1557026576);
|
153
|
+
taskReport.set(ZendeskConstants.Field.END_TIME, 1560309776);
|
151
154
|
|
152
155
|
when(zendeskSupportAPIService.isSupportIncremental()).thenReturn(true);
|
153
156
|
when(zendeskSupportAPIService.addRecordToImporter(anyInt(), any())).thenReturn(taskReport);
|
154
157
|
|
155
158
|
ConfigDiff configDiff = zendeskInputPlugin.transaction(src, new Control());
|
156
159
|
String nextStartTime = configDiff.get(String.class, ZendeskConstants.Field.START_TIME);
|
160
|
+
String nextEndTime = configDiff.get(String.class, ZendeskConstants.Field.END_TIME);
|
157
161
|
verify(pageBuilder, times(1)).finish();
|
158
162
|
assertEquals("2019-05-05 03:22:56 +0000", nextStartTime);
|
163
|
+
assertEquals("2019-06-12 03:22:56 +0000", nextEndTime);
|
159
164
|
}
|
160
165
|
|
161
166
|
@Test
|
@@ -266,8 +271,14 @@ public class TestZendeskInputPlugin
|
|
266
271
|
ConfigSource configSource = ZendeskTestHelper.getConfigSource("base_validator.yml");
|
267
272
|
configSource.set("target", Target.USER_EVENTS.name().toLowerCase());
|
268
273
|
assertValidation(configSource, "Profile Source is required for User Event Target");
|
274
|
+
}
|
269
275
|
|
270
|
-
|
276
|
+
@Test
|
277
|
+
public void validateTimeShouldThrowException()
|
278
|
+
{
|
279
|
+
ConfigSource configSource = ZendeskTestHelper.getConfigSource("base_validator.yml");
|
280
|
+
when(zendeskSupportAPIService.isSupportIncremental()).thenReturn(true);
|
281
|
+
configSource.set("target", Target.TICKETS.name().toLowerCase());
|
271
282
|
configSource.set("start_time", OffsetDateTime.ofInstant(Instant.ofEpochSecond(Instant.now().getEpochSecond()), ZoneOffset.UTC).format(DateTimeFormatter.ISO_INSTANT));
|
272
283
|
configSource.set("end_time", "2019-12-2 22-12-22");
|
273
284
|
assertValidation(configSource, "End Time should follow these format " + ZendeskConstants.Misc.SUPPORT_DATE_TIME_FORMAT.toString());
|
@@ -277,6 +288,43 @@ public class TestZendeskInputPlugin
|
|
277
288
|
assertValidation(configSource, "End Time should be later or equal than Start Time");
|
278
289
|
}
|
279
290
|
|
291
|
+
@Test
|
292
|
+
public void isValidTimeRangeShouldThrowException()
|
293
|
+
{
|
294
|
+
ZendeskTestHelper.setPreviewMode(embulk, true);
|
295
|
+
String expectedMessage = "Invalid End time. End time is greater than current time";
|
296
|
+
ConfigSource configSource = ZendeskTestHelper.getConfigSource("base_validator.yml");
|
297
|
+
when(zendeskSupportAPIService.isSupportIncremental()).thenReturn(true);
|
298
|
+
configSource.set("start_time", OffsetDateTime.ofInstant(Instant.ofEpochSecond(Instant.now().getEpochSecond()), ZoneOffset.UTC).format(DateTimeFormatter.ISO_INSTANT));
|
299
|
+
configSource.set("end_time", OffsetDateTime.ofInstant(Instant.ofEpochSecond(Instant.now().getEpochSecond() + 1000), ZoneOffset.UTC).format(DateTimeFormatter.ISO_INSTANT));
|
300
|
+
|
301
|
+
assertValidation(configSource, expectedMessage);
|
302
|
+
|
303
|
+
try {
|
304
|
+
zendeskInputPlugin.transaction(configSource, new Control());
|
305
|
+
fail("Should not reach here");
|
306
|
+
}
|
307
|
+
catch (final Exception e) {
|
308
|
+
assertEquals(expectedMessage, e.getMessage());
|
309
|
+
}
|
310
|
+
}
|
311
|
+
|
312
|
+
@Test
|
313
|
+
public void runShouldKeepOldStartTimeAndEndTimeInConfigDiff()
|
314
|
+
{
|
315
|
+
ConfigSource configSource = ZendeskTestHelper.getConfigSource("base_validator.yml");
|
316
|
+
ZendeskTestHelper.setPreviewMode(embulk, false);
|
317
|
+
when(zendeskSupportAPIService.isSupportIncremental()).thenReturn(true);
|
318
|
+
configSource.set("start_time", OffsetDateTime.ofInstant(Instant.ofEpochSecond(Instant.now().getEpochSecond()), ZoneOffset.UTC).format(DateTimeFormatter.ISO_INSTANT));
|
319
|
+
configSource.set("end_time", OffsetDateTime.ofInstant(Instant.ofEpochSecond(Instant.now().getEpochSecond() + 1000), ZoneOffset.UTC).format(DateTimeFormatter.ISO_INSTANT));
|
320
|
+
|
321
|
+
ConfigDiff configDiff = zendeskInputPlugin.transaction(configSource, new Control());
|
322
|
+
assertEquals(ZendeskDateUtils.convertToDateTimeFormat(configSource.get(String.class, ZendeskConstants.Field.START_TIME), ZendeskConstants.Misc.RUBY_TIMESTAMP_FORMAT_INPUT),
|
323
|
+
configDiff.get(String.class, ZendeskConstants.Field.START_TIME));
|
324
|
+
assertEquals(ZendeskDateUtils.convertToDateTimeFormat(configSource.get(String.class, ZendeskConstants.Field.END_TIME), ZendeskConstants.Misc.RUBY_TIMESTAMP_FORMAT_INPUT),
|
325
|
+
configDiff.get(String.class, ZendeskConstants.Field.END_TIME));
|
326
|
+
}
|
327
|
+
|
280
328
|
private void assertValidation(final ConfigSource configSource, final String message)
|
281
329
|
{
|
282
330
|
try {
|
@@ -153,7 +153,7 @@ public class TestZendeskNormalService
|
|
153
153
|
}
|
154
154
|
|
155
155
|
@Test
|
156
|
-
public void
|
156
|
+
public void testTicketEventsAddRecordToImporterIncrementalAndAllRecordsShareTheSameTime()
|
157
157
|
{
|
158
158
|
setupSupportAPIService("incremental.yml");
|
159
159
|
ZendeskInputPlugin.PluginTask task = ZendeskTestHelper.getConfigSource("incremental.yml")
|
@@ -168,6 +168,78 @@ public class TestZendeskNormalService
|
|
168
168
|
Assert.assertEquals(1550645444, taskReport.get(JsonNode.class, ZendeskConstants.Field.START_TIME).asLong());
|
169
169
|
}
|
170
170
|
|
171
|
+
@Test
|
172
|
+
public void executeIncrementalContainEndTime()
|
173
|
+
{
|
174
|
+
ConfigSource src = ZendeskTestHelper.getConfigSource("incremental.yml");
|
175
|
+
// same updated_at time of last record
|
176
|
+
src.set("end_time", "2019-02-20T07:17:32Z");
|
177
|
+
ZendeskInputPlugin.PluginTask task = src.loadConfig(ZendeskInputPlugin.PluginTask.class);
|
178
|
+
setupZendeskSupportAPIService(task);
|
179
|
+
loadData("data/tickets.json");
|
180
|
+
|
181
|
+
TaskReport taskReport = zendeskSupportAPIService.addRecordToImporter(0, recordImporter);
|
182
|
+
verify(recordImporter, times(3)).addRecord(any());
|
183
|
+
Assert.assertFalse(taskReport.isEmpty());
|
184
|
+
// start_time = end_time + 1
|
185
|
+
Assert.assertEquals(1550647053, taskReport.get(JsonNode.class, ZendeskConstants.Field.START_TIME).asLong());
|
186
|
+
}
|
187
|
+
|
188
|
+
@Test
|
189
|
+
public void executeIncrementalContainEndTimeFilterOutLastRecord()
|
190
|
+
{
|
191
|
+
ConfigSource src = ZendeskTestHelper.getConfigSource("incremental.yml");
|
192
|
+
// earlier than updated_at time of last record
|
193
|
+
src.set("end_time", "2019-02-20T07:17:32Z");
|
194
|
+
ZendeskInputPlugin.PluginTask task = src.loadConfig(ZendeskInputPlugin.PluginTask.class);
|
195
|
+
setupZendeskSupportAPIService(task);
|
196
|
+
loadData("data/tickets.json");
|
197
|
+
|
198
|
+
TaskReport taskReport = zendeskSupportAPIService.addRecordToImporter(0, recordImporter);
|
199
|
+
verify(recordImporter, times(3)).addRecord(any());
|
200
|
+
Assert.assertFalse(taskReport.isEmpty());
|
201
|
+
//
|
202
|
+
Assert.assertEquals(1550647053, taskReport.get(JsonNode.class, ZendeskConstants.Field.START_TIME).asLong());
|
203
|
+
}
|
204
|
+
|
205
|
+
@Test
|
206
|
+
public void executeIncrementalContainEndTimeFilterOutLastRecordTicketEvents()
|
207
|
+
{
|
208
|
+
ConfigSource src = ZendeskTestHelper.getConfigSource("incremental.yml");
|
209
|
+
src.set("target", Target.TICKET_EVENTS.toString());
|
210
|
+
// earlier than updated_at time of last record
|
211
|
+
// 1550645520
|
212
|
+
src.set("end_time", "2019-02-20T06:52:00Z");
|
213
|
+
ZendeskInputPlugin.PluginTask task = src.loadConfig(ZendeskInputPlugin.PluginTask.class);
|
214
|
+
setupZendeskSupportAPIService(task);
|
215
|
+
loadData("data/ticket_events_updated_by_system_records.json");
|
216
|
+
|
217
|
+
TaskReport taskReport = zendeskSupportAPIService.addRecordToImporter(0, recordImporter);
|
218
|
+
verify(recordImporter, times(3)).addRecord(any());
|
219
|
+
Assert.assertFalse(taskReport.isEmpty());
|
220
|
+
// end_time + 1
|
221
|
+
Assert.assertEquals(1550645521, taskReport.get(JsonNode.class, ZendeskConstants.Field.START_TIME).asLong());
|
222
|
+
}
|
223
|
+
|
224
|
+
@Test
|
225
|
+
public void executeIncrementalContainEndTimeFilterOutLastRecordNPSScore()
|
226
|
+
{
|
227
|
+
ConfigSource src = ZendeskTestHelper.getConfigSource("nps.yml");
|
228
|
+
src.set("target", Target.SCORES.toString());
|
229
|
+
// earlier than updated_at time of last record
|
230
|
+
// 1550645520
|
231
|
+
src.set("end_time", "2019-02-20T06:52:00Z");
|
232
|
+
ZendeskInputPlugin.PluginTask task = src.loadConfig(ZendeskInputPlugin.PluginTask.class);
|
233
|
+
setupZendeskSupportAPIService(task);
|
234
|
+
loadData("data/scores.json");
|
235
|
+
|
236
|
+
TaskReport taskReport = zendeskSupportAPIService.addRecordToImporter(0, recordImporter);
|
237
|
+
verify(recordImporter, times(1)).addRecord(any());
|
238
|
+
Assert.assertFalse(taskReport.isEmpty());
|
239
|
+
// end_time + 1
|
240
|
+
Assert.assertEquals(1550645521, taskReport.get(JsonNode.class, ZendeskConstants.Field.START_TIME).asLong());
|
241
|
+
}
|
242
|
+
|
171
243
|
private void setupSupportAPIService(String file)
|
172
244
|
{
|
173
245
|
ZendeskInputPlugin.PluginTask task = ZendeskTestHelper.getConfigSource(file)
|
@@ -79,7 +79,7 @@ public class TestZendeskDateUtils
|
|
79
79
|
assertEquals(0, actualValue);
|
80
80
|
|
81
81
|
actualValue = ZendeskDateUtils.getStartTime("2019-02-30 06:50:45 +0000");
|
82
|
-
assertEquals(
|
82
|
+
assertEquals(0, actualValue);
|
83
83
|
|
84
84
|
actualValue = ZendeskDateUtils.getStartTime("2019-02-20 06:50:45 +0000");
|
85
85
|
assertEquals(expectedValue, actualValue);
|
@@ -76,4 +76,17 @@ public final class ZendeskTestHelper
|
|
76
76
|
Assert.fail(e.getMessage());
|
77
77
|
}
|
78
78
|
}
|
79
|
+
|
80
|
+
public static void setPreviewMode(final ZendeskPluginTestRuntime runtime, final boolean isPreview)
|
81
|
+
{
|
82
|
+
// A small hack to make the plugin executed in preview mode so
|
83
|
+
try {
|
84
|
+
final Field previewField = ExecSession.class.getDeclaredField("preview");
|
85
|
+
previewField.setAccessible(true);
|
86
|
+
previewField.set(runtime.getExec(), isPreview);
|
87
|
+
}
|
88
|
+
catch (NoSuchFieldException | IllegalAccessException e) {
|
89
|
+
Assert.fail(e.getMessage());
|
90
|
+
}
|
91
|
+
}
|
79
92
|
}
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-zendesk
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- hieu.duong
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-
|
11
|
+
date: 2019-07-01 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -138,7 +138,7 @@ files:
|
|
138
138
|
- src/test/resources/data/util_page.json
|
139
139
|
- classpath/commons-logging-1.2.jar
|
140
140
|
- classpath/httpcore-4.4.10.jar
|
141
|
-
- classpath/embulk-input-zendesk-0.3.
|
141
|
+
- classpath/embulk-input-zendesk-0.3.6.jar
|
142
142
|
- classpath/httpclient-4.5.6.jar
|
143
143
|
- classpath/commons-codec-1.10.jar
|
144
144
|
homepage: https://github.com/treasure-data/embulk-input-zendesk
|