embulk-input-marketo 0.6.1 → 0.6.2.alpha.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -1
- data/build.gradle +1 -1
- data/src/main/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPlugin.java +12 -7
- data/src/test/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPluginTest.java +22 -1
- data/src/test/java/org/embulk/input/marketo/rest/MarketoRestClientTest.java +0 -14
- metadata +15 -15
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 11dcb4f3b03a1a8f44ab526a8904844c66d87939
|
4
|
+
data.tar.gz: 19d86812f3f1d0b129a2b5212b0331455f4b64c9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f6797483eff01213b29cce8b2b23b659419c35a2fba75d499ef55eae2950dcb5d9fa99d6996a5747646b133531bdf3b37755516a1a83c8d9c7d6f80252e0bfc8
|
7
|
+
data.tar.gz: 69f31537e1ca83a97f18a7b2539e2de193c1c31ca90b1637655fb0ac9101b3057ce3f52091f31982efb7c055dde962aa7b97b8e730e9e90dc3c50ee1b4b0a722
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,8 @@
|
|
1
|
+
## 0.6.1 - 2017-10-12
|
2
|
+
* [fixed] OutOfMemeory when run Lead by list, Lead by program [#69](https://github.com/treasure-data/embulk-input-marketo/pull/69)
|
3
|
+
* [fixed] SOAP only field cause NullPointerException [#69](https://github.com/treasure-data/embulk-input-marketo/pull/69)
|
4
|
+
* [enhancement] Implement file download resume in Bulk extract [#69](https://github.com/treasure-data/embulk-input-marketo/pull/69)
|
5
|
+
|
1
6
|
## 0.6.0 - 2017-10-10
|
2
7
|
* [major] Migrate to Java by embulk-base-restclient [#66](https://github.com/treasure-data/embulk-input-marketo/pull/66)
|
3
8
|
* [major] Migrate to REST API [#66](https://github.com/treasure-data/embulk-input-marketo/pull/66)
|
@@ -5,7 +10,6 @@
|
|
5
10
|
* [major] Support Marketo bulk extract API for lead and activity targets [#66](https://github.com/treasure-data/embulk-input-marketo/pull/66)
|
6
11
|
* [major] Support incremental ingestion for lead and activity targets [#66](https://github.com/treasure-data/embulk-input-marketo/pull/66)
|
7
12
|
|
8
|
-
|
9
13
|
## 0.5.6 - 2016-12-14
|
10
14
|
* [maintenance] Enable tcp keepalive [#64](https://github.com/treasure-data/embulk-input-marketo/pull/64)
|
11
15
|
|
data/build.gradle
CHANGED
@@ -43,6 +43,7 @@ import java.text.SimpleDateFormat;
|
|
43
43
|
import java.util.ArrayList;
|
44
44
|
import java.util.Date;
|
45
45
|
import java.util.HashMap;
|
46
|
+
import java.util.HashSet;
|
46
47
|
import java.util.Iterator;
|
47
48
|
import java.util.List;
|
48
49
|
import java.util.Map;
|
@@ -123,6 +124,9 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
|
|
123
124
|
if (task.getFromDate() == null) {
|
124
125
|
throw new ConfigException("From date is required for Bulk Extract");
|
125
126
|
}
|
127
|
+
if (task.getFromDate().getTime() >= task.getJobStartTime().getMillis()) {
|
128
|
+
throw new ConfigException("From date can't not be in future");
|
129
|
+
}
|
126
130
|
//Calculate to date
|
127
131
|
DateTime toDate = getToDate(task);
|
128
132
|
task.setToDate(Optional.of(toDate.toDate()));
|
@@ -134,8 +138,8 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
|
|
134
138
|
DateTime dateTime = new DateTime(fromDate);
|
135
139
|
DateTime toDate = dateTime.plusDays(task.getFetchDays());
|
136
140
|
if (toDate.isAfter(task.getJobStartTime())) {
|
137
|
-
//
|
138
|
-
toDate = task.getJobStartTime()
|
141
|
+
//Lock down to date
|
142
|
+
toDate = task.getJobStartTime();
|
139
143
|
}
|
140
144
|
return toDate;
|
141
145
|
}
|
@@ -145,7 +149,7 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
|
|
145
149
|
{
|
146
150
|
ConfigDiff configDiff = super.buildConfigDiff(task, schema, taskCount, taskReports);
|
147
151
|
Long currentLatestFetchTime = 0L;
|
148
|
-
Set latestUIds =
|
152
|
+
Set latestUIds = new HashSet();
|
149
153
|
String incrementalColumn = task.getIncrementalColumn().orNull();
|
150
154
|
int imported = 0;
|
151
155
|
if (incrementalColumn != null && task.getIncremental()) {
|
@@ -168,11 +172,12 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
|
|
168
172
|
}
|
169
173
|
// in case of we didn't import anything but search range is entirely in the past. Then we should move the the range anyway.
|
170
174
|
if (imported == 0) {
|
171
|
-
|
175
|
+
Date toDate = task.getToDate().orNull();
|
176
|
+
configDiff.set(FROM_DATE, df.format(toDate));
|
172
177
|
}
|
173
178
|
else {
|
174
|
-
// Otherwise it's should start from the currentLastFetchTime
|
175
|
-
configDiff.set(FROM_DATE, df.format(new
|
179
|
+
// Otherwise it's should start from the currentLastFetchTime plus 1 second.
|
180
|
+
configDiff.set(FROM_DATE, df.format(new DateTime(currentLatestFetchTime).plusSeconds(1).toDate()));
|
176
181
|
}
|
177
182
|
configDiff.set(LATEST_FETCH_TIME, currentLatestFetchTime);
|
178
183
|
configDiff.set(LATEST_UID_LIST, latestUIds);
|
@@ -199,7 +204,7 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
|
|
199
204
|
return new CsvRecordIterator(input, task);
|
200
205
|
}
|
201
206
|
}));
|
202
|
-
long currentTimestamp = 0L;
|
207
|
+
long currentTimestamp = task.getLatestFetchTime().or(0L);
|
203
208
|
Set<String> latestUids = task.getPreviousUids();
|
204
209
|
//Keep the preview code here when we can enable real preview
|
205
210
|
if (Exec.isPreview()) {
|
data/src/test/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPluginTest.java
CHANGED
@@ -23,6 +23,7 @@ import java.util.Date;
|
|
23
23
|
import java.util.Set;
|
24
24
|
|
25
25
|
import static org.junit.Assert.assertEquals;
|
26
|
+
import static org.junit.Assert.fail;
|
26
27
|
|
27
28
|
/**
|
28
29
|
* Created by khuutantaitai on 10/3/17.
|
@@ -52,8 +53,10 @@ public class MarketoBaseBulkExtractInputPluginTest
|
|
52
53
|
public void validateInputTaskToDateLessThanJobStartTime()
|
53
54
|
{
|
54
55
|
Date fromDate = new Date(1504224000000L);
|
56
|
+
DateTime jobStartTime = new DateTime(1506842144000L);
|
55
57
|
MarketoBaseBulkExtractInputPlugin.PluginTask pluginTask = Mockito.mock(MarketoBaseBulkExtractInputPlugin.PluginTask.class);
|
56
58
|
Mockito.when(pluginTask.getFromDate()).thenReturn(fromDate);
|
59
|
+
Mockito.when(pluginTask.getJobStartTime()).thenReturn(jobStartTime);
|
57
60
|
Mockito.when(pluginTask.getFetchDays()).thenReturn(7);
|
58
61
|
baseBulkExtractInputPlugin.validateInputTask(pluginTask);
|
59
62
|
ArgumentCaptor<Optional<Date>> argumentCaptor = ArgumentCaptor.forClass(Optional.class);
|
@@ -61,6 +64,24 @@ public class MarketoBaseBulkExtractInputPluginTest
|
|
61
64
|
assertEquals(1504828800000L, argumentCaptor.getValue().get().getTime());
|
62
65
|
}
|
63
66
|
|
67
|
+
@Test()
|
68
|
+
public void validateInputTaskFromDateMoreThanJobStartTime()
|
69
|
+
{
|
70
|
+
Date fromDate = new Date(1507619744000L);
|
71
|
+
DateTime jobStartTime = new DateTime(1506842144000L);
|
72
|
+
MarketoBaseBulkExtractInputPlugin.PluginTask pluginTask = Mockito.mock(MarketoBaseBulkExtractInputPlugin.PluginTask.class);
|
73
|
+
Mockito.when(pluginTask.getFromDate()).thenReturn(fromDate);
|
74
|
+
Mockito.when(pluginTask.getJobStartTime()).thenReturn(jobStartTime);
|
75
|
+
|
76
|
+
try {
|
77
|
+
baseBulkExtractInputPlugin.validateInputTask(pluginTask);
|
78
|
+
}
|
79
|
+
catch (ConfigException ex) {
|
80
|
+
return;
|
81
|
+
}
|
82
|
+
fail();
|
83
|
+
}
|
84
|
+
|
64
85
|
@Test()
|
65
86
|
public void validateInputTaskToDateMoreThanJobStartTime()
|
66
87
|
{
|
@@ -73,7 +94,7 @@ public class MarketoBaseBulkExtractInputPluginTest
|
|
73
94
|
baseBulkExtractInputPlugin.validateInputTask(pluginTask);
|
74
95
|
ArgumentCaptor<Optional<Date>> toDateArgumentCaptor = ArgumentCaptor.forClass(Optional.class);
|
75
96
|
Mockito.verify(pluginTask, Mockito.times(1)).setToDate(toDateArgumentCaptor.capture());
|
76
|
-
assertEquals(jobStartTime.
|
97
|
+
assertEquals(jobStartTime.getMillis(), toDateArgumentCaptor.getValue().get().getTime());
|
77
98
|
}
|
78
99
|
|
79
100
|
@Test
|
@@ -139,20 +139,6 @@ public class MarketoRestClientTest
|
|
139
139
|
Assert.fail();
|
140
140
|
}
|
141
141
|
|
142
|
-
@Test()
|
143
|
-
public void createLeadBulkExtractWithParseError() throws Exception
|
144
|
-
{
|
145
|
-
Mockito.doThrow(JsonProcessingException.class).when(marketoRestClient).doPost(Mockito.eq(END_POINT + MarketoRESTEndpoint.CREATE_LEAD_EXTRACT.getEndpoint()), Mockito.isNull(Map.class), Mockito.isNull(ImmutableListMultimap.class), Mockito.anyString(), Mockito.any(MarketoResponseJetty92EntityReader.class));
|
146
|
-
String filterField = "filterField";
|
147
|
-
try {
|
148
|
-
marketoRestClient.createLeadBulkExtract(new Date(), new Date(), Arrays.asList("extract_field1", "extract_field2"), filterField);
|
149
|
-
}
|
150
|
-
catch (DataException ex) {
|
151
|
-
return;
|
152
|
-
}
|
153
|
-
Assert.fail();
|
154
|
-
}
|
155
|
-
|
156
142
|
@Test
|
157
143
|
public void createActitvityExtract() throws Exception
|
158
144
|
{
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-marketo
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.6.1
|
4
|
+
version: 0.6.2.alpha.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- uu59
|
@@ -10,36 +10,36 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2017-10-
|
13
|
+
date: 2017-10-16 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
|
-
|
16
|
+
name: bundler
|
17
|
+
version_requirements: !ruby/object:Gem::Requirement
|
17
18
|
requirements:
|
18
19
|
- - ~>
|
19
20
|
- !ruby/object:Gem::Version
|
20
21
|
version: '1.0'
|
21
|
-
|
22
|
-
prerelease: false
|
23
|
-
type: :development
|
24
|
-
version_requirements: !ruby/object:Gem::Requirement
|
22
|
+
requirement: !ruby/object:Gem::Requirement
|
25
23
|
requirements:
|
26
24
|
- - ~>
|
27
25
|
- !ruby/object:Gem::Version
|
28
26
|
version: '1.0'
|
27
|
+
prerelease: false
|
28
|
+
type: :development
|
29
29
|
- !ruby/object:Gem::Dependency
|
30
|
-
|
30
|
+
name: rake
|
31
|
+
version_requirements: !ruby/object:Gem::Requirement
|
31
32
|
requirements:
|
32
33
|
- - '>='
|
33
34
|
- !ruby/object:Gem::Version
|
34
35
|
version: '10.0'
|
35
|
-
|
36
|
-
prerelease: false
|
37
|
-
type: :development
|
38
|
-
version_requirements: !ruby/object:Gem::Requirement
|
36
|
+
requirement: !ruby/object:Gem::Requirement
|
39
37
|
requirements:
|
40
38
|
- - '>='
|
41
39
|
- !ruby/object:Gem::Version
|
42
40
|
version: '10.0'
|
41
|
+
prerelease: false
|
42
|
+
type: :development
|
43
43
|
description: Loads records from Marketo.
|
44
44
|
email:
|
45
45
|
- k@uu59.org
|
@@ -125,8 +125,8 @@ files:
|
|
125
125
|
- src/test/resources/fixtures/program_response.json
|
126
126
|
- classpath/jetty-http-9.2.14.v20151106.jar
|
127
127
|
- classpath/embulk-base-restclient-0.5.3.jar
|
128
|
-
- classpath/embulk-input-marketo-0.6.1.jar
|
129
128
|
- classpath/jetty-client-9.2.14.v20151106.jar
|
129
|
+
- classpath/embulk-input-marketo-0.6.2.alpha.1.jar
|
130
130
|
- classpath/jetty-util-9.2.14.v20151106.jar
|
131
131
|
- classpath/embulk-util-retryhelper-jetty92-0.5.3.jar
|
132
132
|
- classpath/jetty-io-9.2.14.v20151106.jar
|
@@ -145,9 +145,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
145
145
|
version: '0'
|
146
146
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
147
147
|
requirements:
|
148
|
-
- - '
|
148
|
+
- - '>'
|
149
149
|
- !ruby/object:Gem::Version
|
150
|
-
version:
|
150
|
+
version: 1.3.1
|
151
151
|
requirements: []
|
152
152
|
rubyforge_project:
|
153
153
|
rubygems_version: 2.1.9
|