embulk-input-marketo 0.6.1 → 0.6.2.alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +5 -1
- data/build.gradle +1 -1
- data/src/main/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPlugin.java +12 -7
- data/src/test/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPluginTest.java +22 -1
- data/src/test/java/org/embulk/input/marketo/rest/MarketoRestClientTest.java +0 -14
- metadata +15 -15
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 11dcb4f3b03a1a8f44ab526a8904844c66d87939
|
4
|
+
data.tar.gz: 19d86812f3f1d0b129a2b5212b0331455f4b64c9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f6797483eff01213b29cce8b2b23b659419c35a2fba75d499ef55eae2950dcb5d9fa99d6996a5747646b133531bdf3b37755516a1a83c8d9c7d6f80252e0bfc8
|
7
|
+
data.tar.gz: 69f31537e1ca83a97f18a7b2539e2de193c1c31ca90b1637655fb0ac9101b3057ce3f52091f31982efb7c055dde962aa7b97b8e730e9e90dc3c50ee1b4b0a722
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,8 @@
|
|
1
|
+
## 0.6.1 - 2017-10-12
|
2
|
+
* [fixed] OutOfMemeory when run Lead by list, Lead by program [#69](https://github.com/treasure-data/embulk-input-marketo/pull/69)
|
3
|
+
* [fixed] SOAP only field cause NullPointerException [#69](https://github.com/treasure-data/embulk-input-marketo/pull/69)
|
4
|
+
* [enhancement] Implement file download resume in Bulk extract [#69](https://github.com/treasure-data/embulk-input-marketo/pull/69)
|
5
|
+
|
1
6
|
## 0.6.0 - 2017-10-10
|
2
7
|
* [major] Migrate to Java by embulk-base-restclient [#66](https://github.com/treasure-data/embulk-input-marketo/pull/66)
|
3
8
|
* [major] Migrate to REST API [#66](https://github.com/treasure-data/embulk-input-marketo/pull/66)
|
@@ -5,7 +10,6 @@
|
|
5
10
|
* [major] Support Marketo bulk extract API for lead and activity targets [#66](https://github.com/treasure-data/embulk-input-marketo/pull/66)
|
6
11
|
* [major] Support incremental ingestion for lead and activity targets [#66](https://github.com/treasure-data/embulk-input-marketo/pull/66)
|
7
12
|
|
8
|
-
|
9
13
|
## 0.5.6 - 2016-12-14
|
10
14
|
* [maintenance] Enable tcp keepalive [#64](https://github.com/treasure-data/embulk-input-marketo/pull/64)
|
11
15
|
|
data/build.gradle
CHANGED
@@ -43,6 +43,7 @@ import java.text.SimpleDateFormat;
|
|
43
43
|
import java.util.ArrayList;
|
44
44
|
import java.util.Date;
|
45
45
|
import java.util.HashMap;
|
46
|
+
import java.util.HashSet;
|
46
47
|
import java.util.Iterator;
|
47
48
|
import java.util.List;
|
48
49
|
import java.util.Map;
|
@@ -123,6 +124,9 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
|
|
123
124
|
if (task.getFromDate() == null) {
|
124
125
|
throw new ConfigException("From date is required for Bulk Extract");
|
125
126
|
}
|
127
|
+
if (task.getFromDate().getTime() >= task.getJobStartTime().getMillis()) {
|
128
|
+
throw new ConfigException("From date can't not be in future");
|
129
|
+
}
|
126
130
|
//Calculate to date
|
127
131
|
DateTime toDate = getToDate(task);
|
128
132
|
task.setToDate(Optional.of(toDate.toDate()));
|
@@ -134,8 +138,8 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
|
|
134
138
|
DateTime dateTime = new DateTime(fromDate);
|
135
139
|
DateTime toDate = dateTime.plusDays(task.getFetchDays());
|
136
140
|
if (toDate.isAfter(task.getJobStartTime())) {
|
137
|
-
//
|
138
|
-
toDate = task.getJobStartTime()
|
141
|
+
//Lock down to date
|
142
|
+
toDate = task.getJobStartTime();
|
139
143
|
}
|
140
144
|
return toDate;
|
141
145
|
}
|
@@ -145,7 +149,7 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
|
|
145
149
|
{
|
146
150
|
ConfigDiff configDiff = super.buildConfigDiff(task, schema, taskCount, taskReports);
|
147
151
|
Long currentLatestFetchTime = 0L;
|
148
|
-
Set latestUIds =
|
152
|
+
Set latestUIds = new HashSet();
|
149
153
|
String incrementalColumn = task.getIncrementalColumn().orNull();
|
150
154
|
int imported = 0;
|
151
155
|
if (incrementalColumn != null && task.getIncremental()) {
|
@@ -168,11 +172,12 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
|
|
168
172
|
}
|
169
173
|
// in case of we didn't import anything but search range is entirely in the past. Then we should move the the range anyway.
|
170
174
|
if (imported == 0) {
|
171
|
-
|
175
|
+
Date toDate = task.getToDate().orNull();
|
176
|
+
configDiff.set(FROM_DATE, df.format(toDate));
|
172
177
|
}
|
173
178
|
else {
|
174
|
-
// Otherwise it's should start from the currentLastFetchTime
|
175
|
-
configDiff.set(FROM_DATE, df.format(new
|
179
|
+
// Otherwise it's should start from the currentLastFetchTime plus 1 second.
|
180
|
+
configDiff.set(FROM_DATE, df.format(new DateTime(currentLatestFetchTime).plusSeconds(1).toDate()));
|
176
181
|
}
|
177
182
|
configDiff.set(LATEST_FETCH_TIME, currentLatestFetchTime);
|
178
183
|
configDiff.set(LATEST_UID_LIST, latestUIds);
|
@@ -199,7 +204,7 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
|
|
199
204
|
return new CsvRecordIterator(input, task);
|
200
205
|
}
|
201
206
|
}));
|
202
|
-
long currentTimestamp = 0L;
|
207
|
+
long currentTimestamp = task.getLatestFetchTime().or(0L);
|
203
208
|
Set<String> latestUids = task.getPreviousUids();
|
204
209
|
//Keep the preview code here when we can enable real preview
|
205
210
|
if (Exec.isPreview()) {
|
data/src/test/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPluginTest.java
CHANGED
@@ -23,6 +23,7 @@ import java.util.Date;
|
|
23
23
|
import java.util.Set;
|
24
24
|
|
25
25
|
import static org.junit.Assert.assertEquals;
|
26
|
+
import static org.junit.Assert.fail;
|
26
27
|
|
27
28
|
/**
|
28
29
|
* Created by khuutantaitai on 10/3/17.
|
@@ -52,8 +53,10 @@ public class MarketoBaseBulkExtractInputPluginTest
|
|
52
53
|
public void validateInputTaskToDateLessThanJobStartTime()
|
53
54
|
{
|
54
55
|
Date fromDate = new Date(1504224000000L);
|
56
|
+
DateTime jobStartTime = new DateTime(1506842144000L);
|
55
57
|
MarketoBaseBulkExtractInputPlugin.PluginTask pluginTask = Mockito.mock(MarketoBaseBulkExtractInputPlugin.PluginTask.class);
|
56
58
|
Mockito.when(pluginTask.getFromDate()).thenReturn(fromDate);
|
59
|
+
Mockito.when(pluginTask.getJobStartTime()).thenReturn(jobStartTime);
|
57
60
|
Mockito.when(pluginTask.getFetchDays()).thenReturn(7);
|
58
61
|
baseBulkExtractInputPlugin.validateInputTask(pluginTask);
|
59
62
|
ArgumentCaptor<Optional<Date>> argumentCaptor = ArgumentCaptor.forClass(Optional.class);
|
@@ -61,6 +64,24 @@ public class MarketoBaseBulkExtractInputPluginTest
|
|
61
64
|
assertEquals(1504828800000L, argumentCaptor.getValue().get().getTime());
|
62
65
|
}
|
63
66
|
|
67
|
+
@Test()
|
68
|
+
public void validateInputTaskFromDateMoreThanJobStartTime()
|
69
|
+
{
|
70
|
+
Date fromDate = new Date(1507619744000L);
|
71
|
+
DateTime jobStartTime = new DateTime(1506842144000L);
|
72
|
+
MarketoBaseBulkExtractInputPlugin.PluginTask pluginTask = Mockito.mock(MarketoBaseBulkExtractInputPlugin.PluginTask.class);
|
73
|
+
Mockito.when(pluginTask.getFromDate()).thenReturn(fromDate);
|
74
|
+
Mockito.when(pluginTask.getJobStartTime()).thenReturn(jobStartTime);
|
75
|
+
|
76
|
+
try {
|
77
|
+
baseBulkExtractInputPlugin.validateInputTask(pluginTask);
|
78
|
+
}
|
79
|
+
catch (ConfigException ex) {
|
80
|
+
return;
|
81
|
+
}
|
82
|
+
fail();
|
83
|
+
}
|
84
|
+
|
64
85
|
@Test()
|
65
86
|
public void validateInputTaskToDateMoreThanJobStartTime()
|
66
87
|
{
|
@@ -73,7 +94,7 @@ public class MarketoBaseBulkExtractInputPluginTest
|
|
73
94
|
baseBulkExtractInputPlugin.validateInputTask(pluginTask);
|
74
95
|
ArgumentCaptor<Optional<Date>> toDateArgumentCaptor = ArgumentCaptor.forClass(Optional.class);
|
75
96
|
Mockito.verify(pluginTask, Mockito.times(1)).setToDate(toDateArgumentCaptor.capture());
|
76
|
-
assertEquals(jobStartTime.
|
97
|
+
assertEquals(jobStartTime.getMillis(), toDateArgumentCaptor.getValue().get().getTime());
|
77
98
|
}
|
78
99
|
|
79
100
|
@Test
|
@@ -139,20 +139,6 @@ public class MarketoRestClientTest
|
|
139
139
|
Assert.fail();
|
140
140
|
}
|
141
141
|
|
142
|
-
@Test()
|
143
|
-
public void createLeadBulkExtractWithParseError() throws Exception
|
144
|
-
{
|
145
|
-
Mockito.doThrow(JsonProcessingException.class).when(marketoRestClient).doPost(Mockito.eq(END_POINT + MarketoRESTEndpoint.CREATE_LEAD_EXTRACT.getEndpoint()), Mockito.isNull(Map.class), Mockito.isNull(ImmutableListMultimap.class), Mockito.anyString(), Mockito.any(MarketoResponseJetty92EntityReader.class));
|
146
|
-
String filterField = "filterField";
|
147
|
-
try {
|
148
|
-
marketoRestClient.createLeadBulkExtract(new Date(), new Date(), Arrays.asList("extract_field1", "extract_field2"), filterField);
|
149
|
-
}
|
150
|
-
catch (DataException ex) {
|
151
|
-
return;
|
152
|
-
}
|
153
|
-
Assert.fail();
|
154
|
-
}
|
155
|
-
|
156
142
|
@Test
|
157
143
|
public void createActitvityExtract() throws Exception
|
158
144
|
{
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-marketo
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.6.1
|
4
|
+
version: 0.6.2.alpha.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- uu59
|
@@ -10,36 +10,36 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2017-10-
|
13
|
+
date: 2017-10-16 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
|
-
|
16
|
+
name: bundler
|
17
|
+
version_requirements: !ruby/object:Gem::Requirement
|
17
18
|
requirements:
|
18
19
|
- - ~>
|
19
20
|
- !ruby/object:Gem::Version
|
20
21
|
version: '1.0'
|
21
|
-
|
22
|
-
prerelease: false
|
23
|
-
type: :development
|
24
|
-
version_requirements: !ruby/object:Gem::Requirement
|
22
|
+
requirement: !ruby/object:Gem::Requirement
|
25
23
|
requirements:
|
26
24
|
- - ~>
|
27
25
|
- !ruby/object:Gem::Version
|
28
26
|
version: '1.0'
|
27
|
+
prerelease: false
|
28
|
+
type: :development
|
29
29
|
- !ruby/object:Gem::Dependency
|
30
|
-
|
30
|
+
name: rake
|
31
|
+
version_requirements: !ruby/object:Gem::Requirement
|
31
32
|
requirements:
|
32
33
|
- - '>='
|
33
34
|
- !ruby/object:Gem::Version
|
34
35
|
version: '10.0'
|
35
|
-
|
36
|
-
prerelease: false
|
37
|
-
type: :development
|
38
|
-
version_requirements: !ruby/object:Gem::Requirement
|
36
|
+
requirement: !ruby/object:Gem::Requirement
|
39
37
|
requirements:
|
40
38
|
- - '>='
|
41
39
|
- !ruby/object:Gem::Version
|
42
40
|
version: '10.0'
|
41
|
+
prerelease: false
|
42
|
+
type: :development
|
43
43
|
description: Loads records from Marketo.
|
44
44
|
email:
|
45
45
|
- k@uu59.org
|
@@ -125,8 +125,8 @@ files:
|
|
125
125
|
- src/test/resources/fixtures/program_response.json
|
126
126
|
- classpath/jetty-http-9.2.14.v20151106.jar
|
127
127
|
- classpath/embulk-base-restclient-0.5.3.jar
|
128
|
-
- classpath/embulk-input-marketo-0.6.1.jar
|
129
128
|
- classpath/jetty-client-9.2.14.v20151106.jar
|
129
|
+
- classpath/embulk-input-marketo-0.6.2.alpha.1.jar
|
130
130
|
- classpath/jetty-util-9.2.14.v20151106.jar
|
131
131
|
- classpath/embulk-util-retryhelper-jetty92-0.5.3.jar
|
132
132
|
- classpath/jetty-io-9.2.14.v20151106.jar
|
@@ -145,9 +145,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
145
145
|
version: '0'
|
146
146
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
147
147
|
requirements:
|
148
|
-
- - '
|
148
|
+
- - '>'
|
149
149
|
- !ruby/object:Gem::Version
|
150
|
-
version:
|
150
|
+
version: 1.3.1
|
151
151
|
requirements: []
|
152
152
|
rubyforge_project:
|
153
153
|
rubygems_version: 2.1.9
|