embulk-input-marketo 0.6.1 → 0.6.2.alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: cea99bba894ff8f3b8bd6abd2b5d089906dc1bbc
4
- data.tar.gz: 4b62273710bd3e6729eceeb73e300bb8490254c3
3
+ metadata.gz: 11dcb4f3b03a1a8f44ab526a8904844c66d87939
4
+ data.tar.gz: 19d86812f3f1d0b129a2b5212b0331455f4b64c9
5
5
  SHA512:
6
- metadata.gz: 4d8f1c4b38e78f8785c22e788ed1b920f0d948401e17c5d9c666cf9c8833f6cdf652782f4486d4d74a4a5d711ddca6379bd0534c0d3a461470d3d25dcdf9b8d3
7
- data.tar.gz: 8976a9a930280e271704729a0b9b85ff26bba1276a6a6504a49605e5f88565bf96a6b35d10de92bade553558a5df5fa1df0554a61430375f11484882a5cace37
6
+ metadata.gz: f6797483eff01213b29cce8b2b23b659419c35a2fba75d499ef55eae2950dcb5d9fa99d6996a5747646b133531bdf3b37755516a1a83c8d9c7d6f80252e0bfc8
7
+ data.tar.gz: 69f31537e1ca83a97f18a7b2539e2de193c1c31ca90b1637655fb0ac9101b3057ce3f52091f31982efb7c055dde962aa7b97b8e730e9e90dc3c50ee1b4b0a722
data/CHANGELOG.md CHANGED
@@ -1,3 +1,8 @@
1
+ ## 0.6.1 - 2017-10-12
2
+ * [fixed] OutOfMemeory when run Lead by list, Lead by program [#69](https://github.com/treasure-data/embulk-input-marketo/pull/69)
3
+ * [fixed] SOAP only field cause NullPointerException [#69](https://github.com/treasure-data/embulk-input-marketo/pull/69)
4
+ * [enhancement] Implement file download resume in Bulk extract [#69](https://github.com/treasure-data/embulk-input-marketo/pull/69)
5
+
1
6
  ## 0.6.0 - 2017-10-10
2
7
  * [major] Migrate to Java by embulk-base-restclient [#66](https://github.com/treasure-data/embulk-input-marketo/pull/66)
3
8
  * [major] Migrate to REST API [#66](https://github.com/treasure-data/embulk-input-marketo/pull/66)
@@ -5,7 +10,6 @@
5
10
  * [major] Support Marketo bulk extract API for lead and activity targets [#66](https://github.com/treasure-data/embulk-input-marketo/pull/66)
6
11
  * [major] Support incremental ingestion for lead and activity targets [#66](https://github.com/treasure-data/embulk-input-marketo/pull/66)
7
12
 
8
-
9
13
  ## 0.5.6 - 2016-12-14
10
14
  * [maintenance] Enable tcp keepalive [#64](https://github.com/treasure-data/embulk-input-marketo/pull/64)
11
15
 
data/build.gradle CHANGED
@@ -16,7 +16,7 @@ repositories {
16
16
  configurations {
17
17
  provided
18
18
  }
19
- version = "0.6.1"
19
+ version = "0.6.2.alpha.1"
20
20
  sourceCompatibility = 1.7
21
21
  targetCompatibility = 1.7
22
22
 
@@ -43,6 +43,7 @@ import java.text.SimpleDateFormat;
43
43
  import java.util.ArrayList;
44
44
  import java.util.Date;
45
45
  import java.util.HashMap;
46
+ import java.util.HashSet;
46
47
  import java.util.Iterator;
47
48
  import java.util.List;
48
49
  import java.util.Map;
@@ -123,6 +124,9 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
123
124
  if (task.getFromDate() == null) {
124
125
  throw new ConfigException("From date is required for Bulk Extract");
125
126
  }
127
+ if (task.getFromDate().getTime() >= task.getJobStartTime().getMillis()) {
128
+ throw new ConfigException("From date can't not be in future");
129
+ }
126
130
  //Calculate to date
127
131
  DateTime toDate = getToDate(task);
128
132
  task.setToDate(Optional.of(toDate.toDate()));
@@ -134,8 +138,8 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
134
138
  DateTime dateTime = new DateTime(fromDate);
135
139
  DateTime toDate = dateTime.plusDays(task.getFetchDays());
136
140
  if (toDate.isAfter(task.getJobStartTime())) {
137
- //Minus 1 hour and lock down toDate
138
- toDate = task.getJobStartTime().minusHours(1);
141
+ //Lock down to date
142
+ toDate = task.getJobStartTime();
139
143
  }
140
144
  return toDate;
141
145
  }
@@ -145,7 +149,7 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
145
149
  {
146
150
  ConfigDiff configDiff = super.buildConfigDiff(task, schema, taskCount, taskReports);
147
151
  Long currentLatestFetchTime = 0L;
148
- Set latestUIds = null;
152
+ Set latestUIds = new HashSet();
149
153
  String incrementalColumn = task.getIncrementalColumn().orNull();
150
154
  int imported = 0;
151
155
  if (incrementalColumn != null && task.getIncremental()) {
@@ -168,11 +172,12 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
168
172
  }
169
173
  // in case of we didn't import anything but search range is entirely in the past. Then we should move the the range anyway.
170
174
  if (imported == 0) {
171
- configDiff.set(FROM_DATE, df.format(task.getToDate().orNull()));
175
+ Date toDate = task.getToDate().orNull();
176
+ configDiff.set(FROM_DATE, df.format(toDate));
172
177
  }
173
178
  else {
174
- // Otherwise it's should start from the currentLastFetchTime. If Marketo could guaranteed that they always return data of full range we could just set it to_date
175
- configDiff.set(FROM_DATE, df.format(new Date(currentLatestFetchTime)));
179
+ // Otherwise it's should start from the currentLastFetchTime plus 1 second.
180
+ configDiff.set(FROM_DATE, df.format(new DateTime(currentLatestFetchTime).plusSeconds(1).toDate()));
176
181
  }
177
182
  configDiff.set(LATEST_FETCH_TIME, currentLatestFetchTime);
178
183
  configDiff.set(LATEST_UID_LIST, latestUIds);
@@ -199,7 +204,7 @@ public abstract class MarketoBaseBulkExtractInputPlugin<T extends MarketoBaseBul
199
204
  return new CsvRecordIterator(input, task);
200
205
  }
201
206
  }));
202
- long currentTimestamp = 0L;
207
+ long currentTimestamp = task.getLatestFetchTime().or(0L);
203
208
  Set<String> latestUids = task.getPreviousUids();
204
209
  //Keep the preview code here when we can enable real preview
205
210
  if (Exec.isPreview()) {
@@ -23,6 +23,7 @@ import java.util.Date;
23
23
  import java.util.Set;
24
24
 
25
25
  import static org.junit.Assert.assertEquals;
26
+ import static org.junit.Assert.fail;
26
27
 
27
28
  /**
28
29
  * Created by khuutantaitai on 10/3/17.
@@ -52,8 +53,10 @@ public class MarketoBaseBulkExtractInputPluginTest
52
53
  public void validateInputTaskToDateLessThanJobStartTime()
53
54
  {
54
55
  Date fromDate = new Date(1504224000000L);
56
+ DateTime jobStartTime = new DateTime(1506842144000L);
55
57
  MarketoBaseBulkExtractInputPlugin.PluginTask pluginTask = Mockito.mock(MarketoBaseBulkExtractInputPlugin.PluginTask.class);
56
58
  Mockito.when(pluginTask.getFromDate()).thenReturn(fromDate);
59
+ Mockito.when(pluginTask.getJobStartTime()).thenReturn(jobStartTime);
57
60
  Mockito.when(pluginTask.getFetchDays()).thenReturn(7);
58
61
  baseBulkExtractInputPlugin.validateInputTask(pluginTask);
59
62
  ArgumentCaptor<Optional<Date>> argumentCaptor = ArgumentCaptor.forClass(Optional.class);
@@ -61,6 +64,24 @@ public class MarketoBaseBulkExtractInputPluginTest
61
64
  assertEquals(1504828800000L, argumentCaptor.getValue().get().getTime());
62
65
  }
63
66
 
67
+ @Test()
68
+ public void validateInputTaskFromDateMoreThanJobStartTime()
69
+ {
70
+ Date fromDate = new Date(1507619744000L);
71
+ DateTime jobStartTime = new DateTime(1506842144000L);
72
+ MarketoBaseBulkExtractInputPlugin.PluginTask pluginTask = Mockito.mock(MarketoBaseBulkExtractInputPlugin.PluginTask.class);
73
+ Mockito.when(pluginTask.getFromDate()).thenReturn(fromDate);
74
+ Mockito.when(pluginTask.getJobStartTime()).thenReturn(jobStartTime);
75
+
76
+ try {
77
+ baseBulkExtractInputPlugin.validateInputTask(pluginTask);
78
+ }
79
+ catch (ConfigException ex) {
80
+ return;
81
+ }
82
+ fail();
83
+ }
84
+
64
85
  @Test()
65
86
  public void validateInputTaskToDateMoreThanJobStartTime()
66
87
  {
@@ -73,7 +94,7 @@ public class MarketoBaseBulkExtractInputPluginTest
73
94
  baseBulkExtractInputPlugin.validateInputTask(pluginTask);
74
95
  ArgumentCaptor<Optional<Date>> toDateArgumentCaptor = ArgumentCaptor.forClass(Optional.class);
75
96
  Mockito.verify(pluginTask, Mockito.times(1)).setToDate(toDateArgumentCaptor.capture());
76
- assertEquals(jobStartTime.minusHours(1).getMillis(), toDateArgumentCaptor.getValue().get().getTime());
97
+ assertEquals(jobStartTime.getMillis(), toDateArgumentCaptor.getValue().get().getTime());
77
98
  }
78
99
 
79
100
  @Test
@@ -139,20 +139,6 @@ public class MarketoRestClientTest
139
139
  Assert.fail();
140
140
  }
141
141
 
142
- @Test()
143
- public void createLeadBulkExtractWithParseError() throws Exception
144
- {
145
- Mockito.doThrow(JsonProcessingException.class).when(marketoRestClient).doPost(Mockito.eq(END_POINT + MarketoRESTEndpoint.CREATE_LEAD_EXTRACT.getEndpoint()), Mockito.isNull(Map.class), Mockito.isNull(ImmutableListMultimap.class), Mockito.anyString(), Mockito.any(MarketoResponseJetty92EntityReader.class));
146
- String filterField = "filterField";
147
- try {
148
- marketoRestClient.createLeadBulkExtract(new Date(), new Date(), Arrays.asList("extract_field1", "extract_field2"), filterField);
149
- }
150
- catch (DataException ex) {
151
- return;
152
- }
153
- Assert.fail();
154
- }
155
-
156
142
  @Test
157
143
  public void createActitvityExtract() throws Exception
158
144
  {
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-input-marketo
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.6.1
4
+ version: 0.6.2.alpha.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - uu59
@@ -10,36 +10,36 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2017-10-12 00:00:00.000000000 Z
13
+ date: 2017-10-16 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
- requirement: !ruby/object:Gem::Requirement
16
+ name: bundler
17
+ version_requirements: !ruby/object:Gem::Requirement
17
18
  requirements:
18
19
  - - ~>
19
20
  - !ruby/object:Gem::Version
20
21
  version: '1.0'
21
- name: bundler
22
- prerelease: false
23
- type: :development
24
- version_requirements: !ruby/object:Gem::Requirement
22
+ requirement: !ruby/object:Gem::Requirement
25
23
  requirements:
26
24
  - - ~>
27
25
  - !ruby/object:Gem::Version
28
26
  version: '1.0'
27
+ prerelease: false
28
+ type: :development
29
29
  - !ruby/object:Gem::Dependency
30
- requirement: !ruby/object:Gem::Requirement
30
+ name: rake
31
+ version_requirements: !ruby/object:Gem::Requirement
31
32
  requirements:
32
33
  - - '>='
33
34
  - !ruby/object:Gem::Version
34
35
  version: '10.0'
35
- name: rake
36
- prerelease: false
37
- type: :development
38
- version_requirements: !ruby/object:Gem::Requirement
36
+ requirement: !ruby/object:Gem::Requirement
39
37
  requirements:
40
38
  - - '>='
41
39
  - !ruby/object:Gem::Version
42
40
  version: '10.0'
41
+ prerelease: false
42
+ type: :development
43
43
  description: Loads records from Marketo.
44
44
  email:
45
45
  - k@uu59.org
@@ -125,8 +125,8 @@ files:
125
125
  - src/test/resources/fixtures/program_response.json
126
126
  - classpath/jetty-http-9.2.14.v20151106.jar
127
127
  - classpath/embulk-base-restclient-0.5.3.jar
128
- - classpath/embulk-input-marketo-0.6.1.jar
129
128
  - classpath/jetty-client-9.2.14.v20151106.jar
129
+ - classpath/embulk-input-marketo-0.6.2.alpha.1.jar
130
130
  - classpath/jetty-util-9.2.14.v20151106.jar
131
131
  - classpath/embulk-util-retryhelper-jetty92-0.5.3.jar
132
132
  - classpath/jetty-io-9.2.14.v20151106.jar
@@ -145,9 +145,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
145
145
  version: '0'
146
146
  required_rubygems_version: !ruby/object:Gem::Requirement
147
147
  requirements:
148
- - - '>='
148
+ - - '>'
149
149
  - !ruby/object:Gem::Version
150
- version: '0'
150
+ version: 1.3.1
151
151
  requirements: []
152
152
  rubyforge_project:
153
153
  rubygems_version: 2.1.9