embulk-input-marketo 0.6.0 → 0.6.1.alpha.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +11 -0
- data/README.md +132 -12
- data/build.gradle +1 -1
- data/src/main/java/org/embulk/input/marketo/MarketoServiceImpl.java +115 -44
- data/src/main/java/org/embulk/input/marketo/MarketoUtils.java +83 -0
- data/src/main/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPlugin.java +17 -3
- data/src/main/java/org/embulk/input/marketo/model/BulkExtractRangeHeader.java +26 -0
- data/src/main/java/org/embulk/input/marketo/rest/MarketoBaseRestClient.java +8 -3
- data/src/main/java/org/embulk/input/marketo/rest/MarketoInputStreamResponseEntityReader.java +1 -1
- data/src/main/java/org/embulk/input/marketo/rest/MarketoResponseJetty92EntityReader.java +2 -4
- data/src/main/java/org/embulk/input/marketo/rest/MarketoRestClient.java +47 -20
- data/src/test/java/org/embulk/input/marketo/MarketoServiceImplTest.java +60 -57
- data/src/test/java/org/embulk/input/marketo/delegate/ActivityBulkExtractInputPluginTest.java +16 -15
- data/src/test/java/org/embulk/input/marketo/delegate/CampaignInputPluginTest.java +10 -10
- data/src/test/java/org/embulk/input/marketo/delegate/LeadBulkExtractInputPluginTest.java +22 -21
- data/src/test/java/org/embulk/input/marketo/delegate/LeadWithListInputPluginTest.java +22 -20
- data/src/test/java/org/embulk/input/marketo/delegate/LeadWithProgramInputPluginTest.java +22 -20
- data/src/test/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPluginTest.java +27 -28
- data/src/test/java/org/embulk/input/marketo/rest/MarketoBaseRestClientTest.java +56 -57
- data/src/test/java/org/embulk/input/marketo/rest/MarketoRestClientTest.java +136 -125
- metadata +6 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ad07d4e0f62200bd2f12b4cdd7ed1dfc17542a7d
|
4
|
+
data.tar.gz: 1a52eb039bc6f9a1ed2f847681d1775e7d7f7d09
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8801d26caac30fab6ce0fbcb5f1e7df7dfaf58712e8273b5d11ae00c23e7cc70edf171efcd6d44e222412a54355b576b64dcdd0187dd94aee02494ada3df57df
|
7
|
+
data.tar.gz: a22b92dd76b7de41b386cf9a8a0763294a3d43d8daf5850797075da57fc95b46b4b1bb8acd3a90179314099d60d203d6cd59279450842978d598f1cde2b97ed0
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,14 @@
|
|
1
|
+
## 0.6.0 - 2017-10-10
|
2
|
+
* [major] Migrate to Java by embulk-base-restclient [#66](https://github.com/treasure-data/embulk-input-marketo/pull/66)
|
3
|
+
* [major] Migrate to REST API [#66](https://github.com/treasure-data/embulk-input-marketo/pull/66)
|
4
|
+
* [major] Add 3 more target, Campaign, Lead by list and Lead by program [#66](https://github.com/treasure-data/embulk-input-marketo/pull/66)
|
5
|
+
* [major] Support Marketo bulk extract API for lead and activity targets [#66](https://github.com/treasure-data/embulk-input-marketo/pull/66)
|
6
|
+
* [major] Support incremental ingestion for lead and activity targets [#66](https://github.com/treasure-data/embulk-input-marketo/pull/66)
|
7
|
+
|
8
|
+
|
9
|
+
## 0.5.6 - 2016-12-14
|
10
|
+
* [maintenance] Enable tcp keepalive [#64](https://github.com/treasure-data/embulk-input-marketo/pull/64)
|
11
|
+
|
1
12
|
## 0.5.6 - 2016-12-14
|
2
13
|
* [maintenance] Enable tcp keepalive [#64](https://github.com/treasure-data/embulk-input-marketo/pull/64)
|
3
14
|
|
data/README.md
CHANGED
@@ -1,32 +1,152 @@
|
|
1
|
+
[![Build Status](https://travis-ci.org/treasure-data/embulk-input-marketo.svg?branch=master)](https://travis-ci.org/treasure-data/embulk-input-marketo)
|
2
|
+
[![Code Climate](https://codeclimate.com/github/treasure-data/embulk-input-marketo/badges/gpa.svg)](https://codeclimate.com/github/treasure-data/embulk-input-marketo)
|
3
|
+
[![Test Coverage](https://codeclimate.com/github/treasure-data/embulk-input-marketo/badges/coverage.svg)](https://codeclimate.com/github/treasure-data/embulk-input-marketo/coverage)
|
4
|
+
[![Gem Version](https://badge.fury.io/rb/embulk-input-marketo.svg)](http://badge.fury.io/rb/embulk-input-marketo)
|
5
|
+
|
1
6
|
# Marketo input plugin for Embulk
|
2
7
|
|
3
|
-
|
8
|
+
embulk-input-marketo is the gem preparing Embulk input plugins for [Marketo](http://www.marketo.com/).
|
9
|
+
|
10
|
+
- Lead(lead)
|
11
|
+
- Activity log(activity)
|
12
|
+
- Lead by list(all_lead_with_list_id)
|
13
|
+
- Lead by program(all_lead_with_program_id)
|
14
|
+
- Campaign(campaign)
|
15
|
+
|
16
|
+
This plugin uses Marketo REST API.
|
4
17
|
|
5
18
|
## Overview
|
6
19
|
|
20
|
+
Required Embulk version >= 0.8.33 (since 0.6.0).
|
21
|
+
|
7
22
|
* **Plugin type**: input
|
8
|
-
* **Resume supported**:
|
9
|
-
* **Cleanup supported**:
|
23
|
+
* **Resume supported**: no
|
24
|
+
* **Cleanup supported**: no
|
10
25
|
* **Guess supported**: no
|
11
26
|
|
27
|
+
## Install
|
28
|
+
|
29
|
+
```
|
30
|
+
$ embulk gem install embulk-input-marketo
|
31
|
+
```
|
32
|
+
|
12
33
|
## Configuration
|
13
34
|
|
14
|
-
|
15
|
-
|
16
|
-
|
35
|
+
### API
|
36
|
+
|
37
|
+
Below parameters are shown in "Admin" > "Web Services" page in Marketo.
|
38
|
+
|
39
|
+
### Base configuration parameter
|
40
|
+
|
41
|
+
All target have this configuration parameters
|
42
|
+
|
43
|
+
| name | required | default value | description |
|
44
|
+
|----------------------------------|----------|---------------|----------------------------------------------------------------------------------------------------------------------------------|
|
45
|
+
| **target** | true | | Marketo targets |
|
46
|
+
| **account_id** | true | | Marketo Muchkin id |
|
47
|
+
| **client_id** | true | | Marketo REST client id |
|
48
|
+
| **client_secret** | true | | Marketo REST client secret |
|
49
|
+
| **marketo_limit_interval_milis** | false | 20 | Marketo have limitation of 100 calls per 20 second. If REST API calls are failed they will wait this amount of time before retry |
|
50
|
+
| **batch_size** | false | 300 | Token paging batch size. Some REST API support batch |
|
51
|
+
| **max_return** | false | 200 | Max return for Endpoint that use offset paging |
|
52
|
+
|
53
|
+
### Bulk extract target configuration parameter (Lead and Activity)
|
54
|
+
|
55
|
+
All bulk extract target use this configuration parameter
|
56
|
+
|
57
|
+
| name | required | default value | description |
|
58
|
+
|-----------------------------|----------|---------------|-------------------------------------------------------------------------------------------------------------------------------|
|
59
|
+
| **from_date** | true | | Import data since this date. Example: 2017-10-11T06:43:24+00:00 |
|
60
|
+
| **fetch_days** | false | 1 | Ammount of days to fetch since from_date |
|
61
|
+
| **polling_interval_second** | false | 60 | Amount of time to wait between pooling job status in second |
|
62
|
+
| **bulk_job_timeout_second** | false | 3600 | Amount of time to wait for bulk job to complete in second |
|
63
|
+
| **incremental** | false | true | If incremental is set to true, next run will have from_date set to the previous to_date(calculated by from_date + fetch_days) |
|
64
|
+
| **incremental_column** | false | createdAt | Column use to filter from_date and to_date |
|
65
|
+
|
66
|
+
### Lead
|
67
|
+
|
68
|
+
Lead target extract all Marketo leads, it use Marketo bulk extract feature. Configuration include bulk extract configuration.
|
69
|
+
|
70
|
+
`target: lead`
|
71
|
+
|
72
|
+
Configuration:
|
73
|
+
|
74
|
+
| name | required | default value | description |
|
75
|
+
|--------------------|----------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
76
|
+
| **use_updated_at** | false | false | Lead data are not immutable so it better to do incremental ingesting with `updateAt` column, but not all Marketo Account have the feature to filter by updatedAt |
|
77
|
+
|
78
|
+
Schema type: Dynamic via describe lead endpoint.
|
79
|
+
|
80
|
+
Incremental support: yes
|
81
|
+
|
82
|
+
Range ingestion: yes
|
83
|
+
|
84
|
+
### Activity
|
85
|
+
|
86
|
+
Activity target extract all Marketo actvity log. Configuration include all bulk extract configuraiont
|
87
|
+
|
88
|
+
`target: activity`
|
89
|
+
|
90
|
+
Schema type: Static schema
|
91
|
+
|
92
|
+
Incremental support: yes
|
93
|
+
|
94
|
+
Range ingestion: yes
|
95
|
+
|
96
|
+
### Campaign
|
97
|
+
|
98
|
+
Campaign extract all campaign data from Marketo
|
99
|
+
|
100
|
+
`target: campaign`
|
101
|
+
|
102
|
+
Schema type: Static schema
|
103
|
+
|
104
|
+
Incremental support: no
|
105
|
+
|
106
|
+
Range ingestion: no
|
107
|
+
|
108
|
+
### Lead by list
|
109
|
+
|
110
|
+
Extract all Lead data including lead's list id
|
111
|
+
|
112
|
+
`target: all_lead_with_list_id`
|
113
|
+
|
114
|
+
Schema type: Dynamic via describe leads. Schema will have 1 addition column name listId that contain the id of the list the lead belong to
|
115
|
+
|
116
|
+
Incremental support: no
|
117
|
+
|
118
|
+
Range ingestion: no
|
119
|
+
|
120
|
+
### Laed by program
|
121
|
+
|
122
|
+
Extract all Lead data including lead's program id
|
123
|
+
|
124
|
+
`target: all_lead_with_program_id`
|
125
|
+
|
126
|
+
Schema type: Dynamic via describe leads. Schema will have 1 addition column name listId that contain the id of the list the lead belong to
|
127
|
+
|
128
|
+
Incremental support: no
|
129
|
+
|
130
|
+
Range ingestion: no
|
17
131
|
|
18
132
|
## Example
|
19
133
|
|
134
|
+
For lead, you have `partial-config.yml` like below:
|
135
|
+
|
20
136
|
```yaml
|
21
137
|
in:
|
22
138
|
type: marketo
|
23
|
-
|
24
|
-
|
139
|
+
target: lead
|
140
|
+
account_id: ACCOUNT_ID
|
141
|
+
client_id: CLIENT_ID
|
142
|
+
client_secret: CLIENT_SECRET
|
143
|
+
from_date: 2017-09-01
|
144
|
+
fetch_days: 1
|
145
|
+
out:
|
146
|
+
type: stdout
|
25
147
|
```
|
26
148
|
|
149
|
+
You can run `embulk guess partial-config.yml -o lead-config.yml` and got `lead-config.yml`. `lead-config.yml` includes a schema for Lead.
|
27
150
|
|
28
|
-
|
151
|
+
Next, you can run `embulk preview lead-config.yml` for preview and `embulk run lead-config.yml` for run.
|
29
152
|
|
30
|
-
```
|
31
|
-
$ ./gradlew gem # -t to watch change of files and rebuild continuously
|
32
|
-
```
|
data/build.gradle
CHANGED
@@ -3,7 +3,8 @@ package org.embulk.input.marketo;
|
|
3
3
|
import com.fasterxml.jackson.databind.node.ObjectNode;
|
4
4
|
import com.google.common.base.Function;
|
5
5
|
import com.google.common.collect.Iterables;
|
6
|
-
import
|
6
|
+
import org.apache.commons.lang3.StringUtils;
|
7
|
+
import org.embulk.input.marketo.model.BulkExtractRangeHeader;
|
7
8
|
import org.embulk.input.marketo.model.MarketoField;
|
8
9
|
import org.embulk.input.marketo.rest.MarketoRestClient;
|
9
10
|
import org.embulk.input.marketo.rest.RecordPagingIterable;
|
@@ -11,14 +12,11 @@ import org.embulk.spi.DataException;
|
|
11
12
|
import org.embulk.spi.Exec;
|
12
13
|
import org.slf4j.Logger;
|
13
14
|
|
14
|
-
import javax.annotation.Nullable;
|
15
|
-
|
16
15
|
import java.io.File;
|
17
16
|
import java.io.FileOutputStream;
|
18
17
|
import java.io.IOException;
|
19
18
|
import java.io.InputStream;
|
20
19
|
import java.io.OutputStream;
|
21
|
-
import java.util.ArrayList;
|
22
20
|
import java.util.Date;
|
23
21
|
import java.util.List;
|
24
22
|
|
@@ -31,6 +29,10 @@ public class MarketoServiceImpl implements MarketoService
|
|
31
29
|
|
32
30
|
private static final String DEFAULT_FILE_FORMAT = "csv";
|
33
31
|
|
32
|
+
private static final int BUF_SIZE = 0x1000;
|
33
|
+
|
34
|
+
private static final int MAX_RESUME_TIME = 50;
|
35
|
+
|
34
36
|
private MarketoRestClient marketoRestClient;
|
35
37
|
|
36
38
|
public MarketoServiceImpl(MarketoRestClient marketoRestClient)
|
@@ -39,9 +41,9 @@ public class MarketoServiceImpl implements MarketoService
|
|
39
41
|
}
|
40
42
|
|
41
43
|
@Override
|
42
|
-
public File extractLead(Date startTime, Date endTime, List<String> extractedFields, String filterField, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond)
|
44
|
+
public File extractLead(final Date startTime, Date endTime, List<String> extractedFields, String filterField, int pollingTimeIntervalSecond, final int bulkJobTimeoutSecond)
|
43
45
|
{
|
44
|
-
String exportID = marketoRestClient.createLeadBulkExtract(startTime, endTime, extractedFields, filterField);
|
46
|
+
final String exportID = marketoRestClient.createLeadBulkExtract(startTime, endTime, extractedFields, filterField);
|
45
47
|
marketoRestClient.startLeadBulkExtract(exportID);
|
46
48
|
try {
|
47
49
|
marketoRestClient.waitLeadExportJobComplete(exportID, pollingTimeIntervalSecond, bulkJobTimeoutSecond);
|
@@ -50,27 +52,41 @@ public class MarketoServiceImpl implements MarketoService
|
|
50
52
|
LOGGER.error("Exception when waiting for export job id: {}", exportID, e);
|
51
53
|
throw new DataException("Error when wait for bulk extract");
|
52
54
|
}
|
53
|
-
|
54
|
-
|
55
|
+
return downloadBulkExtract(new Function<BulkExtractRangeHeader, InputStream>()
|
56
|
+
{
|
57
|
+
@Override
|
58
|
+
public InputStream apply(BulkExtractRangeHeader bulkExtractRangeHeader)
|
59
|
+
{
|
60
|
+
return marketoRestClient.getLeadBulkExtractResult(exportID, bulkExtractRangeHeader);
|
61
|
+
}
|
62
|
+
});
|
55
63
|
}
|
56
64
|
|
57
|
-
private
|
65
|
+
private long saveExtractedFile(InputStream extractResult, File tempFile) throws DownloadBulkExtractException
|
58
66
|
{
|
59
|
-
|
60
|
-
try (OutputStream fileOuputStream = new FileOutputStream(tempFile)) {
|
61
|
-
|
67
|
+
long total = 0;
|
68
|
+
try (OutputStream fileOuputStream = new FileOutputStream(tempFile, true)) {
|
69
|
+
byte[] buf = new byte[BUF_SIZE];
|
70
|
+
while (true) {
|
71
|
+
int r = extractResult.read(buf);
|
72
|
+
if (r == -1) {
|
73
|
+
break;
|
74
|
+
}
|
75
|
+
fileOuputStream.write(buf, 0, r);
|
76
|
+
total += r;
|
77
|
+
}
|
62
78
|
}
|
63
79
|
catch (IOException e) {
|
64
|
-
LOGGER.error("Encounter exception when download bulk extract file
|
65
|
-
throw new
|
80
|
+
LOGGER.error("Encounter exception when download bulk extract file", e);
|
81
|
+
throw new DownloadBulkExtractException("Encounter exception when download bulk extract file", e, total);
|
66
82
|
}
|
67
|
-
return
|
83
|
+
return total;
|
68
84
|
}
|
69
85
|
|
70
86
|
@Override
|
71
87
|
public File extractAllActivity(Date startTime, Date endTime, int pollingTimeIntervalSecond, int bulkJobTimeoutSecond)
|
72
88
|
{
|
73
|
-
String exportID = marketoRestClient.createActivityExtract(startTime, endTime);
|
89
|
+
final String exportID = marketoRestClient.createActivityExtract(startTime, endTime);
|
74
90
|
marketoRestClient.startActitvityBulkExtract(exportID);
|
75
91
|
try {
|
76
92
|
marketoRestClient.waitActitvityExportJobComplete(exportID, pollingTimeIntervalSecond, bulkJobTimeoutSecond);
|
@@ -79,49 +95,82 @@ public class MarketoServiceImpl implements MarketoService
|
|
79
95
|
LOGGER.error("Exception when waiting for export job id: {}", exportID, e);
|
80
96
|
throw new DataException("Error when wait for bulk extract");
|
81
97
|
}
|
82
|
-
|
83
|
-
|
98
|
+
return downloadBulkExtract(new Function<BulkExtractRangeHeader, InputStream>()
|
99
|
+
{
|
100
|
+
@Override
|
101
|
+
public InputStream apply(BulkExtractRangeHeader bulkExtractRangeHeader)
|
102
|
+
{
|
103
|
+
return marketoRestClient.getActivitiesBulkExtractResult(exportID, bulkExtractRangeHeader);
|
104
|
+
}
|
105
|
+
});
|
106
|
+
}
|
107
|
+
private File downloadBulkExtract(Function<BulkExtractRangeHeader, InputStream> getBulkExtractfunction)
|
108
|
+
{
|
109
|
+
final File tempFile = Exec.getTempFileSpace().createTempFile(DEFAULT_FILE_FORMAT);
|
110
|
+
long startByte = 0;
|
111
|
+
int resumeTime = 0;
|
112
|
+
while (resumeTime < MAX_RESUME_TIME) {
|
113
|
+
BulkExtractRangeHeader bulkExtractRangeHeader = new BulkExtractRangeHeader(startByte);
|
114
|
+
InputStream bulkExtractResult = getBulkExtractfunction.apply(bulkExtractRangeHeader);
|
115
|
+
try {
|
116
|
+
saveExtractedFile(bulkExtractResult, tempFile);
|
117
|
+
return tempFile;
|
118
|
+
}
|
119
|
+
catch (DownloadBulkExtractException e) {
|
120
|
+
startByte = startByte + e.getByteWritten();
|
121
|
+
LOGGER.warn("will resume activity bulk extract at byte [{}]", startByte);
|
122
|
+
}
|
123
|
+
resumeTime = resumeTime + 1;
|
124
|
+
}
|
125
|
+
//Too many resume we still can't get the file
|
126
|
+
throw new DataException("Can't down load bulk extract");
|
84
127
|
}
|
85
|
-
|
86
128
|
@Override
|
87
129
|
public Iterable<ObjectNode> getAllListLead(List<String> fieldNames)
|
88
130
|
{
|
89
131
|
RecordPagingIterable<ObjectNode> lists = marketoRestClient.getLists();
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
132
|
+
final String fieldNameString = StringUtils.join(fieldNames, ",");
|
133
|
+
return MarketoUtils.flatMap(lists, new Function<ObjectNode, Iterable<ObjectNode>>()
|
134
|
+
{
|
135
|
+
@Override
|
136
|
+
public Iterable<ObjectNode> apply(ObjectNode input)
|
94
137
|
{
|
95
|
-
|
96
|
-
|
138
|
+
final String id = input.get("id").asText();
|
139
|
+
return Iterables.transform(marketoRestClient.getLeadsByList(id, fieldNameString), new Function<ObjectNode, ObjectNode>()
|
97
140
|
{
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
141
|
+
@Override
|
142
|
+
public ObjectNode apply(ObjectNode input)
|
143
|
+
{
|
144
|
+
input.put(MarketoUtils.LIST_ID_COLUMN_NAME, id);
|
145
|
+
return input;
|
146
|
+
}
|
147
|
+
});
|
148
|
+
}
|
149
|
+
});
|
104
150
|
}
|
105
151
|
|
106
152
|
@Override
|
107
153
|
public Iterable<ObjectNode> getAllProgramLead(List<String> fieldNames)
|
108
154
|
{
|
109
155
|
RecordPagingIterable<ObjectNode> lists = marketoRestClient.getPrograms();
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
156
|
+
final String fieldNameString = StringUtils.join(fieldNames, ",");
|
157
|
+
return MarketoUtils.flatMap(lists, new Function<ObjectNode, Iterable<ObjectNode>>()
|
158
|
+
{
|
159
|
+
@Override
|
160
|
+
public Iterable<ObjectNode> apply(ObjectNode input)
|
114
161
|
{
|
115
|
-
|
116
|
-
|
117
|
-
public ObjectNode apply(@Nullable ObjectNode input)
|
162
|
+
final String id = input.get("id").asText();
|
163
|
+
return Iterables.transform(marketoRestClient.getLeadsByProgram(id, fieldNameString), new Function<ObjectNode, ObjectNode>()
|
118
164
|
{
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
165
|
+
@Override
|
166
|
+
public ObjectNode apply(ObjectNode input)
|
167
|
+
{
|
168
|
+
input.put(MarketoUtils.PROGRAM_ID_COLUMN_NAME, id);
|
169
|
+
return input;
|
170
|
+
}
|
171
|
+
});
|
172
|
+
}
|
173
|
+
});
|
125
174
|
}
|
126
175
|
|
127
176
|
@Override
|
@@ -151,4 +200,26 @@ public class MarketoServiceImpl implements MarketoService
|
|
151
200
|
columns.add(new MarketoField(MarketoUtils.LIST_ID_COLUMN_NAME, MarketoField.MarketoDataType.STRING));
|
152
201
|
return columns;
|
153
202
|
}
|
203
|
+
|
204
|
+
private static class DownloadBulkExtractException extends Exception
|
205
|
+
{
|
206
|
+
private final long byteWritten;
|
207
|
+
|
208
|
+
public DownloadBulkExtractException(String message, Throwable cause, long byteWritten)
|
209
|
+
{
|
210
|
+
super(message, cause);
|
211
|
+
this.byteWritten = byteWritten;
|
212
|
+
}
|
213
|
+
|
214
|
+
public DownloadBulkExtractException(Throwable cause, long byteWritten)
|
215
|
+
{
|
216
|
+
super(cause);
|
217
|
+
this.byteWritten = byteWritten;
|
218
|
+
}
|
219
|
+
|
220
|
+
public long getByteWritten()
|
221
|
+
{
|
222
|
+
return byteWritten;
|
223
|
+
}
|
224
|
+
}
|
154
225
|
}
|
@@ -11,12 +11,17 @@ import org.embulk.base.restclient.jackson.JacksonTopLevelValueLocator;
|
|
11
11
|
import org.embulk.base.restclient.record.ServiceRecord;
|
12
12
|
import org.embulk.base.restclient.record.ValueLocator;
|
13
13
|
import org.embulk.input.marketo.model.MarketoField;
|
14
|
+
import org.embulk.spi.Exec;
|
15
|
+
import org.embulk.spi.util.RetryExecutor;
|
14
16
|
import org.joda.time.DateTime;
|
17
|
+
import org.slf4j.Logger;
|
15
18
|
|
16
19
|
import javax.annotation.Nullable;
|
17
20
|
|
18
21
|
import java.util.ArrayList;
|
22
|
+
import java.util.Iterator;
|
19
23
|
import java.util.List;
|
24
|
+
import java.util.NoSuchElementException;
|
20
25
|
import java.util.Set;
|
21
26
|
|
22
27
|
/**
|
@@ -126,4 +131,82 @@ public class MarketoUtils
|
|
126
131
|
'}';
|
127
132
|
}
|
128
133
|
}
|
134
|
+
|
135
|
+
public static <T> T executeWithRetry(int maximumRetries, int initialRetryIntervalMillis, int maximumRetryIntervalMillis, AlwaysRetryRetryable<T> alwaysRetryRetryable) throws RetryExecutor.RetryGiveupException, InterruptedException
|
136
|
+
{
|
137
|
+
return RetryExecutor
|
138
|
+
.retryExecutor()
|
139
|
+
.withRetryLimit(maximumRetries)
|
140
|
+
.withInitialRetryWait(initialRetryIntervalMillis)
|
141
|
+
.withMaxRetryWait(maximumRetryIntervalMillis)
|
142
|
+
.runInterruptible(alwaysRetryRetryable);
|
143
|
+
}
|
144
|
+
|
145
|
+
public abstract static class AlwaysRetryRetryable<T> implements RetryExecutor.Retryable<T>
|
146
|
+
{
|
147
|
+
private static final Logger LOGGER = Exec.getLogger(AlwaysRetryRetryable.class);
|
148
|
+
|
149
|
+
@Override
|
150
|
+
public abstract T call() throws Exception;
|
151
|
+
|
152
|
+
@Override
|
153
|
+
public boolean isRetryableException(Exception exception)
|
154
|
+
{
|
155
|
+
return true;
|
156
|
+
}
|
157
|
+
|
158
|
+
@Override
|
159
|
+
public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait) throws RetryExecutor.RetryGiveupException
|
160
|
+
{
|
161
|
+
LOGGER.info("Retry [{}]/[{}] with retryWait [{}] on exception {}", retryCount, retryLimit, retryWait, exception.getMessage());
|
162
|
+
}
|
163
|
+
|
164
|
+
@Override
|
165
|
+
public void onGiveup(Exception firstException, Exception lastException) throws RetryExecutor.RetryGiveupException
|
166
|
+
{
|
167
|
+
LOGGER.info("Giving up execution on exception", lastException);
|
168
|
+
}
|
169
|
+
}
|
170
|
+
public static <T, R> Iterable<R> flatMap(final Iterable<T> iterable, final Function<T, Iterable<R>> function)
|
171
|
+
{
|
172
|
+
final Iterator<T> iterator = iterable.iterator();
|
173
|
+
return new Iterable<R>()
|
174
|
+
{
|
175
|
+
@Override
|
176
|
+
public Iterator<R> iterator()
|
177
|
+
{
|
178
|
+
return new Iterator<R>()
|
179
|
+
{
|
180
|
+
Iterator<R> currentIterator;
|
181
|
+
@Override
|
182
|
+
public boolean hasNext()
|
183
|
+
{
|
184
|
+
if (currentIterator != null && currentIterator.hasNext()) {
|
185
|
+
return true;
|
186
|
+
}
|
187
|
+
while (iterator.hasNext()) {
|
188
|
+
currentIterator = function.apply(iterator.next()).iterator();
|
189
|
+
if (currentIterator.hasNext()) {
|
190
|
+
return true;
|
191
|
+
}
|
192
|
+
}
|
193
|
+
return false;
|
194
|
+
}
|
195
|
+
@Override
|
196
|
+
public R next()
|
197
|
+
{
|
198
|
+
if (hasNext()) {
|
199
|
+
return currentIterator.next();
|
200
|
+
}
|
201
|
+
throw new NoSuchElementException();
|
202
|
+
}
|
203
|
+
@Override
|
204
|
+
public void remove()
|
205
|
+
{
|
206
|
+
throw new UnsupportedOperationException();
|
207
|
+
}
|
208
|
+
};
|
209
|
+
}
|
210
|
+
};
|
211
|
+
}
|
129
212
|
}
|