embulk-input-marketo-through-proxy 0.6.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.github/CODEOWNERS +1 -0
- data/.github/PULL_REQUEST_TEMPLATE.md +37 -0
- data/.github/workflows/build.yml +38 -0
- data/.gitignore +14 -0
- data/CHANGELOG.md +178 -0
- data/LICENSE.txt +21 -0
- data/README.md +231 -0
- data/build.gradle +105 -0
- data/config/checkstyle/checkstyle.xml +128 -0
- data/config/checkstyle/default.xml +108 -0
- data/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +6 -0
- data/gradlew +169 -0
- data/gradlew.bat +84 -0
- data/lib/embulk/input/marketo.rb +3 -0
- data/settings.gradle +1 -0
- data/src/main/java/org/embulk/input/marketo/CsvTokenizer.java +695 -0
- data/src/main/java/org/embulk/input/marketo/MarketoInputPlugin.java +15 -0
- data/src/main/java/org/embulk/input/marketo/MarketoInputPluginDelegate.java +100 -0
- data/src/main/java/org/embulk/input/marketo/MarketoService.java +47 -0
- data/src/main/java/org/embulk/input/marketo/MarketoServiceImpl.java +258 -0
- data/src/main/java/org/embulk/input/marketo/MarketoUtils.java +212 -0
- data/src/main/java/org/embulk/input/marketo/delegate/ActivityBulkExtractInputPlugin.java +169 -0
- data/src/main/java/org/embulk/input/marketo/delegate/CampaignInputPlugin.java +48 -0
- data/src/main/java/org/embulk/input/marketo/delegate/CustomObjectInputPlugin.java +124 -0
- data/src/main/java/org/embulk/input/marketo/delegate/CustomObjectResponseMapperBuilder.java +81 -0
- data/src/main/java/org/embulk/input/marketo/delegate/LeadBulkExtractInputPlugin.java +68 -0
- data/src/main/java/org/embulk/input/marketo/delegate/LeadServiceResponseMapperBuilder.java +85 -0
- data/src/main/java/org/embulk/input/marketo/delegate/LeadWithListInputPlugin.java +89 -0
- data/src/main/java/org/embulk/input/marketo/delegate/LeadWithProgramInputPlugin.java +85 -0
- data/src/main/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPlugin.java +448 -0
- data/src/main/java/org/embulk/input/marketo/delegate/MarketoBaseInputPluginDelegate.java +160 -0
- data/src/main/java/org/embulk/input/marketo/delegate/ProgramInputPlugin.java +234 -0
- data/src/main/java/org/embulk/input/marketo/exception/MarketoAPIException.java +30 -0
- data/src/main/java/org/embulk/input/marketo/model/BulkExtractRangeHeader.java +26 -0
- data/src/main/java/org/embulk/input/marketo/model/MarketoAccessTokenResponse.java +92 -0
- data/src/main/java/org/embulk/input/marketo/model/MarketoBulkExtractRequest.java +68 -0
- data/src/main/java/org/embulk/input/marketo/model/MarketoError.java +40 -0
- data/src/main/java/org/embulk/input/marketo/model/MarketoField.java +126 -0
- data/src/main/java/org/embulk/input/marketo/model/MarketoResponse.java +82 -0
- data/src/main/java/org/embulk/input/marketo/model/filter/DateRangeFilter.java +40 -0
- data/src/main/java/org/embulk/input/marketo/rest/MarketoBaseRestClient.java +344 -0
- data/src/main/java/org/embulk/input/marketo/rest/MarketoInputStreamResponseEntityReader.java +69 -0
- data/src/main/java/org/embulk/input/marketo/rest/MarketoRESTEndpoint.java +47 -0
- data/src/main/java/org/embulk/input/marketo/rest/MarketoResponseJetty92EntityReader.java +89 -0
- data/src/main/java/org/embulk/input/marketo/rest/MarketoRestClient.java +601 -0
- data/src/main/java/org/embulk/input/marketo/rest/RecordPagingIterable.java +180 -0
- data/src/test/java/org/embulk/input/marketo/MarketoServiceImplTest.java +147 -0
- data/src/test/java/org/embulk/input/marketo/MarketoUtilsTest.java +89 -0
- data/src/test/java/org/embulk/input/marketo/delegate/ActivityBulkExtractInputPluginTest.java +129 -0
- data/src/test/java/org/embulk/input/marketo/delegate/CampaignInputPluginTest.java +73 -0
- data/src/test/java/org/embulk/input/marketo/delegate/CustomObjectInputPluginTest.java +175 -0
- data/src/test/java/org/embulk/input/marketo/delegate/LeadBulkExtractInputPluginTest.java +102 -0
- data/src/test/java/org/embulk/input/marketo/delegate/LeadServiceResponseMapperBuilderTest.java +119 -0
- data/src/test/java/org/embulk/input/marketo/delegate/LeadWithListInputPluginTest.java +132 -0
- data/src/test/java/org/embulk/input/marketo/delegate/LeadWithProgramInputPluginTest.java +134 -0
- data/src/test/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPluginTest.java +171 -0
- data/src/test/java/org/embulk/input/marketo/delegate/MarketoBaseInputPluginDelegateTest.java +60 -0
- data/src/test/java/org/embulk/input/marketo/delegate/ProgramInputPluginTest.java +325 -0
- data/src/test/java/org/embulk/input/marketo/rest/MarketoBaseRestClientTest.java +368 -0
- data/src/test/java/org/embulk/input/marketo/rest/MarketoRestClientTest.java +649 -0
- data/src/test/resources/config/activity_bulk_extract_config.yaml +7 -0
- data/src/test/resources/config/custom_object_config.yaml +8 -0
- data/src/test/resources/config/lead_bulk_extract_config.yaml +8 -0
- data/src/test/resources/config/rest_config.yaml +3 -0
- data/src/test/resources/fixtures/activity_extract1.csv +35 -0
- data/src/test/resources/fixtures/activity_extract2.csv +22 -0
- data/src/test/resources/fixtures/activity_types.json +22 -0
- data/src/test/resources/fixtures/all_program_full.json +53 -0
- data/src/test/resources/fixtures/campaign_response.json +38 -0
- data/src/test/resources/fixtures/campaign_response_full.json +102 -0
- data/src/test/resources/fixtures/custom_object_describe.json +124 -0
- data/src/test/resources/fixtures/custom_object_describe_marketo_fields_full.json +22 -0
- data/src/test/resources/fixtures/custom_object_expected.json +66 -0
- data/src/test/resources/fixtures/custom_object_response.json +24 -0
- data/src/test/resources/fixtures/custom_object_response_full.json +23 -0
- data/src/test/resources/fixtures/lead_by_list.json +33 -0
- data/src/test/resources/fixtures/lead_by_program_response.json +47 -0
- data/src/test/resources/fixtures/lead_describe.json +221 -0
- data/src/test/resources/fixtures/lead_describe_expected.json +66 -0
- data/src/test/resources/fixtures/lead_describe_marketo_fields_full.json +518 -0
- data/src/test/resources/fixtures/lead_extract1.csv +11 -0
- data/src/test/resources/fixtures/lead_response_full.json +2402 -0
- data/src/test/resources/fixtures/lead_with_program_full.json +17 -0
- data/src/test/resources/fixtures/leads_extract2.csv +10 -0
- data/src/test/resources/fixtures/list_reponse_full.json +191 -0
- data/src/test/resources/fixtures/lists_response.json +31 -0
- data/src/test/resources/fixtures/program_response.json +71 -0
- metadata +173 -0
@@ -0,0 +1,92 @@
|
|
1
|
+
package org.embulk.input.marketo.model;
|
2
|
+
|
3
|
+
import com.fasterxml.jackson.annotation.JsonProperty;
|
4
|
+
|
5
|
+
/**
|
6
|
+
* Created by tai.khuu on 8/25/17.
|
7
|
+
*/
|
8
|
+
public class MarketoAccessTokenResponse
|
9
|
+
{
|
10
|
+
@JsonProperty("error")
|
11
|
+
private String error;
|
12
|
+
|
13
|
+
@JsonProperty("error_description")
|
14
|
+
private String errorDescription;
|
15
|
+
|
16
|
+
@JsonProperty("access_token")
|
17
|
+
private String accessToken;
|
18
|
+
|
19
|
+
@JsonProperty("token_type")
|
20
|
+
private String tokenType;
|
21
|
+
|
22
|
+
@JsonProperty("expires_in")
|
23
|
+
private Integer expiresIn;
|
24
|
+
|
25
|
+
@JsonProperty("scope")
|
26
|
+
private String scope;
|
27
|
+
|
28
|
+
public String getError()
|
29
|
+
{
|
30
|
+
return error;
|
31
|
+
}
|
32
|
+
|
33
|
+
public void setError(String error)
|
34
|
+
{
|
35
|
+
this.error = error;
|
36
|
+
}
|
37
|
+
|
38
|
+
public String getErrorDescription()
|
39
|
+
{
|
40
|
+
return errorDescription;
|
41
|
+
}
|
42
|
+
|
43
|
+
public void setErrorDescription(String errorDescription)
|
44
|
+
{
|
45
|
+
this.errorDescription = errorDescription;
|
46
|
+
}
|
47
|
+
|
48
|
+
public String getAccessToken()
|
49
|
+
{
|
50
|
+
return accessToken;
|
51
|
+
}
|
52
|
+
|
53
|
+
public void setAccessToken(String accessToken)
|
54
|
+
{
|
55
|
+
this.accessToken = accessToken;
|
56
|
+
}
|
57
|
+
|
58
|
+
public String getTokenType()
|
59
|
+
{
|
60
|
+
return tokenType;
|
61
|
+
}
|
62
|
+
|
63
|
+
public void setTokenType(String tokenType)
|
64
|
+
{
|
65
|
+
this.tokenType = tokenType;
|
66
|
+
}
|
67
|
+
|
68
|
+
public Integer getExpiresIn()
|
69
|
+
{
|
70
|
+
return expiresIn;
|
71
|
+
}
|
72
|
+
|
73
|
+
public void setExpiresIn(Integer expiresIn)
|
74
|
+
{
|
75
|
+
this.expiresIn = expiresIn;
|
76
|
+
}
|
77
|
+
|
78
|
+
public String getScope()
|
79
|
+
{
|
80
|
+
return scope;
|
81
|
+
}
|
82
|
+
|
83
|
+
public void setScope(String scope)
|
84
|
+
{
|
85
|
+
this.scope = scope;
|
86
|
+
}
|
87
|
+
|
88
|
+
public boolean hasError()
|
89
|
+
{
|
90
|
+
return error != null;
|
91
|
+
}
|
92
|
+
}
|
@@ -0,0 +1,68 @@
|
|
1
|
+
package org.embulk.input.marketo.model;
|
2
|
+
|
3
|
+
import java.util.HashMap;
|
4
|
+
import java.util.List;
|
5
|
+
import java.util.Map;
|
6
|
+
|
7
|
+
/**
|
8
|
+
* Created by tai.khuu on 8/27/17.
|
9
|
+
*/
|
10
|
+
public class MarketoBulkExtractRequest
|
11
|
+
{
|
12
|
+
private List<String> fields;
|
13
|
+
private String format;
|
14
|
+
|
15
|
+
private Map<String, String> columnHeaderNames;
|
16
|
+
|
17
|
+
private Map<String, Object> filter = new HashMap<>();
|
18
|
+
|
19
|
+
public List<String> getFields()
|
20
|
+
{
|
21
|
+
return fields;
|
22
|
+
}
|
23
|
+
|
24
|
+
public void setFields(List<String> fields)
|
25
|
+
{
|
26
|
+
this.fields = fields;
|
27
|
+
}
|
28
|
+
|
29
|
+
public String getFormat()
|
30
|
+
{
|
31
|
+
return format;
|
32
|
+
}
|
33
|
+
|
34
|
+
public void setFormat(String format)
|
35
|
+
{
|
36
|
+
this.format = format;
|
37
|
+
}
|
38
|
+
|
39
|
+
public Map<String, String> getColumnHeaderNames()
|
40
|
+
{
|
41
|
+
return columnHeaderNames;
|
42
|
+
}
|
43
|
+
|
44
|
+
public void setColumnHeaderNames(Map<String, String> columnHeaderNames)
|
45
|
+
{
|
46
|
+
this.columnHeaderNames = columnHeaderNames;
|
47
|
+
}
|
48
|
+
|
49
|
+
public Map<String, Object> getFilter()
|
50
|
+
{
|
51
|
+
return filter;
|
52
|
+
}
|
53
|
+
|
54
|
+
public void setFilter(Map<String, Object> filter)
|
55
|
+
{
|
56
|
+
this.filter = filter;
|
57
|
+
}
|
58
|
+
|
59
|
+
@Override
|
60
|
+
public String toString()
|
61
|
+
{
|
62
|
+
return "MarketoBulkExtractRequest{" +
|
63
|
+
"format='" + format + '\'' +
|
64
|
+
", columnHeaderNames=" + columnHeaderNames +
|
65
|
+
", filter=" + filter +
|
66
|
+
'}';
|
67
|
+
}
|
68
|
+
}
|
@@ -0,0 +1,40 @@
|
|
1
|
+
package org.embulk.input.marketo.model;
|
2
|
+
|
3
|
+
/**
|
4
|
+
* Created by tai.khuu on 8/25/17.
|
5
|
+
*/
|
6
|
+
public class MarketoError
|
7
|
+
{
|
8
|
+
private String code;
|
9
|
+
|
10
|
+
private String message;
|
11
|
+
|
12
|
+
public String getCode()
|
13
|
+
{
|
14
|
+
return code;
|
15
|
+
}
|
16
|
+
|
17
|
+
public void setCode(String code)
|
18
|
+
{
|
19
|
+
this.code = code;
|
20
|
+
}
|
21
|
+
|
22
|
+
public String getMessage()
|
23
|
+
{
|
24
|
+
return message;
|
25
|
+
}
|
26
|
+
|
27
|
+
public void setMessage(String message)
|
28
|
+
{
|
29
|
+
this.message = message;
|
30
|
+
}
|
31
|
+
|
32
|
+
@Override
|
33
|
+
public String toString()
|
34
|
+
{
|
35
|
+
return "MarketoError{" +
|
36
|
+
"code='" + code + '\'' +
|
37
|
+
", message='" + message + '\'' +
|
38
|
+
'}';
|
39
|
+
}
|
40
|
+
}
|
@@ -0,0 +1,126 @@
|
|
1
|
+
package org.embulk.input.marketo.model;
|
2
|
+
|
3
|
+
import com.google.common.base.Optional;
|
4
|
+
import org.embulk.input.marketo.MarketoUtils;
|
5
|
+
import org.embulk.spi.type.Type;
|
6
|
+
import org.embulk.spi.type.Types;
|
7
|
+
|
8
|
+
/**
|
9
|
+
* Created by tai.khuu on 9/22/17.
|
10
|
+
*/
|
11
|
+
|
12
|
+
public class MarketoField
|
13
|
+
{
|
14
|
+
private String name;
|
15
|
+
|
16
|
+
private MarketoDataType marketoDataType;
|
17
|
+
|
18
|
+
public MarketoField(){}
|
19
|
+
|
20
|
+
public MarketoField(String name, String dataType)
|
21
|
+
{
|
22
|
+
this.name = name;
|
23
|
+
try {
|
24
|
+
marketoDataType = MarketoDataType.valueOf(dataType.toUpperCase());
|
25
|
+
}
|
26
|
+
catch (IllegalArgumentException ex) {
|
27
|
+
marketoDataType = MarketoDataType.STRING;
|
28
|
+
}
|
29
|
+
}
|
30
|
+
|
31
|
+
public MarketoField(String name, MarketoDataType marketoDataType)
|
32
|
+
{
|
33
|
+
this.name = name;
|
34
|
+
this.marketoDataType = marketoDataType;
|
35
|
+
}
|
36
|
+
|
37
|
+
public String getName()
|
38
|
+
{
|
39
|
+
return name;
|
40
|
+
}
|
41
|
+
|
42
|
+
public MarketoDataType getMarketoDataType()
|
43
|
+
{
|
44
|
+
return marketoDataType;
|
45
|
+
}
|
46
|
+
|
47
|
+
@Override
|
48
|
+
public boolean equals(Object o)
|
49
|
+
{
|
50
|
+
if (this == o) {
|
51
|
+
return true;
|
52
|
+
}
|
53
|
+
if (o == null || getClass() != o.getClass()) {
|
54
|
+
return false;
|
55
|
+
}
|
56
|
+
|
57
|
+
MarketoField field = (MarketoField) o;
|
58
|
+
|
59
|
+
if (name != null ? !name.equals(field.name) : field.name != null) {
|
60
|
+
return false;
|
61
|
+
}
|
62
|
+
return marketoDataType == field.marketoDataType;
|
63
|
+
}
|
64
|
+
|
65
|
+
@Override
|
66
|
+
public int hashCode()
|
67
|
+
{
|
68
|
+
int result = name != null ? name.hashCode() : 0;
|
69
|
+
result = 31 * result + (marketoDataType != null ? marketoDataType.hashCode() : 0);
|
70
|
+
return result;
|
71
|
+
}
|
72
|
+
|
73
|
+
public enum MarketoDataType
|
74
|
+
{
|
75
|
+
DATETIME(Types.TIMESTAMP, MarketoUtils.MARKETO_DATE_TIME_FORMAT),
|
76
|
+
EMAIL(Types.STRING),
|
77
|
+
FLOAT(Types.DOUBLE),
|
78
|
+
INTEGER(Types.LONG),
|
79
|
+
FORMULA(Types.STRING),
|
80
|
+
PERCENT(Types.DOUBLE),
|
81
|
+
URL(Types.STRING),
|
82
|
+
PHONE(Types.STRING),
|
83
|
+
TEXTAREA(Types.STRING),
|
84
|
+
TEXT(Types.STRING),
|
85
|
+
STRING(Types.STRING),
|
86
|
+
SCORE(Types.LONG),
|
87
|
+
BOOLEAN(Types.BOOLEAN),
|
88
|
+
CURRENCY(Types.DOUBLE),
|
89
|
+
DATE(Types.TIMESTAMP, MarketoUtils.MARKETO_DATE_FORMAT),
|
90
|
+
REFERENCE(Types.STRING);
|
91
|
+
|
92
|
+
private Type type;
|
93
|
+
|
94
|
+
private String format;
|
95
|
+
|
96
|
+
MarketoDataType(Type type, String format)
|
97
|
+
{
|
98
|
+
this.type = type;
|
99
|
+
this.format = format;
|
100
|
+
}
|
101
|
+
|
102
|
+
MarketoDataType(Type type)
|
103
|
+
{
|
104
|
+
this.type = type;
|
105
|
+
}
|
106
|
+
|
107
|
+
public Type getType()
|
108
|
+
{
|
109
|
+
return type;
|
110
|
+
}
|
111
|
+
|
112
|
+
public Optional<String> getFormat()
|
113
|
+
{
|
114
|
+
return Optional.fromNullable(format);
|
115
|
+
}
|
116
|
+
}
|
117
|
+
|
118
|
+
@Override
|
119
|
+
public String toString()
|
120
|
+
{
|
121
|
+
return "MarketoField{" +
|
122
|
+
"name='" + name + '\'' +
|
123
|
+
", marketoDataType=" + marketoDataType +
|
124
|
+
'}';
|
125
|
+
}
|
126
|
+
}
|
@@ -0,0 +1,82 @@
|
|
1
|
+
package org.embulk.input.marketo.model;
|
2
|
+
|
3
|
+
import java.util.ArrayList;
|
4
|
+
import java.util.List;
|
5
|
+
|
6
|
+
/**
|
7
|
+
* Created by tai.khuu on 8/25/17.
|
8
|
+
*/
|
9
|
+
public class MarketoResponse<T>
|
10
|
+
{
|
11
|
+
private String requestId;
|
12
|
+
|
13
|
+
private boolean success;
|
14
|
+
|
15
|
+
private String nextPageToken;
|
16
|
+
|
17
|
+
private boolean moreResult;
|
18
|
+
|
19
|
+
private List<MarketoError> errors;
|
20
|
+
|
21
|
+
private List<T> result = new ArrayList<>();
|
22
|
+
|
23
|
+
public String getRequestId()
|
24
|
+
{
|
25
|
+
return requestId;
|
26
|
+
}
|
27
|
+
|
28
|
+
public void setRequestId(String requestId)
|
29
|
+
{
|
30
|
+
this.requestId = requestId;
|
31
|
+
}
|
32
|
+
|
33
|
+
public boolean isSuccess()
|
34
|
+
{
|
35
|
+
return success;
|
36
|
+
}
|
37
|
+
|
38
|
+
public void setSuccess(boolean success)
|
39
|
+
{
|
40
|
+
this.success = success;
|
41
|
+
}
|
42
|
+
|
43
|
+
public List<MarketoError> getErrors()
|
44
|
+
{
|
45
|
+
return errors;
|
46
|
+
}
|
47
|
+
|
48
|
+
public void setErrors(List<MarketoError> errors)
|
49
|
+
{
|
50
|
+
this.errors = errors;
|
51
|
+
}
|
52
|
+
|
53
|
+
public List<T> getResult()
|
54
|
+
{
|
55
|
+
return result;
|
56
|
+
}
|
57
|
+
|
58
|
+
public void setResult(List<T> result)
|
59
|
+
{
|
60
|
+
this.result = result;
|
61
|
+
}
|
62
|
+
|
63
|
+
public String getNextPageToken()
|
64
|
+
{
|
65
|
+
return nextPageToken;
|
66
|
+
}
|
67
|
+
|
68
|
+
public void setNextPageToken(String nextPageToken)
|
69
|
+
{
|
70
|
+
this.nextPageToken = nextPageToken;
|
71
|
+
}
|
72
|
+
|
73
|
+
public boolean isMoreResult()
|
74
|
+
{
|
75
|
+
return moreResult;
|
76
|
+
}
|
77
|
+
|
78
|
+
public void setMoreResult(boolean moreResult)
|
79
|
+
{
|
80
|
+
this.moreResult = moreResult;
|
81
|
+
}
|
82
|
+
}
|
@@ -0,0 +1,40 @@
|
|
1
|
+
package org.embulk.input.marketo.model.filter;
|
2
|
+
|
3
|
+
/**
|
4
|
+
* Created by tai.khuu on 8/27/17.
|
5
|
+
*/
|
6
|
+
public class DateRangeFilter
|
7
|
+
{
|
8
|
+
private String startAt;
|
9
|
+
|
10
|
+
private String endAt;
|
11
|
+
|
12
|
+
public String getStartAt()
|
13
|
+
{
|
14
|
+
return startAt;
|
15
|
+
}
|
16
|
+
|
17
|
+
public void setStartAt(String startAt)
|
18
|
+
{
|
19
|
+
this.startAt = startAt;
|
20
|
+
}
|
21
|
+
|
22
|
+
public String getEndAt()
|
23
|
+
{
|
24
|
+
return endAt;
|
25
|
+
}
|
26
|
+
|
27
|
+
public void setEndAt(String endAt)
|
28
|
+
{
|
29
|
+
this.endAt = endAt;
|
30
|
+
}
|
31
|
+
|
32
|
+
@Override
|
33
|
+
public String toString()
|
34
|
+
{
|
35
|
+
return "DateRangeFilter{" +
|
36
|
+
"startAt='" + startAt + '\'' +
|
37
|
+
", endAt='" + endAt + '\'' +
|
38
|
+
'}';
|
39
|
+
}
|
40
|
+
}
|