embulk-input-marketo_extended 0.6.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. checksums.yaml +7 -0
  2. data/.github/PULL_REQUEST_TEMPLATE.md +37 -0
  3. data/.gitignore +14 -0
  4. data/.travis.yml +6 -0
  5. data/CHANGELOG.md +170 -0
  6. data/LICENSE.txt +21 -0
  7. data/README.md +213 -0
  8. data/build.gradle +103 -0
  9. data/config/checkstyle/checkstyle.xml +128 -0
  10. data/config/checkstyle/default.xml +108 -0
  11. data/gradle/wrapper/gradle-wrapper.jar +0 -0
  12. data/gradle/wrapper/gradle-wrapper.properties +6 -0
  13. data/gradlew +169 -0
  14. data/gradlew.bat +84 -0
  15. data/lib/embulk/input/marketo.rb +3 -0
  16. data/settings.gradle +1 -0
  17. data/src/main/java/org/embulk/input/marketo/CsvTokenizer.java +700 -0
  18. data/src/main/java/org/embulk/input/marketo/MarketoInputPlugin.java +15 -0
  19. data/src/main/java/org/embulk/input/marketo/MarketoInputPluginDelegate.java +100 -0
  20. data/src/main/java/org/embulk/input/marketo/MarketoService.java +38 -0
  21. data/src/main/java/org/embulk/input/marketo/MarketoServiceImpl.java +245 -0
  22. data/src/main/java/org/embulk/input/marketo/MarketoUtils.java +212 -0
  23. data/src/main/java/org/embulk/input/marketo/delegate/ActivityBulkExtractInputPlugin.java +167 -0
  24. data/src/main/java/org/embulk/input/marketo/delegate/CampaignInputPlugin.java +48 -0
  25. data/src/main/java/org/embulk/input/marketo/delegate/CustomObjectInputPlugin.java +75 -0
  26. data/src/main/java/org/embulk/input/marketo/delegate/CustomObjectResponseMapperBuilder.java +81 -0
  27. data/src/main/java/org/embulk/input/marketo/delegate/LeadBulkExtractInputPlugin.java +66 -0
  28. data/src/main/java/org/embulk/input/marketo/delegate/LeadServiceResponseMapperBuilder.java +85 -0
  29. data/src/main/java/org/embulk/input/marketo/delegate/LeadWithListInputPlugin.java +64 -0
  30. data/src/main/java/org/embulk/input/marketo/delegate/LeadWithProgramInputPlugin.java +60 -0
  31. data/src/main/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPlugin.java +441 -0
  32. data/src/main/java/org/embulk/input/marketo/delegate/MarketoBaseInputPluginDelegate.java +92 -0
  33. data/src/main/java/org/embulk/input/marketo/delegate/ProgramInputPlugin.java +228 -0
  34. data/src/main/java/org/embulk/input/marketo/exception/MarketoAPIException.java +30 -0
  35. data/src/main/java/org/embulk/input/marketo/model/BulkExtractRangeHeader.java +26 -0
  36. data/src/main/java/org/embulk/input/marketo/model/MarketoAccessTokenResponse.java +92 -0
  37. data/src/main/java/org/embulk/input/marketo/model/MarketoBulkExtractRequest.java +68 -0
  38. data/src/main/java/org/embulk/input/marketo/model/MarketoError.java +40 -0
  39. data/src/main/java/org/embulk/input/marketo/model/MarketoField.java +126 -0
  40. data/src/main/java/org/embulk/input/marketo/model/MarketoResponse.java +82 -0
  41. data/src/main/java/org/embulk/input/marketo/model/filter/DateRangeFilter.java +40 -0
  42. data/src/main/java/org/embulk/input/marketo/rest/MarketoBaseRestClient.java +306 -0
  43. data/src/main/java/org/embulk/input/marketo/rest/MarketoInputStreamResponseEntityReader.java +69 -0
  44. data/src/main/java/org/embulk/input/marketo/rest/MarketoRESTEndpoint.java +47 -0
  45. data/src/main/java/org/embulk/input/marketo/rest/MarketoResponseJetty92EntityReader.java +89 -0
  46. data/src/main/java/org/embulk/input/marketo/rest/MarketoRestClient.java +569 -0
  47. data/src/main/java/org/embulk/input/marketo/rest/RecordPagingIterable.java +180 -0
  48. data/src/test/java/org/embulk/input/marketo/MarketoServiceImplTest.java +140 -0
  49. data/src/test/java/org/embulk/input/marketo/MarketoUtilsTest.java +87 -0
  50. data/src/test/java/org/embulk/input/marketo/delegate/ActivityBulkExtractInputPluginTest.java +128 -0
  51. data/src/test/java/org/embulk/input/marketo/delegate/CampaignInputPluginTest.java +73 -0
  52. data/src/test/java/org/embulk/input/marketo/delegate/CustomObjectInputPluginTest.java +102 -0
  53. data/src/test/java/org/embulk/input/marketo/delegate/LeadBulkExtractInputPluginTest.java +99 -0
  54. data/src/test/java/org/embulk/input/marketo/delegate/LeadServiceResponseMapperBuilderTest.java +119 -0
  55. data/src/test/java/org/embulk/input/marketo/delegate/LeadWithListInputPluginTest.java +101 -0
  56. data/src/test/java/org/embulk/input/marketo/delegate/LeadWithProgramInputPluginTest.java +103 -0
  57. data/src/test/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPluginTest.java +169 -0
  58. data/src/test/java/org/embulk/input/marketo/delegate/ProgramInputPluginTest.java +343 -0
  59. data/src/test/java/org/embulk/input/marketo/rest/MarketoBaseRestClientTest.java +368 -0
  60. data/src/test/java/org/embulk/input/marketo/rest/MarketoRestClientTest.java +584 -0
  61. data/src/test/resources/config/activity_bulk_extract_config.yaml +7 -0
  62. data/src/test/resources/config/custom_object_config.yaml +8 -0
  63. data/src/test/resources/config/lead_bulk_extract_config.yaml +8 -0
  64. data/src/test/resources/config/rest_config.yaml +3 -0
  65. data/src/test/resources/fixtures/activity_extract1.csv +35 -0
  66. data/src/test/resources/fixtures/activity_extract2.csv +22 -0
  67. data/src/test/resources/fixtures/activity_types.json +22 -0
  68. data/src/test/resources/fixtures/all_program_full.json +53 -0
  69. data/src/test/resources/fixtures/campaign_response.json +38 -0
  70. data/src/test/resources/fixtures/campaign_response_full.json +102 -0
  71. data/src/test/resources/fixtures/custom_object_describe.json +124 -0
  72. data/src/test/resources/fixtures/custom_object_describe_marketo_fields_full.json +22 -0
  73. data/src/test/resources/fixtures/custom_object_expected.json +66 -0
  74. data/src/test/resources/fixtures/custom_object_response.json +24 -0
  75. data/src/test/resources/fixtures/custom_object_response_full.json +23 -0
  76. data/src/test/resources/fixtures/lead_by_list.json +33 -0
  77. data/src/test/resources/fixtures/lead_by_program_response.json +47 -0
  78. data/src/test/resources/fixtures/lead_describe.json +221 -0
  79. data/src/test/resources/fixtures/lead_describe_expected.json +66 -0
  80. data/src/test/resources/fixtures/lead_describe_marketo_fields_full.json +518 -0
  81. data/src/test/resources/fixtures/lead_extract1.csv +11 -0
  82. data/src/test/resources/fixtures/lead_response_full.json +2402 -0
  83. data/src/test/resources/fixtures/lead_with_program_full.json +17 -0
  84. data/src/test/resources/fixtures/leads_extract2.csv +10 -0
  85. data/src/test/resources/fixtures/list_reponse_full.json +191 -0
  86. data/src/test/resources/fixtures/lists_response.json +31 -0
  87. data/src/test/resources/fixtures/program_response.json +71 -0
  88. metadata +171 -0
data/src/test/resources/fixtures/custom_object_expected.json ADDED
@@ -0,0 +1,66 @@
1
+ [
2
+ {
3
+ "name": "mKtestboolean",
4
+ "marketoDataType": "BOOLEAN"
5
+ },
6
+ {
7
+ "name": "mKtestcurrency",
8
+ "marketoDataType": "CURRENCY"
9
+ },
10
+ {
11
+ "name": "mKtestdate",
12
+ "marketoDataType": "DATE"
13
+ },
14
+ {
15
+ "name": "mKtestDatetime",
16
+ "marketoDataType": "DATETIME"
17
+ },
18
+ {
19
+ "name": "mKtestemail",
20
+ "marketoDataType": "EMAIL"
21
+ },
22
+ {
23
+ "name": "mKtestfloat",
24
+ "marketoDataType": "FLOAT"
25
+ },
26
+ {
27
+ "name": "mKtestinteger",
28
+ "marketoDataType": "INTEGER"
29
+ },
30
+ {
31
+ "name": "mKtestpercent",
32
+ "marketoDataType": "PERCENT"
33
+ },
34
+ {
35
+ "name": "mKtestphone",
36
+ "marketoDataType": "PHONE"
37
+ },
38
+ {
39
+ "name": "mKtestscore",
40
+ "marketoDataType": "SCORE"
41
+ },
42
+ {
43
+ "name": "mKteststring",
44
+ "marketoDataType": "STRING"
45
+ },
46
+ {
47
+ "name": "mKtesttextarea",
48
+ "marketoDataType": "TEXTAREA"
49
+ },
50
+ {
51
+ "name": "mKtesturl",
52
+ "marketoDataType": "URL"
53
+ },
54
+ {
55
+ "name": "mKTestReference",
56
+ "marketoDataType": "REFERENCE"
57
+ },
58
+ {
59
+ "name": "mKTestFormula",
60
+ "marketoDataType": "FORMULA"
61
+ },
62
+ {
63
+ "name": "mKtesttext",
64
+ "marketoDataType": "TEXT"
65
+ }
66
+ ]
data/src/test/resources/fixtures/custom_object_response.json ADDED
@@ -0,0 +1,24 @@
1
+ {
2
+ "responses":[
3
+ {
4
+ "requestId": "83e9#166f835a592",
5
+ "result": [
6
+ {
7
+ "seq": 0,
8
+ "marketoGUID": "aa168cfa-626d-49b4-b735-526e0b8c069e",
9
+ "id": 1,
10
+ "createdAt": "2018-11-06T12:01:49Z",
11
+ "updatedAt": "2018-11-06T12:01:49Z"
12
+ },
13
+ {
14
+ "seq": 1,
15
+ "marketoGUID": "da7c347d-754b-44f3-9821-d1233f0852f6",
16
+ "id": 2,
17
+ "createdAt": "2018-11-06T12:01:49Z",
18
+ "updatedAt": "2018-11-06T12:01:49Z"
19
+ }
20
+ ],
21
+ "success": true
22
+ }
23
+ ]
24
+ }
data/src/test/resources/fixtures/custom_object_response_full.json ADDED
@@ -0,0 +1,23 @@
1
+ [
2
+ {
3
+ "seq": 0,
4
+ "marketoGUID": "aa168cfa-626d-49b4-b735-526e0b8c069e",
5
+ "id": 1,
6
+ "createdAt": "2018-11-06T12:01:49Z",
7
+ "updatedAt": "2018-11-06T12:01:49Z"
8
+ },
9
+ {
10
+ "seq": 1,
11
+ "marketoGUID": "da7c347d-754b-44f3-9821-d1233f0852f6",
12
+ "id": 2,
13
+ "createdAt": "2018-11-06T12:01:49Z",
14
+ "updatedAt": "2018-11-06T12:01:49Z"
15
+ },
16
+ {
17
+ "seq": 2,
18
+ "marketoGUID": "sdfdc347d-754b-5645-9821-d1233f065456",
19
+ "id": 3,
20
+ "createdAt": "2018-11-06T12:01:49Z",
21
+ "updatedAt": "2018-11-06T12:01:49Z"
22
+ }
23
+ ]
data/src/test/resources/fixtures/lead_by_list.json ADDED
@@ -0,0 +1,33 @@
1
+ {
2
+ "responses": [
3
+ {
4
+ "requestId": "2",
5
+ "result": [
6
+ {
7
+ "id": 103280,
8
+ "updatedAt": "2017-09-25T17:55:39Z",
9
+ "lastName": "Doe10093",
10
+ "email": "JohnDoe_10093@tdtest.combo",
11
+ "createdAt": "2017-09-18T08:16:13Z",
12
+ "firstName": "John10093"
13
+ }
14
+ ],
15
+ "success": true,
16
+ "nextPageToken": "z4MgsIiC5C======"
17
+ },
18
+ {
19
+ "requestId": "1",
20
+ "result": [
21
+ {
22
+ "id": 103281,
23
+ "updatedAt": "2017-09-25T17:55:39Z",
24
+ "lastName": "Doe10094",
25
+ "email": "JohnDoe_10094@tdtest.combo",
26
+ "createdAt": "2017-09-18T08:16:13Z",
27
+ "firstName": "John10094"
28
+ }
29
+ ],
30
+ "success": true
31
+ }
32
+ ]
33
+ }
data/src/test/resources/fixtures/lead_by_program_response.json ADDED
@@ -0,0 +1,47 @@
1
+ {
2
+ "responses": [
3
+ {
4
+ "requestId": "2",
5
+ "result": [
6
+ {
7
+ "id": 102519,
8
+ "updatedAt": "2017-09-11T03:14:12Z",
9
+ "lastName": "Khuu 1",
10
+ "email": "tai1@treasure-data.com",
11
+ "createdAt": "2017-09-11T03:14:13Z",
12
+ "firstName": "Tai 1",
13
+ "membership": {
14
+ "progressionStatus": "Member",
15
+ "isExhausted": false,
16
+ "acquiredBy": true,
17
+ "reachedSuccess": false,
18
+ "membershipDate": "2017-09-11T03:14:13Z"
19
+ }
20
+ }
21
+ ],
22
+ "success": true,
23
+ "nextPageToken": "z4MgsIiC5C======"
24
+ },
25
+ {
26
+ "requestId": "1",
27
+ "result": [
28
+ {
29
+ "id": 102719,
30
+ "updatedAt": "2017-09-11T03:14:12Z",
31
+ "lastName": "Khuu",
32
+ "email": "tai@treasure-data.com",
33
+ "createdAt": "2017-09-11T03:14:13Z",
34
+ "firstName": "Tai",
35
+ "membership": {
36
+ "progressionStatus": "Member",
37
+ "isExhausted": false,
38
+ "acquiredBy": true,
39
+ "reachedSuccess": false,
40
+ "membershipDate": "2017-09-11T03:14:13Z"
41
+ }
42
+ }
43
+ ],
44
+ "success": true
45
+ }
46
+ ]
47
+ }
data/src/test/resources/fixtures/lead_describe.json ADDED
@@ -0,0 +1,221 @@
1
+ {
2
+ "requestId": "121e2#15eebcac5d8",
3
+ "result": [
4
+ {
5
+ "id": 1,
6
+ "displayName": "Marketo_test_boolean",
7
+ "dataType": "boolean",
8
+ "rest": {
9
+ "name": "mKtestboolean",
10
+ "readOnly": false
11
+ },
12
+ "soap": {
13
+ "name": "mKtestboolean",
14
+ "readOnly": false
15
+ }
16
+ },
17
+ {
18
+ "id": 2,
19
+ "displayName": "Marketo_test_currency",
20
+ "dataType": "currency",
21
+ "length": 255,
22
+ "rest": {
23
+ "name": "mKtestcurrency",
24
+ "readOnly": false
25
+ },
26
+ "soap": {
27
+ "name": "mKtestcurrency",
28
+ "readOnly": false
29
+ }
30
+ },
31
+ {
32
+ "id": 3,
33
+ "displayName": "Marketo_test_date",
34
+ "dataType": "date",
35
+ "rest": {
36
+ "name": "mKtestdate",
37
+ "readOnly": false
38
+ },
39
+ "soap": {
40
+ "name": "mKtestdate",
41
+ "readOnly": false
42
+ }
43
+ },
44
+ {
45
+ "id": 4,
46
+ "displayName": "Marketo_test_Datetime",
47
+ "dataType": "datetime",
48
+ "rest": {
49
+ "name": "mKtestDatetime",
50
+ "readOnly": false
51
+ },
52
+ "soap": {
53
+ "name": "mKtestDatetime",
54
+ "readOnly": false
55
+ }
56
+ },
57
+ {
58
+ "id": 5,
59
+ "displayName": "Marketo_test_email",
60
+ "dataType": "email",
61
+ "length": 255,
62
+ "rest": {
63
+ "name": "mKtestemail",
64
+ "readOnly": false
65
+ },
66
+ "soap": {
67
+ "name": "mKtestemail",
68
+ "readOnly": false
69
+ }
70
+ },
71
+ {
72
+ "id": 6,
73
+ "displayName": "Marketo_test_float",
74
+ "dataType": "float",
75
+ "rest": {
76
+ "name": "mKtestfloat",
77
+ "readOnly": false
78
+ },
79
+ "soap": {
80
+ "name": "mKtestfloat",
81
+ "readOnly": false
82
+ }
83
+ },
84
+ {
85
+ "id": 7,
86
+ "displayName": "Marketo_test_integer",
87
+ "dataType": "integer",
88
+ "rest": {
89
+ "name": "mKtestinteger",
90
+ "readOnly": false
91
+ },
92
+ "soap": {
93
+ "name": "mKtestinteger",
94
+ "readOnly": false
95
+ }
96
+ },
97
+ {
98
+ "id": 8,
99
+ "displayName": "Marketo_test_percent",
100
+ "dataType": "percent",
101
+ "rest": {
102
+ "name": "mKtestpercent",
103
+ "readOnly": false
104
+ },
105
+ "soap": {
106
+ "name": "mKtestpercent",
107
+ "readOnly": false
108
+ }
109
+ },
110
+ {
111
+ "id": 9,
112
+ "displayName": "Marketo_test_phone",
113
+ "dataType": "phone",
114
+ "length": 255,
115
+ "rest": {
116
+ "name": "mKtestphone",
117
+ "readOnly": false
118
+ },
119
+ "soap": {
120
+ "name": "mKtestphone",
121
+ "readOnly": false
122
+ }
123
+ },
124
+ {
125
+ "id": 11,
126
+ "displayName": "Marketo_test_score",
127
+ "dataType": "score",
128
+ "rest": {
129
+ "name": "mKtestscore",
130
+ "readOnly": false
131
+ },
132
+ "soap": {
133
+ "name": "mKtestscore",
134
+ "readOnly": false
135
+ }
136
+ },
137
+ {
138
+ "id": 12,
139
+ "displayName": "Marketo_test_string",
140
+ "dataType": "string",
141
+ "length": 255,
142
+ "rest": {
143
+ "name": "mKteststring",
144
+ "readOnly": false
145
+ },
146
+ "soap": {
147
+ "name": "mKteststring",
148
+ "readOnly": false
149
+ }
150
+ },
151
+ {
152
+ "id": 13,
153
+ "displayName": "Marketo_test_textarea",
154
+ "dataType": "textarea",
155
+ "rest": {
156
+ "name": "mKtesttextarea",
157
+ "readOnly": false
158
+ },
159
+ "soap": {
160
+ "name": "mKtesttextarea",
161
+ "readOnly": false
162
+ }
163
+ },
164
+ {
165
+ "id": 14,
166
+ "displayName": "Marketo_test_url",
167
+ "dataType": "url",
168
+ "length": 255,
169
+ "rest": {
170
+ "name": "mKtesturl",
171
+ "readOnly": false
172
+ },
173
+ "soap": {
174
+ "name": "mKtesturl",
175
+ "readOnly": false
176
+ }
177
+ },
178
+ {
179
+ "id": 15,
180
+ "displayName": "Marketo_test_reference",
181
+ "dataType": "reference",
182
+ "length": 255,
183
+ "rest": {
184
+ "name": "mKTestReference",
185
+ "readOnly": true
186
+ },
187
+ "soap": {
188
+ "name": "mKTestReference",
189
+ "readOnly": false
190
+ }
191
+ },
192
+ {
193
+ "id": 16,
194
+ "displayName": "Marketo_test_formula",
195
+ "dataType": "formula",
196
+ "length": 255,
197
+ "rest": {
198
+ "name": "mKTestFormula",
199
+ "readOnly": true
200
+ },
201
+ "soap": {
202
+ "name": "mKTestFormula",
203
+ "readOnly": false
204
+ }
205
+ },{
206
+ "id": 17,
207
+ "displayName": "Marketo_test_text",
208
+ "dataType": "text",
209
+ "rest": {
210
+ "name": "mKtesttext",
211
+ "readOnly": false
212
+ },
213
+ "soap": {
214
+ "name": "mKtesttext",
215
+ "readOnly": false
216
+ }
217
+ }
218
+
219
+ ],
220
+ "success": true
221
+ }
data/src/test/resources/fixtures/lead_describe_expected.json ADDED
@@ -0,0 +1,66 @@
1
+ [
2
+ {
3
+ "name": "mKtestboolean",
4
+ "marketoDataType": "BOOLEAN"
5
+ },
6
+ {
7
+ "name": "mKtestcurrency",
8
+ "marketoDataType": "CURRENCY"
9
+ },
10
+ {
11
+ "name": "mKtestdate",
12
+ "marketoDataType": "DATE"
13
+ },
14
+ {
15
+ "name": "mKtestDatetime",
16
+ "marketoDataType": "DATETIME"
17
+ },
18
+ {
19
+ "name": "mKtestemail",
20
+ "marketoDataType": "EMAIL"
21
+ },
22
+ {
23
+ "name": "mKtestfloat",
24
+ "marketoDataType": "FLOAT"
25
+ },
26
+ {
27
+ "name": "mKtestinteger",
28
+ "marketoDataType": "INTEGER"
29
+ },
30
+ {
31
+ "name": "mKtestpercent",
32
+ "marketoDataType": "PERCENT"
33
+ },
34
+ {
35
+ "name": "mKtestphone",
36
+ "marketoDataType": "PHONE"
37
+ },
38
+ {
39
+ "name": "mKtestscore",
40
+ "marketoDataType": "SCORE"
41
+ },
42
+ {
43
+ "name": "mKteststring",
44
+ "marketoDataType": "STRING"
45
+ },
46
+ {
47
+ "name": "mKtesttextarea",
48
+ "marketoDataType": "TEXTAREA"
49
+ },
50
+ {
51
+ "name": "mKtesturl",
52
+ "marketoDataType": "URL"
53
+ },
54
+ {
55
+ "name": "mKTestReference",
56
+ "marketoDataType": "REFERENCE"
57
+ },
58
+ {
59
+ "name": "mKTestFormula",
60
+ "marketoDataType": "FORMULA"
61
+ },
62
+ {
63
+ "name": "mKtesttext",
64
+ "marketoDataType": "TEXT"
65
+ }
66
+ ]