embulk-input-marketo-extended 0.6.18

Sign up to get free protection for your applications and to get access to all the features.
Files changed (88) hide show
  1. checksums.yaml +7 -0
  2. data/.github/PULL_REQUEST_TEMPLATE.md +37 -0
  3. data/.gitignore +14 -0
  4. data/.travis.yml +6 -0
  5. data/CHANGELOG.md +170 -0
  6. data/LICENSE.txt +21 -0
  7. data/README.md +213 -0
  8. data/build.gradle +103 -0
  9. data/config/checkstyle/checkstyle.xml +128 -0
  10. data/config/checkstyle/default.xml +108 -0
  11. data/gradle/wrapper/gradle-wrapper.jar +0 -0
  12. data/gradle/wrapper/gradle-wrapper.properties +6 -0
  13. data/gradlew +169 -0
  14. data/gradlew.bat +84 -0
  15. data/lib/embulk/input/marketo.rb +3 -0
  16. data/settings.gradle +1 -0
  17. data/src/main/java/org/embulk/input/marketo/CsvTokenizer.java +700 -0
  18. data/src/main/java/org/embulk/input/marketo/MarketoInputPlugin.java +15 -0
  19. data/src/main/java/org/embulk/input/marketo/MarketoInputPluginDelegate.java +100 -0
  20. data/src/main/java/org/embulk/input/marketo/MarketoService.java +38 -0
  21. data/src/main/java/org/embulk/input/marketo/MarketoServiceImpl.java +245 -0
  22. data/src/main/java/org/embulk/input/marketo/MarketoUtils.java +212 -0
  23. data/src/main/java/org/embulk/input/marketo/delegate/ActivityBulkExtractInputPlugin.java +167 -0
  24. data/src/main/java/org/embulk/input/marketo/delegate/CampaignInputPlugin.java +48 -0
  25. data/src/main/java/org/embulk/input/marketo/delegate/CustomObjectInputPlugin.java +75 -0
  26. data/src/main/java/org/embulk/input/marketo/delegate/CustomObjectResponseMapperBuilder.java +81 -0
  27. data/src/main/java/org/embulk/input/marketo/delegate/LeadBulkExtractInputPlugin.java +66 -0
  28. data/src/main/java/org/embulk/input/marketo/delegate/LeadServiceResponseMapperBuilder.java +85 -0
  29. data/src/main/java/org/embulk/input/marketo/delegate/LeadWithListInputPlugin.java +64 -0
  30. data/src/main/java/org/embulk/input/marketo/delegate/LeadWithProgramInputPlugin.java +60 -0
  31. data/src/main/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPlugin.java +441 -0
  32. data/src/main/java/org/embulk/input/marketo/delegate/MarketoBaseInputPluginDelegate.java +92 -0
  33. data/src/main/java/org/embulk/input/marketo/delegate/ProgramInputPlugin.java +228 -0
  34. data/src/main/java/org/embulk/input/marketo/exception/MarketoAPIException.java +30 -0
  35. data/src/main/java/org/embulk/input/marketo/model/BulkExtractRangeHeader.java +26 -0
  36. data/src/main/java/org/embulk/input/marketo/model/MarketoAccessTokenResponse.java +92 -0
  37. data/src/main/java/org/embulk/input/marketo/model/MarketoBulkExtractRequest.java +68 -0
  38. data/src/main/java/org/embulk/input/marketo/model/MarketoError.java +40 -0
  39. data/src/main/java/org/embulk/input/marketo/model/MarketoField.java +126 -0
  40. data/src/main/java/org/embulk/input/marketo/model/MarketoResponse.java +82 -0
  41. data/src/main/java/org/embulk/input/marketo/model/filter/DateRangeFilter.java +40 -0
  42. data/src/main/java/org/embulk/input/marketo/rest/MarketoBaseRestClient.java +306 -0
  43. data/src/main/java/org/embulk/input/marketo/rest/MarketoInputStreamResponseEntityReader.java +69 -0
  44. data/src/main/java/org/embulk/input/marketo/rest/MarketoRESTEndpoint.java +47 -0
  45. data/src/main/java/org/embulk/input/marketo/rest/MarketoResponseJetty92EntityReader.java +89 -0
  46. data/src/main/java/org/embulk/input/marketo/rest/MarketoRestClient.java +569 -0
  47. data/src/main/java/org/embulk/input/marketo/rest/RecordPagingIterable.java +180 -0
  48. data/src/test/java/org/embulk/input/marketo/MarketoServiceImplTest.java +140 -0
  49. data/src/test/java/org/embulk/input/marketo/MarketoUtilsTest.java +87 -0
  50. data/src/test/java/org/embulk/input/marketo/delegate/ActivityBulkExtractInputPluginTest.java +128 -0
  51. data/src/test/java/org/embulk/input/marketo/delegate/CampaignInputPluginTest.java +73 -0
  52. data/src/test/java/org/embulk/input/marketo/delegate/CustomObjectInputPluginTest.java +102 -0
  53. data/src/test/java/org/embulk/input/marketo/delegate/LeadBulkExtractInputPluginTest.java +99 -0
  54. data/src/test/java/org/embulk/input/marketo/delegate/LeadServiceResponseMapperBuilderTest.java +119 -0
  55. data/src/test/java/org/embulk/input/marketo/delegate/LeadWithListInputPluginTest.java +101 -0
  56. data/src/test/java/org/embulk/input/marketo/delegate/LeadWithProgramInputPluginTest.java +103 -0
  57. data/src/test/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPluginTest.java +169 -0
  58. data/src/test/java/org/embulk/input/marketo/delegate/ProgramInputPluginTest.java +343 -0
  59. data/src/test/java/org/embulk/input/marketo/rest/MarketoBaseRestClientTest.java +368 -0
  60. data/src/test/java/org/embulk/input/marketo/rest/MarketoRestClientTest.java +584 -0
  61. data/src/test/resources/config/activity_bulk_extract_config.yaml +7 -0
  62. data/src/test/resources/config/custom_object_config.yaml +8 -0
  63. data/src/test/resources/config/lead_bulk_extract_config.yaml +8 -0
  64. data/src/test/resources/config/rest_config.yaml +3 -0
  65. data/src/test/resources/fixtures/activity_extract1.csv +35 -0
  66. data/src/test/resources/fixtures/activity_extract2.csv +22 -0
  67. data/src/test/resources/fixtures/activity_types.json +22 -0
  68. data/src/test/resources/fixtures/all_program_full.json +53 -0
  69. data/src/test/resources/fixtures/campaign_response.json +38 -0
  70. data/src/test/resources/fixtures/campaign_response_full.json +102 -0
  71. data/src/test/resources/fixtures/custom_object_describe.json +124 -0
  72. data/src/test/resources/fixtures/custom_object_describe_marketo_fields_full.json +22 -0
  73. data/src/test/resources/fixtures/custom_object_expected.json +66 -0
  74. data/src/test/resources/fixtures/custom_object_response.json +24 -0
  75. data/src/test/resources/fixtures/custom_object_response_full.json +23 -0
  76. data/src/test/resources/fixtures/lead_by_list.json +33 -0
  77. data/src/test/resources/fixtures/lead_by_program_response.json +47 -0
  78. data/src/test/resources/fixtures/lead_describe.json +221 -0
  79. data/src/test/resources/fixtures/lead_describe_expected.json +66 -0
  80. data/src/test/resources/fixtures/lead_describe_marketo_fields_full.json +518 -0
  81. data/src/test/resources/fixtures/lead_extract1.csv +11 -0
  82. data/src/test/resources/fixtures/lead_response_full.json +2402 -0
  83. data/src/test/resources/fixtures/lead_with_program_full.json +17 -0
  84. data/src/test/resources/fixtures/leads_extract2.csv +10 -0
  85. data/src/test/resources/fixtures/list_reponse_full.json +191 -0
  86. data/src/test/resources/fixtures/lists_response.json +31 -0
  87. data/src/test/resources/fixtures/program_response.json +71 -0
  88. metadata +171 -0
data/src/test/resources/fixtures/lead_describe_marketo_fields_full.json ADDED
@@ -0,0 +1,518 @@
1
+ [
2
+ {
3
+ "name": "company",
4
+ "marketoDataType": "STRING"
5
+ },
6
+ {
7
+ "name": "site",
8
+ "marketoDataType": "STRING"
9
+ },
10
+ {
11
+ "name": "billingStreet",
12
+ "marketoDataType": "TEXT"
13
+ },
14
+ {
15
+ "name": "billingCity",
16
+ "marketoDataType": "STRING"
17
+ },
18
+ {
19
+ "name": "billingState",
20
+ "marketoDataType": "STRING"
21
+ },
22
+ {
23
+ "name": "billingCountry",
24
+ "marketoDataType": "STRING"
25
+ },
26
+ {
27
+ "name": "billingPostalCode",
28
+ "marketoDataType": "STRING"
29
+ },
30
+ {
31
+ "name": "website",
32
+ "marketoDataType": "URL"
33
+ },
34
+ {
35
+ "name": "mainPhone",
36
+ "marketoDataType": "STRING"
37
+ },
38
+ {
39
+ "name": "annualRevenue",
40
+ "marketoDataType": "CURRENCY"
41
+ },
42
+ {
43
+ "name": "numberOfEmployees",
44
+ "marketoDataType": "INTEGER"
45
+ },
46
+ {
47
+ "name": "industry",
48
+ "marketoDataType": "STRING"
49
+ },
50
+ {
51
+ "name": "sicCode",
52
+ "marketoDataType": "STRING"
53
+ },
54
+ {
55
+ "name": "mktoCompanyNotes",
56
+ "marketoDataType": "TEXT"
57
+ },
58
+ {
59
+ "name": "externalCompanyId",
60
+ "marketoDataType": "STRING"
61
+ },
62
+ {
63
+ "name": "id",
64
+ "marketoDataType": "INTEGER"
65
+ },
66
+ {
67
+ "name": "mktoName",
68
+ "marketoDataType": "STRING"
69
+ },
70
+ {
71
+ "name": "personType",
72
+ "marketoDataType": "STRING"
73
+ },
74
+ {
75
+ "name": "mktoIsPartner",
76
+ "marketoDataType": "BOOLEAN"
77
+ },
78
+ {
79
+ "name": "isLead",
80
+ "marketoDataType": "BOOLEAN"
81
+ },
82
+ {
83
+ "name": "mktoIsCustomer",
84
+ "marketoDataType": "BOOLEAN"
85
+ },
86
+ {
87
+ "name": "isAnonymous",
88
+ "marketoDataType": "BOOLEAN"
89
+ },
90
+ {
91
+ "name": "salutation",
92
+ "marketoDataType": "STRING"
93
+ },
94
+ {
95
+ "name": "firstName",
96
+ "marketoDataType": "STRING"
97
+ },
98
+ {
99
+ "name": "middleName",
100
+ "marketoDataType": "STRING"
101
+ },
102
+ {
103
+ "name": "lastName",
104
+ "marketoDataType": "STRING"
105
+ },
106
+ {
107
+ "name": "email",
108
+ "marketoDataType": "EMAIL"
109
+ },
110
+ {
111
+ "name": "phone",
112
+ "marketoDataType": "PHONE"
113
+ },
114
+ {
115
+ "name": "mobilePhone",
116
+ "marketoDataType": "PHONE"
117
+ },
118
+ {
119
+ "name": "fax",
120
+ "marketoDataType": "PHONE"
121
+ },
122
+ {
123
+ "name": "title",
124
+ "marketoDataType": "STRING"
125
+ },
126
+ {
127
+ "name": "contactCompany",
128
+ "marketoDataType": "REFERENCE"
129
+ },
130
+ {
131
+ "name": "dateOfBirth",
132
+ "marketoDataType": "DATE"
133
+ },
134
+ {
135
+ "name": "address",
136
+ "marketoDataType": "TEXT"
137
+ },
138
+ {
139
+ "name": "city",
140
+ "marketoDataType": "STRING"
141
+ },
142
+ {
143
+ "name": "state",
144
+ "marketoDataType": "STRING"
145
+ },
146
+ {
147
+ "name": "country",
148
+ "marketoDataType": "STRING"
149
+ },
150
+ {
151
+ "name": "postalCode",
152
+ "marketoDataType": "STRING"
153
+ },
154
+ {
155
+ "name": "originalSourceType",
156
+ "marketoDataType": "STRING"
157
+ },
158
+ {
159
+ "name": "originalSourceInfo",
160
+ "marketoDataType": "STRING"
161
+ },
162
+ {
163
+ "name": "registrationSourceType",
164
+ "marketoDataType": "STRING"
165
+ },
166
+ {
167
+ "name": "registrationSourceInfo",
168
+ "marketoDataType": "STRING"
169
+ },
170
+ {
171
+ "name": "originalSearchEngine",
172
+ "marketoDataType": "STRING"
173
+ },
174
+ {
175
+ "name": "originalSearchPhrase",
176
+ "marketoDataType": "STRING"
177
+ },
178
+ {
179
+ "name": "originalReferrer",
180
+ "marketoDataType": "URL"
181
+ },
182
+ {
183
+ "name": "emailInvalid",
184
+ "marketoDataType": "BOOLEAN"
185
+ },
186
+ {
187
+ "name": "emailInvalidCause",
188
+ "marketoDataType": "STRING"
189
+ },
190
+ {
191
+ "name": "unsubscribed",
192
+ "marketoDataType": "BOOLEAN"
193
+ },
194
+ {
195
+ "name": "unsubscribedReason",
196
+ "marketoDataType": "STRING"
197
+ },
198
+ {
199
+ "name": "doNotCall",
200
+ "marketoDataType": "BOOLEAN"
201
+ },
202
+ {
203
+ "name": "mktoDoNotCallCause",
204
+ "marketoDataType": "STRING"
205
+ },
206
+ {
207
+ "name": "doNotCallReason",
208
+ "marketoDataType": "STRING"
209
+ },
210
+ {
211
+ "name": "mktoPersonNotes",
212
+ "marketoDataType": "TEXT"
213
+ },
214
+ {
215
+ "name": "anonymousIP",
216
+ "marketoDataType": "STRING"
217
+ },
218
+ {
219
+ "name": "inferredCompany",
220
+ "marketoDataType": "STRING"
221
+ },
222
+ {
223
+ "name": "inferredCountry",
224
+ "marketoDataType": "STRING"
225
+ },
226
+ {
227
+ "name": "inferredCity",
228
+ "marketoDataType": "STRING"
229
+ },
230
+ {
231
+ "name": "inferredStateRegion",
232
+ "marketoDataType": "STRING"
233
+ },
234
+ {
235
+ "name": "inferredPostalCode",
236
+ "marketoDataType": "STRING"
237
+ },
238
+ {
239
+ "name": "inferredMetropolitanArea",
240
+ "marketoDataType": "STRING"
241
+ },
242
+ {
243
+ "name": "inferredPhoneAreaCode",
244
+ "marketoDataType": "STRING"
245
+ },
246
+ {
247
+ "name": "department",
248
+ "marketoDataType": "STRING"
249
+ },
250
+ {
251
+ "name": "createdAt",
252
+ "marketoDataType": "DATETIME"
253
+ },
254
+ {
255
+ "name": "updatedAt",
256
+ "marketoDataType": "DATETIME"
257
+ },
258
+ {
259
+ "name": "cookies",
260
+ "marketoDataType": "STRING"
261
+ },
262
+ {
263
+ "name": "externalSalesPersonId",
264
+ "marketoDataType": "STRING"
265
+ },
266
+ {
267
+ "name": "leadPerson",
268
+ "marketoDataType": "REFERENCE"
269
+ },
270
+ {
271
+ "name": "leadRole",
272
+ "marketoDataType": "STRING"
273
+ },
274
+ {
275
+ "name": "leadSource",
276
+ "marketoDataType": "STRING"
277
+ },
278
+ {
279
+ "name": "leadStatus",
280
+ "marketoDataType": "STRING"
281
+ },
282
+ {
283
+ "name": "leadScore",
284
+ "marketoDataType": "INTEGER"
285
+ },
286
+ {
287
+ "name": "urgency",
288
+ "marketoDataType": "FLOAT"
289
+ },
290
+ {
291
+ "name": "priority",
292
+ "marketoDataType": "INTEGER"
293
+ },
294
+ {
295
+ "name": "relativeScore",
296
+ "marketoDataType": "INTEGER"
297
+ },
298
+ {
299
+ "name": "relativeUrgency",
300
+ "marketoDataType": "INTEGER"
301
+ },
302
+ {
303
+ "name": "rating",
304
+ "marketoDataType": "STRING"
305
+ },
306
+ {
307
+ "name": "personPrimaryLeadInterest",
308
+ "marketoDataType": "REFERENCE"
309
+ },
310
+ {
311
+ "name": "leadPartitionId",
312
+ "marketoDataType": "REFERENCE"
313
+ },
314
+ {
315
+ "name": "leadRevenueCycleModelId",
316
+ "marketoDataType": "REFERENCE"
317
+ },
318
+ {
319
+ "name": "leadRevenueStageId",
320
+ "marketoDataType": "REFERENCE"
321
+ },
322
+ {
323
+ "name": "gender",
324
+ "marketoDataType": "STRING"
325
+ },
326
+ {
327
+ "name": "facebookDisplayName",
328
+ "marketoDataType": "STRING"
329
+ },
330
+ {
331
+ "name": "twitterDisplayName",
332
+ "marketoDataType": "STRING"
333
+ },
334
+ {
335
+ "name": "linkedInDisplayName",
336
+ "marketoDataType": "STRING"
337
+ },
338
+ {
339
+ "name": "facebookProfileURL",
340
+ "marketoDataType": "STRING"
341
+ },
342
+ {
343
+ "name": "twitterProfileURL",
344
+ "marketoDataType": "STRING"
345
+ },
346
+ {
347
+ "name": "linkedInProfileURL",
348
+ "marketoDataType": "STRING"
349
+ },
350
+ {
351
+ "name": "facebookPhotoURL",
352
+ "marketoDataType": "STRING"
353
+ },
354
+ {
355
+ "name": "twitterPhotoURL",
356
+ "marketoDataType": "STRING"
357
+ },
358
+ {
359
+ "name": "linkedInPhotoURL",
360
+ "marketoDataType": "STRING"
361
+ },
362
+ {
363
+ "name": "facebookReach",
364
+ "marketoDataType": "INTEGER"
365
+ },
366
+ {
367
+ "name": "twitterReach",
368
+ "marketoDataType": "INTEGER"
369
+ },
370
+ {
371
+ "name": "linkedInReach",
372
+ "marketoDataType": "INTEGER"
373
+ },
374
+ {
375
+ "name": "facebookReferredVisits",
376
+ "marketoDataType": "INTEGER"
377
+ },
378
+ {
379
+ "name": "twitterReferredVisits",
380
+ "marketoDataType": "INTEGER"
381
+ },
382
+ {
383
+ "name": "linkedInReferredVisits",
384
+ "marketoDataType": "INTEGER"
385
+ },
386
+ {
387
+ "name": "totalReferredVisits",
388
+ "marketoDataType": "INTEGER"
389
+ },
390
+ {
391
+ "name": "facebookReferredEnrollments",
392
+ "marketoDataType": "INTEGER"
393
+ },
394
+ {
395
+ "name": "twitterReferredEnrollments",
396
+ "marketoDataType": "INTEGER"
397
+ },
398
+ {
399
+ "name": "linkedInReferredEnrollments",
400
+ "marketoDataType": "INTEGER"
401
+ },
402
+ {
403
+ "name": "totalReferredEnrollments",
404
+ "marketoDataType": "INTEGER"
405
+ },
406
+ {
407
+ "name": "lastReferredVisit",
408
+ "marketoDataType": "DATETIME"
409
+ },
410
+ {
411
+ "name": "lastReferredEnrollment",
412
+ "marketoDataType": "DATETIME"
413
+ },
414
+ {
415
+ "name": "syndicationId",
416
+ "marketoDataType": "STRING"
417
+ },
418
+ {
419
+ "name": "facebookId",
420
+ "marketoDataType": "STRING"
421
+ },
422
+ {
423
+ "name": "twitterId",
424
+ "marketoDataType": "STRING"
425
+ },
426
+ {
427
+ "name": "linkedInId",
428
+ "marketoDataType": "STRING"
429
+ },
430
+ {
431
+ "name": "acquisitionProgramId",
432
+ "marketoDataType": "REFERENCE"
433
+ },
434
+ {
435
+ "name": "mktoAcquisitionDate",
436
+ "marketoDataType": "DATETIME"
437
+ },
438
+ {
439
+ "name": "tdglobalid",
440
+ "marketoDataType": "TEXT"
441
+ },
442
+ {
443
+ "name": "tdid",
444
+ "marketoDataType": "INTEGER"
445
+ },
446
+ {
447
+ "name": "mbtesttextarea",
448
+ "marketoDataType": "TEXT"
449
+ },
450
+ {
451
+ "name": "customFieldInt",
452
+ "marketoDataType": "INTEGER"
453
+ },
454
+ {
455
+ "name": "tdidint",
456
+ "marketoDataType": "INTEGER"
457
+ },
458
+ {
459
+ "name": "time",
460
+ "marketoDataType": "INTEGER"
461
+ },
462
+ {
463
+ "name": "mBtestboolean24",
464
+ "marketoDataType": "BOOLEAN"
465
+ },
466
+ {
467
+ "name": "mBtestcurrency24",
468
+ "marketoDataType": "STRING"
469
+ },
470
+ {
471
+ "name": "mBtestdate24",
472
+ "marketoDataType": "DATE"
473
+ },
474
+ {
475
+ "name": "mBtestDatetime24",
476
+ "marketoDataType": "DATETIME"
477
+ },
478
+ {
479
+ "name": "mBtestemail24",
480
+ "marketoDataType": "EMAIL"
481
+ },
482
+ {
483
+ "name": "mBtestfloat24",
484
+ "marketoDataType": "FLOAT"
485
+ },
486
+ {
487
+ "name": "mBtestinteger24",
488
+ "marketoDataType": "INTEGER"
489
+ },
490
+ {
491
+ "name": "mBtestpercent24",
492
+ "marketoDataType": "PERCENT"
493
+ },
494
+ {
495
+ "name": "mBtestphone24",
496
+ "marketoDataType": "PHONE"
497
+ },
498
+ {
499
+ "name": "mBtestscore24",
500
+ "marketoDataType": "INTEGER"
501
+ },
502
+ {
503
+ "name": "mBteststring24",
504
+ "marketoDataType": "STRING"
505
+ },
506
+ {
507
+ "name": "mBtesttextarea24",
508
+ "marketoDataType": "TEXT"
509
+ },
510
+ {
511
+ "name": "mBtesturl24",
512
+ "marketoDataType": "URL"
513
+ },
514
+ {
515
+ "name": "marketoTimeZone",
516
+ "marketoDataType": "STRING"
517
+ }
518
+ ]