embulk-input-marketo_extended 0.6.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. checksums.yaml +7 -0
  2. data/.github/PULL_REQUEST_TEMPLATE.md +37 -0
  3. data/.gitignore +14 -0
  4. data/.travis.yml +6 -0
  5. data/CHANGELOG.md +170 -0
  6. data/LICENSE.txt +21 -0
  7. data/README.md +213 -0
  8. data/build.gradle +103 -0
  9. data/config/checkstyle/checkstyle.xml +128 -0
  10. data/config/checkstyle/default.xml +108 -0
  11. data/gradle/wrapper/gradle-wrapper.jar +0 -0
  12. data/gradle/wrapper/gradle-wrapper.properties +6 -0
  13. data/gradlew +169 -0
  14. data/gradlew.bat +84 -0
  15. data/lib/embulk/input/marketo.rb +3 -0
  16. data/settings.gradle +1 -0
  17. data/src/main/java/org/embulk/input/marketo/CsvTokenizer.java +700 -0
  18. data/src/main/java/org/embulk/input/marketo/MarketoInputPlugin.java +15 -0
  19. data/src/main/java/org/embulk/input/marketo/MarketoInputPluginDelegate.java +100 -0
  20. data/src/main/java/org/embulk/input/marketo/MarketoService.java +38 -0
  21. data/src/main/java/org/embulk/input/marketo/MarketoServiceImpl.java +245 -0
  22. data/src/main/java/org/embulk/input/marketo/MarketoUtils.java +212 -0
  23. data/src/main/java/org/embulk/input/marketo/delegate/ActivityBulkExtractInputPlugin.java +167 -0
  24. data/src/main/java/org/embulk/input/marketo/delegate/CampaignInputPlugin.java +48 -0
  25. data/src/main/java/org/embulk/input/marketo/delegate/CustomObjectInputPlugin.java +75 -0
  26. data/src/main/java/org/embulk/input/marketo/delegate/CustomObjectResponseMapperBuilder.java +81 -0
  27. data/src/main/java/org/embulk/input/marketo/delegate/LeadBulkExtractInputPlugin.java +66 -0
  28. data/src/main/java/org/embulk/input/marketo/delegate/LeadServiceResponseMapperBuilder.java +85 -0
  29. data/src/main/java/org/embulk/input/marketo/delegate/LeadWithListInputPlugin.java +64 -0
  30. data/src/main/java/org/embulk/input/marketo/delegate/LeadWithProgramInputPlugin.java +60 -0
  31. data/src/main/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPlugin.java +441 -0
  32. data/src/main/java/org/embulk/input/marketo/delegate/MarketoBaseInputPluginDelegate.java +92 -0
  33. data/src/main/java/org/embulk/input/marketo/delegate/ProgramInputPlugin.java +228 -0
  34. data/src/main/java/org/embulk/input/marketo/exception/MarketoAPIException.java +30 -0
  35. data/src/main/java/org/embulk/input/marketo/model/BulkExtractRangeHeader.java +26 -0
  36. data/src/main/java/org/embulk/input/marketo/model/MarketoAccessTokenResponse.java +92 -0
  37. data/src/main/java/org/embulk/input/marketo/model/MarketoBulkExtractRequest.java +68 -0
  38. data/src/main/java/org/embulk/input/marketo/model/MarketoError.java +40 -0
  39. data/src/main/java/org/embulk/input/marketo/model/MarketoField.java +126 -0
  40. data/src/main/java/org/embulk/input/marketo/model/MarketoResponse.java +82 -0
  41. data/src/main/java/org/embulk/input/marketo/model/filter/DateRangeFilter.java +40 -0
  42. data/src/main/java/org/embulk/input/marketo/rest/MarketoBaseRestClient.java +306 -0
  43. data/src/main/java/org/embulk/input/marketo/rest/MarketoInputStreamResponseEntityReader.java +69 -0
  44. data/src/main/java/org/embulk/input/marketo/rest/MarketoRESTEndpoint.java +47 -0
  45. data/src/main/java/org/embulk/input/marketo/rest/MarketoResponseJetty92EntityReader.java +89 -0
  46. data/src/main/java/org/embulk/input/marketo/rest/MarketoRestClient.java +569 -0
  47. data/src/main/java/org/embulk/input/marketo/rest/RecordPagingIterable.java +180 -0
  48. data/src/test/java/org/embulk/input/marketo/MarketoServiceImplTest.java +140 -0
  49. data/src/test/java/org/embulk/input/marketo/MarketoUtilsTest.java +87 -0
  50. data/src/test/java/org/embulk/input/marketo/delegate/ActivityBulkExtractInputPluginTest.java +128 -0
  51. data/src/test/java/org/embulk/input/marketo/delegate/CampaignInputPluginTest.java +73 -0
  52. data/src/test/java/org/embulk/input/marketo/delegate/CustomObjectInputPluginTest.java +102 -0
  53. data/src/test/java/org/embulk/input/marketo/delegate/LeadBulkExtractInputPluginTest.java +99 -0
  54. data/src/test/java/org/embulk/input/marketo/delegate/LeadServiceResponseMapperBuilderTest.java +119 -0
  55. data/src/test/java/org/embulk/input/marketo/delegate/LeadWithListInputPluginTest.java +101 -0
  56. data/src/test/java/org/embulk/input/marketo/delegate/LeadWithProgramInputPluginTest.java +103 -0
  57. data/src/test/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPluginTest.java +169 -0
  58. data/src/test/java/org/embulk/input/marketo/delegate/ProgramInputPluginTest.java +343 -0
  59. data/src/test/java/org/embulk/input/marketo/rest/MarketoBaseRestClientTest.java +368 -0
  60. data/src/test/java/org/embulk/input/marketo/rest/MarketoRestClientTest.java +584 -0
  61. data/src/test/resources/config/activity_bulk_extract_config.yaml +7 -0
  62. data/src/test/resources/config/custom_object_config.yaml +8 -0
  63. data/src/test/resources/config/lead_bulk_extract_config.yaml +8 -0
  64. data/src/test/resources/config/rest_config.yaml +3 -0
  65. data/src/test/resources/fixtures/activity_extract1.csv +35 -0
  66. data/src/test/resources/fixtures/activity_extract2.csv +22 -0
  67. data/src/test/resources/fixtures/activity_types.json +22 -0
  68. data/src/test/resources/fixtures/all_program_full.json +53 -0
  69. data/src/test/resources/fixtures/campaign_response.json +38 -0
  70. data/src/test/resources/fixtures/campaign_response_full.json +102 -0
  71. data/src/test/resources/fixtures/custom_object_describe.json +124 -0
  72. data/src/test/resources/fixtures/custom_object_describe_marketo_fields_full.json +22 -0
  73. data/src/test/resources/fixtures/custom_object_expected.json +66 -0
  74. data/src/test/resources/fixtures/custom_object_response.json +24 -0
  75. data/src/test/resources/fixtures/custom_object_response_full.json +23 -0
  76. data/src/test/resources/fixtures/lead_by_list.json +33 -0
  77. data/src/test/resources/fixtures/lead_by_program_response.json +47 -0
  78. data/src/test/resources/fixtures/lead_describe.json +221 -0
  79. data/src/test/resources/fixtures/lead_describe_expected.json +66 -0
  80. data/src/test/resources/fixtures/lead_describe_marketo_fields_full.json +518 -0
  81. data/src/test/resources/fixtures/lead_extract1.csv +11 -0
  82. data/src/test/resources/fixtures/lead_response_full.json +2402 -0
  83. data/src/test/resources/fixtures/lead_with_program_full.json +17 -0
  84. data/src/test/resources/fixtures/leads_extract2.csv +10 -0
  85. data/src/test/resources/fixtures/list_reponse_full.json +191 -0
  86. data/src/test/resources/fixtures/lists_response.json +31 -0
  87. data/src/test/resources/fixtures/program_response.json +71 -0
  88. metadata +171 -0
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 12599394a20752769173478de83e784ad9789281
4
+ data.tar.gz: 92e3492011dfcc74354acc6c0a4cb070e62c2b21
5
+ SHA512:
6
+ metadata.gz: 7e902baac51f0a1fb9fafab08e540bb5acbaa7be69948204b3272b32cb187da3aef3d9572492a965fe692c8e99b7860a66b002c640db5661c4ebb590bae1b1b7
7
+ data.tar.gz: 368d67fa4e00d86cc94bbbdec1a23a68d197cfa91afe28c6bcdea77a54cd14ca261855eae0f0ca255932bcf47a63c7abf5dfc46274e59ab8f38f65b3949da5e5
data/.github/PULL_REQUEST_TEMPLATE.md ADDED
@@ -0,0 +1,37 @@
1
+ ### Are all connections created by the plugin secure?
2
+
3
+ - [ ] Does it opt secure communication standard? Such as HTTPS, SSH, SFTP, SMTP STARTTLS. If not check with CISO to decide we can deploy the plugin.
4
+ - [ ] Does support both authentication and encryption appropriately? Such as: "just encrypting without authentication" that is insecure.
5
+
6
+ ### Does the plugin connect only to its expected external site which the customer explicitly set in their config file?
7
+
8
+ - [ ] Does NOT connect unexpected external site and our internal endpoints? Such as: “v3/job/:id/set_started” callback endpoint.
9
+
10
+ ### Does NOT the plugin persist any customers' private information? Identify the private information beforehand.
11
+
12
+ - [ ] Does NOT include them in (temporary) files?
13
+ - [ ] Does NOT include them in log messages and exception messages?
14
+
15
+ ### What kind of environments does the plugin interact with?
16
+
17
+ - [ ] Does NOT execute any shell command?
18
+ - [ ] Does NOT read any files on the running instance? Such as: "/etc/passwords". It’s ok to read temporary files that the plugin wrote.
19
+ - [ ] Does use to create temporary files by spi.TempFileSpace utility to avoid the conflict of the file names.
20
+ - [ ] Does NOT get environment variables or JVM system properties at runtime? Such as AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in environment variables
21
+
22
+ ### Does NOT the plugin use insecure libraries?
23
+
24
+ - [ ] Line up all depending library so that we can identify the impact of security incident of those library if any.
25
+ - [ ] Check libraries usage of the plugin; all security check list must apply to the library usages. Such as "Are all connections created by the library secure?"
26
+
27
+ ### Does NOT the plugin source code repository contain kinds of credentials
28
+
29
+ - [ ] API keys
30
+ - [ ] Passwords
31
+
32
+ ### Make sure to free up all resources allocated during Embulk transaction “committing” or “rolling back”or before.
33
+
34
+ - [ ] Network (connections, pooled connections)
35
+ - [ ] Memory (cache in static variables)
36
+ - [ ] File (temporary files)
37
+ - [ ] CPU (threads, processes)
data/.gitignore ADDED
@@ -0,0 +1,14 @@
1
+ *~
2
+ /pkg/
3
+ /tmp/
4
+ *.gemspec
5
+ .gradle/
6
+ classpath/
7
+ classes/
8
+ build/
9
+ .idea
10
+ /.settings/
11
+ /.metadata/
12
+ .classpath
13
+ .project
14
+
data/.travis.yml ADDED
@@ -0,0 +1,6 @@
1
+ language: java
2
+ jdk:
3
+ - oraclejdk8
4
+ script:
5
+ - ./gradlew --info check jacocoTestReport
6
+ dist: trusty
data/CHANGELOG.md ADDED
@@ -0,0 +1,170 @@
1
+ ## 0.6.18 - 2020-01-06
2
+ * [enhancement] Support Marketo Partner API Key [#98](https://github.com/treasure-data/embulk-input-marketo/pull/98)
3
+
4
+ ## 0.6.17 - 2019-12-03
5
+ * [hotfix] Fixed issue issue actTypeIds is required [#97](https://github.com/treasure-data/embulk-input-marketo/pull/97)
6
+
7
+ ## 0.6.16 - 2019-12-03
8
+ * [enhancement] Added support for ActivityTypeIds filter PR [#96](https://github.com/treasure-data/embulk-input-marketo/pull/96)
9
+
10
+ ## 0.6.15 - 2019-09-20
11
+ * [enhancement] Raise RuntimeException for temp file error [#94](https://github.com/treasure-data/embulk-input-marketo/pull/94)
12
+
13
+ ## 0.6.14 - 2019-08-19
14
+ * [enhancement] Improve exception handling [#93](https://github.com/treasure-data/embulk-input-marketo/pull/93)
15
+
16
+ ## 0.6.13 - 2019-01-21
17
+ * [enhance] Add more error code to retry [#91](https://github.com/treasure-data/embulk-input-marketo/pull/91)
18
+
19
+ ## 0.6.12 - 2018-11-09
20
+ * [enhance] Implement Custom Object [#90](https://github.com/treasure-data/embulk-input-marketo/pull/90)
21
+
22
+ ## 0.6.11 - 2018-09-10
23
+ * [enhance] Implement Assets Programs [#89](https://github.com/treasure-data/embulk-input-marketo/pull/89)
24
+
25
+ ## 0.6.10 - 2018-05-28
26
+ * [fixed] Add included column option [#87](https://github.com/treasure-data/embulk-input-marketo/pull/87)
27
+
28
+ ## 0.6.9 - 2018-04-16
29
+ * [fixed] Fix wrapped TimeoutException not retry [#85](https://github.com/treasure-data/embulk-input-marketo/pull/85)
30
+ * [enhance] Make read_timeout configurable [#85](https://github.com/treasure-data/embulk-input-marketo/pull/85)
31
+
32
+ ## 0.6.8 - 2018-04-12
33
+ * [fixed] Fix incorrect incorrect retry logic [#84](https://github.com/treasure-data/embulk-input-marketo/pull/84)
34
+
35
+ ## 0.6.7 - 2018-02-26
36
+ * [fixed] Remove de-duplication logic [#83](https://github.com/treasure-data/embulk-input-marketo/pull/83)
37
+
38
+ ## 0.6.6 - 2018-01-30
39
+ * [fixed] Fix JettyRetryHelper not closed [#82](https://github.com/treasure-data/embulk-input-marketo/pull/82)
40
+ ## 0.6.5 - 2017-12-19 [fixed] Fix infinite loop when import non bulk extract targets [#80](https://github.com/treasure-data/embulk-input-marketo/pull/80) ## 0.6.4 - 2017-12-13 [fixed] Fix incorrect job timeout calculation [#78](https://github.com/treasure-data/embulk-input-marketo/pull/78)
41
+ * [enhance] Disable incremental import by updatedAt [#77](https://github.com/treasure-data/embulk-input-marketo/pull/77)
42
+ * [enhance] Add log for exported file size [#76](https://github.com/treasure-data/embulk-input-marketo/pull/76)
43
+
44
+
45
+ ## 0.6.3 - 2017-11-13
46
+ * [enhance] Ignore records with timestamp smaller or equal to latest_fetch_time [#74](https://github.com/treasure-data/embulk-input-marketo/pull/74)
47
+
48
+ ## 0.6.2 - 2017-10-16
49
+ * [fixed] NullPointerException when building config diff [#73](https://github.com/treasure-data/embulk-input-marketo/pull/73)
50
+
51
+ ## 0.6.1 - 2017-10-12
52
+ * [fixed] OutOfMemeory when run Lead by list, Lead by program [#69](https://github.com/treasure-data/embulk-input-marketo/pull/69)
53
+ * [fixed] SOAP only field cause NullPointerException [#69](https://github.com/treasure-data/embulk-input-marketo/pull/69)
54
+ * [enhancement] Implement file download resume in Bulk extract [#69](https://github.com/treasure-data/embulk-input-marketo/pull/69)
55
+
56
+ ## 0.6.0 - 2017-10-10
57
+ * [major] Migrate to Java by embulk-base-restclient [#66](https://github.com/treasure-data/embulk-input-marketo/pull/66)
58
+ * [major] Migrate to REST API [#66](https://github.com/treasure-data/embulk-input-marketo/pull/66)
59
+ * [major] Add 3 more target, Campaign, Lead by list and Lead by program [#66](https://github.com/treasure-data/embulk-input-marketo/pull/66)
60
+ * [major] Support Marketo bulk extract API for lead and activity targets [#66](https://github.com/treasure-data/embulk-input-marketo/pull/66)
61
+ * [major] Support incremental ingestion for lead and activity targets [#66](https://github.com/treasure-data/embulk-input-marketo/pull/66)
62
+
63
+ ## 0.5.6 - 2016-12-14
64
+ * [maintenance] Enable tcp keepalive [#64](https://github.com/treasure-data/embulk-input-marketo/pull/64)
65
+
66
+ ## 0.5.6 - 2016-12-14
67
+ * [maintenance] Enable tcp keepalive [#64](https://github.com/treasure-data/embulk-input-marketo/pull/64)
68
+
69
+ ## 0.5.5 - 2016-11-24
70
+ * [fixed] Generate config_diff even if no records found [#62](https://github.com/treasure-data/embulk-input-marketo/pull/62)
71
+ * [maintenance] Fix to use CodeClimate 0.x [#63](https://github.com/treasure-data/embulk-input-marketo/pull/63)
72
+
73
+ ## 0.5.4 - 2016-10-26
74
+ * [enhancement] Validate wsdl_url and endpoint_url are the valid form [#61](https://github.com/treasure-data/embulk-input-marketo/pull/61)
75
+ * [enhancement] Minor readme change [#58](https://github.com/treasure-data/embulk-input-marketo/pull/58)
76
+ * [fixed] Fix error when retrying on guess/preview [#59](https://github.com/treasure-data/embulk-input-marketo/pull/59)
77
+ * [enhancement] Try newer date at first on preview to avoid miss hit [#60](https://github.com/treasure-data/embulk-input-marketo/pull/60)
78
+
79
+ ## 0.5.3 - 2016-07-01
80
+
81
+ * [enhancement] make concurrent limit exceeded error retryable [#56](https://github.com/treasure-data/embulk-input-marketo/pull/56)
82
+ * [maintenance] Gathering test coverage on CI [#55](https://github.com/treasure-data/embulk-input-marketo/pull/55)
83
+
84
+ ## 0.5.2 - 2016-04-27
85
+ * [enhancement] Make debug easier [#54](https://github.com/treasure-data/embulk-input-marketo/pull/54)
86
+ * [fixed] Recognize empty string as nil value [#53](https://github.com/treasure-data/embulk-input-marketo/pull/53)
87
+
88
+ ## 0.5.1 - 2016-04-06
89
+
90
+ * [maintenance] Relax dependency version
91
+
92
+ ## 0.5.0 - 2016-04-06
93
+
94
+ This version drops old Embulk supports. Embulk 0.8 or later is required since this version.
95
+
96
+ * [enhancement] Add tests for Embulk 0.8 and drop support old Embulk [#52](https://github.com/treasure-data/embulk-input-marketo/pull/52)
97
+ * [maintenance] Refactor lead and activity [#51](https://github.com/treasure-data/embulk-input-marketo/pull/51)
98
+ * [maintenance] Refactor retry [#50](https://github.com/treasure-data/embulk-input-marketo/pull/50)
99
+
100
+ ## 0.4.0 - 2015-10-30
101
+
102
+ This version drops scheduled execution with marketo/lead.
103
+
104
+ * [enhancement] Append processed time column [#49](https://github.com/treasure-data/embulk-input-marketo/pull/49)
105
+ * [enhancement] Exponential backoff retry [#48](https://github.com/treasure-data/embulk-input-marketo/pull/48)
106
+ * [fixed] Fix preview didn't stop after fetched if multiple ranges have [#45](https://github.com/treasure-data/embulk-input-marketo/pull/45)
107
+ * [enhancement] activity_log: Use from..from+30m range for guess [#47](https://github.com/treasure-data/embulk-input-marketo/pull/47)
108
+ * [enhancement] Unsupport scheduled execution for lead [#46](https://github.com/treasure-data/embulk-input-marketo/pull/46) [#41](https://github.com/treasure-data/embulk-input-marketo/pull/41) [Reported by @muga. Thanks!]
109
+
110
+ ## 0.3.2 - 2015-10-13
111
+
112
+ * [fixed] Prevent memoize in class [#44](https://github.com/treasure-data/embulk-input-marketo/pull/44)
113
+
114
+ ## 0.3.1 - 2015-10-06
115
+
116
+ * [enhancement] Supports embulk.0.7 [#43](https://github.com/treasure-data/embulk-input-marketo/pull/43)
117
+ * [maintenance] Refactor [#40](https://github.com/treasure-data/embulk-input-marketo/pull/40)
118
+
119
+ ## 0.3.0 - 2015-09-30
120
+
121
+ This version breaks backword compatibility of marketo/activity_log. Please check README.md to modify your config.
122
+
123
+ * [enhancement] Also activity_log uses from_datetime/to_datetime same as lead [#39](https://github.com/treasure-data/embulk-input-marketo/pull/39)
124
+
125
+ ## 0.2.5 - 2015-09-28
126
+
127
+ * [fixed] lead: Fix the bug when `from_datetime` and `to_datetime` are same [#37](https://github.com/treasure-data/embulk-input-marketo/pull/37)
128
+
129
+ ## 0.2.4 - 2015-09-17
130
+
131
+ * [enhancement] Retry to call API until 5 times when Timeout [#36](https://github.com/treasure-data/embulk-input-marketo/pull/36) [[Reported by @muga](https://github.com/treasure-data/embulk-input-marketo/issues/34). Thanks!]
132
+
133
+ ## 0.2.3 - 2015-09-14
134
+
135
+ * [enhancement] Catch config error [#33](https://github.com/treasure-data/embulk-input-marketo/pull/33)
136
+ * [enhancement] Concurrent worker [#31](https://github.com/treasure-data/embulk-input-marketo/pull/31)
137
+
138
+ ## 0.2.2 - 2015-09-08
139
+
140
+ * [fixed] Fix handling for activity_date_time [#32](https://github.com/treasure-data/embulk-input-marketo/pull/32)
141
+
142
+ ## 0.2.1 - 2015-09-01
143
+
144
+ * [fixed] activity_log: Avoid to cast values unexpectedly [#29](https://github.com/treasure-data/embulk-input-marketo/pull/29)
145
+ * [maintenance] Add a minor comment [#28](https://github.com/treasure-data/embulk-input-marketo/pull/28) [Requested by @muga. Thanks!!]
146
+ * [maintenance] Fix minor issues [#27](https://github.com/treasure-data/embulk-input-marketo/pull/27) [[Reviewed by @muga.](https://github.com/treasure-data/embulk-input-marketo/pull/25#issuecomment-135570967) Thanks!!]
147
+
148
+ ## 0.2.0 - 2015-08-27
149
+
150
+ This version breaks backword compatibility of marketo/lead. Please check README.md to modify your config.
151
+
152
+ * [enhancement] Avoid timeout for marketo/lead input [#25](https://github.com/treasure-data/embulk-input-marketo/pull/25)
153
+ * [fixed] Fix timestamp column [#24](https://github.com/treasure-data/embulk-input-marketo/pull/24)
154
+ * [enhancement] Raise ConfigError if unretryable error occured [#23](https://github.com/treasure-data/embulk-input-marketo/pull/23)
155
+ * [fixed] Fix the bug that the default value for wsdl is broken [#22](https://github.com/treasure-data/embulk-input-marketo/pull/22)
156
+
157
+ ## 0.1.1 - 2015-08-19
158
+
159
+ * [enhanement] Support scheduled execution [#20](https://github.com/treasure-data/embulk-input-marketo/pull/20) [[Reported by @muga](https://github.com/treasure-data/embulk-input-marketo/issues/18). Thanks!]
160
+ * [maintenance] Use everyleaf-embulk_helper [#19](https://github.com/treasure-data/embulk-input-marketo/pull/19)
161
+
162
+ ## 0.1.0 - 2015-07-15
163
+
164
+ We implemented activity_log plugin for marketo, so config generated from 0.0.1 should be modified. Please check README.md to do it.
165
+
166
+ * [enhancement] Implement activity_log plugin [#13](https://github.com/treasure-data/embulk-input-marketo/pull/13) [#14](https://github.com/treasure-data/embulk-input-marketo/pull/14) [#15](https://github.com/treasure-data/embulk-input-marketo/pull/15)
167
+
168
+ ## 0.0.1 - 2015-07-06
169
+
170
+ The first release!!
data/LICENSE.txt ADDED
@@ -0,0 +1,21 @@
1
+
2
+ MIT License
3
+
4
+ Permission is hereby granted, free of charge, to any person obtaining
5
+ a copy of this software and associated documentation files (the
6
+ "Software"), to deal in the Software without restriction, including
7
+ without limitation the rights to use, copy, modify, merge, publish,
8
+ distribute, sublicense, and/or sell copies of the Software, and to
9
+ permit persons to whom the Software is furnished to do so, subject to
10
+ the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be
13
+ included in all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
18
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
19
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
20
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
21
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,213 @@
1
+ [![Build Status](https://travis-ci.org/treasure-data/embulk-input-marketo.svg?branch=master)](https://travis-ci.org/treasure-data/embulk-input-marketo)
2
+ [![Code Climate](https://codeclimate.com/github/treasure-data/embulk-input-marketo/badges/gpa.svg)](https://codeclimate.com/github/treasure-data/embulk-input-marketo)
3
+ [![Test Coverage](https://codeclimate.com/github/treasure-data/embulk-input-marketo/badges/coverage.svg)](https://codeclimate.com/github/treasure-data/embulk-input-marketo/coverage)
4
+ [![Gem Version](https://badge.fury.io/rb/embulk-input-marketo.svg)](http://badge.fury.io/rb/embulk-input-marketo)
5
+
6
+ # Marketo input plugin for Embulk
7
+
8
+ embulk-input-marketo is the gem preparing Embulk input plugins for [Marketo](http://www.marketo.com/).
9
+
10
+ - Lead(lead)
11
+ - Activity log(activity)
12
+ - Lead by list(all_lead_with_list_id)
13
+ - Lead by program(all_lead_with_program_id)
14
+ - Campaign(campaign)
15
+ - Assets Programs (program)
16
+
17
+ This plugin uses Marketo REST API.
18
+
19
+ ## Overview
20
+
21
+ Required Embulk version >= 0.8.33 (since 0.6.0).
22
+
23
+ * **Plugin type**: input
24
+ * **Resume supported**: no
25
+ * **Cleanup supported**: no
26
+ * **Guess supported**: no
27
+
28
+ ## Install
29
+
30
+ ```
31
+ $ embulk gem install embulk-input-marketo
32
+ ```
33
+
34
+ ## Configuration
35
+
36
+ ### API
37
+
38
+ Below parameters are shown in "Admin" > "Web Services" page in Marketo.
39
+
40
+ ### Base configuration parameter
41
+
42
+ All target have this configuration parameters
43
+
44
+ | name | required | default value | description |
45
+ |----------------------------------|----------|---------------|----------------------------------------------------------------------------------------------------------------------------------|
46
+ | **target** | true | | Marketo targets |
47
+ | **account_id** | true | | Marketo Muchkin id |
48
+ | **client_id** | true | | Marketo REST client id |
49
+ | **client_secret** | true | | Marketo REST client secret |
50
+ | **marketo_limit_interval_milis** | false | 20 | Marketo have limitation of 100 calls per 20 second. If REST API calls are failed they will wait this amount of time before retry |
51
+ | **batch_size** | false | 300 | Token paging batch size. Some REST API support batch |
52
+ | **max_return** | false | 200 | Max return for Endpoint that use offset paging |
53
+ | **partner_api_key** | false | | Set Marketo Partner API Key see: http://developers.marketo.com/support/Marketo_LaunchPoint_Technology_Partner_API_Key.pdf |
54
+
55
+ ### Bulk extract target configuration parameter (Lead and Activity)
56
+
57
+ All bulk extract target use this configuration parameter
58
+
59
+ | name | required | default value | description |
60
+ |-----------------------------|----------|---------------|-------------------------------------------------------------------------------------------------------------------------------|
61
+ | **from_date** | true | | Import data since this date. Example: 2017-10-11T06:43:24+00:00 |
62
+ | **fetch_days** | false | 1 | Amount of days to fetch since from_date |
63
+ | **polling_interval_second** | false | 60 | Amount of time to wait between pooling job status in second |
64
+ | **bulk_job_timeout_second** | false | 3600 | Amount of time to wait for bulk job to complete in second |
65
+ | **incremental** | false | true | If incremental is set to true, next run will have from_date set to the previous to_date(calculated by from_date + fetch_days) |
66
+ | **incremental_column** | false | createdAt | Column use to filter from_date and to_date |
67
+
68
+ ### Lead
69
+
70
+ Lead target extract all Marketo leads, it use Marketo bulk extract feature. Configuration include bulk extract configuration.
71
+
72
+ `target: lead`
73
+
74
+ Configuration:
75
+
76
+ | name | required | default value | description |
77
+ |---------------------|----------|---------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------|
78
+ | **use_updated_at** | false | false | Support filter with `updateAt` column, but not all Marketo Account have the feature to filter by updatedAt, updatedAt don't support incremental ingestion |
79
+ | **included_fields** | false | null | List of lead fields to included in export request sent to Marketo, can be used to reduce the size of BulkExtract file |
80
+
81
+ Schema type: Dynamic via describe lead endpoint.
82
+
83
+ Incremental support: yes
84
+
85
+ Range ingestion: yes
86
+
87
+ ### Activity
88
+
89
+ Activity target extract all Marketo activity log. Configuration include all bulk extract configuration
90
+
91
+ `target: activity`
92
+
93
+ Schema type: Static schema
94
+
95
+ Incremental support: yes
96
+
97
+ Range ingestion: yes
98
+
99
+ Filter by specific activity type ids: yes. See [#95](https://github.com/treasure-data/embulk-input-marketo/issues/95)
100
+
101
+ ### Campaign
102
+
103
+ Campaign extract all campaign data from Marketo
104
+
105
+ `target: campaign`
106
+
107
+ Schema type: Static schema
108
+
109
+ Incremental support: no
110
+
111
+ Range ingestion: no
112
+
113
+ ### Lead by list
114
+
115
+ Extract all Lead data including lead's list id
116
+
117
+ `target: all_lead_with_list_id`
118
+
119
+ Configuration:
120
+
121
+ | name | required | default value | description |
122
+ |---------------------|----------|---------------|-----------------------------------------------------------------------------------------------------------------|
123
+ | **included_fields** | false | null | List of lead fields to included in export request sent to Marketo, can be used to reduce request, response size |
124
+
125
+ Schema type: Dynamic via describe leads. Schema will have 1 addition column name listId that contain the id of the list the lead belong to
126
+
127
+ Incremental support: no
128
+
129
+ Range ingestion: no
130
+
131
+ ### Lead by program
132
+
133
+ Extract all Lead data including lead's program id
134
+
135
+ `target: all_lead_with_program_id`
136
+
137
+ Configuration:
138
+
139
+ | name | required | default value | description |
140
+ |---------------------|----------|---------------|-----------------------------------------------------------------------------------------------------------------|
141
+ | **included_fields** | false | null | List of lead fields to included in export request sent to Marketo, can be used to reduce request, response size |
142
+
143
+ Schema type: Dynamic via describe leads. Schema will have 1 addition column name listId that contain the id of the list the lead belong to
144
+
145
+ Incremental support: no
146
+
147
+ Range ingestion: no
148
+
149
+ ### Assets programs
150
+
151
+ Get Assets Programs by Query Tag type, Date range or all if no query by specified.
152
+
153
+ `target: program`
154
+
155
+ Configuration:
156
+
157
+ | name | required | default value | description |
158
+ |-----------------------------|----------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------|
159
+ | **query_by** | false | null | Get assets programs by query, supported values `date_range`, `tag_type` leave unset to fetch all programs |
160
+ | **earliest_updated_at** | false | null | Required if query by `date_range` is selected. Exclude programs prior to this date. Must be valid ISO-8601 string |
161
+ | **latest_updated_at** | false | null | Required if query by `date_range` is selected. Exclude programs after this date. Must be valid ISO-8601 string |
162
+ | **filter_type** | false | null | Optional value send with query by `date_range` is selected to filter out the result from Marketo. Supported values `id`, `programId`, `folderId`, `workspace`|
163
+ | **filter_values** | false | null | Set the values associated with `filter_type` |
164
+ | **tag_type** | false | null | Required if query by `tag_type` is selected. Type of program tag |
165
+ | **tag_value** | false | null | Required if query by `tag_type` is selected. Value of the tag |
166
+ | **report_duration** | false | null | Amount of milliseconds to fetch from `earliest_updated_at`. If `incremental = true` this value will automatically calculated for the first run by `latest_updated_at` - `earliest_updated_at` |
167
+ | **incremental** | false | true | If incremental is set to true, next run will have `earliest_updated_at` set to the previous `latest_updated_at` + `report_duration`. Incremental import only support by query `date_range` |
168
+
169
+ Schema type: Static schema
170
+
171
+ Incremental support: yes (Query by `date_range` only)
172
+
173
+ Range ingestion: yes
174
+
175
+ ## Example
176
+
177
+ For lead, you have `partial-config.yml` like below:
178
+
179
+ ```yaml
180
+ in:
181
+ type: marketo
182
+ target: lead
183
+ account_id: ACCOUNT_ID
184
+ client_id: CLIENT_ID
185
+ client_secret: CLIENT_SECRET
186
+ from_date: 2017-09-01
187
+ fetch_days: 1
188
+ out:
189
+ type: stdout
190
+ ```
191
+
192
+ You can run `embulk guess partial-config.yml -o lead-config.yml` and got `lead-config.yml`. `lead-config.yml` includes a schema for Lead.
193
+
194
+ Next, you can run `embulk preview lead-config.yml` for preview and `embulk run lead-config.yml` for run.
195
+
196
+ Example of Assets Programs config
197
+ ```yaml
198
+ in:
199
+ account_id: ACCOUNT_ID
200
+ client_id: CLIENT_ID
201
+ client_secret: CLIENT_SECRET
202
+ target: program
203
+ type: marketo
204
+ query_by: date_range
205
+ filter_type: folderId
206
+ filter_values:
207
+ - 2598
208
+ - 1001
209
+ earliest_updated_at: 2018-08-20T00:00:00.000Z
210
+ latest_updated_at: 2018-08-31T00:00:00.000Z
211
+ incremental: true
212
+ ```
213
+