embulk-input-marketo-extended 0.6.18

Sign up to get free protection for your applications and to get access to all the features.
Files changed (88) hide show
  1. checksums.yaml +7 -0
  2. data/.github/PULL_REQUEST_TEMPLATE.md +37 -0
  3. data/.gitignore +14 -0
  4. data/.travis.yml +6 -0
  5. data/CHANGELOG.md +170 -0
  6. data/LICENSE.txt +21 -0
  7. data/README.md +213 -0
  8. data/build.gradle +103 -0
  9. data/config/checkstyle/checkstyle.xml +128 -0
  10. data/config/checkstyle/default.xml +108 -0
  11. data/gradle/wrapper/gradle-wrapper.jar +0 -0
  12. data/gradle/wrapper/gradle-wrapper.properties +6 -0
  13. data/gradlew +169 -0
  14. data/gradlew.bat +84 -0
  15. data/lib/embulk/input/marketo.rb +3 -0
  16. data/settings.gradle +1 -0
  17. data/src/main/java/org/embulk/input/marketo/CsvTokenizer.java +700 -0
  18. data/src/main/java/org/embulk/input/marketo/MarketoInputPlugin.java +15 -0
  19. data/src/main/java/org/embulk/input/marketo/MarketoInputPluginDelegate.java +100 -0
  20. data/src/main/java/org/embulk/input/marketo/MarketoService.java +38 -0
  21. data/src/main/java/org/embulk/input/marketo/MarketoServiceImpl.java +245 -0
  22. data/src/main/java/org/embulk/input/marketo/MarketoUtils.java +212 -0
  23. data/src/main/java/org/embulk/input/marketo/delegate/ActivityBulkExtractInputPlugin.java +167 -0
  24. data/src/main/java/org/embulk/input/marketo/delegate/CampaignInputPlugin.java +48 -0
  25. data/src/main/java/org/embulk/input/marketo/delegate/CustomObjectInputPlugin.java +75 -0
  26. data/src/main/java/org/embulk/input/marketo/delegate/CustomObjectResponseMapperBuilder.java +81 -0
  27. data/src/main/java/org/embulk/input/marketo/delegate/LeadBulkExtractInputPlugin.java +66 -0
  28. data/src/main/java/org/embulk/input/marketo/delegate/LeadServiceResponseMapperBuilder.java +85 -0
  29. data/src/main/java/org/embulk/input/marketo/delegate/LeadWithListInputPlugin.java +64 -0
  30. data/src/main/java/org/embulk/input/marketo/delegate/LeadWithProgramInputPlugin.java +60 -0
  31. data/src/main/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPlugin.java +441 -0
  32. data/src/main/java/org/embulk/input/marketo/delegate/MarketoBaseInputPluginDelegate.java +92 -0
  33. data/src/main/java/org/embulk/input/marketo/delegate/ProgramInputPlugin.java +228 -0
  34. data/src/main/java/org/embulk/input/marketo/exception/MarketoAPIException.java +30 -0
  35. data/src/main/java/org/embulk/input/marketo/model/BulkExtractRangeHeader.java +26 -0
  36. data/src/main/java/org/embulk/input/marketo/model/MarketoAccessTokenResponse.java +92 -0
  37. data/src/main/java/org/embulk/input/marketo/model/MarketoBulkExtractRequest.java +68 -0
  38. data/src/main/java/org/embulk/input/marketo/model/MarketoError.java +40 -0
  39. data/src/main/java/org/embulk/input/marketo/model/MarketoField.java +126 -0
  40. data/src/main/java/org/embulk/input/marketo/model/MarketoResponse.java +82 -0
  41. data/src/main/java/org/embulk/input/marketo/model/filter/DateRangeFilter.java +40 -0
  42. data/src/main/java/org/embulk/input/marketo/rest/MarketoBaseRestClient.java +306 -0
  43. data/src/main/java/org/embulk/input/marketo/rest/MarketoInputStreamResponseEntityReader.java +69 -0
  44. data/src/main/java/org/embulk/input/marketo/rest/MarketoRESTEndpoint.java +47 -0
  45. data/src/main/java/org/embulk/input/marketo/rest/MarketoResponseJetty92EntityReader.java +89 -0
  46. data/src/main/java/org/embulk/input/marketo/rest/MarketoRestClient.java +569 -0
  47. data/src/main/java/org/embulk/input/marketo/rest/RecordPagingIterable.java +180 -0
  48. data/src/test/java/org/embulk/input/marketo/MarketoServiceImplTest.java +140 -0
  49. data/src/test/java/org/embulk/input/marketo/MarketoUtilsTest.java +87 -0
  50. data/src/test/java/org/embulk/input/marketo/delegate/ActivityBulkExtractInputPluginTest.java +128 -0
  51. data/src/test/java/org/embulk/input/marketo/delegate/CampaignInputPluginTest.java +73 -0
  52. data/src/test/java/org/embulk/input/marketo/delegate/CustomObjectInputPluginTest.java +102 -0
  53. data/src/test/java/org/embulk/input/marketo/delegate/LeadBulkExtractInputPluginTest.java +99 -0
  54. data/src/test/java/org/embulk/input/marketo/delegate/LeadServiceResponseMapperBuilderTest.java +119 -0
  55. data/src/test/java/org/embulk/input/marketo/delegate/LeadWithListInputPluginTest.java +101 -0
  56. data/src/test/java/org/embulk/input/marketo/delegate/LeadWithProgramInputPluginTest.java +103 -0
  57. data/src/test/java/org/embulk/input/marketo/delegate/MarketoBaseBulkExtractInputPluginTest.java +169 -0
  58. data/src/test/java/org/embulk/input/marketo/delegate/ProgramInputPluginTest.java +343 -0
  59. data/src/test/java/org/embulk/input/marketo/rest/MarketoBaseRestClientTest.java +368 -0
  60. data/src/test/java/org/embulk/input/marketo/rest/MarketoRestClientTest.java +584 -0
  61. data/src/test/resources/config/activity_bulk_extract_config.yaml +7 -0
  62. data/src/test/resources/config/custom_object_config.yaml +8 -0
  63. data/src/test/resources/config/lead_bulk_extract_config.yaml +8 -0
  64. data/src/test/resources/config/rest_config.yaml +3 -0
  65. data/src/test/resources/fixtures/activity_extract1.csv +35 -0
  66. data/src/test/resources/fixtures/activity_extract2.csv +22 -0
  67. data/src/test/resources/fixtures/activity_types.json +22 -0
  68. data/src/test/resources/fixtures/all_program_full.json +53 -0
  69. data/src/test/resources/fixtures/campaign_response.json +38 -0
  70. data/src/test/resources/fixtures/campaign_response_full.json +102 -0
  71. data/src/test/resources/fixtures/custom_object_describe.json +124 -0
  72. data/src/test/resources/fixtures/custom_object_describe_marketo_fields_full.json +22 -0
  73. data/src/test/resources/fixtures/custom_object_expected.json +66 -0
  74. data/src/test/resources/fixtures/custom_object_response.json +24 -0
  75. data/src/test/resources/fixtures/custom_object_response_full.json +23 -0
  76. data/src/test/resources/fixtures/lead_by_list.json +33 -0
  77. data/src/test/resources/fixtures/lead_by_program_response.json +47 -0
  78. data/src/test/resources/fixtures/lead_describe.json +221 -0
  79. data/src/test/resources/fixtures/lead_describe_expected.json +66 -0
  80. data/src/test/resources/fixtures/lead_describe_marketo_fields_full.json +518 -0
  81. data/src/test/resources/fixtures/lead_extract1.csv +11 -0
  82. data/src/test/resources/fixtures/lead_response_full.json +2402 -0
  83. data/src/test/resources/fixtures/lead_with_program_full.json +17 -0
  84. data/src/test/resources/fixtures/leads_extract2.csv +10 -0
  85. data/src/test/resources/fixtures/list_reponse_full.json +191 -0
  86. data/src/test/resources/fixtures/lists_response.json +31 -0
  87. data/src/test/resources/fixtures/program_response.json +71 -0
  88. metadata +171 -0
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: e6ad4c25bc2ed707b4f9620e774fb70d3e7fafd2
4
+ data.tar.gz: e2b0cc5f99a54312fb068fb21f508bd114e37f2c
5
+ SHA512:
6
+ metadata.gz: 1a9b47ee0ec75dfa3e36480f00df8383c2d31419d43fdbfa4293438b201dd231f40df34eb4159ca7f905f37bdf1fda93d248f1d63b6592a2e052844eccb12801
7
+ data.tar.gz: f4f5d4725c923745ba1124d8d0ad8ec616fbc1e8f62c371030092c44e7aac778706e84877921846b6acb9a33b485138aed7a633349d1a24328461ae7076fc488
data/.github/PULL_REQUEST_TEMPLATE.md ADDED
@@ -0,0 +1,37 @@
1
+ ### Are all connections created by the plugin secure?
2
+
3
+ - [ ] Does it opt secure communication standard? Such as HTTPS, SSH, SFTP, SMTP STARTTLS. If not check with CISO to decide we can deploy the plugin.
4
+ - [ ] Does support both authentication and encryption appropriately? Such as: "just encrypting without authentication" that is insecure.
5
+
6
+ ### Does the plugin connect only to its expected external site which the customer explicitly set in their config file?
7
+
8
+ - [ ] Does NOT connect unexpected external site and our internal endpoints? Such as: “v3/job/:id/set_started” callback endpoint.
9
+
10
+ ### Does NOT the plugin persist any customers' private information? Identify the private information beforehand.
11
+
12
+ - [ ] Does NOT include them in (temporary) files?
13
+ - [ ] Does NOT include them in log messages and exception messages?
14
+
15
+ ### What kind of environments does the plugin interact with?
16
+
17
+ - [ ] Does NOT execute any shell command?
18
+ - [ ] Does NOT read any files on the running instance? Such as: "/etc/passwords". It’s ok to read temporary files that the plugin wrote.
19
+ - [ ] Does use to create temporary files by spi.TempFileSpace utility to avoid the conflict of the file names.
20
+ - [ ] Does NOT get environment variables or JVM system properties at runtime? Such as AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY in environment variables
21
+
22
+ ### Does NOT the plugin use insecure libraries?
23
+
24
+ - [ ] Line up all depending library so that we can identify the impact of security incident of those library if any.
25
+ - [ ] Check libraries usage of the plugin; all security check list must apply to the library usages. Such as "Are all connections created by the library secure?"
26
+
27
+ ### Does NOT the plugin source code repository contain kinds of credentials
28
+
29
+ - [ ] API keys
30
+ - [ ] Passwords
31
+
32
+ ### Make sure to free up all resources allocated during Embulk transaction “committing” or “rolling back”or before.
33
+
34
+ - [ ] Network (connections, pooled connections)
35
+ - [ ] Memory (cache in static variables)
36
+ - [ ] File (temporary files)
37
+ - [ ] CPU (threads, processes)
data/.gitignore ADDED
@@ -0,0 +1,14 @@
1
+ *~
2
+ /pkg/
3
+ /tmp/
4
+ *.gemspec
5
+ .gradle/
6
+ classpath/
7
+ classes/
8
+ build/
9
+ .idea
10
+ /.settings/
11
+ /.metadata/
12
+ .classpath
13
+ .project
14
+
data/.travis.yml ADDED
@@ -0,0 +1,6 @@
1
+ language: java
2
+ jdk:
3
+ - oraclejdk8
4
+ script:
5
+ - ./gradlew --info check jacocoTestReport
6
+ dist: trusty
data/CHANGELOG.md ADDED
@@ -0,0 +1,170 @@
1
+ ## 0.6.18 - 2020-01-06
2
+ * [enhancement] Support Marketo Partner API Key [#98](https://github.com/treasure-data/embulk-input-marketo/pull/98)
3
+
4
+ ## 0.6.17 - 2019-12-03
5
+ * [hotfix] Fixed issue issue actTypeIds is required [#97](https://github.com/treasure-data/embulk-input-marketo/pull/97)
6
+
7
+ ## 0.6.16 - 2019-12-03
8
+ * [enhancement] Added support for ActivityTypeIds filter PR [#96](https://github.com/treasure-data/embulk-input-marketo/pull/96)
9
+
10
+ ## 0.6.15 - 2019-09-20
11
+ * [enhancement] Raise RuntimeException for temp file error [#94](https://github.com/treasure-data/embulk-input-marketo/pull/94)
12
+
13
+ ## 0.6.14 - 2019-08-19
14
+ * [enhancement] Improve exception handling [#93](https://github.com/treasure-data/embulk-input-marketo/pull/93)
15
+
16
+ ## 0.6.13 - 2019-01-21
17
+ * [enhance] Add more error code to retry [#91](https://github.com/treasure-data/embulk-input-marketo/pull/91)
18
+
19
+ ## 0.6.12 - 2018-11-09
20
+ * [enhance] Implement Custom Object [#90](https://github.com/treasure-data/embulk-input-marketo/pull/90)
21
+
22
+ ## 0.6.11 - 2018-09-10
23
+ * [enhance] Implement Assets Programs [#89](https://github.com/treasure-data/embulk-input-marketo/pull/89)
24
+
25
+ ## 0.6.10 - 2018-05-28
26
+ * [fixed] Add included column option [#87](https://github.com/treasure-data/embulk-input-marketo/pull/87)
27
+
28
+ ## 0.6.9 - 2018-04-16
29
+ * [fixed] Fix wrapped TimeoutException not retry [#85](https://github.com/treasure-data/embulk-input-marketo/pull/85)
30
+ * [enhance] Make read_timeout configurable [#85](https://github.com/treasure-data/embulk-input-marketo/pull/85)
31
+
32
+ ## 0.6.8 - 2018-04-12
33
+ * [fixed] Fix incorrect incorrect retry logic [#84](https://github.com/treasure-data/embulk-input-marketo/pull/84)
34
+
35
+ ## 0.6.7 - 2018-02-26
36
+ * [fixed] Remove de-duplication logic [#83](https://github.com/treasure-data/embulk-input-marketo/pull/83)
37
+
38
+ ## 0.6.6 - 2018-01-30
39
+ * [fixed] Fix JettyRetryHelper not closed [#82](https://github.com/treasure-data/embulk-input-marketo/pull/82)
40
+ ## 0.6.5 - 2017-12-19 [fixed] Fix infinite loop when import non bulk extract targets [#80](https://github.com/treasure-data/embulk-input-marketo/pull/80) ## 0.6.4 - 2017-12-13 [fixed] Fix incorrect job timeout calculation [#78](https://github.com/treasure-data/embulk-input-marketo/pull/78)
41
+ * [enhance] Disable incremental import by updatedAt [#77](https://github.com/treasure-data/embulk-input-marketo/pull/77)
42
+ * [enhance] Add log for exported file size [#76](https://github.com/treasure-data/embulk-input-marketo/pull/76)
43
+
44
+
45
+ ## 0.6.3 - 2017-11-13
46
+ * [enhance] Ignore records with timestamp smaller or equal to latest_fetch_time [#74](https://github.com/treasure-data/embulk-input-marketo/pull/74)
47
+
48
+ ## 0.6.2 - 2017-10-16
49
+ * [fixed] NullPointerException when building config diff [#73](https://github.com/treasure-data/embulk-input-marketo/pull/73)
50
+
51
+ ## 0.6.1 - 2017-10-12
52
+ * [fixed] OutOfMemeory when run Lead by list, Lead by program [#69](https://github.com/treasure-data/embulk-input-marketo/pull/69)
53
+ * [fixed] SOAP only field cause NullPointerException [#69](https://github.com/treasure-data/embulk-input-marketo/pull/69)
54
+ * [enhancement] Implement file download resume in Bulk extract [#69](https://github.com/treasure-data/embulk-input-marketo/pull/69)
55
+
56
+ ## 0.6.0 - 2017-10-10
57
+ * [major] Migrate to Java by embulk-base-restclient [#66](https://github.com/treasure-data/embulk-input-marketo/pull/66)
58
+ * [major] Migrate to REST API [#66](https://github.com/treasure-data/embulk-input-marketo/pull/66)
59
+ * [major] Add 3 more target, Campaign, Lead by list and Lead by program [#66](https://github.com/treasure-data/embulk-input-marketo/pull/66)
60
+ * [major] Support Marketo bulk extract API for lead and activity targets [#66](https://github.com/treasure-data/embulk-input-marketo/pull/66)
61
+ * [major] Support incremental ingestion for lead and activity targets [#66](https://github.com/treasure-data/embulk-input-marketo/pull/66)
62
+
63
+ ## 0.5.6 - 2016-12-14
64
+ * [maintenance] Enable tcp keepalive [#64](https://github.com/treasure-data/embulk-input-marketo/pull/64)
65
+
66
+ ## 0.5.6 - 2016-12-14
67
+ * [maintenance] Enable tcp keepalive [#64](https://github.com/treasure-data/embulk-input-marketo/pull/64)
68
+
69
+ ## 0.5.5 - 2016-11-24
70
+ * [fixed] Generate config_diff even if no records found [#62](https://github.com/treasure-data/embulk-input-marketo/pull/62)
71
+ * [maintenance] Fix to use CodeClimate 0.x [#63](https://github.com/treasure-data/embulk-input-marketo/pull/63)
72
+
73
+ ## 0.5.4 - 2016-10-26
74
+ * [enhancement] Validate wsdl_url and endpoint_url are the valid form [#61](https://github.com/treasure-data/embulk-input-marketo/pull/61)
75
+ * [enhancement] Minor readme change [#58](https://github.com/treasure-data/embulk-input-marketo/pull/58)
76
+ * [fixed] Fix error when retrying on guess/preview [#59](https://github.com/treasure-data/embulk-input-marketo/pull/59)
77
+ * [enhancement] Try newer date at first on preview to avoid miss hit [#60](https://github.com/treasure-data/embulk-input-marketo/pull/60)
78
+
79
+ ## 0.5.3 - 2016-07-01
80
+
81
+ * [enhancement] make concurrent limit exceeded error retryable [#56](https://github.com/treasure-data/embulk-input-marketo/pull/56)
82
+ * [maintenance] Gathering test coverage on CI [#55](https://github.com/treasure-data/embulk-input-marketo/pull/55)
83
+
84
+ ## 0.5.2 - 2016-04-27
85
+ * [enhancement] Make debug easier [#54](https://github.com/treasure-data/embulk-input-marketo/pull/54)
86
+ * [fixed] Recognize empty string as nil value [#53](https://github.com/treasure-data/embulk-input-marketo/pull/53)
87
+
88
+ ## 0.5.1 - 2016-04-06
89
+
90
+ * [maintenance] Relax dependency version
91
+
92
+ ## 0.5.0 - 2016-04-06
93
+
94
+ This version drops old Embulk supports. Embulk 0.8 or later is required since this version.
95
+
96
+ * [enhancement] Add tests for Embulk 0.8 and drop support old Embulk [#52](https://github.com/treasure-data/embulk-input-marketo/pull/52)
97
+ * [maintenance] Refactor lead and activity [#51](https://github.com/treasure-data/embulk-input-marketo/pull/51)
98
+ * [maintenance] Refactor retry [#50](https://github.com/treasure-data/embulk-input-marketo/pull/50)
99
+
100
+ ## 0.4.0 - 2015-10-30
101
+
102
+ This version drops scheduled execution with marketo/lead.
103
+
104
+ * [enhancement] Append processed time column [#49](https://github.com/treasure-data/embulk-input-marketo/pull/49)
105
+ * [enhancement] Exponential backoff retry [#48](https://github.com/treasure-data/embulk-input-marketo/pull/48)
106
+ * [fixed] Fix preview didn't stop after fetched if multiple ranges have [#45](https://github.com/treasure-data/embulk-input-marketo/pull/45)
107
+ * [enhancement] activity_log: Use from..from+30m range for guess [#47](https://github.com/treasure-data/embulk-input-marketo/pull/47)
108
+ * [enhancement] Unsupport scheduled execution for lead [#46](https://github.com/treasure-data/embulk-input-marketo/pull/46) [#41](https://github.com/treasure-data/embulk-input-marketo/pull/41) [Reported by @muga. Thanks!]
109
+
110
+ ## 0.3.2 - 2015-10-13
111
+
112
+ * [fixed] Prevent memoize in class [#44](https://github.com/treasure-data/embulk-input-marketo/pull/44)
113
+
114
+ ## 0.3.1 - 2015-10-06
115
+
116
+ * [enhancement] Supports embulk.0.7 [#43](https://github.com/treasure-data/embulk-input-marketo/pull/43)
117
+ * [maintenance] Refactor [#40](https://github.com/treasure-data/embulk-input-marketo/pull/40)
118
+
119
+ ## 0.3.0 - 2015-09-30
120
+
121
+ This version breaks backword compatibility of marketo/activity_log. Please check README.md to modify your config.
122
+
123
+ * [enhancement] Also activity_log uses from_datetime/to_datetime same as lead [#39](https://github.com/treasure-data/embulk-input-marketo/pull/39)
124
+
125
+ ## 0.2.5 - 2015-09-28
126
+
127
+ * [fixed] lead: Fix the bug when `from_datetime` and `to_datetime` are same [#37](https://github.com/treasure-data/embulk-input-marketo/pull/37)
128
+
129
+ ## 0.2.4 - 2015-09-17
130
+
131
+ * [enhancement] Retry to call API until 5 times when Timeout [#36](https://github.com/treasure-data/embulk-input-marketo/pull/36) [[Reported by @muga](https://github.com/treasure-data/embulk-input-marketo/issues/34). Thanks!]
132
+
133
+ ## 0.2.3 - 2015-09-14
134
+
135
+ * [enhancement] Catch config error [#33](https://github.com/treasure-data/embulk-input-marketo/pull/33)
136
+ * [enhancement] Concurrent worker [#31](https://github.com/treasure-data/embulk-input-marketo/pull/31)
137
+
138
+ ## 0.2.2 - 2015-09-08
139
+
140
+ * [fixed] Fix handling for activity_date_time [#32](https://github.com/treasure-data/embulk-input-marketo/pull/32)
141
+
142
+ ## 0.2.1 - 2015-09-01
143
+
144
+ * [fixed] activity_log: Avoid to cast values unexpectedly [#29](https://github.com/treasure-data/embulk-input-marketo/pull/29)
145
+ * [maintenance] Add a minor comment [#28](https://github.com/treasure-data/embulk-input-marketo/pull/28) [Requested by @muga. Thanks!!]
146
+ * [maintenance] Fix minor issues [#27](https://github.com/treasure-data/embulk-input-marketo/pull/27) [[Reviewed by @muga.](https://github.com/treasure-data/embulk-input-marketo/pull/25#issuecomment-135570967) Thanks!!]
147
+
148
+ ## 0.2.0 - 2015-08-27
149
+
150
+ This version breaks backword compatibility of marketo/lead. Please check README.md to modify your config.
151
+
152
+ * [enhancement] Avoid timeout for marketo/lead input [#25](https://github.com/treasure-data/embulk-input-marketo/pull/25)
153
+ * [fixed] Fix timestamp column [#24](https://github.com/treasure-data/embulk-input-marketo/pull/24)
154
+ * [enhancement] Raise ConfigError if unretryable error occured [#23](https://github.com/treasure-data/embulk-input-marketo/pull/23)
155
+ * [fixed] Fix the bug that the default value for wsdl is broken [#22](https://github.com/treasure-data/embulk-input-marketo/pull/22)
156
+
157
+ ## 0.1.1 - 2015-08-19
158
+
159
+ * [enhanement] Support scheduled execution [#20](https://github.com/treasure-data/embulk-input-marketo/pull/20) [[Reported by @muga](https://github.com/treasure-data/embulk-input-marketo/issues/18). Thanks!]
160
+ * [maintenance] Use everyleaf-embulk_helper [#19](https://github.com/treasure-data/embulk-input-marketo/pull/19)
161
+
162
+ ## 0.1.0 - 2015-07-15
163
+
164
+ We implemented activity_log plugin for marketo, so config generated from 0.0.1 should be modified. Please check README.md to do it.
165
+
166
+ * [enhancement] Implement activity_log plugin [#13](https://github.com/treasure-data/embulk-input-marketo/pull/13) [#14](https://github.com/treasure-data/embulk-input-marketo/pull/14) [#15](https://github.com/treasure-data/embulk-input-marketo/pull/15)
167
+
168
+ ## 0.0.1 - 2015-07-06
169
+
170
+ The first release!!
data/LICENSE.txt ADDED
@@ -0,0 +1,21 @@
1
+
2
+ MIT License
3
+
4
+ Permission is hereby granted, free of charge, to any person obtaining
5
+ a copy of this software and associated documentation files (the
6
+ "Software"), to deal in the Software without restriction, including
7
+ without limitation the rights to use, copy, modify, merge, publish,
8
+ distribute, sublicense, and/or sell copies of the Software, and to
9
+ permit persons to whom the Software is furnished to do so, subject to
10
+ the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be
13
+ included in all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
18
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
19
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
20
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
21
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,213 @@
1
+ [![Build Status](https://travis-ci.org/treasure-data/embulk-input-marketo.svg?branch=master)](https://travis-ci.org/treasure-data/embulk-input-marketo)
2
+ [![Code Climate](https://codeclimate.com/github/treasure-data/embulk-input-marketo/badges/gpa.svg)](https://codeclimate.com/github/treasure-data/embulk-input-marketo)
3
+ [![Test Coverage](https://codeclimate.com/github/treasure-data/embulk-input-marketo/badges/coverage.svg)](https://codeclimate.com/github/treasure-data/embulk-input-marketo/coverage)
4
+ [![Gem Version](https://badge.fury.io/rb/embulk-input-marketo.svg)](http://badge.fury.io/rb/embulk-input-marketo)
5
+
6
+ # Marketo input plugin for Embulk
7
+
8
+ embulk-input-marketo is the gem preparing Embulk input plugins for [Marketo](http://www.marketo.com/).
9
+
10
+ - Lead(lead)
11
+ - Activity log(activity)
12
+ - Lead by list(all_lead_with_list_id)
13
+ - Lead by program(all_lead_with_program_id)
14
+ - Campaign(campaign)
15
+ - Assets Programs (program)
16
+
17
+ This plugin uses Marketo REST API.
18
+
19
+ ## Overview
20
+
21
+ Required Embulk version >= 0.8.33 (since 0.6.0).
22
+
23
+ * **Plugin type**: input
24
+ * **Resume supported**: no
25
+ * **Cleanup supported**: no
26
+ * **Guess supported**: no
27
+
28
+ ## Install
29
+
30
+ ```
31
+ $ embulk gem install embulk-input-marketo
32
+ ```
33
+
34
+ ## Configuration
35
+
36
+ ### API
37
+
38
+ Below parameters are shown in "Admin" > "Web Services" page in Marketo.
39
+
40
+ ### Base configuration parameter
41
+
42
+ All target have this configuration parameters
43
+
44
+ | name | required | default value | description |
45
+ |----------------------------------|----------|---------------|----------------------------------------------------------------------------------------------------------------------------------|
46
+ | **target** | true | | Marketo targets |
47
+ | **account_id** | true | | Marketo Muchkin id |
48
+ | **client_id** | true | | Marketo REST client id |
49
+ | **client_secret** | true | | Marketo REST client secret |
50
+ | **marketo_limit_interval_milis** | false | 20 | Marketo have limitation of 100 calls per 20 second. If REST API calls are failed they will wait this amount of time before retry |
51
+ | **batch_size** | false | 300 | Token paging batch size. Some REST API support batch |
52
+ | **max_return** | false | 200 | Max return for Endpoint that use offset paging |
53
+ | **partner_api_key** | false | | Set Marketo Partner API Key see: http://developers.marketo.com/support/Marketo_LaunchPoint_Technology_Partner_API_Key.pdf |
54
+
55
+ ### Bulk extract target configuration parameter (Lead and Activity)
56
+
57
+ All bulk extract target use this configuration parameter
58
+
59
+ | name | required | default value | description |
60
+ |-----------------------------|----------|---------------|-------------------------------------------------------------------------------------------------------------------------------|
61
+ | **from_date** | true | | Import data since this date. Example: 2017-10-11T06:43:24+00:00 |
62
+ | **fetch_days** | false | 1 | Amount of days to fetch since from_date |
63
+ | **polling_interval_second** | false | 60 | Amount of time to wait between pooling job status in second |
64
+ | **bulk_job_timeout_second** | false | 3600 | Amount of time to wait for bulk job to complete in second |
65
+ | **incremental** | false | true | If incremental is set to true, next run will have from_date set to the previous to_date(calculated by from_date + fetch_days) |
66
+ | **incremental_column** | false | createdAt | Column use to filter from_date and to_date |
67
+
68
+ ### Lead
69
+
70
+ Lead target extract all Marketo leads, it use Marketo bulk extract feature. Configuration include bulk extract configuration.
71
+
72
+ `target: lead`
73
+
74
+ Configuration:
75
+
76
+ | name | required | default value | description |
77
+ |---------------------|----------|---------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------|
78
+ | **use_updated_at** | false | false | Support filter with `updateAt` column, but not all Marketo Account have the feature to filter by updatedAt, updatedAt don't support incremental ingestion |
79
+ | **included_fields** | false | null | List of lead fields to included in export request sent to Marketo, can be used to reduce the size of BulkExtract file |
80
+
81
+ Schema type: Dynamic via describe lead endpoint.
82
+
83
+ Incremental support: yes
84
+
85
+ Range ingestion: yes
86
+
87
+ ### Activity
88
+
89
+ Activity target extract all Marketo activity log. Configuration include all bulk extract configuration
90
+
91
+ `target: activity`
92
+
93
+ Schema type: Static schema
94
+
95
+ Incremental support: yes
96
+
97
+ Range ingestion: yes
98
+
99
+ Filter by specific activity type ids: yes. See [#95](https://github.com/treasure-data/embulk-input-marketo/issues/95)
100
+
101
+ ### Campaign
102
+
103
+ Campaign extract all campaign data from Marketo
104
+
105
+ `target: campaign`
106
+
107
+ Schema type: Static schema
108
+
109
+ Incremental support: no
110
+
111
+ Range ingestion: no
112
+
113
+ ### Lead by list
114
+
115
+ Extract all Lead data including lead's list id
116
+
117
+ `target: all_lead_with_list_id`
118
+
119
+ Configuration:
120
+
121
+ | name | required | default value | description |
122
+ |---------------------|----------|---------------|-----------------------------------------------------------------------------------------------------------------|
123
+ | **included_fields** | false | null | List of lead fields to included in export request sent to Marketo, can be used to reduce request, response size |
124
+
125
+ Schema type: Dynamic via describe leads. Schema will have 1 addition column name listId that contain the id of the list the lead belong to
126
+
127
+ Incremental support: no
128
+
129
+ Range ingestion: no
130
+
131
+ ### Lead by program
132
+
133
+ Extract all Lead data including lead's program id
134
+
135
+ `target: all_lead_with_program_id`
136
+
137
+ Configuration:
138
+
139
+ | name | required | default value | description |
140
+ |---------------------|----------|---------------|-----------------------------------------------------------------------------------------------------------------|
141
+ | **included_fields** | false | null | List of lead fields to included in export request sent to Marketo, can be used to reduce request, response size |
142
+
143
+ Schema type: Dynamic via describe leads. Schema will have 1 addition column name listId that contain the id of the list the lead belong to
144
+
145
+ Incremental support: no
146
+
147
+ Range ingestion: no
148
+
149
+ ### Assets programs
150
+
151
+ Get Assets Programs by Query Tag type, Date range or all if no query by specified.
152
+
153
+ `target: program`
154
+
155
+ Configuration:
156
+
157
+ | name | required | default value | description |
158
+ |-----------------------------|----------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------|
159
+ | **query_by** | false | null | Get assets programs by query, supported values `date_range`, `tag_type` leave unset to fetch all programs |
160
+ | **earliest_updated_at** | false | null | Required if query by `date_range` is selected. Exclude programs prior to this date. Must be valid ISO-8601 string |
161
+ | **latest_updated_at** | false | null | Required if query by `date_range` is selected. Exclude programs after this date. Must be valid ISO-8601 string |
162
+ | **filter_type** | false | null | Optional value send with query by `date_range` is selected to filter out the result from Marketo. Supported values `id`, `programId`, `folderId`, `workspace`|
163
+ | **filter_values** | false | null | Set the values associated with `filter_type` |
164
+ | **tag_type** | false | null | Required if query by `tag_type` is selected. Type of program tag |
165
+ | **tag_value** | false | null | Required if query by `tag_type` is selected. Value of the tag |
166
+ | **report_duration** | false | null | Amount of milliseconds to fetch from `earliest_updated_at`. If `incremental = true` this value will automatically calculated for the first run by `latest_updated_at` - `earliest_updated_at` |
167
+ | **incremental** | false | true | If incremental is set to true, next run will have `earliest_updated_at` set to the previous `latest_updated_at` + `report_duration`. Incremental import only support by query `date_range` |
168
+
169
+ Schema type: Static schema
170
+
171
+ Incremental support: yes (Query by `date_range` only)
172
+
173
+ Range ingestion: yes
174
+
175
+ ## Example
176
+
177
+ For lead, you have `partial-config.yml` like below:
178
+
179
+ ```yaml
180
+ in:
181
+ type: marketo
182
+ target: lead
183
+ account_id: ACCOUNT_ID
184
+ client_id: CLIENT_ID
185
+ client_secret: CLIENT_SECRET
186
+ from_date: 2017-09-01
187
+ fetch_days: 1
188
+ out:
189
+ type: stdout
190
+ ```
191
+
192
+ You can run `embulk guess partial-config.yml -o lead-config.yml` and got `lead-config.yml`. `lead-config.yml` includes a schema for Lead.
193
+
194
+ Next, you can run `embulk preview lead-config.yml` for preview and `embulk run lead-config.yml` for run.
195
+
196
+ Example of Assets Programs config
197
+ ```yaml
198
+ in:
199
+ account_id: ACCOUNT_ID
200
+ client_id: CLIENT_ID
201
+ client_secret: CLIENT_SECRET
202
+ target: program
203
+ type: marketo
204
+ query_by: date_range
205
+ filter_type: folderId
206
+ filter_values:
207
+ - 2598
208
+ - 1001
209
+ earliest_updated_at: 2018-08-20T00:00:00.000Z
210
+ latest_updated_at: 2018-08-31T00:00:00.000Z
211
+ incremental: true
212
+ ```
213
+