etl-integrations 0.1.81 → 0.1.85
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +9 -9
- data/LICENSE.txt +1 -1
- data/README.md +13 -13
- data/etl-integrations.gemspec +65 -0
- data/lib/{multiwoven → etl}/integrations/config.rb +1 -1
- data/lib/{multiwoven → etl}/integrations/core/base_connector.rb +2 -2
- data/lib/{multiwoven → etl}/integrations/core/constants.rb +1 -1
- data/lib/{multiwoven → etl}/integrations/core/destination_connector.rb +1 -1
- data/lib/{multiwoven → etl}/integrations/core/fullrefresher.rb +2 -2
- data/lib/{multiwoven → etl}/integrations/core/http_client.rb +1 -1
- data/lib/{multiwoven → etl}/integrations/core/query_builder.rb +1 -1
- data/lib/{multiwoven → etl}/integrations/core/rate_limiter.rb +1 -1
- data/lib/{multiwoven → etl}/integrations/core/source_connector.rb +1 -1
- data/lib/{multiwoven → etl}/integrations/core/utils.rb +3 -3
- data/lib/{multiwoven → etl}/integrations/destination/airtable/client.rb +8 -8
- data/lib/{multiwoven → etl}/integrations/destination/airtable/schema_helper.rb +1 -1
- data/lib/{multiwoven → etl}/integrations/destination/facebook_custom_audience/client.rb +6 -6
- data/lib/{multiwoven → etl}/integrations/destination/google_sheets/client.rb +5 -5
- data/lib/{multiwoven → etl}/integrations/destination/hubspot/client.rb +5 -5
- data/lib/{multiwoven → etl}/integrations/destination/klaviyo/client.rb +6 -6
- data/lib/{multiwoven → etl}/integrations/destination/postgresql/client.rb +5 -5
- data/lib/{multiwoven → etl}/integrations/destination/salesforce_consumer_goods_cloud/client.rb +5 -5
- data/lib/{multiwoven → etl}/integrations/destination/salesforce_consumer_goods_cloud/schema_helper.rb +1 -1
- data/lib/{multiwoven → etl}/integrations/destination/salesforce_crm/client.rb +5 -5
- data/lib/{multiwoven → etl}/integrations/destination/sftp/client.rb +5 -5
- data/lib/{multiwoven → etl}/integrations/destination/slack/client.rb +4 -4
- data/lib/{multiwoven → etl}/integrations/destination/stripe/client.rb +5 -5
- data/lib/{multiwoven → etl}/integrations/destination/tally/client.rb +5 -5
- data/lib/{multiwoven → etl}/integrations/protocol/protocol.json +2 -2
- data/lib/{multiwoven → etl}/integrations/protocol/protocol.rb +2 -2
- data/lib/{multiwoven → etl}/integrations/rollout.rb +2 -2
- data/lib/{multiwoven → etl}/integrations/service.rb +3 -3
- data/lib/{multiwoven → etl}/integrations/source/bigquery/client.rb +3 -3
- data/lib/{multiwoven → etl}/integrations/source/databricks/client.rb +3 -3
- data/lib/{multiwoven → etl}/integrations/source/postgresql/client.rb +3 -3
- data/lib/{multiwoven → etl}/integrations/source/redshift/client.rb +3 -3
- data/lib/{multiwoven → etl}/integrations/source/salesforce_consumer_goods_cloud/client.rb +5 -5
- data/lib/{multiwoven → etl}/integrations/source/salesforce_consumer_goods_cloud/schema_helper.rb +1 -1
- data/lib/{multiwoven → etl}/integrations/source/snowflake/client.rb +3 -3
- data/lib/{multiwoven → etl}/integrations/source/zoho_books/client.rb +5 -5
- data/lib/{multiwoven → etl}/integrations.rb +1 -1
- data/sig/{multiwoven → etl}/integrations.rbs +1 -1
- metadata +119 -118
- /data/{multiwoven-integrations-0.1.68.gem → etl-integrations-0.1.68.gem} +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/airtable/config/catalog.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/airtable/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/airtable/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/airtable/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/facebook_custom_audience/config/catalog.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/facebook_custom_audience/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/facebook_custom_audience/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/facebook_custom_audience/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/google_sheets/config/catalog.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/google_sheets/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/google_sheets/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/google_sheets/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/hubspot/config/catalog.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/hubspot/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/hubspot/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/hubspot/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/klaviyo/config/catalog.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/klaviyo/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/klaviyo/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/klaviyo/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/postgresql/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/postgresql/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/postgresql/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/salesforce_consumer_goods_cloud/config/catalog.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/salesforce_consumer_goods_cloud/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/salesforce_consumer_goods_cloud/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/salesforce_consumer_goods_cloud/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/salesforce_crm/config/catalog.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/salesforce_crm/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/salesforce_crm/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/salesforce_crm/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/sftp/config/catalog.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/sftp/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/sftp/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/sftp/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/slack/config/catalog.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/slack/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/slack/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/slack/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/stripe/config/catalog.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/stripe/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/stripe/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/stripe/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/tally/config/catalog.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/tally/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/tally/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/tally/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/bigquery/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/bigquery/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/bigquery/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/databricks/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/databricks/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/databricks/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/postgresql/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/postgresql/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/postgresql/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/redshift/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/redshift/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/redshift/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/salesforce_consumer_goods_cloud/config/catalog.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/salesforce_consumer_goods_cloud/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/salesforce_consumer_goods_cloud/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/salesforce_consumer_goods_cloud/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/snowflake/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/snowflake/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/snowflake/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/zoho_books/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/zoho_books/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/zoho_books/icon.svg +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a1b9a7a2e4780b4830cded5259e52ea8e630ac7cd27347df88c1a4fc06285daf
|
4
|
+
data.tar.gz: '088f80483ec5f386aa11ae8bc735e4a9f518af56494b51242b92860216d4d339'
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: cb77f7c5f9e29d772fcafa6f944f9ded6204a21f8bcbe42486546b8c713a1453a0df82716b70a424637e16626874753c77de1944877f1b622a4622e72187ef76
|
7
|
+
data.tar.gz: 7b3ce9767ff9dfefefdee24ff117f528c8ee5d7458da2e1e8700a677cec87235ec3281764d9599c3aad6f51181413464460cc3c5c76291997b158e8834baa071
|
data/CHANGELOG.md
CHANGED
@@ -1,37 +1,37 @@
|
|
1
1
|
## [Unreleased]
|
2
2
|
|
3
|
-
### [0.4.1](https://www.github.com/
|
3
|
+
### [0.4.1](https://www.github.com/Etl/multiwoven-integrations/compare/v0.4.0...v0.4.1) (2024-02-15)
|
4
4
|
|
5
5
|
|
6
6
|
### Bug Fixes
|
7
7
|
|
8
|
-
* Add batch params to stream ([#68](https://www.github.com/
|
8
|
+
* Add batch params to stream ([#68](https://www.github.com/Etl/multiwoven-integrations/issues/68)) ([080530f](https://www.github.com/Etl/multiwoven-integrations/commit/080530f8280b467c8e40a3b6429f178ad18fa05e))
|
9
9
|
|
10
|
-
## [0.4.0](https://www.github.com/
|
10
|
+
## [0.4.0](https://www.github.com/Etl/multiwoven-integrations/compare/v0.3.0...v0.4.0) (2024-02-14)
|
11
11
|
|
12
12
|
|
13
13
|
### Features
|
14
14
|
|
15
|
-
* Bump version to 1.21 ([#62](https://www.github.com/
|
15
|
+
* Bump version to 1.21 ([#62](https://www.github.com/Etl/multiwoven-integrations/issues/62)) ([7b1476d](https://www.github.com/Etl/multiwoven-integrations/commit/7b1476d4b04a073ab060971019095d0a13ebd153))
|
16
16
|
|
17
|
-
## [0.3.0](https://www.github.com/
|
17
|
+
## [0.3.0](https://www.github.com/Etl/multiwoven-integrations/compare/v0.2.0...v0.3.0) (2024-02-14)
|
18
18
|
|
19
19
|
|
20
20
|
### Features
|
21
21
|
|
22
|
-
* Publish gem to rubygems ([#60](https://www.github.com/
|
22
|
+
* Publish gem to rubygems ([#60](https://www.github.com/Etl/multiwoven-integrations/issues/60)) ([04cfbb6](https://www.github.com/Etl/multiwoven-integrations/commit/04cfbb60c78bc90d886ce691d5094fed9c6f078f))
|
23
23
|
|
24
|
-
## [0.2.0](https://www.github.com/
|
24
|
+
## [0.2.0](https://www.github.com/Etl/multiwoven-integrations/compare/v0.1.0...v0.2.0) (2024-02-14)
|
25
25
|
|
26
26
|
|
27
27
|
### Features
|
28
28
|
|
29
|
-
* Automate gem release ([#55](https://www.github.com/
|
29
|
+
* Automate gem release ([#55](https://www.github.com/Etl/multiwoven-integrations/issues/55)) ([1c3373b](https://www.github.com/Etl/multiwoven-integrations/commit/1c3373bcedae60c6fb8171c54ce40de94f1537ce))
|
30
30
|
|
31
31
|
|
32
32
|
### Bug Fixes
|
33
33
|
|
34
|
-
* Klaviyo discover ([#23](https://www.github.com/
|
34
|
+
* Klaviyo discover ([#23](https://www.github.com/Etl/multiwoven-integrations/issues/23)) ([7c43db6](https://www.github.com/Etl/multiwoven-integrations/commit/7c43db6a369403748ed29085bb5254405ad1c95e))
|
35
35
|
|
36
36
|
## [0.1.0] - 2023-12-01
|
37
37
|
|
data/LICENSE.txt
CHANGED
data/README.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
<div align="center">
|
2
2
|
<a href="https://multiwoven.com?utm_source=github" target="_blank">
|
3
|
-
<img src="https://res.cloudinary.com/dspflukeu/image/upload/v1706696350/
|
3
|
+
<img src="https://res.cloudinary.com/dspflukeu/image/upload/v1706696350/Etl/Logo_hrkaxj.png" alt="Etl" width="240" />
|
4
4
|
</a>
|
5
5
|
</div>
|
6
6
|
<br />
|
@@ -8,8 +8,8 @@
|
|
8
8
|
<a href="http://badge.fury.io/rb/multiwoven-integrations">
|
9
9
|
<img src="https://badge.fury.io/rb/multiwoven-integrations.svg" alt="Gem Version">
|
10
10
|
</a>
|
11
|
-
<a href="https://github.com/
|
12
|
-
<img src="https://github.com/
|
11
|
+
<a href="https://github.com/Etl/multiwoven-integrations/actions/workflows/ci.yml">
|
12
|
+
<img src="https://github.com/Etl/multiwoven-integrations/actions/workflows/ci.yml/badge.svg" alt="CI">
|
13
13
|
</a>
|
14
14
|
<a href="https://codeclimate.com/repos/657d0a2a60265a2f2155ffca/maintainability">
|
15
15
|
<img src="https://api.codeclimate.com/v1/badges/d841270f1f7a966043c1/maintainability" alt="Maintainability">
|
@@ -31,16 +31,16 @@
|
|
31
31
|
·
|
32
32
|
<a href="https://multiwoven.com">Website</a>
|
33
33
|
·
|
34
|
-
<a href="https://github.com/
|
34
|
+
<a href="https://github.com/Etl/multiwoven-integrations/issues">Report Issues</a>
|
35
35
|
·
|
36
|
-
<a href="https://github.com/orgs/
|
36
|
+
<a href="https://github.com/orgs/Etl/projects/4">Roadmap</a>
|
37
37
|
</p>
|
38
38
|
|
39
|
-
##
|
39
|
+
## Etl Integrations
|
40
40
|
|
41
|
-
|
41
|
+
Etl integrations is the collection of connectors built on top of [Etl protocol](https://docs.multiwoven.com/guides/architecture/multiwoven-protocol).
|
42
42
|
|
43
|
-
|
43
|
+
Etl protocol is an open source standard for moving data between data sources to any third-part destinations.
|
44
44
|
Anyone can build a connetor with basic ruby knowledge using the protocol.
|
45
45
|
|
46
46
|
### Installation
|
@@ -54,18 +54,18 @@ Install the gem and add to the application's Gemfile by executing:
|
|
54
54
|
|
55
55
|
#### Source
|
56
56
|
```
|
57
|
-
source =
|
57
|
+
source = Etl::Integrations::Source::[CONNECTOR_NAME]::Client.new
|
58
58
|
source.read(sync_config)
|
59
59
|
```
|
60
60
|
#### Destination
|
61
61
|
|
62
62
|
```
|
63
|
-
destination =
|
63
|
+
destination = Etl::Integrations::Destination::[CONNECTOR_NAME]::Client.new
|
64
64
|
destination.write(sync_config, records)
|
65
65
|
```
|
66
66
|
|
67
67
|
#### Supported methods
|
68
|
-
Please refer [
|
68
|
+
Please refer [Etl Protocol](https://docs.multiwoven.com/guides/architecture/multiwoven-protocol) to understand more about the supported methods on source and destination
|
69
69
|
|
70
70
|
## Development
|
71
71
|
|
@@ -94,11 +94,11 @@ Please refer [Multiwoven Protocol](https://docs.multiwoven.com/guides/architectu
|
|
94
94
|
|
95
95
|
## Contributing
|
96
96
|
|
97
|
-
Bug reports and pull requests are welcome on [GitHub](https://github.com/
|
97
|
+
Bug reports and pull requests are welcome on [GitHub](https://github.com/Etl/multiwoven-integrations). This project aims to be a safe, welcoming space for collaboration.
|
98
98
|
|
99
99
|
## Code of Conduct
|
100
100
|
|
101
|
-
Contributors are expected to adhere to the project's [code of conduct](https://github.com/
|
101
|
+
Contributors are expected to adhere to the project's [code of conduct](https://github.com/Etl/multiwoven-integrations/blob/main/CODE_OF_CONDUCT.md)
|
102
102
|
|
103
103
|
## License
|
104
104
|
|
@@ -0,0 +1,65 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require_relative "lib/etl/integrations/rollout"
|
4
|
+
|
5
|
+
Gem::Specification.new do |spec|
|
6
|
+
spec.name = "etl-integrations"
|
7
|
+
spec.version = Etl::Integrations::VERSION
|
8
|
+
spec.authors = ["Anas Ahmed"]
|
9
|
+
spec.email = ["anas31197@gmail.com"]
|
10
|
+
|
11
|
+
spec.summary = "Integration suite for open source reverse ETL platform"
|
12
|
+
spec.description = "Etl Integrations is a comprehensive Ruby gem designed to facilitate seamless connectivity between various data sources and SaaS platforms."
|
13
|
+
|
14
|
+
spec.homepage = "https://github.com/anasValign/etl-tool"
|
15
|
+
spec.license = "MIT"
|
16
|
+
spec.required_ruby_version = ">= 2.6.0"
|
17
|
+
|
18
|
+
# spec.metadata["allowed_push_host"] = nil
|
19
|
+
spec.metadata["github_repo"] = "https://github.com/anasValign/etl-tool"
|
20
|
+
spec.metadata["homepage_uri"] = spec.homepage
|
21
|
+
spec.metadata["source_code_uri"] = "https://github.com/Etl/multiwoven/tree/main/integrations"
|
22
|
+
spec.metadata["changelog_uri"] = "https://github.com/Etl/multiwoven/blob/main/integrations/CHANGELOG.md"
|
23
|
+
|
24
|
+
# Specify which files should be added to the gem when it is released.
|
25
|
+
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
26
|
+
spec.files = Dir.chdir(__dir__) do
|
27
|
+
`git ls-files -z`.split("\x0").reject do |f|
|
28
|
+
(File.expand_path(f) == __FILE__) ||
|
29
|
+
f.start_with?(*%w[bin/ test/ spec/ features/ .git .circleci appveyor Gemfile])
|
30
|
+
end
|
31
|
+
end
|
32
|
+
spec.bindir = "exe"
|
33
|
+
spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
|
34
|
+
spec.require_paths = ["lib"]
|
35
|
+
|
36
|
+
spec.add_runtime_dependency "activesupport"
|
37
|
+
spec.add_runtime_dependency "async-websocket"
|
38
|
+
spec.add_runtime_dependency "csv"
|
39
|
+
spec.add_runtime_dependency "dry-schema"
|
40
|
+
spec.add_runtime_dependency "dry-struct"
|
41
|
+
spec.add_runtime_dependency "dry-types"
|
42
|
+
spec.add_runtime_dependency "git"
|
43
|
+
spec.add_runtime_dependency "google-apis-sheets_v4"
|
44
|
+
spec.add_runtime_dependency "google-cloud-bigquery"
|
45
|
+
spec.add_runtime_dependency "hubspot-api-client"
|
46
|
+
spec.add_runtime_dependency "net-sftp"
|
47
|
+
spec.add_runtime_dependency "pg"
|
48
|
+
spec.add_runtime_dependency "rake"
|
49
|
+
spec.add_runtime_dependency "restforce"
|
50
|
+
spec.add_runtime_dependency "ruby-limiter"
|
51
|
+
spec.add_runtime_dependency "ruby-odbc"
|
52
|
+
spec.add_runtime_dependency "sequel"
|
53
|
+
spec.add_runtime_dependency "slack-ruby-client"
|
54
|
+
spec.add_runtime_dependency "stripe"
|
55
|
+
|
56
|
+
spec.add_development_dependency "byebug"
|
57
|
+
spec.add_development_dependency "rspec"
|
58
|
+
spec.add_development_dependency "rubocop"
|
59
|
+
spec.add_development_dependency "simplecov"
|
60
|
+
spec.add_development_dependency "simplecov_json_formatter"
|
61
|
+
spec.add_development_dependency "webmock"
|
62
|
+
|
63
|
+
# For more information and examples about making a new gem, check out our
|
64
|
+
# guide at: https://bundler.io/guides/creating_gem.html
|
65
|
+
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
module
|
3
|
+
module Etl
|
4
4
|
module Integrations::Core
|
5
5
|
class BaseConnector
|
6
6
|
include Integrations::Protocol
|
@@ -18,7 +18,7 @@ module Multiwoven
|
|
18
18
|
def meta_data
|
19
19
|
client_meta_data = read_json(META_DATA_PATH).deep_symbolize_keys
|
20
20
|
icon_name = client_meta_data[:data][:icon]
|
21
|
-
icon_url = "https://raw.githubusercontent.com/
|
21
|
+
icon_url = "https://raw.githubusercontent.com/Etl/multiwoven/main/integrations#{relative_path}/#{icon_name}"
|
22
22
|
client_meta_data[:data][:icon] = icon_url
|
23
23
|
# returns hash
|
24
24
|
@meta_data ||= client_meta_data
|
@@ -1,13 +1,13 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
module
|
3
|
+
module Etl
|
4
4
|
module Integrations::Core
|
5
5
|
module Fullrefresher
|
6
6
|
def write(sync_config, records, action = "insert")
|
7
7
|
if sync_config && sync_config.sync_mode == "full_refresh" && !@full_refreshed
|
8
8
|
response = clear_all_records(sync_config)
|
9
9
|
return response unless response &&
|
10
|
-
response.control.status ==
|
10
|
+
response.control.status == Etl::Integrations::Protocol::ConnectionStatusType["succeeded"]
|
11
11
|
|
12
12
|
@full_refreshed = true
|
13
13
|
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
module
|
3
|
+
module Etl
|
4
4
|
module Integrations::Core
|
5
5
|
module Utils
|
6
6
|
def keys_to_symbols(hash)
|
@@ -75,7 +75,7 @@ module Multiwoven
|
|
75
75
|
|
76
76
|
def build_catalog(catalog_json)
|
77
77
|
streams = catalog_json["streams"].map { |stream_json| build_stream(stream_json) }
|
78
|
-
|
78
|
+
Etl::Integrations::Protocol::Catalog.new(
|
79
79
|
streams: streams,
|
80
80
|
request_rate_limit: catalog_json["request_rate_limit"] || 60,
|
81
81
|
request_rate_limit_unit: catalog_json["request_rate_limit_unit"] || "minute",
|
@@ -85,7 +85,7 @@ module Multiwoven
|
|
85
85
|
end
|
86
86
|
|
87
87
|
def build_stream(stream_json)
|
88
|
-
|
88
|
+
Etl::Integrations::Protocol::Stream.new(
|
89
89
|
name: stream_json["name"],
|
90
90
|
url: stream_json["url"],
|
91
91
|
action: stream_json["action"],
|
@@ -1,17 +1,17 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require_relative "schema_helper"
|
4
|
-
module
|
4
|
+
module Etl
|
5
5
|
module Integrations
|
6
6
|
module Destination
|
7
7
|
module Airtable
|
8
|
-
include
|
8
|
+
include Etl::Integrations::Core
|
9
9
|
class Client < DestinationConnector # rubocop:disable Metrics/ClassLength
|
10
|
-
prepend
|
10
|
+
prepend Etl::Integrations::Core::RateLimiter
|
11
11
|
MAX_CHUNK_SIZE = 10
|
12
12
|
def check_connection(connection_config)
|
13
13
|
connection_config = connection_config.with_indifferent_access
|
14
|
-
bases =
|
14
|
+
bases = Etl::Integrations::Core::HttpClient.request(
|
15
15
|
AIRTABLE_BASES_ENDPOINT,
|
16
16
|
HTTP_GET,
|
17
17
|
headers: auth_headers(connection_config[:api_key])
|
@@ -31,7 +31,7 @@ module Multiwoven
|
|
31
31
|
base_id = connection_config[:base_id]
|
32
32
|
api_key = connection_config[:api_key]
|
33
33
|
|
34
|
-
bases =
|
34
|
+
bases = Etl::Integrations::Core::HttpClient.request(
|
35
35
|
AIRTABLE_BASES_ENDPOINT,
|
36
36
|
HTTP_GET,
|
37
37
|
headers: auth_headers(api_key)
|
@@ -40,7 +40,7 @@ module Multiwoven
|
|
40
40
|
base = extract_bases(bases).find { |b| b["id"] == base_id }
|
41
41
|
base_name = base["name"]
|
42
42
|
|
43
|
-
schema =
|
43
|
+
schema = Etl::Integrations::Core::HttpClient.request(
|
44
44
|
AIRTABLE_GET_BASE_SCHEMA_ENDPOINT.gsub("{baseId}", base_id),
|
45
45
|
HTTP_GET,
|
46
46
|
headers: auth_headers(api_key)
|
@@ -60,7 +60,7 @@ module Multiwoven
|
|
60
60
|
write_failure = 0
|
61
61
|
records.each_slice(MAX_CHUNK_SIZE) do |chunk|
|
62
62
|
payload = create_payload(chunk)
|
63
|
-
response =
|
63
|
+
response = Etl::Integrations::Core::HttpClient.request(
|
64
64
|
url,
|
65
65
|
sync_config.stream.request_method,
|
66
66
|
payload: payload,
|
@@ -76,7 +76,7 @@ module Multiwoven
|
|
76
76
|
write_failure += chunk.size
|
77
77
|
end
|
78
78
|
|
79
|
-
tracker =
|
79
|
+
tracker = Etl::Integrations::Protocol::TrackingMessage.new(
|
80
80
|
success: write_success,
|
81
81
|
failed: write_failure
|
82
82
|
)
|
@@ -1,15 +1,15 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
module
|
3
|
+
module Etl::Integrations::Destination
|
4
4
|
module FacebookCustomAudience
|
5
|
-
include
|
5
|
+
include Etl::Integrations::Core
|
6
6
|
class Client < DestinationConnector # rubocop:disable Metrics/ClassLength
|
7
|
-
prepend
|
7
|
+
prepend Etl::Integrations::Core::RateLimiter
|
8
8
|
MAX_CHUNK_SIZE = 10_000
|
9
9
|
def check_connection(connection_config)
|
10
10
|
connection_config = connection_config.with_indifferent_access
|
11
11
|
access_token = connection_config[:access_token]
|
12
|
-
response =
|
12
|
+
response = Etl::Integrations::Core::HttpClient.request(
|
13
13
|
FACEBOOK_AUDIENCE_GET_ALL_ACCOUNTS,
|
14
14
|
HTTP_GET,
|
15
15
|
headers: auth_headers(access_token)
|
@@ -44,7 +44,7 @@ module Multiwoven::Integrations::Destination
|
|
44
44
|
write_failure = 0
|
45
45
|
records.each_slice(MAX_CHUNK_SIZE) do |chunk|
|
46
46
|
payload = create_payload(chunk, sync_config.stream.json_schema.with_indifferent_access)
|
47
|
-
response =
|
47
|
+
response = Etl::Integrations::Core::HttpClient.request(
|
48
48
|
url,
|
49
49
|
sync_config.stream.request_method,
|
50
50
|
payload: payload,
|
@@ -60,7 +60,7 @@ module Multiwoven::Integrations::Destination
|
|
60
60
|
write_failure += chunk.size
|
61
61
|
end
|
62
62
|
|
63
|
-
tracker =
|
63
|
+
tracker = Etl::Integrations::Protocol::TrackingMessage.new(
|
64
64
|
success: write_success,
|
65
65
|
failed: write_failure
|
66
66
|
)
|
@@ -1,14 +1,14 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
module
|
3
|
+
module Etl
|
4
4
|
module Integrations
|
5
5
|
module Destination
|
6
6
|
module GoogleSheets
|
7
|
-
include
|
7
|
+
include Etl::Integrations::Core
|
8
8
|
|
9
9
|
class Client < DestinationConnector # rubocop:disable Metrics/ClassLength
|
10
|
-
prepend
|
11
|
-
prepend
|
10
|
+
prepend Etl::Integrations::Core::Fullrefresher
|
11
|
+
prepend Etl::Integrations::Core::RateLimiter
|
12
12
|
MAX_CHUNK_SIZE = 10_000
|
13
13
|
|
14
14
|
def check_connection(connection_config)
|
@@ -195,7 +195,7 @@ module Multiwoven
|
|
195
195
|
end
|
196
196
|
|
197
197
|
def tracking_message(success, failure)
|
198
|
-
|
198
|
+
Etl::Integrations::Protocol::TrackingMessage.new(
|
199
199
|
success: success, failed: failure
|
200
200
|
).to_multiwoven_message
|
201
201
|
end
|
@@ -2,14 +2,14 @@
|
|
2
2
|
|
3
3
|
require "stringio"
|
4
4
|
|
5
|
-
module
|
5
|
+
module Etl
|
6
6
|
module Integrations
|
7
7
|
module Destination
|
8
8
|
module Hubspot
|
9
|
-
include
|
9
|
+
include Etl::Integrations::Core
|
10
10
|
|
11
11
|
class Client < DestinationConnector
|
12
|
-
prepend
|
12
|
+
prepend Etl::Integrations::Core::RateLimiter
|
13
13
|
def check_connection(connection_config)
|
14
14
|
connection_config = connection_config.with_indifferent_access
|
15
15
|
initialize_client(connection_config)
|
@@ -92,13 +92,13 @@ module Multiwoven
|
|
92
92
|
end
|
93
93
|
|
94
94
|
def tracking_message(success, failure)
|
95
|
-
|
95
|
+
Etl::Integrations::Protocol::TrackingMessage.new(
|
96
96
|
success: success, failed: failure
|
97
97
|
).to_multiwoven_message
|
98
98
|
end
|
99
99
|
|
100
100
|
def log_debug(message)
|
101
|
-
|
101
|
+
Etl::Integrations::Service.logger.debug(message)
|
102
102
|
end
|
103
103
|
end
|
104
104
|
end
|
@@ -1,15 +1,15 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
module
|
3
|
+
module Etl::Integrations::Destination
|
4
4
|
module Klaviyo
|
5
|
-
include
|
5
|
+
include Etl::Integrations::Core
|
6
6
|
class Client < DestinationConnector
|
7
|
-
prepend
|
7
|
+
prepend Etl::Integrations::Core::RateLimiter
|
8
8
|
def check_connection(connection_config)
|
9
9
|
connection_config = connection_config.with_indifferent_access
|
10
10
|
api_key = connection_config[:private_api_key]
|
11
11
|
|
12
|
-
response =
|
12
|
+
response = Etl::Integrations::Core::HttpClient.request(
|
13
13
|
KLAVIYO_AUTH_ENDPOINT,
|
14
14
|
HTTP_POST,
|
15
15
|
payload: KLAVIYO_AUTH_PAYLOAD,
|
@@ -47,7 +47,7 @@ module Multiwoven::Integrations::Destination
|
|
47
47
|
record["data"] ||= {}
|
48
48
|
record["data"]["type"] = sync_config.stream.name
|
49
49
|
|
50
|
-
response =
|
50
|
+
response = Etl::Integrations::Core::HttpClient.request(
|
51
51
|
url,
|
52
52
|
request_method,
|
53
53
|
payload: record,
|
@@ -64,7 +64,7 @@ module Multiwoven::Integrations::Destination
|
|
64
64
|
)
|
65
65
|
write_failure += 1
|
66
66
|
end
|
67
|
-
tracker =
|
67
|
+
tracker = Etl::Integrations::Protocol::TrackingMessage.new(
|
68
68
|
success: write_success,
|
69
69
|
failed: write_failure
|
70
70
|
)
|
@@ -2,9 +2,9 @@
|
|
2
2
|
|
3
3
|
require "pg"
|
4
4
|
|
5
|
-
module
|
5
|
+
module Etl::Integrations::Destination
|
6
6
|
module Postgresql
|
7
|
-
include
|
7
|
+
include Etl::Integrations::Core
|
8
8
|
class Client < DestinationConnector
|
9
9
|
def check_connection(connection_config)
|
10
10
|
connection_config = connection_config.with_indifferent_access
|
@@ -52,7 +52,7 @@ module Multiwoven::Integrations::Destination
|
|
52
52
|
write_failure = 0
|
53
53
|
|
54
54
|
records.each do |record|
|
55
|
-
query =
|
55
|
+
query = Etl::Integrations::Core::QueryBuilder.perform(action, table_name, record)
|
56
56
|
begin
|
57
57
|
db.exec(query)
|
58
58
|
write_success += 1
|
@@ -94,7 +94,7 @@ module Multiwoven::Integrations::Destination
|
|
94
94
|
|
95
95
|
def create_streams(records)
|
96
96
|
group_by_table(records).map do |r|
|
97
|
-
|
97
|
+
Etl::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
|
98
98
|
end
|
99
99
|
end
|
100
100
|
|
@@ -114,7 +114,7 @@ module Multiwoven::Integrations::Destination
|
|
114
114
|
end
|
115
115
|
|
116
116
|
def tracking_message(success, failure)
|
117
|
-
|
117
|
+
Etl::Integrations::Protocol::TrackingMessage.new(
|
118
118
|
success: success, failed: failure
|
119
119
|
).to_multiwoven_message
|
120
120
|
end
|
data/lib/{multiwoven → etl}/integrations/destination/salesforce_consumer_goods_cloud/client.rb
RENAMED
@@ -3,17 +3,17 @@
|
|
3
3
|
require "stringio"
|
4
4
|
require_relative "schema_helper"
|
5
5
|
|
6
|
-
module
|
6
|
+
module Etl
|
7
7
|
module Integrations
|
8
8
|
module Destination
|
9
9
|
module SalesforceConsumerGoodsCloud
|
10
|
-
include
|
10
|
+
include Etl::Integrations::Core
|
11
11
|
|
12
12
|
API_VERSION = "59.0"
|
13
13
|
SALESFORCE_OBJECTS = %w[Account User Visit RetailStore RecordType].freeze
|
14
14
|
|
15
15
|
class Client < DestinationConnector
|
16
|
-
prepend
|
16
|
+
prepend Etl::Integrations::Core::RateLimiter
|
17
17
|
def check_connection(connection_config)
|
18
18
|
connection_config = connection_config.with_indifferent_access
|
19
19
|
initialize_client(connection_config)
|
@@ -99,13 +99,13 @@ module Multiwoven
|
|
99
99
|
end
|
100
100
|
|
101
101
|
def tracking_message(success, failure)
|
102
|
-
|
102
|
+
Etl::Integrations::Protocol::TrackingMessage.new(
|
103
103
|
success: success, failed: failure
|
104
104
|
).to_multiwoven_message
|
105
105
|
end
|
106
106
|
|
107
107
|
def log_debug(message)
|
108
|
-
|
108
|
+
Etl::Integrations::Service.logger.debug(message)
|
109
109
|
end
|
110
110
|
end
|
111
111
|
end
|