etl-integrations 0.1.82 → 0.1.85
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +9 -9
- data/LICENSE.txt +1 -1
- data/README.md +13 -13
- data/{multiwoven-integrations.gemspec → etl-integrations.gemspec} +5 -5
- data/lib/{multiwoven → etl}/integrations/config.rb +1 -1
- data/lib/{multiwoven → etl}/integrations/core/base_connector.rb +2 -2
- data/lib/{multiwoven → etl}/integrations/core/constants.rb +1 -1
- data/lib/{multiwoven → etl}/integrations/core/destination_connector.rb +1 -1
- data/lib/{multiwoven → etl}/integrations/core/fullrefresher.rb +2 -2
- data/lib/{multiwoven → etl}/integrations/core/http_client.rb +1 -1
- data/lib/{multiwoven → etl}/integrations/core/query_builder.rb +1 -1
- data/lib/{multiwoven → etl}/integrations/core/rate_limiter.rb +1 -1
- data/lib/{multiwoven → etl}/integrations/core/source_connector.rb +1 -1
- data/lib/{multiwoven → etl}/integrations/core/utils.rb +3 -3
- data/lib/{multiwoven → etl}/integrations/destination/airtable/client.rb +8 -8
- data/lib/{multiwoven → etl}/integrations/destination/airtable/schema_helper.rb +1 -1
- data/lib/{multiwoven → etl}/integrations/destination/facebook_custom_audience/client.rb +6 -6
- data/lib/{multiwoven → etl}/integrations/destination/google_sheets/client.rb +5 -5
- data/lib/{multiwoven → etl}/integrations/destination/hubspot/client.rb +5 -5
- data/lib/{multiwoven → etl}/integrations/destination/klaviyo/client.rb +6 -6
- data/lib/{multiwoven → etl}/integrations/destination/postgresql/client.rb +5 -5
- data/lib/{multiwoven → etl}/integrations/destination/salesforce_consumer_goods_cloud/client.rb +5 -5
- data/lib/{multiwoven → etl}/integrations/destination/salesforce_consumer_goods_cloud/schema_helper.rb +1 -1
- data/lib/{multiwoven → etl}/integrations/destination/salesforce_crm/client.rb +5 -5
- data/lib/{multiwoven → etl}/integrations/destination/sftp/client.rb +5 -5
- data/lib/{multiwoven → etl}/integrations/destination/slack/client.rb +4 -4
- data/lib/{multiwoven → etl}/integrations/destination/stripe/client.rb +5 -5
- data/lib/{multiwoven → etl}/integrations/destination/tally/client.rb +5 -5
- data/lib/{multiwoven → etl}/integrations/protocol/protocol.json +2 -2
- data/lib/{multiwoven → etl}/integrations/protocol/protocol.rb +2 -2
- data/lib/{multiwoven → etl}/integrations/rollout.rb +2 -2
- data/lib/{multiwoven → etl}/integrations/service.rb +3 -3
- data/lib/{multiwoven → etl}/integrations/source/bigquery/client.rb +3 -3
- data/lib/{multiwoven → etl}/integrations/source/databricks/client.rb +3 -3
- data/lib/{multiwoven → etl}/integrations/source/postgresql/client.rb +3 -3
- data/lib/{multiwoven → etl}/integrations/source/redshift/client.rb +3 -3
- data/lib/{multiwoven → etl}/integrations/source/salesforce_consumer_goods_cloud/client.rb +5 -5
- data/lib/{multiwoven → etl}/integrations/source/salesforce_consumer_goods_cloud/schema_helper.rb +1 -1
- data/lib/{multiwoven → etl}/integrations/source/snowflake/client.rb +3 -3
- data/lib/{multiwoven → etl}/integrations/source/zoho_books/client.rb +5 -5
- data/lib/{multiwoven → etl}/integrations.rb +1 -1
- data/sig/{multiwoven → etl}/integrations.rbs +1 -1
- metadata +115 -115
- /data/{multiwoven-integrations-0.1.68.gem → etl-integrations-0.1.68.gem} +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/airtable/config/catalog.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/airtable/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/airtable/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/airtable/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/facebook_custom_audience/config/catalog.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/facebook_custom_audience/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/facebook_custom_audience/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/facebook_custom_audience/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/google_sheets/config/catalog.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/google_sheets/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/google_sheets/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/google_sheets/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/hubspot/config/catalog.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/hubspot/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/hubspot/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/hubspot/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/klaviyo/config/catalog.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/klaviyo/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/klaviyo/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/klaviyo/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/postgresql/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/postgresql/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/postgresql/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/salesforce_consumer_goods_cloud/config/catalog.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/salesforce_consumer_goods_cloud/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/salesforce_consumer_goods_cloud/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/salesforce_consumer_goods_cloud/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/salesforce_crm/config/catalog.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/salesforce_crm/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/salesforce_crm/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/salesforce_crm/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/sftp/config/catalog.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/sftp/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/sftp/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/sftp/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/slack/config/catalog.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/slack/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/slack/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/slack/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/stripe/config/catalog.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/stripe/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/stripe/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/stripe/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/tally/config/catalog.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/tally/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/tally/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/tally/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/bigquery/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/bigquery/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/bigquery/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/databricks/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/databricks/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/databricks/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/postgresql/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/postgresql/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/postgresql/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/redshift/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/redshift/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/redshift/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/salesforce_consumer_goods_cloud/config/catalog.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/salesforce_consumer_goods_cloud/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/salesforce_consumer_goods_cloud/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/salesforce_consumer_goods_cloud/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/snowflake/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/snowflake/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/snowflake/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/zoho_books/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/zoho_books/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/zoho_books/icon.svg +0 -0
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: a1b9a7a2e4780b4830cded5259e52ea8e630ac7cd27347df88c1a4fc06285daf
|
|
4
|
+
data.tar.gz: '088f80483ec5f386aa11ae8bc735e4a9f518af56494b51242b92860216d4d339'
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: cb77f7c5f9e29d772fcafa6f944f9ded6204a21f8bcbe42486546b8c713a1453a0df82716b70a424637e16626874753c77de1944877f1b622a4622e72187ef76
|
|
7
|
+
data.tar.gz: 7b3ce9767ff9dfefefdee24ff117f528c8ee5d7458da2e1e8700a677cec87235ec3281764d9599c3aad6f51181413464460cc3c5c76291997b158e8834baa071
|
data/CHANGELOG.md
CHANGED
|
@@ -1,37 +1,37 @@
|
|
|
1
1
|
## [Unreleased]
|
|
2
2
|
|
|
3
|
-
### [0.4.1](https://www.github.com/
|
|
3
|
+
### [0.4.1](https://www.github.com/Etl/multiwoven-integrations/compare/v0.4.0...v0.4.1) (2024-02-15)
|
|
4
4
|
|
|
5
5
|
|
|
6
6
|
### Bug Fixes
|
|
7
7
|
|
|
8
|
-
* Add batch params to stream ([#68](https://www.github.com/
|
|
8
|
+
* Add batch params to stream ([#68](https://www.github.com/Etl/multiwoven-integrations/issues/68)) ([080530f](https://www.github.com/Etl/multiwoven-integrations/commit/080530f8280b467c8e40a3b6429f178ad18fa05e))
|
|
9
9
|
|
|
10
|
-
## [0.4.0](https://www.github.com/
|
|
10
|
+
## [0.4.0](https://www.github.com/Etl/multiwoven-integrations/compare/v0.3.0...v0.4.0) (2024-02-14)
|
|
11
11
|
|
|
12
12
|
|
|
13
13
|
### Features
|
|
14
14
|
|
|
15
|
-
* Bump version to 1.21 ([#62](https://www.github.com/
|
|
15
|
+
* Bump version to 1.21 ([#62](https://www.github.com/Etl/multiwoven-integrations/issues/62)) ([7b1476d](https://www.github.com/Etl/multiwoven-integrations/commit/7b1476d4b04a073ab060971019095d0a13ebd153))
|
|
16
16
|
|
|
17
|
-
## [0.3.0](https://www.github.com/
|
|
17
|
+
## [0.3.0](https://www.github.com/Etl/multiwoven-integrations/compare/v0.2.0...v0.3.0) (2024-02-14)
|
|
18
18
|
|
|
19
19
|
|
|
20
20
|
### Features
|
|
21
21
|
|
|
22
|
-
* Publish gem to rubygems ([#60](https://www.github.com/
|
|
22
|
+
* Publish gem to rubygems ([#60](https://www.github.com/Etl/multiwoven-integrations/issues/60)) ([04cfbb6](https://www.github.com/Etl/multiwoven-integrations/commit/04cfbb60c78bc90d886ce691d5094fed9c6f078f))
|
|
23
23
|
|
|
24
|
-
## [0.2.0](https://www.github.com/
|
|
24
|
+
## [0.2.0](https://www.github.com/Etl/multiwoven-integrations/compare/v0.1.0...v0.2.0) (2024-02-14)
|
|
25
25
|
|
|
26
26
|
|
|
27
27
|
### Features
|
|
28
28
|
|
|
29
|
-
* Automate gem release ([#55](https://www.github.com/
|
|
29
|
+
* Automate gem release ([#55](https://www.github.com/Etl/multiwoven-integrations/issues/55)) ([1c3373b](https://www.github.com/Etl/multiwoven-integrations/commit/1c3373bcedae60c6fb8171c54ce40de94f1537ce))
|
|
30
30
|
|
|
31
31
|
|
|
32
32
|
### Bug Fixes
|
|
33
33
|
|
|
34
|
-
* Klaviyo discover ([#23](https://www.github.com/
|
|
34
|
+
* Klaviyo discover ([#23](https://www.github.com/Etl/multiwoven-integrations/issues/23)) ([7c43db6](https://www.github.com/Etl/multiwoven-integrations/commit/7c43db6a369403748ed29085bb5254405ad1c95e))
|
|
35
35
|
|
|
36
36
|
## [0.1.0] - 2023-12-01
|
|
37
37
|
|
data/LICENSE.txt
CHANGED
data/README.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
<div align="center">
|
|
2
2
|
<a href="https://multiwoven.com?utm_source=github" target="_blank">
|
|
3
|
-
<img src="https://res.cloudinary.com/dspflukeu/image/upload/v1706696350/
|
|
3
|
+
<img src="https://res.cloudinary.com/dspflukeu/image/upload/v1706696350/Etl/Logo_hrkaxj.png" alt="Etl" width="240" />
|
|
4
4
|
</a>
|
|
5
5
|
</div>
|
|
6
6
|
<br />
|
|
@@ -8,8 +8,8 @@
|
|
|
8
8
|
<a href="http://badge.fury.io/rb/multiwoven-integrations">
|
|
9
9
|
<img src="https://badge.fury.io/rb/multiwoven-integrations.svg" alt="Gem Version">
|
|
10
10
|
</a>
|
|
11
|
-
<a href="https://github.com/
|
|
12
|
-
<img src="https://github.com/
|
|
11
|
+
<a href="https://github.com/Etl/multiwoven-integrations/actions/workflows/ci.yml">
|
|
12
|
+
<img src="https://github.com/Etl/multiwoven-integrations/actions/workflows/ci.yml/badge.svg" alt="CI">
|
|
13
13
|
</a>
|
|
14
14
|
<a href="https://codeclimate.com/repos/657d0a2a60265a2f2155ffca/maintainability">
|
|
15
15
|
<img src="https://api.codeclimate.com/v1/badges/d841270f1f7a966043c1/maintainability" alt="Maintainability">
|
|
@@ -31,16 +31,16 @@
|
|
|
31
31
|
·
|
|
32
32
|
<a href="https://multiwoven.com">Website</a>
|
|
33
33
|
·
|
|
34
|
-
<a href="https://github.com/
|
|
34
|
+
<a href="https://github.com/Etl/multiwoven-integrations/issues">Report Issues</a>
|
|
35
35
|
·
|
|
36
|
-
<a href="https://github.com/orgs/
|
|
36
|
+
<a href="https://github.com/orgs/Etl/projects/4">Roadmap</a>
|
|
37
37
|
</p>
|
|
38
38
|
|
|
39
|
-
##
|
|
39
|
+
## Etl Integrations
|
|
40
40
|
|
|
41
|
-
|
|
41
|
+
Etl integrations is the collection of connectors built on top of [Etl protocol](https://docs.multiwoven.com/guides/architecture/multiwoven-protocol).
|
|
42
42
|
|
|
43
|
-
|
|
43
|
+
Etl protocol is an open source standard for moving data between data sources to any third-part destinations.
|
|
44
44
|
Anyone can build a connetor with basic ruby knowledge using the protocol.
|
|
45
45
|
|
|
46
46
|
### Installation
|
|
@@ -54,18 +54,18 @@ Install the gem and add to the application's Gemfile by executing:
|
|
|
54
54
|
|
|
55
55
|
#### Source
|
|
56
56
|
```
|
|
57
|
-
source =
|
|
57
|
+
source = Etl::Integrations::Source::[CONNECTOR_NAME]::Client.new
|
|
58
58
|
source.read(sync_config)
|
|
59
59
|
```
|
|
60
60
|
#### Destination
|
|
61
61
|
|
|
62
62
|
```
|
|
63
|
-
destination =
|
|
63
|
+
destination = Etl::Integrations::Destination::[CONNECTOR_NAME]::Client.new
|
|
64
64
|
destination.write(sync_config, records)
|
|
65
65
|
```
|
|
66
66
|
|
|
67
67
|
#### Supported methods
|
|
68
|
-
Please refer [
|
|
68
|
+
Please refer [Etl Protocol](https://docs.multiwoven.com/guides/architecture/multiwoven-protocol) to understand more about the supported methods on source and destination
|
|
69
69
|
|
|
70
70
|
## Development
|
|
71
71
|
|
|
@@ -94,11 +94,11 @@ Please refer [Multiwoven Protocol](https://docs.multiwoven.com/guides/architectu
|
|
|
94
94
|
|
|
95
95
|
## Contributing
|
|
96
96
|
|
|
97
|
-
Bug reports and pull requests are welcome on [GitHub](https://github.com/
|
|
97
|
+
Bug reports and pull requests are welcome on [GitHub](https://github.com/Etl/multiwoven-integrations). This project aims to be a safe, welcoming space for collaboration.
|
|
98
98
|
|
|
99
99
|
## Code of Conduct
|
|
100
100
|
|
|
101
|
-
Contributors are expected to adhere to the project's [code of conduct](https://github.com/
|
|
101
|
+
Contributors are expected to adhere to the project's [code of conduct](https://github.com/Etl/multiwoven-integrations/blob/main/CODE_OF_CONDUCT.md)
|
|
102
102
|
|
|
103
103
|
## License
|
|
104
104
|
|
|
@@ -1,25 +1,25 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require_relative "lib/
|
|
3
|
+
require_relative "lib/etl/integrations/rollout"
|
|
4
4
|
|
|
5
5
|
Gem::Specification.new do |spec|
|
|
6
6
|
spec.name = "etl-integrations"
|
|
7
|
-
spec.version =
|
|
7
|
+
spec.version = Etl::Integrations::VERSION
|
|
8
8
|
spec.authors = ["Anas Ahmed"]
|
|
9
9
|
spec.email = ["anas31197@gmail.com"]
|
|
10
10
|
|
|
11
11
|
spec.summary = "Integration suite for open source reverse ETL platform"
|
|
12
12
|
spec.description = "Etl Integrations is a comprehensive Ruby gem designed to facilitate seamless connectivity between various data sources and SaaS platforms."
|
|
13
13
|
|
|
14
|
-
spec.homepage = "https://
|
|
14
|
+
spec.homepage = "https://github.com/anasValign/etl-tool"
|
|
15
15
|
spec.license = "MIT"
|
|
16
16
|
spec.required_ruby_version = ">= 2.6.0"
|
|
17
17
|
|
|
18
18
|
# spec.metadata["allowed_push_host"] = nil
|
|
19
19
|
spec.metadata["github_repo"] = "https://github.com/anasValign/etl-tool"
|
|
20
20
|
spec.metadata["homepage_uri"] = spec.homepage
|
|
21
|
-
spec.metadata["source_code_uri"] = "https://github.com/
|
|
22
|
-
spec.metadata["changelog_uri"] = "https://github.com/
|
|
21
|
+
spec.metadata["source_code_uri"] = "https://github.com/Etl/multiwoven/tree/main/integrations"
|
|
22
|
+
spec.metadata["changelog_uri"] = "https://github.com/Etl/multiwoven/blob/main/integrations/CHANGELOG.md"
|
|
23
23
|
|
|
24
24
|
# Specify which files should be added to the gem when it is released.
|
|
25
25
|
# The `git ls-files -z` loads the files in the RubyGem that have been added into git.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
module
|
|
3
|
+
module Etl
|
|
4
4
|
module Integrations::Core
|
|
5
5
|
class BaseConnector
|
|
6
6
|
include Integrations::Protocol
|
|
@@ -18,7 +18,7 @@ module Multiwoven
|
|
|
18
18
|
def meta_data
|
|
19
19
|
client_meta_data = read_json(META_DATA_PATH).deep_symbolize_keys
|
|
20
20
|
icon_name = client_meta_data[:data][:icon]
|
|
21
|
-
icon_url = "https://raw.githubusercontent.com/
|
|
21
|
+
icon_url = "https://raw.githubusercontent.com/Etl/multiwoven/main/integrations#{relative_path}/#{icon_name}"
|
|
22
22
|
client_meta_data[:data][:icon] = icon_url
|
|
23
23
|
# returns hash
|
|
24
24
|
@meta_data ||= client_meta_data
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
module
|
|
3
|
+
module Etl
|
|
4
4
|
module Integrations::Core
|
|
5
5
|
module Fullrefresher
|
|
6
6
|
def write(sync_config, records, action = "insert")
|
|
7
7
|
if sync_config && sync_config.sync_mode == "full_refresh" && !@full_refreshed
|
|
8
8
|
response = clear_all_records(sync_config)
|
|
9
9
|
return response unless response &&
|
|
10
|
-
response.control.status ==
|
|
10
|
+
response.control.status == Etl::Integrations::Protocol::ConnectionStatusType["succeeded"]
|
|
11
11
|
|
|
12
12
|
@full_refreshed = true
|
|
13
13
|
end
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
module
|
|
3
|
+
module Etl
|
|
4
4
|
module Integrations::Core
|
|
5
5
|
module Utils
|
|
6
6
|
def keys_to_symbols(hash)
|
|
@@ -75,7 +75,7 @@ module Multiwoven
|
|
|
75
75
|
|
|
76
76
|
def build_catalog(catalog_json)
|
|
77
77
|
streams = catalog_json["streams"].map { |stream_json| build_stream(stream_json) }
|
|
78
|
-
|
|
78
|
+
Etl::Integrations::Protocol::Catalog.new(
|
|
79
79
|
streams: streams,
|
|
80
80
|
request_rate_limit: catalog_json["request_rate_limit"] || 60,
|
|
81
81
|
request_rate_limit_unit: catalog_json["request_rate_limit_unit"] || "minute",
|
|
@@ -85,7 +85,7 @@ module Multiwoven
|
|
|
85
85
|
end
|
|
86
86
|
|
|
87
87
|
def build_stream(stream_json)
|
|
88
|
-
|
|
88
|
+
Etl::Integrations::Protocol::Stream.new(
|
|
89
89
|
name: stream_json["name"],
|
|
90
90
|
url: stream_json["url"],
|
|
91
91
|
action: stream_json["action"],
|
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative "schema_helper"
|
|
4
|
-
module
|
|
4
|
+
module Etl
|
|
5
5
|
module Integrations
|
|
6
6
|
module Destination
|
|
7
7
|
module Airtable
|
|
8
|
-
include
|
|
8
|
+
include Etl::Integrations::Core
|
|
9
9
|
class Client < DestinationConnector # rubocop:disable Metrics/ClassLength
|
|
10
|
-
prepend
|
|
10
|
+
prepend Etl::Integrations::Core::RateLimiter
|
|
11
11
|
MAX_CHUNK_SIZE = 10
|
|
12
12
|
def check_connection(connection_config)
|
|
13
13
|
connection_config = connection_config.with_indifferent_access
|
|
14
|
-
bases =
|
|
14
|
+
bases = Etl::Integrations::Core::HttpClient.request(
|
|
15
15
|
AIRTABLE_BASES_ENDPOINT,
|
|
16
16
|
HTTP_GET,
|
|
17
17
|
headers: auth_headers(connection_config[:api_key])
|
|
@@ -31,7 +31,7 @@ module Multiwoven
|
|
|
31
31
|
base_id = connection_config[:base_id]
|
|
32
32
|
api_key = connection_config[:api_key]
|
|
33
33
|
|
|
34
|
-
bases =
|
|
34
|
+
bases = Etl::Integrations::Core::HttpClient.request(
|
|
35
35
|
AIRTABLE_BASES_ENDPOINT,
|
|
36
36
|
HTTP_GET,
|
|
37
37
|
headers: auth_headers(api_key)
|
|
@@ -40,7 +40,7 @@ module Multiwoven
|
|
|
40
40
|
base = extract_bases(bases).find { |b| b["id"] == base_id }
|
|
41
41
|
base_name = base["name"]
|
|
42
42
|
|
|
43
|
-
schema =
|
|
43
|
+
schema = Etl::Integrations::Core::HttpClient.request(
|
|
44
44
|
AIRTABLE_GET_BASE_SCHEMA_ENDPOINT.gsub("{baseId}", base_id),
|
|
45
45
|
HTTP_GET,
|
|
46
46
|
headers: auth_headers(api_key)
|
|
@@ -60,7 +60,7 @@ module Multiwoven
|
|
|
60
60
|
write_failure = 0
|
|
61
61
|
records.each_slice(MAX_CHUNK_SIZE) do |chunk|
|
|
62
62
|
payload = create_payload(chunk)
|
|
63
|
-
response =
|
|
63
|
+
response = Etl::Integrations::Core::HttpClient.request(
|
|
64
64
|
url,
|
|
65
65
|
sync_config.stream.request_method,
|
|
66
66
|
payload: payload,
|
|
@@ -76,7 +76,7 @@ module Multiwoven
|
|
|
76
76
|
write_failure += chunk.size
|
|
77
77
|
end
|
|
78
78
|
|
|
79
|
-
tracker =
|
|
79
|
+
tracker = Etl::Integrations::Protocol::TrackingMessage.new(
|
|
80
80
|
success: write_success,
|
|
81
81
|
failed: write_failure
|
|
82
82
|
)
|
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
module
|
|
3
|
+
module Etl::Integrations::Destination
|
|
4
4
|
module FacebookCustomAudience
|
|
5
|
-
include
|
|
5
|
+
include Etl::Integrations::Core
|
|
6
6
|
class Client < DestinationConnector # rubocop:disable Metrics/ClassLength
|
|
7
|
-
prepend
|
|
7
|
+
prepend Etl::Integrations::Core::RateLimiter
|
|
8
8
|
MAX_CHUNK_SIZE = 10_000
|
|
9
9
|
def check_connection(connection_config)
|
|
10
10
|
connection_config = connection_config.with_indifferent_access
|
|
11
11
|
access_token = connection_config[:access_token]
|
|
12
|
-
response =
|
|
12
|
+
response = Etl::Integrations::Core::HttpClient.request(
|
|
13
13
|
FACEBOOK_AUDIENCE_GET_ALL_ACCOUNTS,
|
|
14
14
|
HTTP_GET,
|
|
15
15
|
headers: auth_headers(access_token)
|
|
@@ -44,7 +44,7 @@ module Multiwoven::Integrations::Destination
|
|
|
44
44
|
write_failure = 0
|
|
45
45
|
records.each_slice(MAX_CHUNK_SIZE) do |chunk|
|
|
46
46
|
payload = create_payload(chunk, sync_config.stream.json_schema.with_indifferent_access)
|
|
47
|
-
response =
|
|
47
|
+
response = Etl::Integrations::Core::HttpClient.request(
|
|
48
48
|
url,
|
|
49
49
|
sync_config.stream.request_method,
|
|
50
50
|
payload: payload,
|
|
@@ -60,7 +60,7 @@ module Multiwoven::Integrations::Destination
|
|
|
60
60
|
write_failure += chunk.size
|
|
61
61
|
end
|
|
62
62
|
|
|
63
|
-
tracker =
|
|
63
|
+
tracker = Etl::Integrations::Protocol::TrackingMessage.new(
|
|
64
64
|
success: write_success,
|
|
65
65
|
failed: write_failure
|
|
66
66
|
)
|
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
module
|
|
3
|
+
module Etl
|
|
4
4
|
module Integrations
|
|
5
5
|
module Destination
|
|
6
6
|
module GoogleSheets
|
|
7
|
-
include
|
|
7
|
+
include Etl::Integrations::Core
|
|
8
8
|
|
|
9
9
|
class Client < DestinationConnector # rubocop:disable Metrics/ClassLength
|
|
10
|
-
prepend
|
|
11
|
-
prepend
|
|
10
|
+
prepend Etl::Integrations::Core::Fullrefresher
|
|
11
|
+
prepend Etl::Integrations::Core::RateLimiter
|
|
12
12
|
MAX_CHUNK_SIZE = 10_000
|
|
13
13
|
|
|
14
14
|
def check_connection(connection_config)
|
|
@@ -195,7 +195,7 @@ module Multiwoven
|
|
|
195
195
|
end
|
|
196
196
|
|
|
197
197
|
def tracking_message(success, failure)
|
|
198
|
-
|
|
198
|
+
Etl::Integrations::Protocol::TrackingMessage.new(
|
|
199
199
|
success: success, failed: failure
|
|
200
200
|
).to_multiwoven_message
|
|
201
201
|
end
|
|
@@ -2,14 +2,14 @@
|
|
|
2
2
|
|
|
3
3
|
require "stringio"
|
|
4
4
|
|
|
5
|
-
module
|
|
5
|
+
module Etl
|
|
6
6
|
module Integrations
|
|
7
7
|
module Destination
|
|
8
8
|
module Hubspot
|
|
9
|
-
include
|
|
9
|
+
include Etl::Integrations::Core
|
|
10
10
|
|
|
11
11
|
class Client < DestinationConnector
|
|
12
|
-
prepend
|
|
12
|
+
prepend Etl::Integrations::Core::RateLimiter
|
|
13
13
|
def check_connection(connection_config)
|
|
14
14
|
connection_config = connection_config.with_indifferent_access
|
|
15
15
|
initialize_client(connection_config)
|
|
@@ -92,13 +92,13 @@ module Multiwoven
|
|
|
92
92
|
end
|
|
93
93
|
|
|
94
94
|
def tracking_message(success, failure)
|
|
95
|
-
|
|
95
|
+
Etl::Integrations::Protocol::TrackingMessage.new(
|
|
96
96
|
success: success, failed: failure
|
|
97
97
|
).to_multiwoven_message
|
|
98
98
|
end
|
|
99
99
|
|
|
100
100
|
def log_debug(message)
|
|
101
|
-
|
|
101
|
+
Etl::Integrations::Service.logger.debug(message)
|
|
102
102
|
end
|
|
103
103
|
end
|
|
104
104
|
end
|
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
module
|
|
3
|
+
module Etl::Integrations::Destination
|
|
4
4
|
module Klaviyo
|
|
5
|
-
include
|
|
5
|
+
include Etl::Integrations::Core
|
|
6
6
|
class Client < DestinationConnector
|
|
7
|
-
prepend
|
|
7
|
+
prepend Etl::Integrations::Core::RateLimiter
|
|
8
8
|
def check_connection(connection_config)
|
|
9
9
|
connection_config = connection_config.with_indifferent_access
|
|
10
10
|
api_key = connection_config[:private_api_key]
|
|
11
11
|
|
|
12
|
-
response =
|
|
12
|
+
response = Etl::Integrations::Core::HttpClient.request(
|
|
13
13
|
KLAVIYO_AUTH_ENDPOINT,
|
|
14
14
|
HTTP_POST,
|
|
15
15
|
payload: KLAVIYO_AUTH_PAYLOAD,
|
|
@@ -47,7 +47,7 @@ module Multiwoven::Integrations::Destination
|
|
|
47
47
|
record["data"] ||= {}
|
|
48
48
|
record["data"]["type"] = sync_config.stream.name
|
|
49
49
|
|
|
50
|
-
response =
|
|
50
|
+
response = Etl::Integrations::Core::HttpClient.request(
|
|
51
51
|
url,
|
|
52
52
|
request_method,
|
|
53
53
|
payload: record,
|
|
@@ -64,7 +64,7 @@ module Multiwoven::Integrations::Destination
|
|
|
64
64
|
)
|
|
65
65
|
write_failure += 1
|
|
66
66
|
end
|
|
67
|
-
tracker =
|
|
67
|
+
tracker = Etl::Integrations::Protocol::TrackingMessage.new(
|
|
68
68
|
success: write_success,
|
|
69
69
|
failed: write_failure
|
|
70
70
|
)
|
|
@@ -2,9 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
require "pg"
|
|
4
4
|
|
|
5
|
-
module
|
|
5
|
+
module Etl::Integrations::Destination
|
|
6
6
|
module Postgresql
|
|
7
|
-
include
|
|
7
|
+
include Etl::Integrations::Core
|
|
8
8
|
class Client < DestinationConnector
|
|
9
9
|
def check_connection(connection_config)
|
|
10
10
|
connection_config = connection_config.with_indifferent_access
|
|
@@ -52,7 +52,7 @@ module Multiwoven::Integrations::Destination
|
|
|
52
52
|
write_failure = 0
|
|
53
53
|
|
|
54
54
|
records.each do |record|
|
|
55
|
-
query =
|
|
55
|
+
query = Etl::Integrations::Core::QueryBuilder.perform(action, table_name, record)
|
|
56
56
|
begin
|
|
57
57
|
db.exec(query)
|
|
58
58
|
write_success += 1
|
|
@@ -94,7 +94,7 @@ module Multiwoven::Integrations::Destination
|
|
|
94
94
|
|
|
95
95
|
def create_streams(records)
|
|
96
96
|
group_by_table(records).map do |r|
|
|
97
|
-
|
|
97
|
+
Etl::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
|
|
98
98
|
end
|
|
99
99
|
end
|
|
100
100
|
|
|
@@ -114,7 +114,7 @@ module Multiwoven::Integrations::Destination
|
|
|
114
114
|
end
|
|
115
115
|
|
|
116
116
|
def tracking_message(success, failure)
|
|
117
|
-
|
|
117
|
+
Etl::Integrations::Protocol::TrackingMessage.new(
|
|
118
118
|
success: success, failed: failure
|
|
119
119
|
).to_multiwoven_message
|
|
120
120
|
end
|
data/lib/{multiwoven → etl}/integrations/destination/salesforce_consumer_goods_cloud/client.rb
RENAMED
|
@@ -3,17 +3,17 @@
|
|
|
3
3
|
require "stringio"
|
|
4
4
|
require_relative "schema_helper"
|
|
5
5
|
|
|
6
|
-
module
|
|
6
|
+
module Etl
|
|
7
7
|
module Integrations
|
|
8
8
|
module Destination
|
|
9
9
|
module SalesforceConsumerGoodsCloud
|
|
10
|
-
include
|
|
10
|
+
include Etl::Integrations::Core
|
|
11
11
|
|
|
12
12
|
API_VERSION = "59.0"
|
|
13
13
|
SALESFORCE_OBJECTS = %w[Account User Visit RetailStore RecordType].freeze
|
|
14
14
|
|
|
15
15
|
class Client < DestinationConnector
|
|
16
|
-
prepend
|
|
16
|
+
prepend Etl::Integrations::Core::RateLimiter
|
|
17
17
|
def check_connection(connection_config)
|
|
18
18
|
connection_config = connection_config.with_indifferent_access
|
|
19
19
|
initialize_client(connection_config)
|
|
@@ -99,13 +99,13 @@ module Multiwoven
|
|
|
99
99
|
end
|
|
100
100
|
|
|
101
101
|
def tracking_message(success, failure)
|
|
102
|
-
|
|
102
|
+
Etl::Integrations::Protocol::TrackingMessage.new(
|
|
103
103
|
success: success, failed: failure
|
|
104
104
|
).to_multiwoven_message
|
|
105
105
|
end
|
|
106
106
|
|
|
107
107
|
def log_debug(message)
|
|
108
|
-
|
|
108
|
+
Etl::Integrations::Service.logger.debug(message)
|
|
109
109
|
end
|
|
110
110
|
end
|
|
111
111
|
end
|
|
@@ -2,16 +2,16 @@
|
|
|
2
2
|
|
|
3
3
|
require "stringio"
|
|
4
4
|
|
|
5
|
-
module
|
|
5
|
+
module Etl
|
|
6
6
|
module Integrations
|
|
7
7
|
module Destination
|
|
8
8
|
module SalesforceCrm
|
|
9
|
-
include
|
|
9
|
+
include Etl::Integrations::Core
|
|
10
10
|
|
|
11
11
|
API_VERSION = "59.0"
|
|
12
12
|
|
|
13
13
|
class Client < DestinationConnector
|
|
14
|
-
prepend
|
|
14
|
+
prepend Etl::Integrations::Core::RateLimiter
|
|
15
15
|
def check_connection(connection_config)
|
|
16
16
|
connection_config = connection_config.with_indifferent_access
|
|
17
17
|
initialize_client(connection_config)
|
|
@@ -102,13 +102,13 @@ module Multiwoven
|
|
|
102
102
|
end
|
|
103
103
|
|
|
104
104
|
def tracking_message(success, failure)
|
|
105
|
-
|
|
105
|
+
Etl::Integrations::Protocol::TrackingMessage.new(
|
|
106
106
|
success: success, failed: failure
|
|
107
107
|
).to_multiwoven_message
|
|
108
108
|
end
|
|
109
109
|
|
|
110
110
|
def log_debug(message)
|
|
111
|
-
|
|
111
|
+
Etl::Integrations::Service.logger.debug(message)
|
|
112
112
|
end
|
|
113
113
|
end
|
|
114
114
|
end
|