etl-integrations 0.1.81 → 0.1.85
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +9 -9
- data/LICENSE.txt +1 -1
- data/README.md +13 -13
- data/etl-integrations.gemspec +65 -0
- data/lib/{multiwoven → etl}/integrations/config.rb +1 -1
- data/lib/{multiwoven → etl}/integrations/core/base_connector.rb +2 -2
- data/lib/{multiwoven → etl}/integrations/core/constants.rb +1 -1
- data/lib/{multiwoven → etl}/integrations/core/destination_connector.rb +1 -1
- data/lib/{multiwoven → etl}/integrations/core/fullrefresher.rb +2 -2
- data/lib/{multiwoven → etl}/integrations/core/http_client.rb +1 -1
- data/lib/{multiwoven → etl}/integrations/core/query_builder.rb +1 -1
- data/lib/{multiwoven → etl}/integrations/core/rate_limiter.rb +1 -1
- data/lib/{multiwoven → etl}/integrations/core/source_connector.rb +1 -1
- data/lib/{multiwoven → etl}/integrations/core/utils.rb +3 -3
- data/lib/{multiwoven → etl}/integrations/destination/airtable/client.rb +8 -8
- data/lib/{multiwoven → etl}/integrations/destination/airtable/schema_helper.rb +1 -1
- data/lib/{multiwoven → etl}/integrations/destination/facebook_custom_audience/client.rb +6 -6
- data/lib/{multiwoven → etl}/integrations/destination/google_sheets/client.rb +5 -5
- data/lib/{multiwoven → etl}/integrations/destination/hubspot/client.rb +5 -5
- data/lib/{multiwoven → etl}/integrations/destination/klaviyo/client.rb +6 -6
- data/lib/{multiwoven → etl}/integrations/destination/postgresql/client.rb +5 -5
- data/lib/{multiwoven → etl}/integrations/destination/salesforce_consumer_goods_cloud/client.rb +5 -5
- data/lib/{multiwoven → etl}/integrations/destination/salesforce_consumer_goods_cloud/schema_helper.rb +1 -1
- data/lib/{multiwoven → etl}/integrations/destination/salesforce_crm/client.rb +5 -5
- data/lib/{multiwoven → etl}/integrations/destination/sftp/client.rb +5 -5
- data/lib/{multiwoven → etl}/integrations/destination/slack/client.rb +4 -4
- data/lib/{multiwoven → etl}/integrations/destination/stripe/client.rb +5 -5
- data/lib/{multiwoven → etl}/integrations/destination/tally/client.rb +5 -5
- data/lib/{multiwoven → etl}/integrations/protocol/protocol.json +2 -2
- data/lib/{multiwoven → etl}/integrations/protocol/protocol.rb +2 -2
- data/lib/{multiwoven → etl}/integrations/rollout.rb +2 -2
- data/lib/{multiwoven → etl}/integrations/service.rb +3 -3
- data/lib/{multiwoven → etl}/integrations/source/bigquery/client.rb +3 -3
- data/lib/{multiwoven → etl}/integrations/source/databricks/client.rb +3 -3
- data/lib/{multiwoven → etl}/integrations/source/postgresql/client.rb +3 -3
- data/lib/{multiwoven → etl}/integrations/source/redshift/client.rb +3 -3
- data/lib/{multiwoven → etl}/integrations/source/salesforce_consumer_goods_cloud/client.rb +5 -5
- data/lib/{multiwoven → etl}/integrations/source/salesforce_consumer_goods_cloud/schema_helper.rb +1 -1
- data/lib/{multiwoven → etl}/integrations/source/snowflake/client.rb +3 -3
- data/lib/{multiwoven → etl}/integrations/source/zoho_books/client.rb +5 -5
- data/lib/{multiwoven → etl}/integrations.rb +1 -1
- data/sig/{multiwoven → etl}/integrations.rbs +1 -1
- metadata +119 -118
- /data/{multiwoven-integrations-0.1.68.gem → etl-integrations-0.1.68.gem} +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/airtable/config/catalog.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/airtable/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/airtable/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/airtable/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/facebook_custom_audience/config/catalog.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/facebook_custom_audience/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/facebook_custom_audience/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/facebook_custom_audience/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/google_sheets/config/catalog.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/google_sheets/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/google_sheets/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/google_sheets/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/hubspot/config/catalog.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/hubspot/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/hubspot/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/hubspot/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/klaviyo/config/catalog.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/klaviyo/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/klaviyo/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/klaviyo/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/postgresql/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/postgresql/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/postgresql/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/salesforce_consumer_goods_cloud/config/catalog.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/salesforce_consumer_goods_cloud/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/salesforce_consumer_goods_cloud/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/salesforce_consumer_goods_cloud/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/salesforce_crm/config/catalog.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/salesforce_crm/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/salesforce_crm/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/salesforce_crm/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/sftp/config/catalog.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/sftp/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/sftp/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/sftp/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/slack/config/catalog.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/slack/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/slack/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/slack/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/stripe/config/catalog.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/stripe/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/stripe/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/stripe/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/tally/config/catalog.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/tally/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/tally/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/destination/tally/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/bigquery/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/bigquery/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/bigquery/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/databricks/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/databricks/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/databricks/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/postgresql/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/postgresql/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/postgresql/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/redshift/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/redshift/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/redshift/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/salesforce_consumer_goods_cloud/config/catalog.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/salesforce_consumer_goods_cloud/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/salesforce_consumer_goods_cloud/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/salesforce_consumer_goods_cloud/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/snowflake/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/snowflake/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/snowflake/icon.svg +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/zoho_books/config/meta.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/zoho_books/config/spec.json +0 -0
- /data/lib/{multiwoven → etl}/integrations/source/zoho_books/icon.svg +0 -0
|
@@ -2,16 +2,16 @@
|
|
|
2
2
|
|
|
3
3
|
require "stringio"
|
|
4
4
|
|
|
5
|
-
module
|
|
5
|
+
module Etl
|
|
6
6
|
module Integrations
|
|
7
7
|
module Destination
|
|
8
8
|
module SalesforceCrm
|
|
9
|
-
include
|
|
9
|
+
include Etl::Integrations::Core
|
|
10
10
|
|
|
11
11
|
API_VERSION = "59.0"
|
|
12
12
|
|
|
13
13
|
class Client < DestinationConnector
|
|
14
|
-
prepend
|
|
14
|
+
prepend Etl::Integrations::Core::RateLimiter
|
|
15
15
|
def check_connection(connection_config)
|
|
16
16
|
connection_config = connection_config.with_indifferent_access
|
|
17
17
|
initialize_client(connection_config)
|
|
@@ -102,13 +102,13 @@ module Multiwoven
|
|
|
102
102
|
end
|
|
103
103
|
|
|
104
104
|
def tracking_message(success, failure)
|
|
105
|
-
|
|
105
|
+
Etl::Integrations::Protocol::TrackingMessage.new(
|
|
106
106
|
success: success, failed: failure
|
|
107
107
|
).to_multiwoven_message
|
|
108
108
|
end
|
|
109
109
|
|
|
110
110
|
def log_debug(message)
|
|
111
|
-
|
|
111
|
+
Etl::Integrations::Service.logger.debug(message)
|
|
112
112
|
end
|
|
113
113
|
end
|
|
114
114
|
end
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
module
|
|
3
|
+
module Etl::Integrations::Destination
|
|
4
4
|
module Sftp
|
|
5
|
-
include
|
|
5
|
+
include Etl::Integrations::Core
|
|
6
6
|
class Client < DestinationConnector # rubocop:disable Metrics/ClassLength
|
|
7
|
-
prepend
|
|
8
|
-
prepend
|
|
7
|
+
prepend Etl::Integrations::Core::Fullrefresher
|
|
8
|
+
prepend Etl::Integrations::Core::RateLimiter
|
|
9
9
|
|
|
10
10
|
def check_connection(connection_config)
|
|
11
11
|
connection_config = connection_config.with_indifferent_access
|
|
@@ -101,7 +101,7 @@ module Multiwoven::Integrations::Destination
|
|
|
101
101
|
end
|
|
102
102
|
|
|
103
103
|
def tracking_message(success, failure)
|
|
104
|
-
|
|
104
|
+
Etl::Integrations::Protocol::TrackingMessage.new(
|
|
105
105
|
success: success, failed: failure
|
|
106
106
|
).to_multiwoven_message
|
|
107
107
|
end
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
module
|
|
3
|
+
module Etl
|
|
4
4
|
module Integrations
|
|
5
5
|
module Destination
|
|
6
6
|
module Slack
|
|
7
|
-
include
|
|
7
|
+
include Etl::Integrations::Core
|
|
8
8
|
|
|
9
9
|
class Client < DestinationConnector
|
|
10
|
-
prepend
|
|
10
|
+
prepend Etl::Integrations::Core::RateLimiter
|
|
11
11
|
attr_accessor :channel_id
|
|
12
12
|
|
|
13
13
|
def check_connection(connection_config)
|
|
@@ -103,7 +103,7 @@ module Multiwoven
|
|
|
103
103
|
end
|
|
104
104
|
|
|
105
105
|
def tracking_message(success, failure)
|
|
106
|
-
|
|
106
|
+
Etl::Integrations::Protocol::TrackingMessage.new(
|
|
107
107
|
success: success, failed: failure
|
|
108
108
|
).to_multiwoven_message
|
|
109
109
|
end
|
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
module
|
|
3
|
+
module Etl
|
|
4
4
|
module Integrations
|
|
5
5
|
module Destination
|
|
6
6
|
module Stripe
|
|
7
|
-
include
|
|
7
|
+
include Etl::Integrations::Core
|
|
8
8
|
|
|
9
9
|
API_VERSION = "59.0"
|
|
10
10
|
|
|
11
11
|
class Client < DestinationConnector
|
|
12
|
-
prepend
|
|
12
|
+
prepend Etl::Integrations::Core::RateLimiter
|
|
13
13
|
def check_connection(connection_config)
|
|
14
14
|
connection_config = connection_config.with_indifferent_access
|
|
15
15
|
initialize_client(connection_config)
|
|
@@ -67,13 +67,13 @@ module Multiwoven
|
|
|
67
67
|
end
|
|
68
68
|
|
|
69
69
|
def tracking_message(success, failure)
|
|
70
|
-
|
|
70
|
+
Etl::Integrations::Protocol::TrackingMessage.new(
|
|
71
71
|
success: success, failed: failure
|
|
72
72
|
).to_multiwoven_message
|
|
73
73
|
end
|
|
74
74
|
|
|
75
75
|
def log_debug(message)
|
|
76
|
-
|
|
76
|
+
Etl::Integrations::Service.logger.debug(message)
|
|
77
77
|
end
|
|
78
78
|
end
|
|
79
79
|
end
|
|
@@ -6,16 +6,16 @@ require "uri"
|
|
|
6
6
|
require "json"
|
|
7
7
|
require 'builder' # Add builder gem for XML generation
|
|
8
8
|
|
|
9
|
-
module
|
|
9
|
+
module Etl
|
|
10
10
|
module Integrations
|
|
11
11
|
module Destination
|
|
12
12
|
module Tally
|
|
13
|
-
include
|
|
13
|
+
include Etl::Integrations::Core
|
|
14
14
|
|
|
15
15
|
API_VERSION = "1.0"
|
|
16
16
|
|
|
17
17
|
class Client < DestinationConnector
|
|
18
|
-
prepend
|
|
18
|
+
prepend Etl::Integrations::Core::RateLimiter
|
|
19
19
|
|
|
20
20
|
def check_connection(connection_config)
|
|
21
21
|
connection_config = connection_config.with_indifferent_access
|
|
@@ -136,13 +136,13 @@ module Multiwoven
|
|
|
136
136
|
end
|
|
137
137
|
|
|
138
138
|
def tracking_message(success, failure)
|
|
139
|
-
|
|
139
|
+
Etl::Integrations::Protocol::TrackingMessage.new(
|
|
140
140
|
success: success, failed: failure
|
|
141
141
|
).to_multiwoven_message
|
|
142
142
|
end
|
|
143
143
|
|
|
144
144
|
def log_debug(message)
|
|
145
|
-
|
|
145
|
+
Etl::Integrations::Service.logger.debug(message)
|
|
146
146
|
end
|
|
147
147
|
end
|
|
148
148
|
end
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
3
|
-
"title": "
|
|
3
|
+
"title": "Etl Protocol",
|
|
4
4
|
"type": "object",
|
|
5
|
-
"description": "
|
|
5
|
+
"description": "Etl protocol schema",
|
|
6
6
|
"version": "1.0.0",
|
|
7
7
|
"definitions": {
|
|
8
8
|
"SyncMode": {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
module
|
|
3
|
+
module Etl
|
|
4
4
|
module Integrations::Protocol
|
|
5
5
|
module Types
|
|
6
6
|
include Dry.Types()
|
|
@@ -28,7 +28,7 @@ module Multiwoven
|
|
|
28
28
|
SchemaMode = Types::String.enum("schema", "schemaless")
|
|
29
29
|
|
|
30
30
|
class ProtocolModel < Dry::Struct
|
|
31
|
-
extend
|
|
31
|
+
extend Etl::Integrations::Core::Utils
|
|
32
32
|
class << self
|
|
33
33
|
def from_json(json_string)
|
|
34
34
|
data = JSON.parse(json_string)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
module
|
|
3
|
+
module Etl
|
|
4
4
|
module Integrations
|
|
5
5
|
class Service
|
|
6
6
|
def initialize
|
|
@@ -21,12 +21,12 @@ module Multiwoven
|
|
|
21
21
|
def connector_class(connector_type, connector_name)
|
|
22
22
|
# *args
|
|
23
23
|
Object.const_get(
|
|
24
|
-
"
|
|
24
|
+
"Etl::Integrations::#{connector_type}::#{connector_name}::Client"
|
|
25
25
|
)
|
|
26
26
|
|
|
27
27
|
# # Retrieve the class dynamically
|
|
28
28
|
# klass = Object.const_get(
|
|
29
|
-
# "
|
|
29
|
+
# "Etl::Integrations::#{connector_type}::#{connector_name}::Client"
|
|
30
30
|
# )
|
|
31
31
|
|
|
32
32
|
# # Instantiate the class (assuming it's not a singleton)
|
|
@@ -2,9 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
require "google/cloud/bigquery"
|
|
4
4
|
|
|
5
|
-
module
|
|
5
|
+
module Etl::Integrations::Source
|
|
6
6
|
module Bigquery
|
|
7
|
-
include
|
|
7
|
+
include Etl::Integrations::Core
|
|
8
8
|
class Client < SourceConnector
|
|
9
9
|
def check_connection(connection_config)
|
|
10
10
|
connection_config = connection_config.with_indifferent_access
|
|
@@ -81,7 +81,7 @@ module Multiwoven::Integrations::Source
|
|
|
81
81
|
|
|
82
82
|
def create_streams(records)
|
|
83
83
|
group_by_table(records).map do |r|
|
|
84
|
-
|
|
84
|
+
Etl::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
|
|
85
85
|
end
|
|
86
86
|
end
|
|
87
87
|
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
module
|
|
3
|
+
module Etl::Integrations::Source
|
|
4
4
|
module Databricks
|
|
5
|
-
include
|
|
5
|
+
include Etl::Integrations::Core
|
|
6
6
|
class Client < SourceConnector
|
|
7
7
|
def check_connection(connection_config)
|
|
8
8
|
connection_config = connection_config.with_indifferent_access
|
|
@@ -81,7 +81,7 @@ module Multiwoven::Integrations::Source
|
|
|
81
81
|
|
|
82
82
|
def create_streams(records)
|
|
83
83
|
group_by_table(records).map do |r|
|
|
84
|
-
|
|
84
|
+
Etl::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
|
|
85
85
|
end
|
|
86
86
|
end
|
|
87
87
|
|
|
@@ -2,9 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
require "pg"
|
|
4
4
|
|
|
5
|
-
module
|
|
5
|
+
module Etl::Integrations::Source
|
|
6
6
|
module Postgresql
|
|
7
|
-
include
|
|
7
|
+
include Etl::Integrations::Core
|
|
8
8
|
class Client < SourceConnector
|
|
9
9
|
def check_connection(connection_config)
|
|
10
10
|
connection_config = connection_config.with_indifferent_access
|
|
@@ -86,7 +86,7 @@ module Multiwoven::Integrations::Source
|
|
|
86
86
|
|
|
87
87
|
def create_streams(records)
|
|
88
88
|
group_by_table(records).map do |r|
|
|
89
|
-
|
|
89
|
+
Etl::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
|
|
90
90
|
end
|
|
91
91
|
end
|
|
92
92
|
|
|
@@ -2,9 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
require "pg"
|
|
4
4
|
|
|
5
|
-
module
|
|
5
|
+
module Etl::Integrations::Source
|
|
6
6
|
module Redshift
|
|
7
|
-
include
|
|
7
|
+
include Etl::Integrations::Core
|
|
8
8
|
class Client < SourceConnector
|
|
9
9
|
def check_connection(connection_config)
|
|
10
10
|
connection_config = connection_config.with_indifferent_access
|
|
@@ -86,7 +86,7 @@ module Multiwoven::Integrations::Source
|
|
|
86
86
|
|
|
87
87
|
def create_streams(records)
|
|
88
88
|
group_by_table(records).map do |r|
|
|
89
|
-
|
|
89
|
+
Etl::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
|
|
90
90
|
end
|
|
91
91
|
end
|
|
92
92
|
|
|
@@ -3,17 +3,17 @@
|
|
|
3
3
|
require "stringio"
|
|
4
4
|
require_relative "schema_helper"
|
|
5
5
|
|
|
6
|
-
module
|
|
6
|
+
module Etl
|
|
7
7
|
module Integrations
|
|
8
8
|
module Source
|
|
9
9
|
module SalesforceConsumerGoodsCloud
|
|
10
|
-
include
|
|
10
|
+
include Etl::Integrations::Core
|
|
11
11
|
|
|
12
12
|
API_VERSION = "59.0"
|
|
13
13
|
SALESFORCE_OBJECTS = %w[Account User Visit RetailStore RecordType].freeze
|
|
14
14
|
|
|
15
15
|
class Client < SourceConnector
|
|
16
|
-
prepend
|
|
16
|
+
prepend Etl::Integrations::Core::RateLimiter
|
|
17
17
|
def check_connection(connection_config)
|
|
18
18
|
connection_config = connection_config.with_indifferent_access
|
|
19
19
|
initialize_client(connection_config)
|
|
@@ -108,13 +108,13 @@ module Multiwoven
|
|
|
108
108
|
end
|
|
109
109
|
|
|
110
110
|
def tracking_message(success, failure)
|
|
111
|
-
|
|
111
|
+
Etl::Integrations::Protocol::TrackingMessage.new(
|
|
112
112
|
success: success, failed: failure
|
|
113
113
|
).to_multiwoven_message
|
|
114
114
|
end
|
|
115
115
|
|
|
116
116
|
def log_debug(message)
|
|
117
|
-
|
|
117
|
+
Etl::Integrations::Service.logger.debug(message)
|
|
118
118
|
end
|
|
119
119
|
end
|
|
120
120
|
end
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
module
|
|
3
|
+
module Etl::Integrations::Source
|
|
4
4
|
module Snowflake
|
|
5
|
-
include
|
|
5
|
+
include Etl::Integrations::Core
|
|
6
6
|
class Client < SourceConnector
|
|
7
7
|
def check_connection(connection_config)
|
|
8
8
|
connection_config = connection_config.with_indifferent_access
|
|
@@ -75,7 +75,7 @@ module Multiwoven::Integrations::Source
|
|
|
75
75
|
|
|
76
76
|
def create_streams(records)
|
|
77
77
|
group_by_table(records).map do |r|
|
|
78
|
-
|
|
78
|
+
Etl::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
|
|
79
79
|
end
|
|
80
80
|
end
|
|
81
81
|
|
|
@@ -3,14 +3,14 @@
|
|
|
3
3
|
# require "zoho_books" # or the correct gem for Zoho Books
|
|
4
4
|
require "stringio"
|
|
5
5
|
|
|
6
|
-
module
|
|
6
|
+
module Etl
|
|
7
7
|
module Integrations
|
|
8
8
|
module Source
|
|
9
9
|
module ZohoBooks
|
|
10
|
-
include
|
|
10
|
+
include Etl::Integrations::Core
|
|
11
11
|
|
|
12
12
|
class Client < DestinationConnector
|
|
13
|
-
prepend
|
|
13
|
+
prepend Etl::Integrations::Core::RateLimiter
|
|
14
14
|
|
|
15
15
|
def check_connection(connection_config)
|
|
16
16
|
connection_config = connection_config.with_indifferent_access
|
|
@@ -140,13 +140,13 @@ module Multiwoven
|
|
|
140
140
|
end
|
|
141
141
|
|
|
142
142
|
def tracking_message(success, failure)
|
|
143
|
-
|
|
143
|
+
Etl::Integrations::Protocol::TrackingMessage.new(
|
|
144
144
|
success: success, failed: failure
|
|
145
145
|
).to_multiwoven_message
|
|
146
146
|
end
|
|
147
147
|
|
|
148
148
|
def log_debug(message)
|
|
149
|
-
|
|
149
|
+
Etl::Integrations::Service.logger.debug(message)
|
|
150
150
|
end
|
|
151
151
|
end
|
|
152
152
|
end
|
|
@@ -63,7 +63,7 @@ require_relative "integrations/destination/sftp/client"
|
|
|
63
63
|
require_relative "integrations/destination/postgresql/client"
|
|
64
64
|
require_relative "integrations/destination/tally/client"
|
|
65
65
|
|
|
66
|
-
module
|
|
66
|
+
module Etl
|
|
67
67
|
module Integrations
|
|
68
68
|
class Error < StandardError; end
|
|
69
69
|
# Your code goes here...
|