multiwoven-integrations 0.1.70 → 0.1.72
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +9 -0
- data/lib/multiwoven/integrations/core/utils.rb +10 -6
- data/lib/multiwoven/integrations/destination/airtable/client.rb +17 -4
- data/lib/multiwoven/integrations/destination/facebook_custom_audience/client.rb +17 -8
- data/lib/multiwoven/integrations/destination/google_sheets/client.rb +17 -5
- data/lib/multiwoven/integrations/destination/http/client.rb +20 -8
- data/lib/multiwoven/integrations/destination/http/config/meta.json +1 -1
- data/lib/multiwoven/integrations/destination/hubspot/client.rb +21 -4
- data/lib/multiwoven/integrations/destination/klaviyo/client.rb +16 -13
- data/lib/multiwoven/integrations/destination/postgresql/client.rb +17 -11
- data/lib/multiwoven/integrations/destination/salesforce_consumer_goods_cloud/client.rb +19 -6
- data/lib/multiwoven/integrations/destination/salesforce_consumer_goods_cloud/schema_helper.rb +1 -1
- data/lib/multiwoven/integrations/destination/salesforce_crm/client.rb +18 -3
- data/lib/multiwoven/integrations/destination/sftp/client.rb +30 -14
- data/lib/multiwoven/integrations/destination/slack/client.rb +17 -4
- data/lib/multiwoven/integrations/destination/stripe/client.rb +17 -3
- data/lib/multiwoven/integrations/destination/zendesk/client.rb +21 -4
- data/lib/multiwoven/integrations/protocol/protocol.rb +3 -3
- data/lib/multiwoven/integrations/rollout.rb +1 -1
- data/lib/multiwoven/integrations/source/aws_athena/client.rb +10 -10
- data/lib/multiwoven/integrations/source/bigquery/client.rb +10 -10
- data/lib/multiwoven/integrations/source/clickhouse/client.rb +10 -10
- data/lib/multiwoven/integrations/source/databricks/client.rb +10 -10
- data/lib/multiwoven/integrations/source/postgresql/client.rb +10 -10
- data/lib/multiwoven/integrations/source/redshift/client.rb +10 -10
- data/lib/multiwoven/integrations/source/salesforce_consumer_goods_cloud/client.rb +10 -2
- data/lib/multiwoven/integrations/source/salesforce_consumer_goods_cloud/schema_helper.rb +1 -1
- data/lib/multiwoven/integrations/source/snowflake/client.rb +10 -10
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 145e9dbf372280297517b8ca9ae1e7aa5b36807d3e22b7935565a150e952498e
|
|
4
|
+
data.tar.gz: dabf94ddf1c14e138a5c07378484aaa5bb58194cd58532eb70ad2cdd7d9e0361
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: c26f8408f8e525c7c607ecc5d78a31ccde1ff6e518e30859c2247622c05e12d681fde3d863bb59c6cec29494e242f7a08cdde4dc59e334e3d2fc5f4090f2ceae
|
|
7
|
+
data.tar.gz: 16072217e59ce1af0e019dde252282515219420a3428fa57a0e0fdce51d754602bdbb6c657fd1030650f29fd401f4e4deffd210f28bd1e02e4354e02ce8fff80
|
data/.rubocop.yml
CHANGED
|
@@ -48,9 +48,9 @@ module Multiwoven
|
|
|
48
48
|
Integrations::Service.logger
|
|
49
49
|
end
|
|
50
50
|
|
|
51
|
-
def report_exception(exception)
|
|
51
|
+
def report_exception(exception, meta = {})
|
|
52
52
|
reporter = Integrations::Service.exception_reporter
|
|
53
|
-
reporter&.report(exception)
|
|
53
|
+
reporter&.report(exception, meta)
|
|
54
54
|
end
|
|
55
55
|
|
|
56
56
|
def create_log_message(context, type, exception)
|
|
@@ -61,12 +61,16 @@ module Multiwoven
|
|
|
61
61
|
).to_multiwoven_message
|
|
62
62
|
end
|
|
63
63
|
|
|
64
|
-
def handle_exception(
|
|
64
|
+
def handle_exception(exception, meta = {})
|
|
65
65
|
logger.error(
|
|
66
|
-
"#{
|
|
66
|
+
"#{hash_to_string(meta)}: #{exception.message}"
|
|
67
67
|
)
|
|
68
|
-
report_exception(exception)
|
|
69
|
-
create_log_message(context, type, exception)
|
|
68
|
+
report_exception(exception, meta)
|
|
69
|
+
create_log_message(meta[:context], meta[:type], exception)
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def hash_to_string(hash)
|
|
73
|
+
hash.map { |key, value| "#{key} = #{value}" }.join(", ")
|
|
70
74
|
end
|
|
71
75
|
|
|
72
76
|
def extract_data(record_object, properties)
|
|
@@ -6,7 +6,7 @@ module Multiwoven
|
|
|
6
6
|
module Destination
|
|
7
7
|
module Airtable
|
|
8
8
|
include Multiwoven::Integrations::Core
|
|
9
|
-
class Client < DestinationConnector
|
|
9
|
+
class Client < DestinationConnector
|
|
10
10
|
prepend Multiwoven::Integrations::Core::RateLimiter
|
|
11
11
|
MAX_CHUNK_SIZE = 10
|
|
12
12
|
def check_connection(connection_config)
|
|
@@ -49,7 +49,10 @@ module Multiwoven
|
|
|
49
49
|
catalog = build_catalog_from_schema(extract_body(schema), base_id, base_name)
|
|
50
50
|
catalog.to_multiwoven_message
|
|
51
51
|
rescue StandardError => e
|
|
52
|
-
handle_exception(
|
|
52
|
+
handle_exception(e, {
|
|
53
|
+
context: "AIRTABLE:DISCOVER:EXCEPTION",
|
|
54
|
+
type: "error"
|
|
55
|
+
})
|
|
53
56
|
end
|
|
54
57
|
|
|
55
58
|
def write(sync_config, records, _action = "create")
|
|
@@ -72,7 +75,12 @@ module Multiwoven
|
|
|
72
75
|
write_failure += chunk.size
|
|
73
76
|
end
|
|
74
77
|
rescue StandardError => e
|
|
75
|
-
handle_exception(
|
|
78
|
+
handle_exception(e, {
|
|
79
|
+
context: "AIRTABLE:RECORD:WRITE:EXCEPTION",
|
|
80
|
+
type: "error",
|
|
81
|
+
sync_id: sync_config.sync_id,
|
|
82
|
+
sync_run_id: sync_config.sync_run_id
|
|
83
|
+
})
|
|
76
84
|
write_failure += chunk.size
|
|
77
85
|
end
|
|
78
86
|
|
|
@@ -82,7 +90,12 @@ module Multiwoven
|
|
|
82
90
|
)
|
|
83
91
|
tracker.to_multiwoven_message
|
|
84
92
|
rescue StandardError => e
|
|
85
|
-
handle_exception(
|
|
93
|
+
handle_exception(e, {
|
|
94
|
+
context: "AIRTABLE:RECORD:WRITE:EXCEPTION",
|
|
95
|
+
type: "error",
|
|
96
|
+
sync_id: sync_config.sync_id,
|
|
97
|
+
sync_run_id: sync_config.sync_run_id
|
|
98
|
+
})
|
|
86
99
|
end
|
|
87
100
|
|
|
88
101
|
private
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
module Multiwoven::Integrations::Destination
|
|
4
4
|
module FacebookCustomAudience
|
|
5
5
|
include Multiwoven::Integrations::Core
|
|
6
|
-
class Client < DestinationConnector
|
|
6
|
+
class Client < DestinationConnector
|
|
7
7
|
prepend Multiwoven::Integrations::Core::RateLimiter
|
|
8
8
|
MAX_CHUNK_SIZE = 10_000
|
|
9
9
|
def check_connection(connection_config)
|
|
@@ -29,11 +29,10 @@ module Multiwoven::Integrations::Destination
|
|
|
29
29
|
catalog = build_catalog(catalog_json)
|
|
30
30
|
catalog.to_multiwoven_message
|
|
31
31
|
rescue StandardError => e
|
|
32
|
-
handle_exception(
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
)
|
|
32
|
+
handle_exception(e, {
|
|
33
|
+
context: "FACEBOOK AUDIENCE:DISCOVER:EXCEPTION",
|
|
34
|
+
type: "error"
|
|
35
|
+
})
|
|
37
36
|
end
|
|
38
37
|
|
|
39
38
|
def write(sync_config, records, _action = "insert")
|
|
@@ -56,7 +55,12 @@ module Multiwoven::Integrations::Destination
|
|
|
56
55
|
write_failure += chunk.size
|
|
57
56
|
end
|
|
58
57
|
rescue StandardError => e
|
|
59
|
-
handle_exception(
|
|
58
|
+
handle_exception(e, {
|
|
59
|
+
context: "FACEBOOK:RECORD:WRITE:EXCEPTION",
|
|
60
|
+
type: "error",
|
|
61
|
+
sync_id: sync_config.sync_id,
|
|
62
|
+
sync_run_id: sync_config.sync_run_id
|
|
63
|
+
})
|
|
60
64
|
write_failure += chunk.size
|
|
61
65
|
end
|
|
62
66
|
|
|
@@ -66,7 +70,12 @@ module Multiwoven::Integrations::Destination
|
|
|
66
70
|
)
|
|
67
71
|
tracker.to_multiwoven_message
|
|
68
72
|
rescue StandardError => e
|
|
69
|
-
handle_exception(
|
|
73
|
+
handle_exception(e, {
|
|
74
|
+
context: "FACEBOOK:RECORD:WRITE:EXCEPTION",
|
|
75
|
+
type: "error",
|
|
76
|
+
sync_id: sync_config.sync_id,
|
|
77
|
+
sync_run_id: sync_config.sync_run_id
|
|
78
|
+
})
|
|
70
79
|
end
|
|
71
80
|
|
|
72
81
|
private
|
|
@@ -6,7 +6,7 @@ module Multiwoven
|
|
|
6
6
|
module GoogleSheets
|
|
7
7
|
include Multiwoven::Integrations::Core
|
|
8
8
|
|
|
9
|
-
class Client < DestinationConnector
|
|
9
|
+
class Client < DestinationConnector
|
|
10
10
|
prepend Multiwoven::Integrations::Core::Fullrefresher
|
|
11
11
|
prepend Multiwoven::Integrations::Core::RateLimiter
|
|
12
12
|
MAX_CHUNK_SIZE = 10_000
|
|
@@ -17,7 +17,6 @@ module Multiwoven
|
|
|
17
17
|
fetch_google_spread_sheets(connection_config)
|
|
18
18
|
success_status
|
|
19
19
|
rescue StandardError => e
|
|
20
|
-
handle_exception("GOOGLE_SHEETS:CRM:DISCOVER:EXCEPTION", "error", e)
|
|
21
20
|
failure_status(e)
|
|
22
21
|
end
|
|
23
22
|
|
|
@@ -28,14 +27,22 @@ module Multiwoven
|
|
|
28
27
|
catalog = build_catalog_from_spreadsheets(spreadsheets, connection_config)
|
|
29
28
|
catalog.to_multiwoven_message
|
|
30
29
|
rescue StandardError => e
|
|
31
|
-
handle_exception(
|
|
30
|
+
handle_exception(e, {
|
|
31
|
+
context: "GOOGLE_SHEETS:CRM:DISCOVER:EXCEPTION",
|
|
32
|
+
type: "error"
|
|
33
|
+
})
|
|
32
34
|
end
|
|
33
35
|
|
|
34
36
|
def write(sync_config, records, action = "create")
|
|
35
37
|
setup_write_environment(sync_config, action)
|
|
36
38
|
process_record_chunks(records, sync_config)
|
|
37
39
|
rescue StandardError => e
|
|
38
|
-
handle_exception(
|
|
40
|
+
handle_exception(e, {
|
|
41
|
+
context: "GOOGLE_SHEETS:CRM:WRITE:EXCEPTION",
|
|
42
|
+
type: "error",
|
|
43
|
+
sync_id: sync_config.sync_id,
|
|
44
|
+
sync_run_id: sync_config.sync_run_id
|
|
45
|
+
})
|
|
39
46
|
end
|
|
40
47
|
|
|
41
48
|
def clear_all_records(sync_config)
|
|
@@ -154,7 +161,12 @@ module Multiwoven
|
|
|
154
161
|
update_sheet_values(values, sync_config.stream.name)
|
|
155
162
|
write_success += values.size
|
|
156
163
|
rescue StandardError => e
|
|
157
|
-
handle_exception(
|
|
164
|
+
handle_exception(e, {
|
|
165
|
+
context: "GOOGLE_SHEETS:RECORD:WRITE:EXCEPTION",
|
|
166
|
+
type: "error",
|
|
167
|
+
sync_id: sync_config.sync_id,
|
|
168
|
+
sync_run_id: sync_config.sync_run_id
|
|
169
|
+
})
|
|
158
170
|
write_failure += chunk.size
|
|
159
171
|
end
|
|
160
172
|
|
|
@@ -23,7 +23,10 @@ module Multiwoven
|
|
|
23
23
|
failure_status(nil)
|
|
24
24
|
end
|
|
25
25
|
rescue StandardError => e
|
|
26
|
-
handle_exception(
|
|
26
|
+
handle_exception(e, {
|
|
27
|
+
context: "HTTP:CHECK_CONNECTION:EXCEPTION",
|
|
28
|
+
type: "error"
|
|
29
|
+
})
|
|
27
30
|
failure_status(e)
|
|
28
31
|
end
|
|
29
32
|
|
|
@@ -32,11 +35,10 @@ module Multiwoven
|
|
|
32
35
|
catalog = build_catalog(catalog_json)
|
|
33
36
|
catalog.to_multiwoven_message
|
|
34
37
|
rescue StandardError => e
|
|
35
|
-
handle_exception(
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
)
|
|
38
|
+
handle_exception(e, {
|
|
39
|
+
context: "HTTP:DISCOVER:EXCEPTION",
|
|
40
|
+
type: "error"
|
|
41
|
+
})
|
|
40
42
|
end
|
|
41
43
|
|
|
42
44
|
def write(sync_config, records, _action = "create")
|
|
@@ -59,7 +61,12 @@ module Multiwoven
|
|
|
59
61
|
write_failure += chunk.size
|
|
60
62
|
end
|
|
61
63
|
rescue StandardError => e
|
|
62
|
-
handle_exception(
|
|
64
|
+
handle_exception(e, {
|
|
65
|
+
context: "HTTP:RECORD:WRITE:EXCEPTION",
|
|
66
|
+
type: "error",
|
|
67
|
+
sync_id: sync_config.sync_id,
|
|
68
|
+
sync_run_id: sync_config.sync_run_id
|
|
69
|
+
})
|
|
63
70
|
write_failure += chunk.size
|
|
64
71
|
end
|
|
65
72
|
|
|
@@ -69,7 +76,12 @@ module Multiwoven
|
|
|
69
76
|
)
|
|
70
77
|
tracker.to_multiwoven_message
|
|
71
78
|
rescue StandardError => e
|
|
72
|
-
handle_exception(
|
|
79
|
+
handle_exception(e, {
|
|
80
|
+
context: "HTTP:RECORD:WRITE:EXCEPTION",
|
|
81
|
+
type: "error",
|
|
82
|
+
sync_id: sync_config.sync_id,
|
|
83
|
+
sync_run_id: sync_config.sync_run_id
|
|
84
|
+
})
|
|
73
85
|
end
|
|
74
86
|
|
|
75
87
|
private
|
|
@@ -16,7 +16,10 @@ module Multiwoven
|
|
|
16
16
|
authenticate_client
|
|
17
17
|
success_status
|
|
18
18
|
rescue StandardError => e
|
|
19
|
-
handle_exception(
|
|
19
|
+
handle_exception(e, {
|
|
20
|
+
context: "HUBSPOT:CRM:CHECK_CONNECTION:EXCEPTION",
|
|
21
|
+
type: "error"
|
|
22
|
+
})
|
|
20
23
|
failure_status(e)
|
|
21
24
|
end
|
|
22
25
|
|
|
@@ -24,15 +27,24 @@ module Multiwoven
|
|
|
24
27
|
catalog = build_catalog(load_catalog)
|
|
25
28
|
catalog.to_multiwoven_message
|
|
26
29
|
rescue StandardError => e
|
|
27
|
-
handle_exception(
|
|
30
|
+
handle_exception(e, {
|
|
31
|
+
context: "HUBSPOT:CRM:DISCOVER:EXCEPTION",
|
|
32
|
+
type: "error"
|
|
33
|
+
})
|
|
28
34
|
end
|
|
29
35
|
|
|
30
36
|
def write(sync_config, records, action = "create")
|
|
31
37
|
@action = sync_config.stream.action || action
|
|
38
|
+
@sync_config = sync_config
|
|
32
39
|
initialize_client(sync_config.destination.connection_specification)
|
|
33
40
|
process_records(records, sync_config.stream)
|
|
34
41
|
rescue StandardError => e
|
|
35
|
-
handle_exception(
|
|
42
|
+
handle_exception(e, {
|
|
43
|
+
context: "HUBSPOT:CRM:WRITE:EXCEPTION",
|
|
44
|
+
type: "error",
|
|
45
|
+
sync_id: @sync_config.sync_id,
|
|
46
|
+
sync_run_id: @sync_config.sync_run_id
|
|
47
|
+
})
|
|
36
48
|
end
|
|
37
49
|
|
|
38
50
|
private
|
|
@@ -51,7 +63,12 @@ module Multiwoven
|
|
|
51
63
|
send_data_to_hubspot(stream.name, record)
|
|
52
64
|
write_success += 1
|
|
53
65
|
rescue StandardError => e
|
|
54
|
-
handle_exception(
|
|
66
|
+
handle_exception(e, {
|
|
67
|
+
context: "HUBSPOT:CRM:WRITE:EXCEPTION",
|
|
68
|
+
type: "error",
|
|
69
|
+
sync_id: @sync_config.sync_id,
|
|
70
|
+
sync_run_id: @sync_config.sync_run_id
|
|
71
|
+
})
|
|
55
72
|
write_failure += 1
|
|
56
73
|
end
|
|
57
74
|
tracking_message(write_success, write_failure)
|
|
@@ -25,11 +25,10 @@ module Multiwoven::Integrations::Destination
|
|
|
25
25
|
|
|
26
26
|
catalog.to_multiwoven_message
|
|
27
27
|
rescue StandardError => e
|
|
28
|
-
handle_exception(
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
)
|
|
28
|
+
handle_exception(e, {
|
|
29
|
+
context: "KLAVIYO:DISCOVER:EXCEPTION",
|
|
30
|
+
type: "error"
|
|
31
|
+
})
|
|
33
32
|
end
|
|
34
33
|
|
|
35
34
|
def write(sync_config, records, _action = "insert")
|
|
@@ -59,9 +58,12 @@ module Multiwoven::Integrations::Destination
|
|
|
59
58
|
write_failure += 1
|
|
60
59
|
end
|
|
61
60
|
rescue StandardError => e
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
61
|
+
handle_exception(e, {
|
|
62
|
+
context: "KLAVIYO:RECORD:WRITE:FAILURE",
|
|
63
|
+
type: "error",
|
|
64
|
+
sync_id: sync_config.sync_id,
|
|
65
|
+
sync_run_id: sync_config.sync_run_id
|
|
66
|
+
})
|
|
65
67
|
write_failure += 1
|
|
66
68
|
end
|
|
67
69
|
tracker = Multiwoven::Integrations::Protocol::TrackingMessage.new(
|
|
@@ -71,11 +73,12 @@ module Multiwoven::Integrations::Destination
|
|
|
71
73
|
tracker.to_multiwoven_message
|
|
72
74
|
rescue StandardError => e
|
|
73
75
|
# TODO: Handle rate limiting seperately
|
|
74
|
-
handle_exception(
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
76
|
+
handle_exception(e, {
|
|
77
|
+
context: "KLAVIYO:RECORD:WRITE:FAILURE",
|
|
78
|
+
type: "error",
|
|
79
|
+
sync_id: sync_config.sync_id,
|
|
80
|
+
sync_run_id: sync_config.sync_run_id
|
|
81
|
+
})
|
|
79
82
|
end
|
|
80
83
|
|
|
81
84
|
private
|
|
@@ -34,11 +34,10 @@ module Multiwoven::Integrations::Destination
|
|
|
34
34
|
catalog = Catalog.new(streams: create_streams(records))
|
|
35
35
|
catalog.to_multiwoven_message
|
|
36
36
|
rescue StandardError => e
|
|
37
|
-
handle_exception(
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
)
|
|
37
|
+
handle_exception(e, {
|
|
38
|
+
context: "POSTGRESQL:DISCOVER:EXCEPTION",
|
|
39
|
+
type: "error"
|
|
40
|
+
})
|
|
42
41
|
ensure
|
|
43
42
|
db&.close
|
|
44
43
|
end
|
|
@@ -53,21 +52,28 @@ module Multiwoven::Integrations::Destination
|
|
|
53
52
|
|
|
54
53
|
records.each do |record|
|
|
55
54
|
query = Multiwoven::Integrations::Core::QueryBuilder.perform(action, table_name, record)
|
|
55
|
+
logger.debug("POSTGRESQL:WRITE:QUERY query = #{query} sync_id = #{sync_config.sync_id} sync_run_id = #{sync_config.sync_run_id}")
|
|
56
56
|
begin
|
|
57
57
|
db.exec(query)
|
|
58
58
|
write_success += 1
|
|
59
59
|
rescue StandardError => e
|
|
60
|
-
handle_exception(
|
|
60
|
+
handle_exception(e, {
|
|
61
|
+
context: "POSTGRESQL:RECORD:WRITE:EXCEPTION",
|
|
62
|
+
type: "error",
|
|
63
|
+
sync_id: sync_config.sync_id,
|
|
64
|
+
sync_run_id: sync_config.sync_run_id
|
|
65
|
+
})
|
|
61
66
|
write_failure += 1
|
|
62
67
|
end
|
|
63
68
|
end
|
|
64
69
|
tracking_message(write_success, write_failure)
|
|
65
70
|
rescue StandardError => e
|
|
66
|
-
handle_exception(
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
+
handle_exception(e, {
|
|
72
|
+
context: "POSTGRESQL:RECORD:WRITE:EXCEPTION",
|
|
73
|
+
type: "error",
|
|
74
|
+
sync_id: sync_config.sync_id,
|
|
75
|
+
sync_run_id: sync_config.sync_run_id
|
|
76
|
+
})
|
|
71
77
|
end
|
|
72
78
|
|
|
73
79
|
private
|
|
@@ -44,17 +44,24 @@ module Multiwoven
|
|
|
44
44
|
end
|
|
45
45
|
catalog.to_multiwoven_message
|
|
46
46
|
rescue StandardError => e
|
|
47
|
-
handle_exception(
|
|
47
|
+
handle_exception(e, {
|
|
48
|
+
context: "SALESFORCE:CONSUMER:GOODS:ClOUD:DISCOVER:EXCEPTION",
|
|
49
|
+
type: "error"
|
|
50
|
+
})
|
|
48
51
|
end
|
|
49
52
|
|
|
50
53
|
def write(sync_config, records, action = "create")
|
|
51
54
|
@action = sync_config.stream.action || action
|
|
52
|
-
@
|
|
53
|
-
@sync_run_id = sync_config.sync_run_id
|
|
55
|
+
@sync_config = sync_config
|
|
54
56
|
initialize_client(sync_config.destination.connection_specification)
|
|
55
57
|
process_records(records, sync_config.stream)
|
|
56
58
|
rescue StandardError => e
|
|
57
|
-
handle_exception(
|
|
59
|
+
handle_exception(e, {
|
|
60
|
+
context: "SALESFORCE:CONSUMER:GOODS:ClOUD:WRITE:EXCEPTION",
|
|
61
|
+
type: "error",
|
|
62
|
+
sync_id: @sync_config.sync_id,
|
|
63
|
+
sync_run_id: @sync_config.sync_run_id
|
|
64
|
+
})
|
|
58
65
|
end
|
|
59
66
|
|
|
60
67
|
private
|
|
@@ -78,7 +85,13 @@ module Multiwoven
|
|
|
78
85
|
process_record(stream, record)
|
|
79
86
|
write_success += 1
|
|
80
87
|
rescue StandardError => e
|
|
81
|
-
|
|
88
|
+
# TODO: add sync_id and sync run id to the logs
|
|
89
|
+
handle_exception(e, {
|
|
90
|
+
context: "SALESFORCE:CONSUMER:GOODS:ClOUD:WRITE:EXCEPTION",
|
|
91
|
+
type: "error",
|
|
92
|
+
sync_id: @sync_config.sync_id,
|
|
93
|
+
sync_run_id: @sync_config.sync_run_id
|
|
94
|
+
})
|
|
82
95
|
write_failure += 1
|
|
83
96
|
end
|
|
84
97
|
tracking_message(write_success, write_failure)
|
|
@@ -91,7 +104,7 @@ module Multiwoven
|
|
|
91
104
|
def send_data_to_salesforce(stream_name, record = {})
|
|
92
105
|
method_name = "upsert!"
|
|
93
106
|
args = [stream_name, "Id", record]
|
|
94
|
-
@logger.debug("sync_id: #{@sync_id}, sync_run_id: #{@sync_run_id}, record: #{record}")
|
|
107
|
+
@logger.debug("sync_id: #{@sync_config.sync_id}, sync_run_id: #{@sync_config.sync_run_id}, record: #{record}")
|
|
95
108
|
@client.send(method_name, *args)
|
|
96
109
|
end
|
|
97
110
|
|
data/lib/multiwoven/integrations/destination/salesforce_consumer_goods_cloud/schema_helper.rb
CHANGED
|
@@ -9,7 +9,7 @@ module Multiwoven
|
|
|
9
9
|
|
|
10
10
|
module_function
|
|
11
11
|
|
|
12
|
-
def salesforce_field_to_json_schema_type(sf_field) # rubocop:disable Metrics/
|
|
12
|
+
def salesforce_field_to_json_schema_type(sf_field) # rubocop:disable Metrics/MethodLength, Metrics/PerceivedComplexity
|
|
13
13
|
case sf_field["type"]
|
|
14
14
|
when "string", "Email", "Phone", "Text", "TextArea", "TextEncrypted", "URL", "Picklist (Single)"
|
|
15
15
|
if sf_field["nillable"]
|
|
@@ -25,15 +25,24 @@ module Multiwoven
|
|
|
25
25
|
catalog = build_catalog(load_catalog)
|
|
26
26
|
catalog.to_multiwoven_message
|
|
27
27
|
rescue StandardError => e
|
|
28
|
-
handle_exception(
|
|
28
|
+
handle_exception(e, {
|
|
29
|
+
context: "SALESFORCE:CRM:DISCOVER:EXCEPTION",
|
|
30
|
+
type: "error"
|
|
31
|
+
})
|
|
29
32
|
end
|
|
30
33
|
|
|
31
34
|
def write(sync_config, records, action = "create")
|
|
32
35
|
@action = sync_config.stream.action || action
|
|
36
|
+
@sync_config = sync_config
|
|
33
37
|
initialize_client(sync_config.destination.connection_specification)
|
|
34
38
|
process_records(records, sync_config.stream)
|
|
35
39
|
rescue StandardError => e
|
|
36
|
-
handle_exception(
|
|
40
|
+
handle_exception(e, {
|
|
41
|
+
context: "SALESFORCE:CRM:WRITE:EXCEPTION",
|
|
42
|
+
type: "error",
|
|
43
|
+
sync_id: @sync_config.sync_id,
|
|
44
|
+
sync_run_id: @sync_config.sync_run_id
|
|
45
|
+
})
|
|
37
46
|
end
|
|
38
47
|
|
|
39
48
|
private
|
|
@@ -58,7 +67,13 @@ module Multiwoven
|
|
|
58
67
|
process_record(stream, record)
|
|
59
68
|
write_success += 1
|
|
60
69
|
rescue StandardError => e
|
|
61
|
-
|
|
70
|
+
# TODO: add sync_id and sync_run_id to the logs
|
|
71
|
+
handle_exception(e, {
|
|
72
|
+
context: "SALESFORCE:CRM:WRITE:EXCEPTION",
|
|
73
|
+
type: "error",
|
|
74
|
+
sync_id: @sync_config.sync_id,
|
|
75
|
+
sync_run_id: @sync_config.sync_run_id
|
|
76
|
+
})
|
|
62
77
|
write_failure += 1
|
|
63
78
|
end
|
|
64
79
|
tracking_message(write_success, write_failure)
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
module Multiwoven::Integrations::Destination
|
|
4
4
|
module Sftp
|
|
5
5
|
include Multiwoven::Integrations::Core
|
|
6
|
-
class Client < DestinationConnector
|
|
6
|
+
class Client < DestinationConnector
|
|
7
7
|
prepend Multiwoven::Integrations::Core::Fullrefresher
|
|
8
8
|
prepend Multiwoven::Integrations::Core::RateLimiter
|
|
9
9
|
|
|
@@ -16,7 +16,10 @@ module Multiwoven::Integrations::Destination
|
|
|
16
16
|
return success_status
|
|
17
17
|
end
|
|
18
18
|
rescue StandardError => e
|
|
19
|
-
handle_exception(
|
|
19
|
+
handle_exception(e, {
|
|
20
|
+
context: "SFTP:CHECK_CONNECTION:EXCEPTION",
|
|
21
|
+
type: "error"
|
|
22
|
+
})
|
|
20
23
|
failure_status(e)
|
|
21
24
|
end
|
|
22
25
|
|
|
@@ -27,14 +30,14 @@ module Multiwoven::Integrations::Destination
|
|
|
27
30
|
|
|
28
31
|
catalog.to_multiwoven_message
|
|
29
32
|
rescue StandardError => e
|
|
30
|
-
handle_exception(
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
)
|
|
33
|
+
handle_exception(e, {
|
|
34
|
+
context: "SFTP:DISCOVER:EXCEPTION",
|
|
35
|
+
type: "error"
|
|
36
|
+
})
|
|
35
37
|
end
|
|
36
38
|
|
|
37
39
|
def write(sync_config, records, _action = "insert")
|
|
40
|
+
@sync_config = sync_config
|
|
38
41
|
connection_config = sync_config.destination.connection_specification.with_indifferent_access
|
|
39
42
|
file_path = generate_file_path(sync_config)
|
|
40
43
|
local_file_name = generate_local_file_name(sync_config)
|
|
@@ -53,11 +56,12 @@ module Multiwoven::Integrations::Destination
|
|
|
53
56
|
write_failure = records.size - write_success
|
|
54
57
|
tracking_message(write_success, write_failure)
|
|
55
58
|
rescue StandardError => e
|
|
56
|
-
handle_exception(
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
59
|
+
handle_exception(e, {
|
|
60
|
+
context: "SFTP:WRITE:EXCEPTION",
|
|
61
|
+
type: "error",
|
|
62
|
+
sync_id: @sync_config.sync_id,
|
|
63
|
+
sync_run_id: @sync_config.sync_run_id
|
|
64
|
+
})
|
|
61
65
|
end
|
|
62
66
|
|
|
63
67
|
def write_compressed_data(connection_config, file_path, local_file_name, csv_content, records_size)
|
|
@@ -70,7 +74,13 @@ module Multiwoven::Integrations::Destination
|
|
|
70
74
|
sftp.upload!(tempfile.path, file_path)
|
|
71
75
|
write_success = records_size
|
|
72
76
|
rescue StandardError => e
|
|
73
|
-
|
|
77
|
+
# TODO: add sync_id and sync_run_id to the log
|
|
78
|
+
handle_exception(e, {
|
|
79
|
+
context: "SFTP:RECORD:WRITE:EXCEPTION",
|
|
80
|
+
type: "error",
|
|
81
|
+
sync_id: @sync_config.sync_id,
|
|
82
|
+
sync_run_id: @sync_config.sync_run_id
|
|
83
|
+
})
|
|
74
84
|
write_success = 0
|
|
75
85
|
end
|
|
76
86
|
end
|
|
@@ -86,7 +96,13 @@ module Multiwoven::Integrations::Destination
|
|
|
86
96
|
sftp.upload!(tempfile.path, file_path)
|
|
87
97
|
write_success = records_size
|
|
88
98
|
rescue StandardError => e
|
|
89
|
-
|
|
99
|
+
# TODO: add sync_id and sync_run_id to the log
|
|
100
|
+
handle_exception(e, {
|
|
101
|
+
context: "SFTP:RECORD:WRITE:EXCEPTION",
|
|
102
|
+
type: "error",
|
|
103
|
+
sync_id: @sync_config.sync_id,
|
|
104
|
+
sync_run_id: @sync_config.sync_run_id
|
|
105
|
+
})
|
|
90
106
|
write_success = 0
|
|
91
107
|
end
|
|
92
108
|
end
|
|
@@ -23,13 +23,16 @@ module Multiwoven
|
|
|
23
23
|
catalog = build_catalog(load_catalog)
|
|
24
24
|
catalog.to_multiwoven_message
|
|
25
25
|
rescue StandardError => e
|
|
26
|
-
handle_exception(
|
|
26
|
+
handle_exception(e, {
|
|
27
|
+
context: "SLACK:DISCOVER:EXCEPTION",
|
|
28
|
+
type: "error"
|
|
29
|
+
})
|
|
27
30
|
end
|
|
28
31
|
|
|
29
32
|
def write(sync_config, records, action = "create")
|
|
30
33
|
# Currently as we only create a message for each record in slack, we are not using actions.
|
|
31
34
|
# This will be changed in future.
|
|
32
|
-
|
|
35
|
+
@sync_config = sync_config
|
|
33
36
|
@action = sync_config.stream.action || action
|
|
34
37
|
connection_config = sync_config.destination.connection_specification.with_indifferent_access
|
|
35
38
|
configure_slack(connection_config[:api_token])
|
|
@@ -37,7 +40,12 @@ module Multiwoven
|
|
|
37
40
|
@channel_id = connection_config[:channel_id]
|
|
38
41
|
process_records(records, sync_config.stream)
|
|
39
42
|
rescue StandardError => e
|
|
40
|
-
handle_exception(
|
|
43
|
+
handle_exception(e, {
|
|
44
|
+
context: "SLACK:WRITE:EXCEPTION",
|
|
45
|
+
type: "error",
|
|
46
|
+
sync_id: @sync_config.sync_id,
|
|
47
|
+
sync_run_id: @sync_config.sync_run_id
|
|
48
|
+
})
|
|
41
49
|
end
|
|
42
50
|
|
|
43
51
|
private
|
|
@@ -56,7 +64,12 @@ module Multiwoven
|
|
|
56
64
|
write_success += 1
|
|
57
65
|
rescue StandardError => e
|
|
58
66
|
write_failure += 1
|
|
59
|
-
handle_exception(
|
|
67
|
+
handle_exception(e, {
|
|
68
|
+
context: "SLACK:WRITE:EXCEPTION",
|
|
69
|
+
type: "error",
|
|
70
|
+
sync_id: @sync_config.sync_id,
|
|
71
|
+
sync_run_id: @sync_config.sync_run_id
|
|
72
|
+
})
|
|
60
73
|
end
|
|
61
74
|
tracking_message(write_success, write_failure)
|
|
62
75
|
end
|
|
@@ -23,15 +23,24 @@ module Multiwoven
|
|
|
23
23
|
catalog = build_catalog(load_catalog)
|
|
24
24
|
catalog.to_multiwoven_message
|
|
25
25
|
rescue StandardError => e
|
|
26
|
-
handle_exception(
|
|
26
|
+
handle_exception(e, {
|
|
27
|
+
context: "STRIPE:CRM:DISCOVER:EXCEPTION",
|
|
28
|
+
type: "error"
|
|
29
|
+
})
|
|
27
30
|
end
|
|
28
31
|
|
|
29
32
|
def write(sync_config, records, action = "create")
|
|
33
|
+
@sync_config = sync_config
|
|
30
34
|
@action = sync_config.stream.action || action
|
|
31
35
|
initialize_client(sync_config.destination.connection_specification)
|
|
32
36
|
process_records(records, sync_config.stream)
|
|
33
37
|
rescue StandardError => e
|
|
34
|
-
handle_exception(
|
|
38
|
+
handle_exception(e, {
|
|
39
|
+
context: "STRIPE:CRM:WRITE:EXCEPTION",
|
|
40
|
+
type: "error",
|
|
41
|
+
sync_id: @sync_config.sync_id,
|
|
42
|
+
sync_run_id: @sync_config.sync_run_id
|
|
43
|
+
})
|
|
35
44
|
end
|
|
36
45
|
|
|
37
46
|
private
|
|
@@ -52,7 +61,12 @@ module Multiwoven
|
|
|
52
61
|
klass.send(@action, record)
|
|
53
62
|
write_success += 1
|
|
54
63
|
rescue StandardError => e
|
|
55
|
-
handle_exception(
|
|
64
|
+
handle_exception(e, {
|
|
65
|
+
context: "STRIPE:CRM:WRITE:EXCEPTION",
|
|
66
|
+
type: "error",
|
|
67
|
+
sync_id: @sync_config.sync_id,
|
|
68
|
+
sync_run_id: @sync_config.sync_run_id
|
|
69
|
+
})
|
|
56
70
|
write_failure += 1
|
|
57
71
|
end
|
|
58
72
|
tracking_message(write_success, write_failure)
|
|
@@ -13,7 +13,10 @@ module Multiwoven
|
|
|
13
13
|
authenticate_client
|
|
14
14
|
success_status
|
|
15
15
|
rescue StandardError => e
|
|
16
|
-
handle_exception(
|
|
16
|
+
handle_exception(e, {
|
|
17
|
+
context: "ZENDESK:CHECK_CONNECTION:EXCEPTION",
|
|
18
|
+
type: "error"
|
|
19
|
+
})
|
|
17
20
|
failure_status(e)
|
|
18
21
|
end
|
|
19
22
|
|
|
@@ -21,16 +24,25 @@ module Multiwoven
|
|
|
21
24
|
catalog = build_catalog(load_catalog)
|
|
22
25
|
catalog.to_multiwoven_message
|
|
23
26
|
rescue StandardError => e
|
|
24
|
-
handle_exception(
|
|
27
|
+
handle_exception(e, {
|
|
28
|
+
context: "ZENDESK:DISCOVER:EXCEPTION",
|
|
29
|
+
type: "error"
|
|
30
|
+
})
|
|
25
31
|
failure_status(e)
|
|
26
32
|
end
|
|
27
33
|
|
|
28
34
|
def write(sync_config, records, action = "create")
|
|
35
|
+
@sync_config = sync_config
|
|
29
36
|
@action = sync_config.stream.action || action
|
|
30
37
|
initialize_client(sync_config.destination.connection_specification)
|
|
31
38
|
process_records(records, sync_config.stream)
|
|
32
39
|
rescue StandardError => e
|
|
33
|
-
handle_exception(
|
|
40
|
+
handle_exception(e, {
|
|
41
|
+
context: "ZENDESK:WRITE:EXCEPTION",
|
|
42
|
+
type: "error",
|
|
43
|
+
sync_id: @sync_config.sync_id,
|
|
44
|
+
sync_run_id: @sync_config.sync_run_id
|
|
45
|
+
})
|
|
34
46
|
failure_status(e)
|
|
35
47
|
end
|
|
36
48
|
|
|
@@ -68,7 +80,12 @@ module Multiwoven
|
|
|
68
80
|
|
|
69
81
|
write_success += 1
|
|
70
82
|
rescue StandardError => e
|
|
71
|
-
handle_exception(
|
|
83
|
+
handle_exception(e, {
|
|
84
|
+
context: "ZENDESK:WRITE:EXCEPTION",
|
|
85
|
+
type: "error",
|
|
86
|
+
sync_id: @sync_config.sync_id,
|
|
87
|
+
sync_run_id: @sync_config.sync_run_id
|
|
88
|
+
})
|
|
72
89
|
write_failure += 1
|
|
73
90
|
end
|
|
74
91
|
|
|
@@ -165,7 +165,7 @@ module Multiwoven
|
|
|
165
165
|
end
|
|
166
166
|
|
|
167
167
|
class SyncConfig < ProtocolModel
|
|
168
|
-
attr_accessor :offset, :limit
|
|
168
|
+
attr_accessor :offset, :limit, :sync_run_id
|
|
169
169
|
|
|
170
170
|
attribute :source, Connector
|
|
171
171
|
attribute :destination, Connector
|
|
@@ -175,8 +175,8 @@ module Multiwoven
|
|
|
175
175
|
attribute? :cursor_field, Types::String.optional
|
|
176
176
|
attribute? :current_cursor_field, Types::String.optional
|
|
177
177
|
attribute :destination_sync_mode, DestinationSyncMode
|
|
178
|
-
|
|
179
|
-
attribute
|
|
178
|
+
# reference ids
|
|
179
|
+
attribute :sync_id, Types::String.default("unknown")
|
|
180
180
|
end
|
|
181
181
|
|
|
182
182
|
class ControlMessage < ProtocolModel
|
|
@@ -21,11 +21,10 @@ module Multiwoven::Integrations::Source
|
|
|
21
21
|
catalog = Catalog.new(streams: create_streams(results))
|
|
22
22
|
catalog.to_multiwoven_message
|
|
23
23
|
rescue StandardError => e
|
|
24
|
-
handle_exception(
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
)
|
|
24
|
+
handle_exception(e, {
|
|
25
|
+
context: "AWS:ATHENA:DISCOVER:EXCEPTION",
|
|
26
|
+
type: "error"
|
|
27
|
+
})
|
|
29
28
|
end
|
|
30
29
|
|
|
31
30
|
def read(sync_config)
|
|
@@ -36,11 +35,12 @@ module Multiwoven::Integrations::Source
|
|
|
36
35
|
db = create_connection(connection_config)
|
|
37
36
|
query(db, query)
|
|
38
37
|
rescue StandardError => e
|
|
39
|
-
handle_exception(
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
38
|
+
handle_exception(e, {
|
|
39
|
+
context: "AWS:ATHENA:READ:EXCEPTION",
|
|
40
|
+
type: "error",
|
|
41
|
+
sync_id: sync_config.sync_id,
|
|
42
|
+
sync_run_id: sync_config.sync_run_id
|
|
43
|
+
})
|
|
44
44
|
end
|
|
45
45
|
|
|
46
46
|
private
|
|
@@ -36,11 +36,10 @@ module Multiwoven::Integrations::Source
|
|
|
36
36
|
catalog = Catalog.new(streams: create_streams(records))
|
|
37
37
|
catalog.to_multiwoven_message
|
|
38
38
|
rescue StandardError => e
|
|
39
|
-
handle_exception(
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
)
|
|
39
|
+
handle_exception(e, {
|
|
40
|
+
context: "BIGQUERY:DISCOVER:EXCEPTION",
|
|
41
|
+
type: "error"
|
|
42
|
+
})
|
|
44
43
|
end
|
|
45
44
|
|
|
46
45
|
def read(sync_config)
|
|
@@ -54,11 +53,12 @@ module Multiwoven::Integrations::Source
|
|
|
54
53
|
|
|
55
54
|
query(bigquery, query)
|
|
56
55
|
rescue StandardError => e
|
|
57
|
-
handle_exception(
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
56
|
+
handle_exception(e, {
|
|
57
|
+
context: "BIGQUERY:READ:EXCEPTION",
|
|
58
|
+
type: "error",
|
|
59
|
+
sync_id: sync_config.sync_id,
|
|
60
|
+
sync_run_id: sync_config.sync_run_id
|
|
61
|
+
})
|
|
62
62
|
end
|
|
63
63
|
|
|
64
64
|
private
|
|
@@ -24,11 +24,10 @@ module Multiwoven::Integrations::Source
|
|
|
24
24
|
catalog = Catalog.new(streams: create_streams(records))
|
|
25
25
|
catalog.to_multiwoven_message
|
|
26
26
|
rescue StandardError => e
|
|
27
|
-
handle_exception(
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
)
|
|
27
|
+
handle_exception(e, {
|
|
28
|
+
context: "CLICKHOUSE:DISCOVER:EXCEPTION",
|
|
29
|
+
type: "error"
|
|
30
|
+
})
|
|
32
31
|
end
|
|
33
32
|
|
|
34
33
|
def read(sync_config)
|
|
@@ -39,11 +38,12 @@ module Multiwoven::Integrations::Source
|
|
|
39
38
|
db = create_connection(connection_config)
|
|
40
39
|
query(db, query)
|
|
41
40
|
rescue StandardError => e
|
|
42
|
-
handle_exception(
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
41
|
+
handle_exception(e, {
|
|
42
|
+
context: "CLICKHOUSE:READ:EXCEPTION",
|
|
43
|
+
type: "error",
|
|
44
|
+
sync_id: sync_config.sync_id,
|
|
45
|
+
sync_run_id: sync_config.sync_run_id
|
|
46
|
+
})
|
|
47
47
|
end
|
|
48
48
|
|
|
49
49
|
private
|
|
@@ -28,11 +28,10 @@ module Multiwoven::Integrations::Source
|
|
|
28
28
|
catalog = Catalog.new(streams: create_streams(records))
|
|
29
29
|
catalog.to_multiwoven_message
|
|
30
30
|
rescue StandardError => e
|
|
31
|
-
handle_exception(
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
)
|
|
31
|
+
handle_exception(e, {
|
|
32
|
+
context: "DATABRICKS:DISCOVER:EXCEPTION",
|
|
33
|
+
type: "error"
|
|
34
|
+
})
|
|
36
35
|
end
|
|
37
36
|
|
|
38
37
|
def read(sync_config)
|
|
@@ -45,11 +44,12 @@ module Multiwoven::Integrations::Source
|
|
|
45
44
|
|
|
46
45
|
query(db, query)
|
|
47
46
|
rescue StandardError => e
|
|
48
|
-
handle_exception(
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
47
|
+
handle_exception(e, {
|
|
48
|
+
context: "DATABRICKS:READ:EXCEPTION",
|
|
49
|
+
type: "error",
|
|
50
|
+
sync_id: sync_config.sync_id,
|
|
51
|
+
sync_run_id: sync_config.sync_run_id
|
|
52
|
+
})
|
|
53
53
|
end
|
|
54
54
|
|
|
55
55
|
private
|
|
@@ -34,11 +34,10 @@ module Multiwoven::Integrations::Source
|
|
|
34
34
|
catalog = Catalog.new(streams: create_streams(records))
|
|
35
35
|
catalog.to_multiwoven_message
|
|
36
36
|
rescue StandardError => e
|
|
37
|
-
handle_exception(
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
)
|
|
37
|
+
handle_exception(e, {
|
|
38
|
+
context: "POSTGRESQL:DISCOVER:EXCEPTION",
|
|
39
|
+
type: "error"
|
|
40
|
+
})
|
|
42
41
|
ensure
|
|
43
42
|
db&.close
|
|
44
43
|
end
|
|
@@ -53,11 +52,12 @@ module Multiwoven::Integrations::Source
|
|
|
53
52
|
|
|
54
53
|
query(db, query)
|
|
55
54
|
rescue StandardError => e
|
|
56
|
-
handle_exception(
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
55
|
+
handle_exception(e, {
|
|
56
|
+
context: "POSTGRESQL:READ:EXCEPTION",
|
|
57
|
+
type: "error",
|
|
58
|
+
sync_id: sync_config.sync_id,
|
|
59
|
+
sync_run_id: sync_config.sync_run_id
|
|
60
|
+
})
|
|
61
61
|
ensure
|
|
62
62
|
db&.close
|
|
63
63
|
end
|
|
@@ -34,11 +34,10 @@ module Multiwoven::Integrations::Source
|
|
|
34
34
|
catalog = Catalog.new(streams: create_streams(records))
|
|
35
35
|
catalog.to_multiwoven_message
|
|
36
36
|
rescue StandardError => e
|
|
37
|
-
handle_exception(
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
)
|
|
37
|
+
handle_exception(e, {
|
|
38
|
+
context: "REDSHIFT:DISCOVER:EXCEPTION",
|
|
39
|
+
type: "error"
|
|
40
|
+
})
|
|
42
41
|
ensure
|
|
43
42
|
db&.close
|
|
44
43
|
end
|
|
@@ -53,11 +52,12 @@ module Multiwoven::Integrations::Source
|
|
|
53
52
|
|
|
54
53
|
query(db, query)
|
|
55
54
|
rescue StandardError => e
|
|
56
|
-
handle_exception(
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
55
|
+
handle_exception(e, {
|
|
56
|
+
context: "REDSHIFT:READ:EXCEPTION",
|
|
57
|
+
type: "error",
|
|
58
|
+
sync_id: sync_config.sync_id,
|
|
59
|
+
sync_run_id: sync_config.sync_run_id
|
|
60
|
+
})
|
|
61
61
|
ensure
|
|
62
62
|
db&.close
|
|
63
63
|
end
|
|
@@ -34,7 +34,10 @@ module Multiwoven
|
|
|
34
34
|
end
|
|
35
35
|
catalog.to_multiwoven_message
|
|
36
36
|
rescue StandardError => e
|
|
37
|
-
handle_exception(
|
|
37
|
+
handle_exception(e, {
|
|
38
|
+
context: "SALESFORCE:CONSUMER:GOODS:ClOUD:DISCOVER:EXCEPTION",
|
|
39
|
+
type: "error"
|
|
40
|
+
})
|
|
38
41
|
end
|
|
39
42
|
|
|
40
43
|
def read(sync_config)
|
|
@@ -50,7 +53,12 @@ module Multiwoven
|
|
|
50
53
|
RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
|
|
51
54
|
end
|
|
52
55
|
rescue StandardError => e
|
|
53
|
-
handle_exception(
|
|
56
|
+
handle_exception(e, {
|
|
57
|
+
context: "SALESFORCE:CONSUMER:GOODS:ClOUD:WRITE:EXCEPTION",
|
|
58
|
+
type: "error",
|
|
59
|
+
sync_id: sync_config.sync_id,
|
|
60
|
+
sync_run_id: sync_config.sync_run_id
|
|
61
|
+
})
|
|
54
62
|
end
|
|
55
63
|
|
|
56
64
|
private
|
|
@@ -9,7 +9,7 @@ module Multiwoven
|
|
|
9
9
|
|
|
10
10
|
module_function
|
|
11
11
|
|
|
12
|
-
def salesforce_field_to_json_schema_type(sf_field) # rubocop:disable Metrics/
|
|
12
|
+
def salesforce_field_to_json_schema_type(sf_field) # rubocop:disable Metrics/MethodLength, Metrics/PerceivedComplexity
|
|
13
13
|
case sf_field["type"]
|
|
14
14
|
when "string", "Email", "Phone", "Text", "TextArea", "TextEncrypted", "URL", "Picklist (Single)"
|
|
15
15
|
if sf_field["nillable"]
|
|
@@ -28,11 +28,10 @@ module Multiwoven::Integrations::Source
|
|
|
28
28
|
catalog = Catalog.new(streams: create_streams(records))
|
|
29
29
|
catalog.to_multiwoven_message
|
|
30
30
|
rescue StandardError => e
|
|
31
|
-
handle_exception(
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
)
|
|
31
|
+
handle_exception(e, {
|
|
32
|
+
context: "SNOWFLAKE:DISCOVER:EXCEPTION",
|
|
33
|
+
type: "error"
|
|
34
|
+
})
|
|
36
35
|
end
|
|
37
36
|
|
|
38
37
|
def read(sync_config)
|
|
@@ -45,11 +44,12 @@ module Multiwoven::Integrations::Source
|
|
|
45
44
|
|
|
46
45
|
query(db, query)
|
|
47
46
|
rescue StandardError => e
|
|
48
|
-
handle_exception(
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
47
|
+
handle_exception(e, {
|
|
48
|
+
context: "SNOWFLAKE:READ:EXCEPTION",
|
|
49
|
+
type: "error",
|
|
50
|
+
sync_id: sync_config.sync_id,
|
|
51
|
+
sync_run_id: sync_config.sync_run_id
|
|
52
|
+
})
|
|
53
53
|
end
|
|
54
54
|
|
|
55
55
|
private
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: multiwoven-integrations
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.72
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Subin T P
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2024-05-
|
|
11
|
+
date: 2024-05-29 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activesupport
|