multiwoven-integrations 0.1.69 → 0.1.71
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +9 -0
- data/lib/multiwoven/integrations/core/utils.rb +10 -6
- data/lib/multiwoven/integrations/destination/airtable/client.rb +17 -4
- data/lib/multiwoven/integrations/destination/facebook_custom_audience/client.rb +17 -8
- data/lib/multiwoven/integrations/destination/google_sheets/client.rb +17 -5
- data/lib/multiwoven/integrations/destination/http/client.rb +20 -8
- data/lib/multiwoven/integrations/destination/hubspot/client.rb +21 -4
- data/lib/multiwoven/integrations/destination/klaviyo/client.rb +16 -13
- data/lib/multiwoven/integrations/destination/postgresql/client.rb +16 -11
- data/lib/multiwoven/integrations/destination/salesforce_consumer_goods_cloud/client.rb +29 -3
- data/lib/multiwoven/integrations/destination/salesforce_consumer_goods_cloud/schema_helper.rb +1 -1
- data/lib/multiwoven/integrations/destination/salesforce_crm/client.rb +18 -3
- data/lib/multiwoven/integrations/destination/sftp/client.rb +30 -14
- data/lib/multiwoven/integrations/destination/slack/client.rb +17 -4
- data/lib/multiwoven/integrations/destination/stripe/client.rb +17 -3
- data/lib/multiwoven/integrations/destination/zendesk/client.rb +21 -4
- data/lib/multiwoven/integrations/protocol/protocol.rb +3 -1
- data/lib/multiwoven/integrations/rollout.rb +1 -1
- data/lib/multiwoven/integrations/source/aws_athena/client.rb +10 -10
- data/lib/multiwoven/integrations/source/bigquery/client.rb +10 -10
- data/lib/multiwoven/integrations/source/clickhouse/client.rb +10 -10
- data/lib/multiwoven/integrations/source/databricks/client.rb +10 -10
- data/lib/multiwoven/integrations/source/postgresql/client.rb +10 -10
- data/lib/multiwoven/integrations/source/redshift/client.rb +10 -10
- data/lib/multiwoven/integrations/source/salesforce_consumer_goods_cloud/client.rb +10 -2
- data/lib/multiwoven/integrations/source/salesforce_consumer_goods_cloud/schema_helper.rb +1 -1
- data/lib/multiwoven/integrations/source/snowflake/client.rb +10 -10
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: a65066efeada8a6555fc8e23f80862ebfddb8c615be9542e0fc500c45f67fa01
|
|
4
|
+
data.tar.gz: 6f996c02a65311d517780cc34342f9af1639c8aa25fbe5fa6fb4cd56dbb587bd
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 9b476f1e2eb4f7560138b35ef34469e466a87e718841415ad6974dcb060ef4e908307089360ec4f3715d8b51f1291cf4fd4d0f76f0af7199a0366d0a7d5316cc
|
|
7
|
+
data.tar.gz: 12442e52ba12b7abfac10b63bb57432fc070b2b422827364cbb60982db4ce1afe05abab04d1c8f03b5cfc386d4f7dad2fe66852d093f86fafc0491c636b74308
|
data/.rubocop.yml
CHANGED
|
@@ -48,9 +48,9 @@ module Multiwoven
|
|
|
48
48
|
Integrations::Service.logger
|
|
49
49
|
end
|
|
50
50
|
|
|
51
|
-
def report_exception(exception)
|
|
51
|
+
def report_exception(exception, meta = {})
|
|
52
52
|
reporter = Integrations::Service.exception_reporter
|
|
53
|
-
reporter&.report(exception)
|
|
53
|
+
reporter&.report(exception, meta)
|
|
54
54
|
end
|
|
55
55
|
|
|
56
56
|
def create_log_message(context, type, exception)
|
|
@@ -61,12 +61,16 @@ module Multiwoven
|
|
|
61
61
|
).to_multiwoven_message
|
|
62
62
|
end
|
|
63
63
|
|
|
64
|
-
def handle_exception(
|
|
64
|
+
def handle_exception(exception, meta = {})
|
|
65
65
|
logger.error(
|
|
66
|
-
"#{
|
|
66
|
+
"#{hash_to_string(meta)}: #{exception.message}"
|
|
67
67
|
)
|
|
68
|
-
report_exception(exception)
|
|
69
|
-
create_log_message(context, type, exception)
|
|
68
|
+
report_exception(exception, meta)
|
|
69
|
+
create_log_message(meta[:context], meta[:type], exception)
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def hash_to_string(hash)
|
|
73
|
+
hash.map { |key, value| "#{key} = #{value}" }.join(", ")
|
|
70
74
|
end
|
|
71
75
|
|
|
72
76
|
def extract_data(record_object, properties)
|
|
@@ -6,7 +6,7 @@ module Multiwoven
|
|
|
6
6
|
module Destination
|
|
7
7
|
module Airtable
|
|
8
8
|
include Multiwoven::Integrations::Core
|
|
9
|
-
class Client < DestinationConnector
|
|
9
|
+
class Client < DestinationConnector
|
|
10
10
|
prepend Multiwoven::Integrations::Core::RateLimiter
|
|
11
11
|
MAX_CHUNK_SIZE = 10
|
|
12
12
|
def check_connection(connection_config)
|
|
@@ -49,7 +49,10 @@ module Multiwoven
|
|
|
49
49
|
catalog = build_catalog_from_schema(extract_body(schema), base_id, base_name)
|
|
50
50
|
catalog.to_multiwoven_message
|
|
51
51
|
rescue StandardError => e
|
|
52
|
-
handle_exception(
|
|
52
|
+
handle_exception(e, {
|
|
53
|
+
context: "AIRTABLE:DISCOVER:EXCEPTION",
|
|
54
|
+
type: "error"
|
|
55
|
+
})
|
|
53
56
|
end
|
|
54
57
|
|
|
55
58
|
def write(sync_config, records, _action = "create")
|
|
@@ -72,7 +75,12 @@ module Multiwoven
|
|
|
72
75
|
write_failure += chunk.size
|
|
73
76
|
end
|
|
74
77
|
rescue StandardError => e
|
|
75
|
-
handle_exception(
|
|
78
|
+
handle_exception(e, {
|
|
79
|
+
context: "AIRTABLE:RECORD:WRITE:EXCEPTION",
|
|
80
|
+
type: "error",
|
|
81
|
+
sync_id: sync_config.sync_id,
|
|
82
|
+
sync_run_id: sync_config.sync_run_id
|
|
83
|
+
})
|
|
76
84
|
write_failure += chunk.size
|
|
77
85
|
end
|
|
78
86
|
|
|
@@ -82,7 +90,12 @@ module Multiwoven
|
|
|
82
90
|
)
|
|
83
91
|
tracker.to_multiwoven_message
|
|
84
92
|
rescue StandardError => e
|
|
85
|
-
handle_exception(
|
|
93
|
+
handle_exception(e, {
|
|
94
|
+
context: "AIRTABLE:RECORD:WRITE:EXCEPTION",
|
|
95
|
+
type: "error",
|
|
96
|
+
sync_id: sync_config.sync_id,
|
|
97
|
+
sync_run_id: sync_config.sync_run_id
|
|
98
|
+
})
|
|
86
99
|
end
|
|
87
100
|
|
|
88
101
|
private
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
module Multiwoven::Integrations::Destination
|
|
4
4
|
module FacebookCustomAudience
|
|
5
5
|
include Multiwoven::Integrations::Core
|
|
6
|
-
class Client < DestinationConnector
|
|
6
|
+
class Client < DestinationConnector
|
|
7
7
|
prepend Multiwoven::Integrations::Core::RateLimiter
|
|
8
8
|
MAX_CHUNK_SIZE = 10_000
|
|
9
9
|
def check_connection(connection_config)
|
|
@@ -29,11 +29,10 @@ module Multiwoven::Integrations::Destination
|
|
|
29
29
|
catalog = build_catalog(catalog_json)
|
|
30
30
|
catalog.to_multiwoven_message
|
|
31
31
|
rescue StandardError => e
|
|
32
|
-
handle_exception(
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
)
|
|
32
|
+
handle_exception(e, {
|
|
33
|
+
context: "FACEBOOK AUDIENCE:DISCOVER:EXCEPTION",
|
|
34
|
+
type: "error"
|
|
35
|
+
})
|
|
37
36
|
end
|
|
38
37
|
|
|
39
38
|
def write(sync_config, records, _action = "insert")
|
|
@@ -56,7 +55,12 @@ module Multiwoven::Integrations::Destination
|
|
|
56
55
|
write_failure += chunk.size
|
|
57
56
|
end
|
|
58
57
|
rescue StandardError => e
|
|
59
|
-
handle_exception(
|
|
58
|
+
handle_exception(e, {
|
|
59
|
+
context: "FACEBOOK:RECORD:WRITE:EXCEPTION",
|
|
60
|
+
type: "error",
|
|
61
|
+
sync_id: sync_config.sync_id,
|
|
62
|
+
sync_run_id: sync_config.sync_run_id
|
|
63
|
+
})
|
|
60
64
|
write_failure += chunk.size
|
|
61
65
|
end
|
|
62
66
|
|
|
@@ -66,7 +70,12 @@ module Multiwoven::Integrations::Destination
|
|
|
66
70
|
)
|
|
67
71
|
tracker.to_multiwoven_message
|
|
68
72
|
rescue StandardError => e
|
|
69
|
-
handle_exception(
|
|
73
|
+
handle_exception(e, {
|
|
74
|
+
context: "FACEBOOK:RECORD:WRITE:EXCEPTION",
|
|
75
|
+
type: "error",
|
|
76
|
+
sync_id: sync_config.sync_id,
|
|
77
|
+
sync_run_id: sync_config.sync_run_id
|
|
78
|
+
})
|
|
70
79
|
end
|
|
71
80
|
|
|
72
81
|
private
|
|
@@ -6,7 +6,7 @@ module Multiwoven
|
|
|
6
6
|
module GoogleSheets
|
|
7
7
|
include Multiwoven::Integrations::Core
|
|
8
8
|
|
|
9
|
-
class Client < DestinationConnector
|
|
9
|
+
class Client < DestinationConnector
|
|
10
10
|
prepend Multiwoven::Integrations::Core::Fullrefresher
|
|
11
11
|
prepend Multiwoven::Integrations::Core::RateLimiter
|
|
12
12
|
MAX_CHUNK_SIZE = 10_000
|
|
@@ -17,7 +17,6 @@ module Multiwoven
|
|
|
17
17
|
fetch_google_spread_sheets(connection_config)
|
|
18
18
|
success_status
|
|
19
19
|
rescue StandardError => e
|
|
20
|
-
handle_exception("GOOGLE_SHEETS:CRM:DISCOVER:EXCEPTION", "error", e)
|
|
21
20
|
failure_status(e)
|
|
22
21
|
end
|
|
23
22
|
|
|
@@ -28,14 +27,22 @@ module Multiwoven
|
|
|
28
27
|
catalog = build_catalog_from_spreadsheets(spreadsheets, connection_config)
|
|
29
28
|
catalog.to_multiwoven_message
|
|
30
29
|
rescue StandardError => e
|
|
31
|
-
handle_exception(
|
|
30
|
+
handle_exception(e, {
|
|
31
|
+
context: "GOOGLE_SHEETS:CRM:DISCOVER:EXCEPTION",
|
|
32
|
+
type: "error"
|
|
33
|
+
})
|
|
32
34
|
end
|
|
33
35
|
|
|
34
36
|
def write(sync_config, records, action = "create")
|
|
35
37
|
setup_write_environment(sync_config, action)
|
|
36
38
|
process_record_chunks(records, sync_config)
|
|
37
39
|
rescue StandardError => e
|
|
38
|
-
handle_exception(
|
|
40
|
+
handle_exception(e, {
|
|
41
|
+
context: "GOOGLE_SHEETS:CRM:WRITE:EXCEPTION",
|
|
42
|
+
type: "error",
|
|
43
|
+
sync_id: sync_config.sync_id,
|
|
44
|
+
sync_run_id: sync_config.sync_run_id
|
|
45
|
+
})
|
|
39
46
|
end
|
|
40
47
|
|
|
41
48
|
def clear_all_records(sync_config)
|
|
@@ -154,7 +161,12 @@ module Multiwoven
|
|
|
154
161
|
update_sheet_values(values, sync_config.stream.name)
|
|
155
162
|
write_success += values.size
|
|
156
163
|
rescue StandardError => e
|
|
157
|
-
handle_exception(
|
|
164
|
+
handle_exception(e, {
|
|
165
|
+
context: "GOOGLE_SHEETS:RECORD:WRITE:EXCEPTION",
|
|
166
|
+
type: "error",
|
|
167
|
+
sync_id: sync_config.sync_id,
|
|
168
|
+
sync_run_id: sync_config.sync_run_id
|
|
169
|
+
})
|
|
158
170
|
write_failure += chunk.size
|
|
159
171
|
end
|
|
160
172
|
|
|
@@ -23,7 +23,10 @@ module Multiwoven
|
|
|
23
23
|
failure_status(nil)
|
|
24
24
|
end
|
|
25
25
|
rescue StandardError => e
|
|
26
|
-
handle_exception(
|
|
26
|
+
handle_exception(e, {
|
|
27
|
+
context: "HTTP:CHECK_CONNECTION:EXCEPTION",
|
|
28
|
+
type: "error"
|
|
29
|
+
})
|
|
27
30
|
failure_status(e)
|
|
28
31
|
end
|
|
29
32
|
|
|
@@ -32,11 +35,10 @@ module Multiwoven
|
|
|
32
35
|
catalog = build_catalog(catalog_json)
|
|
33
36
|
catalog.to_multiwoven_message
|
|
34
37
|
rescue StandardError => e
|
|
35
|
-
handle_exception(
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
)
|
|
38
|
+
handle_exception(e, {
|
|
39
|
+
context: "HTTP:DISCOVER:EXCEPTION",
|
|
40
|
+
type: "error"
|
|
41
|
+
})
|
|
40
42
|
end
|
|
41
43
|
|
|
42
44
|
def write(sync_config, records, _action = "create")
|
|
@@ -59,7 +61,12 @@ module Multiwoven
|
|
|
59
61
|
write_failure += chunk.size
|
|
60
62
|
end
|
|
61
63
|
rescue StandardError => e
|
|
62
|
-
handle_exception(
|
|
64
|
+
handle_exception(e, {
|
|
65
|
+
context: "HTTP:RECORD:WRITE:EXCEPTION",
|
|
66
|
+
type: "error",
|
|
67
|
+
sync_id: sync_config.sync_id,
|
|
68
|
+
sync_run_id: sync_config.sync_run_id
|
|
69
|
+
})
|
|
63
70
|
write_failure += chunk.size
|
|
64
71
|
end
|
|
65
72
|
|
|
@@ -69,7 +76,12 @@ module Multiwoven
|
|
|
69
76
|
)
|
|
70
77
|
tracker.to_multiwoven_message
|
|
71
78
|
rescue StandardError => e
|
|
72
|
-
handle_exception(
|
|
79
|
+
handle_exception(e, {
|
|
80
|
+
context: "HTTP:RECORD:WRITE:EXCEPTION",
|
|
81
|
+
type: "error",
|
|
82
|
+
sync_id: sync_config.sync_id,
|
|
83
|
+
sync_run_id: sync_config.sync_run_id
|
|
84
|
+
})
|
|
73
85
|
end
|
|
74
86
|
|
|
75
87
|
private
|
|
@@ -16,7 +16,10 @@ module Multiwoven
|
|
|
16
16
|
authenticate_client
|
|
17
17
|
success_status
|
|
18
18
|
rescue StandardError => e
|
|
19
|
-
handle_exception(
|
|
19
|
+
handle_exception(e, {
|
|
20
|
+
context: "HUBSPOT:CRM:CHECK_CONNECTION:EXCEPTION",
|
|
21
|
+
type: "error"
|
|
22
|
+
})
|
|
20
23
|
failure_status(e)
|
|
21
24
|
end
|
|
22
25
|
|
|
@@ -24,15 +27,24 @@ module Multiwoven
|
|
|
24
27
|
catalog = build_catalog(load_catalog)
|
|
25
28
|
catalog.to_multiwoven_message
|
|
26
29
|
rescue StandardError => e
|
|
27
|
-
handle_exception(
|
|
30
|
+
handle_exception(e, {
|
|
31
|
+
context: "HUBSPOT:CRM:DISCOVER:EXCEPTION",
|
|
32
|
+
type: "error"
|
|
33
|
+
})
|
|
28
34
|
end
|
|
29
35
|
|
|
30
36
|
def write(sync_config, records, action = "create")
|
|
31
37
|
@action = sync_config.stream.action || action
|
|
38
|
+
@sync_config = sync_config
|
|
32
39
|
initialize_client(sync_config.destination.connection_specification)
|
|
33
40
|
process_records(records, sync_config.stream)
|
|
34
41
|
rescue StandardError => e
|
|
35
|
-
handle_exception(
|
|
42
|
+
handle_exception(e, {
|
|
43
|
+
context: "HUBSPOT:CRM:WRITE:EXCEPTION",
|
|
44
|
+
type: "error",
|
|
45
|
+
sync_id: @sync_config.sync_id,
|
|
46
|
+
sync_run_id: @sync_config.sync_run_id
|
|
47
|
+
})
|
|
36
48
|
end
|
|
37
49
|
|
|
38
50
|
private
|
|
@@ -51,7 +63,12 @@ module Multiwoven
|
|
|
51
63
|
send_data_to_hubspot(stream.name, record)
|
|
52
64
|
write_success += 1
|
|
53
65
|
rescue StandardError => e
|
|
54
|
-
handle_exception(
|
|
66
|
+
handle_exception(e, {
|
|
67
|
+
context: "HUBSPOT:CRM:WRITE:EXCEPTION",
|
|
68
|
+
type: "error",
|
|
69
|
+
sync_id: @sync_config.sync_id,
|
|
70
|
+
sync_run_id: @sync_config.sync_run_id
|
|
71
|
+
})
|
|
55
72
|
write_failure += 1
|
|
56
73
|
end
|
|
57
74
|
tracking_message(write_success, write_failure)
|
|
@@ -25,11 +25,10 @@ module Multiwoven::Integrations::Destination
|
|
|
25
25
|
|
|
26
26
|
catalog.to_multiwoven_message
|
|
27
27
|
rescue StandardError => e
|
|
28
|
-
handle_exception(
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
)
|
|
28
|
+
handle_exception(e, {
|
|
29
|
+
context: "KLAVIYO:DISCOVER:EXCEPTION",
|
|
30
|
+
type: "error"
|
|
31
|
+
})
|
|
33
32
|
end
|
|
34
33
|
|
|
35
34
|
def write(sync_config, records, _action = "insert")
|
|
@@ -59,9 +58,12 @@ module Multiwoven::Integrations::Destination
|
|
|
59
58
|
write_failure += 1
|
|
60
59
|
end
|
|
61
60
|
rescue StandardError => e
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
61
|
+
handle_exception(e, {
|
|
62
|
+
context: "KLAVIYO:RECORD:WRITE:FAILURE",
|
|
63
|
+
type: "error",
|
|
64
|
+
sync_id: sync_config.sync_id,
|
|
65
|
+
sync_run_id: sync_config.sync_run_id
|
|
66
|
+
})
|
|
65
67
|
write_failure += 1
|
|
66
68
|
end
|
|
67
69
|
tracker = Multiwoven::Integrations::Protocol::TrackingMessage.new(
|
|
@@ -71,11 +73,12 @@ module Multiwoven::Integrations::Destination
|
|
|
71
73
|
tracker.to_multiwoven_message
|
|
72
74
|
rescue StandardError => e
|
|
73
75
|
# TODO: Handle rate limiting seperately
|
|
74
|
-
handle_exception(
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
76
|
+
handle_exception(e, {
|
|
77
|
+
context: "KLAVIYO:RECORD:WRITE:FAILURE",
|
|
78
|
+
type: "error",
|
|
79
|
+
sync_id: sync_config.sync_id,
|
|
80
|
+
sync_run_id: sync_config.sync_run_id
|
|
81
|
+
})
|
|
79
82
|
end
|
|
80
83
|
|
|
81
84
|
private
|
|
@@ -34,11 +34,10 @@ module Multiwoven::Integrations::Destination
|
|
|
34
34
|
catalog = Catalog.new(streams: create_streams(records))
|
|
35
35
|
catalog.to_multiwoven_message
|
|
36
36
|
rescue StandardError => e
|
|
37
|
-
handle_exception(
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
)
|
|
37
|
+
handle_exception(e, {
|
|
38
|
+
context: "POSTGRESQL:DISCOVER:EXCEPTION",
|
|
39
|
+
type: "error"
|
|
40
|
+
})
|
|
42
41
|
ensure
|
|
43
42
|
db&.close
|
|
44
43
|
end
|
|
@@ -57,17 +56,23 @@ module Multiwoven::Integrations::Destination
|
|
|
57
56
|
db.exec(query)
|
|
58
57
|
write_success += 1
|
|
59
58
|
rescue StandardError => e
|
|
60
|
-
handle_exception(
|
|
59
|
+
handle_exception(e, {
|
|
60
|
+
context: "POSTGRESQL:RECORD:WRITE:EXCEPTION",
|
|
61
|
+
type: "error",
|
|
62
|
+
sync_id: sync_config.sync_id,
|
|
63
|
+
sync_run_id: sync_config.sync_run_id
|
|
64
|
+
})
|
|
61
65
|
write_failure += 1
|
|
62
66
|
end
|
|
63
67
|
end
|
|
64
68
|
tracking_message(write_success, write_failure)
|
|
65
69
|
rescue StandardError => e
|
|
66
|
-
handle_exception(
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
70
|
+
handle_exception(e, {
|
|
71
|
+
context: "POSTGRESQL:RECORD:WRITE:EXCEPTION",
|
|
72
|
+
type: "error",
|
|
73
|
+
sync_id: sync_config.sync_id,
|
|
74
|
+
sync_run_id: sync_config.sync_run_id
|
|
75
|
+
})
|
|
71
76
|
end
|
|
72
77
|
|
|
73
78
|
private
|
|
@@ -14,6 +14,16 @@ module Multiwoven
|
|
|
14
14
|
|
|
15
15
|
class Client < DestinationConnector
|
|
16
16
|
prepend Multiwoven::Integrations::Core::RateLimiter
|
|
17
|
+
|
|
18
|
+
def initialize
|
|
19
|
+
super
|
|
20
|
+
@logger = Integrations::Service.logger
|
|
21
|
+
Restforce.configure do |config|
|
|
22
|
+
config.logger = @logger
|
|
23
|
+
end
|
|
24
|
+
Restforce.log = true
|
|
25
|
+
end
|
|
26
|
+
|
|
17
27
|
def check_connection(connection_config)
|
|
18
28
|
connection_config = connection_config.with_indifferent_access
|
|
19
29
|
initialize_client(connection_config)
|
|
@@ -34,15 +44,24 @@ module Multiwoven
|
|
|
34
44
|
end
|
|
35
45
|
catalog.to_multiwoven_message
|
|
36
46
|
rescue StandardError => e
|
|
37
|
-
handle_exception(
|
|
47
|
+
handle_exception(e, {
|
|
48
|
+
context: "SALESFORCE:CONSUMER:GOODS:ClOUD:DISCOVER:EXCEPTION",
|
|
49
|
+
type: "error"
|
|
50
|
+
})
|
|
38
51
|
end
|
|
39
52
|
|
|
40
53
|
def write(sync_config, records, action = "create")
|
|
41
54
|
@action = sync_config.stream.action || action
|
|
55
|
+
@sync_config = sync_config
|
|
42
56
|
initialize_client(sync_config.destination.connection_specification)
|
|
43
57
|
process_records(records, sync_config.stream)
|
|
44
58
|
rescue StandardError => e
|
|
45
|
-
handle_exception(
|
|
59
|
+
handle_exception(e, {
|
|
60
|
+
context: "SALESFORCE:CONSUMER:GOODS:ClOUD:WRITE:EXCEPTION",
|
|
61
|
+
type: "error",
|
|
62
|
+
sync_id: @sync_config.sync_id,
|
|
63
|
+
sync_run_id: @sync_config.sync_run_id
|
|
64
|
+
})
|
|
46
65
|
end
|
|
47
66
|
|
|
48
67
|
private
|
|
@@ -66,7 +85,13 @@ module Multiwoven
|
|
|
66
85
|
process_record(stream, record)
|
|
67
86
|
write_success += 1
|
|
68
87
|
rescue StandardError => e
|
|
69
|
-
|
|
88
|
+
# TODO: add sync_id and sync run id to the logs
|
|
89
|
+
handle_exception(e, {
|
|
90
|
+
context: "SALESFORCE:CONSUMER:GOODS:ClOUD:WRITE:EXCEPTION",
|
|
91
|
+
type: "error",
|
|
92
|
+
sync_id: @sync_config.sync_id,
|
|
93
|
+
sync_run_id: @sync_config.sync_run_id
|
|
94
|
+
})
|
|
70
95
|
write_failure += 1
|
|
71
96
|
end
|
|
72
97
|
tracking_message(write_success, write_failure)
|
|
@@ -79,6 +104,7 @@ module Multiwoven
|
|
|
79
104
|
def send_data_to_salesforce(stream_name, record = {})
|
|
80
105
|
method_name = "upsert!"
|
|
81
106
|
args = [stream_name, "Id", record]
|
|
107
|
+
@logger.debug("sync_id: #{@sync_config.sync_id}, sync_run_id: #{@sync_config.sync_run_id}, record: #{record}")
|
|
82
108
|
@client.send(method_name, *args)
|
|
83
109
|
end
|
|
84
110
|
|
data/lib/multiwoven/integrations/destination/salesforce_consumer_goods_cloud/schema_helper.rb
CHANGED
|
@@ -9,7 +9,7 @@ module Multiwoven
|
|
|
9
9
|
|
|
10
10
|
module_function
|
|
11
11
|
|
|
12
|
-
def salesforce_field_to_json_schema_type(sf_field) # rubocop:disable Metrics/
|
|
12
|
+
def salesforce_field_to_json_schema_type(sf_field) # rubocop:disable Metrics/MethodLength, Metrics/PerceivedComplexity
|
|
13
13
|
case sf_field["type"]
|
|
14
14
|
when "string", "Email", "Phone", "Text", "TextArea", "TextEncrypted", "URL", "Picklist (Single)"
|
|
15
15
|
if sf_field["nillable"]
|
|
@@ -25,15 +25,24 @@ module Multiwoven
|
|
|
25
25
|
catalog = build_catalog(load_catalog)
|
|
26
26
|
catalog.to_multiwoven_message
|
|
27
27
|
rescue StandardError => e
|
|
28
|
-
handle_exception(
|
|
28
|
+
handle_exception(e, {
|
|
29
|
+
context: "SALESFORCE:CRM:DISCOVER:EXCEPTION",
|
|
30
|
+
type: "error"
|
|
31
|
+
})
|
|
29
32
|
end
|
|
30
33
|
|
|
31
34
|
def write(sync_config, records, action = "create")
|
|
32
35
|
@action = sync_config.stream.action || action
|
|
36
|
+
@sync_config = sync_config
|
|
33
37
|
initialize_client(sync_config.destination.connection_specification)
|
|
34
38
|
process_records(records, sync_config.stream)
|
|
35
39
|
rescue StandardError => e
|
|
36
|
-
handle_exception(
|
|
40
|
+
handle_exception(e, {
|
|
41
|
+
context: "SALESFORCE:CRM:WRITE:EXCEPTION",
|
|
42
|
+
type: "error",
|
|
43
|
+
sync_id: @sync_config.sync_id,
|
|
44
|
+
sync_run_id: @sync_config.sync_run_id
|
|
45
|
+
})
|
|
37
46
|
end
|
|
38
47
|
|
|
39
48
|
private
|
|
@@ -58,7 +67,13 @@ module Multiwoven
|
|
|
58
67
|
process_record(stream, record)
|
|
59
68
|
write_success += 1
|
|
60
69
|
rescue StandardError => e
|
|
61
|
-
|
|
70
|
+
# TODO: add sync_id and sync_run_id to the logs
|
|
71
|
+
handle_exception(e, {
|
|
72
|
+
context: "SALESFORCE:CRM:WRITE:EXCEPTION",
|
|
73
|
+
type: "error",
|
|
74
|
+
sync_id: @sync_config.sync_id,
|
|
75
|
+
sync_run_id: @sync_config.sync_run_id
|
|
76
|
+
})
|
|
62
77
|
write_failure += 1
|
|
63
78
|
end
|
|
64
79
|
tracking_message(write_success, write_failure)
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
module Multiwoven::Integrations::Destination
|
|
4
4
|
module Sftp
|
|
5
5
|
include Multiwoven::Integrations::Core
|
|
6
|
-
class Client < DestinationConnector
|
|
6
|
+
class Client < DestinationConnector
|
|
7
7
|
prepend Multiwoven::Integrations::Core::Fullrefresher
|
|
8
8
|
prepend Multiwoven::Integrations::Core::RateLimiter
|
|
9
9
|
|
|
@@ -16,7 +16,10 @@ module Multiwoven::Integrations::Destination
|
|
|
16
16
|
return success_status
|
|
17
17
|
end
|
|
18
18
|
rescue StandardError => e
|
|
19
|
-
handle_exception(
|
|
19
|
+
handle_exception(e, {
|
|
20
|
+
context: "SFTP:CHECK_CONNECTION:EXCEPTION",
|
|
21
|
+
type: "error"
|
|
22
|
+
})
|
|
20
23
|
failure_status(e)
|
|
21
24
|
end
|
|
22
25
|
|
|
@@ -27,14 +30,14 @@ module Multiwoven::Integrations::Destination
|
|
|
27
30
|
|
|
28
31
|
catalog.to_multiwoven_message
|
|
29
32
|
rescue StandardError => e
|
|
30
|
-
handle_exception(
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
)
|
|
33
|
+
handle_exception(e, {
|
|
34
|
+
context: "SFTP:DISCOVER:EXCEPTION",
|
|
35
|
+
type: "error"
|
|
36
|
+
})
|
|
35
37
|
end
|
|
36
38
|
|
|
37
39
|
def write(sync_config, records, _action = "insert")
|
|
40
|
+
@sync_config = sync_config
|
|
38
41
|
connection_config = sync_config.destination.connection_specification.with_indifferent_access
|
|
39
42
|
file_path = generate_file_path(sync_config)
|
|
40
43
|
local_file_name = generate_local_file_name(sync_config)
|
|
@@ -53,11 +56,12 @@ module Multiwoven::Integrations::Destination
|
|
|
53
56
|
write_failure = records.size - write_success
|
|
54
57
|
tracking_message(write_success, write_failure)
|
|
55
58
|
rescue StandardError => e
|
|
56
|
-
handle_exception(
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
59
|
+
handle_exception(e, {
|
|
60
|
+
context: "SFTP:WRITE:EXCEPTION",
|
|
61
|
+
type: "error",
|
|
62
|
+
sync_id: @sync_config.sync_id,
|
|
63
|
+
sync_run_id: @sync_config.sync_run_id
|
|
64
|
+
})
|
|
61
65
|
end
|
|
62
66
|
|
|
63
67
|
def write_compressed_data(connection_config, file_path, local_file_name, csv_content, records_size)
|
|
@@ -70,7 +74,13 @@ module Multiwoven::Integrations::Destination
|
|
|
70
74
|
sftp.upload!(tempfile.path, file_path)
|
|
71
75
|
write_success = records_size
|
|
72
76
|
rescue StandardError => e
|
|
73
|
-
|
|
77
|
+
# TODO: add sync_id and sync_run_id to the log
|
|
78
|
+
handle_exception(e, {
|
|
79
|
+
context: "SFTP:RECORD:WRITE:EXCEPTION",
|
|
80
|
+
type: "error",
|
|
81
|
+
sync_id: @sync_config.sync_id,
|
|
82
|
+
sync_run_id: @sync_config.sync_run_id
|
|
83
|
+
})
|
|
74
84
|
write_success = 0
|
|
75
85
|
end
|
|
76
86
|
end
|
|
@@ -86,7 +96,13 @@ module Multiwoven::Integrations::Destination
|
|
|
86
96
|
sftp.upload!(tempfile.path, file_path)
|
|
87
97
|
write_success = records_size
|
|
88
98
|
rescue StandardError => e
|
|
89
|
-
|
|
99
|
+
# TODO: add sync_id and sync_run_id to the log
|
|
100
|
+
handle_exception(e, {
|
|
101
|
+
context: "SFTP:RECORD:WRITE:EXCEPTION",
|
|
102
|
+
type: "error",
|
|
103
|
+
sync_id: @sync_config.sync_id,
|
|
104
|
+
sync_run_id: @sync_config.sync_run_id
|
|
105
|
+
})
|
|
90
106
|
write_success = 0
|
|
91
107
|
end
|
|
92
108
|
end
|
|
@@ -23,13 +23,16 @@ module Multiwoven
|
|
|
23
23
|
catalog = build_catalog(load_catalog)
|
|
24
24
|
catalog.to_multiwoven_message
|
|
25
25
|
rescue StandardError => e
|
|
26
|
-
handle_exception(
|
|
26
|
+
handle_exception(e, {
|
|
27
|
+
context: "SLACK:DISCOVER:EXCEPTION",
|
|
28
|
+
type: "error"
|
|
29
|
+
})
|
|
27
30
|
end
|
|
28
31
|
|
|
29
32
|
def write(sync_config, records, action = "create")
|
|
30
33
|
# Currently as we only create a message for each record in slack, we are not using actions.
|
|
31
34
|
# This will be changed in future.
|
|
32
|
-
|
|
35
|
+
@sync_config = sync_config
|
|
33
36
|
@action = sync_config.stream.action || action
|
|
34
37
|
connection_config = sync_config.destination.connection_specification.with_indifferent_access
|
|
35
38
|
configure_slack(connection_config[:api_token])
|
|
@@ -37,7 +40,12 @@ module Multiwoven
|
|
|
37
40
|
@channel_id = connection_config[:channel_id]
|
|
38
41
|
process_records(records, sync_config.stream)
|
|
39
42
|
rescue StandardError => e
|
|
40
|
-
handle_exception(
|
|
43
|
+
handle_exception(e, {
|
|
44
|
+
context: "SLACK:WRITE:EXCEPTION",
|
|
45
|
+
type: "error",
|
|
46
|
+
sync_id: @sync_config.sync_id,
|
|
47
|
+
sync_run_id: @sync_config.sync_run_id
|
|
48
|
+
})
|
|
41
49
|
end
|
|
42
50
|
|
|
43
51
|
private
|
|
@@ -56,7 +64,12 @@ module Multiwoven
|
|
|
56
64
|
write_success += 1
|
|
57
65
|
rescue StandardError => e
|
|
58
66
|
write_failure += 1
|
|
59
|
-
handle_exception(
|
|
67
|
+
handle_exception(e, {
|
|
68
|
+
context: "SLACK:WRITE:EXCEPTION",
|
|
69
|
+
type: "error",
|
|
70
|
+
sync_id: @sync_config.sync_id,
|
|
71
|
+
sync_run_id: @sync_config.sync_run_id
|
|
72
|
+
})
|
|
60
73
|
end
|
|
61
74
|
tracking_message(write_success, write_failure)
|
|
62
75
|
end
|
|
@@ -23,15 +23,24 @@ module Multiwoven
|
|
|
23
23
|
catalog = build_catalog(load_catalog)
|
|
24
24
|
catalog.to_multiwoven_message
|
|
25
25
|
rescue StandardError => e
|
|
26
|
-
handle_exception(
|
|
26
|
+
handle_exception(e, {
|
|
27
|
+
context: "STRIPE:CRM:DISCOVER:EXCEPTION",
|
|
28
|
+
type: "error"
|
|
29
|
+
})
|
|
27
30
|
end
|
|
28
31
|
|
|
29
32
|
def write(sync_config, records, action = "create")
|
|
33
|
+
@sync_config = sync_config
|
|
30
34
|
@action = sync_config.stream.action || action
|
|
31
35
|
initialize_client(sync_config.destination.connection_specification)
|
|
32
36
|
process_records(records, sync_config.stream)
|
|
33
37
|
rescue StandardError => e
|
|
34
|
-
handle_exception(
|
|
38
|
+
handle_exception(e, {
|
|
39
|
+
context: "STRIPE:CRM:WRITE:EXCEPTION",
|
|
40
|
+
type: "error",
|
|
41
|
+
sync_id: @sync_config.sync_id,
|
|
42
|
+
sync_run_id: @sync_config.sync_run_id
|
|
43
|
+
})
|
|
35
44
|
end
|
|
36
45
|
|
|
37
46
|
private
|
|
@@ -52,7 +61,12 @@ module Multiwoven
|
|
|
52
61
|
klass.send(@action, record)
|
|
53
62
|
write_success += 1
|
|
54
63
|
rescue StandardError => e
|
|
55
|
-
handle_exception(
|
|
64
|
+
handle_exception(e, {
|
|
65
|
+
context: "STRIPE:CRM:WRITE:EXCEPTION",
|
|
66
|
+
type: "error",
|
|
67
|
+
sync_id: @sync_config.sync_id,
|
|
68
|
+
sync_run_id: @sync_config.sync_run_id
|
|
69
|
+
})
|
|
56
70
|
write_failure += 1
|
|
57
71
|
end
|
|
58
72
|
tracking_message(write_success, write_failure)
|
|
@@ -13,7 +13,10 @@ module Multiwoven
|
|
|
13
13
|
authenticate_client
|
|
14
14
|
success_status
|
|
15
15
|
rescue StandardError => e
|
|
16
|
-
handle_exception(
|
|
16
|
+
handle_exception(e, {
|
|
17
|
+
context: "ZENDESK:CHECK_CONNECTION:EXCEPTION",
|
|
18
|
+
type: "error"
|
|
19
|
+
})
|
|
17
20
|
failure_status(e)
|
|
18
21
|
end
|
|
19
22
|
|
|
@@ -21,16 +24,25 @@ module Multiwoven
|
|
|
21
24
|
catalog = build_catalog(load_catalog)
|
|
22
25
|
catalog.to_multiwoven_message
|
|
23
26
|
rescue StandardError => e
|
|
24
|
-
handle_exception(
|
|
27
|
+
handle_exception(e, {
|
|
28
|
+
context: "ZENDESK:DISCOVER:EXCEPTION",
|
|
29
|
+
type: "error"
|
|
30
|
+
})
|
|
25
31
|
failure_status(e)
|
|
26
32
|
end
|
|
27
33
|
|
|
28
34
|
def write(sync_config, records, action = "create")
|
|
35
|
+
@sync_config = sync_config
|
|
29
36
|
@action = sync_config.stream.action || action
|
|
30
37
|
initialize_client(sync_config.destination.connection_specification)
|
|
31
38
|
process_records(records, sync_config.stream)
|
|
32
39
|
rescue StandardError => e
|
|
33
|
-
handle_exception(
|
|
40
|
+
handle_exception(e, {
|
|
41
|
+
context: "ZENDESK:WRITE:EXCEPTION",
|
|
42
|
+
type: "error",
|
|
43
|
+
sync_id: @sync_config.sync_id,
|
|
44
|
+
sync_run_id: @sync_config.sync_run_id
|
|
45
|
+
})
|
|
34
46
|
failure_status(e)
|
|
35
47
|
end
|
|
36
48
|
|
|
@@ -68,7 +80,12 @@ module Multiwoven
|
|
|
68
80
|
|
|
69
81
|
write_success += 1
|
|
70
82
|
rescue StandardError => e
|
|
71
|
-
handle_exception(
|
|
83
|
+
handle_exception(e, {
|
|
84
|
+
context: "ZENDESK:WRITE:EXCEPTION",
|
|
85
|
+
type: "error",
|
|
86
|
+
sync_id: @sync_config.sync_id,
|
|
87
|
+
sync_run_id: @sync_config.sync_run_id
|
|
88
|
+
})
|
|
72
89
|
write_failure += 1
|
|
73
90
|
end
|
|
74
91
|
|
|
@@ -165,7 +165,7 @@ module Multiwoven
|
|
|
165
165
|
end
|
|
166
166
|
|
|
167
167
|
class SyncConfig < ProtocolModel
|
|
168
|
-
attr_accessor :offset, :limit
|
|
168
|
+
attr_accessor :offset, :limit, :sync_run_id
|
|
169
169
|
|
|
170
170
|
attribute :source, Connector
|
|
171
171
|
attribute :destination, Connector
|
|
@@ -175,6 +175,8 @@ module Multiwoven
|
|
|
175
175
|
attribute? :cursor_field, Types::String.optional
|
|
176
176
|
attribute? :current_cursor_field, Types::String.optional
|
|
177
177
|
attribute :destination_sync_mode, DestinationSyncMode
|
|
178
|
+
# reference ids
|
|
179
|
+
attribute :sync_id, Types::String.default("unknown")
|
|
178
180
|
end
|
|
179
181
|
|
|
180
182
|
class ControlMessage < ProtocolModel
|
|
@@ -21,11 +21,10 @@ module Multiwoven::Integrations::Source
|
|
|
21
21
|
catalog = Catalog.new(streams: create_streams(results))
|
|
22
22
|
catalog.to_multiwoven_message
|
|
23
23
|
rescue StandardError => e
|
|
24
|
-
handle_exception(
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
)
|
|
24
|
+
handle_exception(e, {
|
|
25
|
+
context: "AWS:ATHENA:DISCOVER:EXCEPTION",
|
|
26
|
+
type: "error"
|
|
27
|
+
})
|
|
29
28
|
end
|
|
30
29
|
|
|
31
30
|
def read(sync_config)
|
|
@@ -36,11 +35,12 @@ module Multiwoven::Integrations::Source
|
|
|
36
35
|
db = create_connection(connection_config)
|
|
37
36
|
query(db, query)
|
|
38
37
|
rescue StandardError => e
|
|
39
|
-
handle_exception(
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
38
|
+
handle_exception(e, {
|
|
39
|
+
context: "AWS:ATHENA:READ:EXCEPTION",
|
|
40
|
+
type: "error",
|
|
41
|
+
sync_id: sync_config.sync_id,
|
|
42
|
+
sync_run_id: sync_config.sync_run_id
|
|
43
|
+
})
|
|
44
44
|
end
|
|
45
45
|
|
|
46
46
|
private
|
|
@@ -36,11 +36,10 @@ module Multiwoven::Integrations::Source
|
|
|
36
36
|
catalog = Catalog.new(streams: create_streams(records))
|
|
37
37
|
catalog.to_multiwoven_message
|
|
38
38
|
rescue StandardError => e
|
|
39
|
-
handle_exception(
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
)
|
|
39
|
+
handle_exception(e, {
|
|
40
|
+
context: "BIGQUERY:DISCOVER:EXCEPTION",
|
|
41
|
+
type: "error"
|
|
42
|
+
})
|
|
44
43
|
end
|
|
45
44
|
|
|
46
45
|
def read(sync_config)
|
|
@@ -54,11 +53,12 @@ module Multiwoven::Integrations::Source
|
|
|
54
53
|
|
|
55
54
|
query(bigquery, query)
|
|
56
55
|
rescue StandardError => e
|
|
57
|
-
handle_exception(
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
56
|
+
handle_exception(e, {
|
|
57
|
+
context: "BIGQUERY:READ:EXCEPTION",
|
|
58
|
+
type: "error",
|
|
59
|
+
sync_id: sync_config.sync_id,
|
|
60
|
+
sync_run_id: sync_config.sync_run_id
|
|
61
|
+
})
|
|
62
62
|
end
|
|
63
63
|
|
|
64
64
|
private
|
|
@@ -24,11 +24,10 @@ module Multiwoven::Integrations::Source
|
|
|
24
24
|
catalog = Catalog.new(streams: create_streams(records))
|
|
25
25
|
catalog.to_multiwoven_message
|
|
26
26
|
rescue StandardError => e
|
|
27
|
-
handle_exception(
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
)
|
|
27
|
+
handle_exception(e, {
|
|
28
|
+
context: "CLICKHOUSE:DISCOVER:EXCEPTION",
|
|
29
|
+
type: "error"
|
|
30
|
+
})
|
|
32
31
|
end
|
|
33
32
|
|
|
34
33
|
def read(sync_config)
|
|
@@ -39,11 +38,12 @@ module Multiwoven::Integrations::Source
|
|
|
39
38
|
db = create_connection(connection_config)
|
|
40
39
|
query(db, query)
|
|
41
40
|
rescue StandardError => e
|
|
42
|
-
handle_exception(
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
41
|
+
handle_exception(e, {
|
|
42
|
+
context: "CLICKHOUSE:READ:EXCEPTION",
|
|
43
|
+
type: "error",
|
|
44
|
+
sync_id: sync_config.sync_id,
|
|
45
|
+
sync_run_id: sync_config.sync_run_id
|
|
46
|
+
})
|
|
47
47
|
end
|
|
48
48
|
|
|
49
49
|
private
|
|
@@ -28,11 +28,10 @@ module Multiwoven::Integrations::Source
|
|
|
28
28
|
catalog = Catalog.new(streams: create_streams(records))
|
|
29
29
|
catalog.to_multiwoven_message
|
|
30
30
|
rescue StandardError => e
|
|
31
|
-
handle_exception(
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
)
|
|
31
|
+
handle_exception(e, {
|
|
32
|
+
context: "DATABRICKS:DISCOVER:EXCEPTION",
|
|
33
|
+
type: "error"
|
|
34
|
+
})
|
|
36
35
|
end
|
|
37
36
|
|
|
38
37
|
def read(sync_config)
|
|
@@ -45,11 +44,12 @@ module Multiwoven::Integrations::Source
|
|
|
45
44
|
|
|
46
45
|
query(db, query)
|
|
47
46
|
rescue StandardError => e
|
|
48
|
-
handle_exception(
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
47
|
+
handle_exception(e, {
|
|
48
|
+
context: "DATABRICKS:READ:EXCEPTION",
|
|
49
|
+
type: "error",
|
|
50
|
+
sync_id: sync_config.sync_id,
|
|
51
|
+
sync_run_id: sync_config.sync_run_id
|
|
52
|
+
})
|
|
53
53
|
end
|
|
54
54
|
|
|
55
55
|
private
|
|
@@ -34,11 +34,10 @@ module Multiwoven::Integrations::Source
|
|
|
34
34
|
catalog = Catalog.new(streams: create_streams(records))
|
|
35
35
|
catalog.to_multiwoven_message
|
|
36
36
|
rescue StandardError => e
|
|
37
|
-
handle_exception(
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
)
|
|
37
|
+
handle_exception(e, {
|
|
38
|
+
context: "POSTGRESQL:DISCOVER:EXCEPTION",
|
|
39
|
+
type: "error"
|
|
40
|
+
})
|
|
42
41
|
ensure
|
|
43
42
|
db&.close
|
|
44
43
|
end
|
|
@@ -53,11 +52,12 @@ module Multiwoven::Integrations::Source
|
|
|
53
52
|
|
|
54
53
|
query(db, query)
|
|
55
54
|
rescue StandardError => e
|
|
56
|
-
handle_exception(
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
55
|
+
handle_exception(e, {
|
|
56
|
+
context: "POSTGRESQL:READ:EXCEPTION",
|
|
57
|
+
type: "error",
|
|
58
|
+
sync_id: sync_config.sync_id,
|
|
59
|
+
sync_run_id: sync_config.sync_run_id
|
|
60
|
+
})
|
|
61
61
|
ensure
|
|
62
62
|
db&.close
|
|
63
63
|
end
|
|
@@ -34,11 +34,10 @@ module Multiwoven::Integrations::Source
|
|
|
34
34
|
catalog = Catalog.new(streams: create_streams(records))
|
|
35
35
|
catalog.to_multiwoven_message
|
|
36
36
|
rescue StandardError => e
|
|
37
|
-
handle_exception(
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
)
|
|
37
|
+
handle_exception(e, {
|
|
38
|
+
context: "REDSHIFT:DISCOVER:EXCEPTION",
|
|
39
|
+
type: "error"
|
|
40
|
+
})
|
|
42
41
|
ensure
|
|
43
42
|
db&.close
|
|
44
43
|
end
|
|
@@ -53,11 +52,12 @@ module Multiwoven::Integrations::Source
|
|
|
53
52
|
|
|
54
53
|
query(db, query)
|
|
55
54
|
rescue StandardError => e
|
|
56
|
-
handle_exception(
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
55
|
+
handle_exception(e, {
|
|
56
|
+
context: "REDSHIFT:READ:EXCEPTION",
|
|
57
|
+
type: "error",
|
|
58
|
+
sync_id: sync_config.sync_id,
|
|
59
|
+
sync_run_id: sync_config.sync_run_id
|
|
60
|
+
})
|
|
61
61
|
ensure
|
|
62
62
|
db&.close
|
|
63
63
|
end
|
|
@@ -34,7 +34,10 @@ module Multiwoven
|
|
|
34
34
|
end
|
|
35
35
|
catalog.to_multiwoven_message
|
|
36
36
|
rescue StandardError => e
|
|
37
|
-
handle_exception(
|
|
37
|
+
handle_exception(e, {
|
|
38
|
+
context: "SALESFORCE:CONSUMER:GOODS:ClOUD:DISCOVER:EXCEPTION",
|
|
39
|
+
type: "error"
|
|
40
|
+
})
|
|
38
41
|
end
|
|
39
42
|
|
|
40
43
|
def read(sync_config)
|
|
@@ -50,7 +53,12 @@ module Multiwoven
|
|
|
50
53
|
RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
|
|
51
54
|
end
|
|
52
55
|
rescue StandardError => e
|
|
53
|
-
handle_exception(
|
|
56
|
+
handle_exception(e, {
|
|
57
|
+
context: "SALESFORCE:CONSUMER:GOODS:ClOUD:WRITE:EXCEPTION",
|
|
58
|
+
type: "error",
|
|
59
|
+
sync_id: sync_config.sync_id,
|
|
60
|
+
sync_run_id: sync_config.sync_run_id
|
|
61
|
+
})
|
|
54
62
|
end
|
|
55
63
|
|
|
56
64
|
private
|
|
@@ -9,7 +9,7 @@ module Multiwoven
|
|
|
9
9
|
|
|
10
10
|
module_function
|
|
11
11
|
|
|
12
|
-
def salesforce_field_to_json_schema_type(sf_field) # rubocop:disable Metrics/
|
|
12
|
+
def salesforce_field_to_json_schema_type(sf_field) # rubocop:disable Metrics/MethodLength, Metrics/PerceivedComplexity
|
|
13
13
|
case sf_field["type"]
|
|
14
14
|
when "string", "Email", "Phone", "Text", "TextArea", "TextEncrypted", "URL", "Picklist (Single)"
|
|
15
15
|
if sf_field["nillable"]
|
|
@@ -28,11 +28,10 @@ module Multiwoven::Integrations::Source
|
|
|
28
28
|
catalog = Catalog.new(streams: create_streams(records))
|
|
29
29
|
catalog.to_multiwoven_message
|
|
30
30
|
rescue StandardError => e
|
|
31
|
-
handle_exception(
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
)
|
|
31
|
+
handle_exception(e, {
|
|
32
|
+
context: "SNOWFLAKE:DISCOVER:EXCEPTION",
|
|
33
|
+
type: "error"
|
|
34
|
+
})
|
|
36
35
|
end
|
|
37
36
|
|
|
38
37
|
def read(sync_config)
|
|
@@ -45,11 +44,12 @@ module Multiwoven::Integrations::Source
|
|
|
45
44
|
|
|
46
45
|
query(db, query)
|
|
47
46
|
rescue StandardError => e
|
|
48
|
-
handle_exception(
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
47
|
+
handle_exception(e, {
|
|
48
|
+
context: "SNOWFLAKE:READ:EXCEPTION",
|
|
49
|
+
type: "error",
|
|
50
|
+
sync_id: sync_config.sync_id,
|
|
51
|
+
sync_run_id: sync_config.sync_run_id
|
|
52
|
+
})
|
|
53
53
|
end
|
|
54
54
|
|
|
55
55
|
private
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: multiwoven-integrations
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.71
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Subin T P
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2024-05-
|
|
11
|
+
date: 2024-05-27 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activesupport
|