multiwoven-integrations 0.3.4 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: cb9f117bf574599b93ccfd2238f1629166c473184837ec48a16a447650b1dc54
4
- data.tar.gz: efa39262c596d2b50bbd04d2d8a9d98889b708a9e2859e6ec3801b2c6eb79169
3
+ metadata.gz: 70c20cbce32a5325dde37e8cb8e1930905d3d9003300d9d552db1f50f63817b0
4
+ data.tar.gz: 54e03d9a4dd254d20fbee73ca9479b849beab979b5d9f7b76a708c34fa4881cb
5
5
  SHA512:
6
- metadata.gz: 959b53319cb4581fd8aeb2731891f967031a23096c42e1e855fa79802bf81e799a746ba62c1c08f20484c3a443418d2175e1d9218abd28939eceaf89b6f87964
7
- data.tar.gz: 22c706c8d47d9803cbdee5fa91a8f6862f7d0efb11c13466e2b0b0e07753a5e784e43443487a939a05f0cee0457bf310d12ce274e24ad322e5aa2cf8767c05ea
6
+ metadata.gz: f13156eac84b28e95904d9e32025f3e9943da758cfb14d69561ba1d69b40fd25c4eb9f1d97e91d34013eddf723d17ac579a35ab713a7272a6da78fa10d56c298
7
+ data.tar.gz: 35345fcd1d52d6237be64829092611ae69c629c13ca8964f38de7cf0fd4e133914c54265f87d3c9259dbf5948aa67ea4b017e31be5e2faecec3cd85d56493552
@@ -9,6 +9,12 @@ module Multiwoven
9
9
  raise "Not implemented"
10
10
  # return Protocol::TrackingMessage
11
11
  end
12
+
13
+ def tracking_message(success, failure, log_message_array)
14
+ Multiwoven::Integrations::Protocol::TrackingMessage.new(
15
+ success: success, failed: failure, logs: log_message_array
16
+ ).to_multiwoven_message
17
+ end
12
18
  end
13
19
  end
14
20
  end
@@ -53,6 +53,14 @@ module Multiwoven
53
53
  reporter&.report(exception, meta)
54
54
  end
55
55
 
56
+ def log_request_response(level, request, response)
57
+ Integrations::Protocol::LogMessage.new(
58
+ name: self.class.name,
59
+ level: level,
60
+ message: { request: request.to_s, response: response.to_s, level: level }.to_json
61
+ )
62
+ end
63
+
56
64
  def create_log_message(context, type, exception)
57
65
  Integrations::Protocol::LogMessage.new(
58
66
  name: context,
@@ -0,0 +1,115 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Multiwoven::Integrations::Destination
4
+ module MariaDB
5
+ include Multiwoven::Integrations::Core
6
+ class Client < DestinationConnector
7
+ def check_connection(connection_config)
8
+ connection_config = connection_config.with_indifferent_access
9
+ create_connection(connection_config)
10
+ ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_multiwoven_message
11
+ rescue StandardError => e
12
+ ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_multiwoven_message
13
+ end
14
+
15
+ def discover(connection_config)
16
+ connection_config = connection_config.with_indifferent_access
17
+ query = "SELECT table_name, column_name, data_type, is_nullable
18
+ FROM information_schema.columns
19
+ WHERE table_schema = '#{connection_config[:database]}'
20
+ ORDER BY table_name, ordinal_position;"
21
+
22
+ db = create_connection(connection_config)
23
+ records = db.fetch(query) do |result|
24
+ result.map do |row|
25
+ row
26
+ end
27
+ end
28
+ catalog = Catalog.new(streams: create_streams(records))
29
+ catalog.to_multiwoven_message
30
+ rescue StandardError => e
31
+ handle_exception(
32
+ "MARIA:DB:DISCOVER:EXCEPTION",
33
+ "error",
34
+ e
35
+ )
36
+ end
37
+
38
+ def write(sync_config, records, action = "destination_insert")
39
+ connection_config = sync_config.destination.connection_specification.with_indifferent_access
40
+ table_name = sync_config.stream.name
41
+ primary_key = sync_config.model.primary_key
42
+ db = create_connection(connection_config)
43
+
44
+ write_success = 0
45
+ write_failure = 0
46
+
47
+ records.each do |record|
48
+ query = Multiwoven::Integrations::Core::QueryBuilder.perform(action, table_name, record, primary_key)
49
+ logger.debug("MARIA:DB:WRITE:QUERY query = #{query} sync_id = #{sync_config.sync_id} sync_run_id = #{sync_config.sync_run_id}")
50
+ begin
51
+ db.run(query)
52
+ write_success += 1
53
+ rescue StandardError => e
54
+ handle_exception(e, {
55
+ context: "MARIA:DB:RECORD:WRITE:EXCEPTION",
56
+ type: "error",
57
+ sync_id: sync_config.sync_id,
58
+ sync_run_id: sync_config.sync_run_id
59
+ })
60
+ write_failure += 1
61
+ end
62
+ end
63
+ tracking_message(write_success, write_failure)
64
+ rescue StandardError => e
65
+ handle_exception(e, {
66
+ context: "MARIA:DB:RECORD:WRITE:EXCEPTION",
67
+ type: "error",
68
+ sync_id: sync_config.sync_id,
69
+ sync_run_id: sync_config.sync_run_id
70
+ })
71
+ end
72
+
73
+ private
74
+
75
+ def create_connection(connection_config)
76
+ Sequel.connect(
77
+ adapter: "mysql2",
78
+ host: connection_config[:host],
79
+ port: connection_config[:port],
80
+ user: connection_config[:username],
81
+ password: connection_config[:password],
82
+ database: connection_config[:database]
83
+ )
84
+ end
85
+
86
+ def create_streams(records)
87
+ group_by_table(records).map do |_, r|
88
+ Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
89
+ end
90
+ end
91
+
92
+ def group_by_table(records)
93
+ result = {}
94
+ records.each_with_index do |entry, index|
95
+ table_name = entry[:table_name]
96
+ column_data = {
97
+ column_name: entry[:column_name],
98
+ data_type: entry[:data_type],
99
+ is_nullable: entry[:is_nullable] == "YES"
100
+ }
101
+ result[index] ||= {}
102
+ result[index][:tablename] = table_name
103
+ result[index][:columns] = [column_data]
104
+ end
105
+ result
106
+ end
107
+
108
+ def tracking_message(success, failure)
109
+ Multiwoven::Integrations::Protocol::TrackingMessage.new(
110
+ success: success, failed: failure
111
+ ).to_multiwoven_message
112
+ end
113
+ end
114
+ end
115
+ end
@@ -0,0 +1,15 @@
1
+ {
2
+ "data": {
3
+ "name": "MariaDB",
4
+ "title": "Maria DB",
5
+ "connector_type": "destination",
6
+ "category": "Database",
7
+ "documentation_url": "https://docs.squared.ai/guides/data-integration/destination/mariadb",
8
+ "github_issue_label": "destination-maria-db",
9
+ "icon": "icon.svg",
10
+ "license": "MIT",
11
+ "release_stage": "alpha",
12
+ "support_level": "community",
13
+ "tags": ["language:ruby", "multiwoven"]
14
+ }
15
+ }
@@ -0,0 +1,48 @@
1
+ {
2
+ "documentation_url": "https://docs.squared.ai/guides/data-integration/destination/mariadb",
3
+ "stream_type": "dynamic",
4
+ "connector_query_type": "raw_sql",
5
+ "connection_specification": {
6
+ "$schema": "http://json-schema.org/draft-07/schema#",
7
+ "title": "Maria DB",
8
+ "type": "object",
9
+ "required": ["host", "port", "username", "password", "database"],
10
+ "properties": {
11
+ "host": {
12
+ "description": "The hostname or IP address of the server where the MariaDB database is hosted.",
13
+ "examples": ["localhost"],
14
+ "type": "string",
15
+ "title": "Host",
16
+ "order": 0
17
+ },
18
+ "port": {
19
+ "description": "The port number on which the MariaDB server is listening for connections.",
20
+ "examples": ["3306"],
21
+ "type": "string",
22
+ "title": "Port",
23
+ "order": 1
24
+ },
25
+ "username": {
26
+ "description": "The username used to authenticate and connect to the MariaDB database.",
27
+ "examples": ["root"],
28
+ "type": "string",
29
+ "title": "Username",
30
+ "order": 2
31
+ },
32
+ "password": {
33
+ "description": "The password corresponding to the username used for authentication.",
34
+ "type": "string",
35
+ "multiwoven_secret": true,
36
+ "title": "Password",
37
+ "order": 3
38
+ },
39
+ "database": {
40
+ "description": "The name of the specific database within the MariaDB server to connect to.",
41
+ "examples": ["test"],
42
+ "type": "string",
43
+ "title": "Database",
44
+ "order": 4
45
+ }
46
+ }
47
+ }
48
+ }
@@ -0,0 +1,15 @@
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <svg xmlns="http://www.w3.org/2000/svg" id="Layer_1" data-name="Layer 1" viewBox="0 0 309.88 252.72">
3
+ <defs>
4
+ <style>.cls-1{fill:#003545;}</style>
5
+ </defs>
6
+ <title>MDB-VLogo_RGB</title>
7
+ <path class="cls-1" d="M61.74,214.91,73,258.46h-8.5L57,227.09,42.86,258.46H35.68L21.62,227.27,13.9,258.46H5.58l11.16-43.55H24.1l15.2,33.72,15.14-33.72Z" transform="translate(-5.58 -6.4)"></path>
8
+ <path class="cls-1" d="M105.11,231.74v-5.07h8.21v31.79h-8.21v-5.12c-2.17,3.68-6.63,5.79-12.06,5.79-11.35,0-17.68-8-17.68-17.2,0-8.87,6-16,16.47-16C97.81,226,102.76,228,105.11,231.74Zm-21,10.62c0,5.85,3.68,10.86,10.8,10.86,6.87,0,10.61-4.89,10.61-10.68s-3.86-10.74-11-10.74C87.68,231.8,84.12,236.74,84.12,242.36Z" transform="translate(-5.58 -6.4)"></path>
9
+ <path class="cls-1" d="M131.83,258.46h-8.2V226.67h8.2v7.12A12.58,12.58,0,0,1,143.29,226a14.84,14.84,0,0,1,5.13.84l-1.75,6a18,18,0,0,0-4.34-.6c-5.92,0-10.5,4.46-10.5,11Z" transform="translate(-5.58 -6.4)"></path>
10
+ <path class="cls-1" d="M152.52,218.28a4.17,4.17,0,0,1,4.4-4.28,4.33,4.33,0,0,1,4.47,4.41,4.15,4.15,0,0,1-4.47,4.22A4.22,4.22,0,0,1,152.52,218.28Zm.36,8.39h8.21V249.9c0,1.44.3,2.47,1.5,2.47a8.85,8.85,0,0,0,1.57-.18l1.27,6a14.36,14.36,0,0,1-5.43,1c-3.44,0-7.12-1-7.12-8.81Z" transform="translate(-5.58 -6.4)"></path>
11
+ <path class="cls-1" d="M197.76,231.74v-5.07H206v31.79h-8.21v-5.12c-2.17,3.68-6.63,5.79-12.06,5.79-11.34,0-17.68-8-17.68-17.2,0-8.87,6-16,16.47-16C190.46,226,195.41,228,197.76,231.74Zm-21,10.62c0,5.85,3.68,10.86,10.8,10.86,6.88,0,10.62-4.89,10.62-10.68s-3.87-10.74-11-10.74C180.33,231.8,176.77,236.74,176.77,242.36Z" transform="translate(-5.58 -6.4)"></path>
12
+ <path class="cls-1" d="M218.57,214.91h19.67c16.53,0,24.79,9.11,24.67,21.77.12,13.16-9,21.78-23.22,21.78H218.57Zm5.43,3.86v35.89h15c13.15,0,18.16-8.87,18.16-18.1,0-10.43-6.27-17.79-18.16-17.79Z" transform="translate(-5.58 -6.4)"></path>
13
+ <path class="cls-1" d="M296.45,258.46h-25V214.91H294c8.62,0,16.83,1.62,16.71,11.28,0,6.81-4.23,8.69-8.69,9.41,6.33.54,10.14,4.58,10.14,11.1C312.2,256.47,303.63,258.46,296.45,258.46Zm-1.87-24.55c8.63,0,10.56-3.32,10.56-7.54,0-6.34-3.86-7.78-10.56-7.78H276.66v15.32Zm.24,3.68H276.66v17.19H295.6c5.31,0,10.92-1.75,10.92-8.44C306.52,238.62,300.07,237.59,294.82,237.59Z" transform="translate(-5.58 -6.4)"></path>
14
+ <path class="cls-1" d="M314.08,7.35a4.18,4.18,0,0,0-2.84-.95c-2.83,0-6.49,1.92-8.46,2.95l-.78.4a26.86,26.86,0,0,1-10.57,2.66c-3.76.12-7,.34-11.22.78-25,2.57-36.15,21.73-46.89,40.26C227.47,63.53,221.43,74,213.15,82a54.4,54.4,0,0,1-5.45,4.63C199.13,93,188.37,97.55,180,100.77c-8.06,3.08-16.86,5.85-25.36,8.53-7.79,2.45-15.14,4.77-21.9,7.28-3.05,1.13-5.64,2-7.93,2.76-6.16,2-10.6,3.53-17.09,8-2.53,1.73-5.07,3.6-6.79,5a71.62,71.62,0,0,0-13.55,14.27A84.25,84.25,0,0,1,76,160.27c-1.37,1.34-3.8,2-7.44,2-4.26,0-9.43-.88-14.9-1.81-5.64-1-11.47-1.95-16.47-1.95-4.06,0-7.17.66-9.49,2,0,0-3.91,2.28-5.56,5.23l1.62.73a33.21,33.21,0,0,1,6.92,5,34.72,34.72,0,0,0,7.2,5.12A6.18,6.18,0,0,1,40.1,178c-.68,1-1.68,2.29-2.73,3.67-5.77,7.55-9.14,12.32-7.21,14.92a6.07,6.07,0,0,0,3,.68c12.58,0,19.34-3.27,27.89-7.41,2.48-1.2,5-2.43,8-3.7,5-2.17,10.38-5.63,16.09-9.29C92.61,172,100.42,167,108,164.59a62.3,62.3,0,0,1,19.23-2.7c8,0,16.42,1.07,24.54,2.11,6.05.78,12.32,1.58,18.47,1.95,2.39.14,4.6.21,6.75.21a78.21,78.21,0,0,0,8.61-.45l.69-.24c4.31-2.65,6.33-8.34,8.29-13.84,1.26-3.54,2.32-6.72,4-8.74a2.55,2.55,0,0,1,.32-.27.4.4,0,0,1,.49.08s0,.05,0,.16c-1,21.51-9.66,35.17-18.42,47.31l-5.85,6.27s8.19,0,12.85-1.8c17-5.08,29.83-16.28,39.17-34.14a145.7,145.7,0,0,0,6.17-14.09c.16-.4,1.63-1.14,1.49.93-.05.61-.09,1.29-.14,2h0c0,.42,0,.85-.08,1.28-.24,3-.95,9.34-.95,9.34l5.25-2.81c12.66-8,22.42-24.14,29.82-49.25,3.08-10.46,5.34-20.85,7.33-30,2.38-11,4.43-20.43,6.78-24.09,3.69-5.74,9.32-9.62,14.77-13.39.74-.51,1.49-1,2.22-1.54,6.85-4.81,13.66-10.36,15.16-20.71l0-.23C316.05,10.22,315.13,8.25,314.08,7.35Z" transform="translate(-5.58 -6.4)"></path>
15
+ </svg>
@@ -46,6 +46,7 @@ module Multiwoven::Integrations::Destination
46
46
  connection_config = sync_config.destination.connection_specification.with_indifferent_access
47
47
  table_name = sync_config.stream.name
48
48
  primary_key = sync_config.model.primary_key
49
+ log_message_array = []
49
50
  db = create_connection(connection_config)
50
51
 
51
52
  write_success = 0
@@ -55,8 +56,9 @@ module Multiwoven::Integrations::Destination
55
56
  query = Multiwoven::Integrations::Core::QueryBuilder.perform(action, table_name, record, primary_key)
56
57
  logger.debug("POSTGRESQL:WRITE:QUERY query = #{query} sync_id = #{sync_config.sync_id} sync_run_id = #{sync_config.sync_run_id}")
57
58
  begin
58
- db.exec(query)
59
+ response = db.exec(query)
59
60
  write_success += 1
61
+ log_message_array << log_request_response("info", query, response)
60
62
  rescue StandardError => e
61
63
  handle_exception(e, {
62
64
  context: "POSTGRESQL:RECORD:WRITE:EXCEPTION",
@@ -65,9 +67,10 @@ module Multiwoven::Integrations::Destination
65
67
  sync_run_id: sync_config.sync_run_id
66
68
  })
67
69
  write_failure += 1
70
+ log_message_array << log_request_response("error", query, e.message)
68
71
  end
69
72
  end
70
- tracking_message(write_success, write_failure)
73
+ tracking_message(write_success, write_failure, log_message_array)
71
74
  rescue StandardError => e
72
75
  handle_exception(e, {
73
76
  context: "POSTGRESQL:RECORD:WRITE:EXCEPTION",
@@ -119,12 +122,6 @@ module Multiwoven::Integrations::Destination
119
122
  }
120
123
  end
121
124
  end
122
-
123
- def tracking_message(success, failure)
124
- Multiwoven::Integrations::Protocol::TrackingMessage.new(
125
- success: success, failed: failure
126
- ).to_multiwoven_message
127
- end
128
125
  end
129
126
  end
130
127
  end
@@ -80,10 +80,14 @@ module Multiwoven
80
80
  write_success = 0
81
81
  write_failure = 0
82
82
  properties = stream.json_schema[:properties]
83
+ log_message_array = []
84
+
83
85
  records.each do |record_object|
84
86
  record = extract_data(record_object, properties)
85
- process_record(stream, record)
87
+ args = [stream.name, "Id", record]
88
+ response = send_data_to_salesforce(args)
86
89
  write_success += 1
90
+ log_message_array << log_request_response("info", args, response)
87
91
  rescue StandardError => e
88
92
  # TODO: add sync_id and sync run id to the logs
89
93
  handle_exception(e, {
@@ -93,18 +97,14 @@ module Multiwoven
93
97
  sync_run_id: @sync_config.sync_run_id
94
98
  })
95
99
  write_failure += 1
100
+ log_message_array << log_request_response("error", args, e.message)
96
101
  end
97
- tracking_message(write_success, write_failure)
98
- end
99
-
100
- def process_record(stream, record)
101
- send_data_to_salesforce(stream.name, record)
102
+ tracking_message(write_success, write_failure, log_message_array)
102
103
  end
103
104
 
104
- def send_data_to_salesforce(stream_name, record = {})
105
+ def send_data_to_salesforce(args)
105
106
  method_name = "upsert!"
106
- args = [stream_name, "Id", record]
107
- @logger.debug("sync_id: #{@sync_config.sync_id}, sync_run_id: #{@sync_config.sync_run_id}, record: #{record}")
107
+ @logger.debug("sync_id: #{@sync_config.sync_id}, sync_run_id: #{@sync_config.sync_run_id}, args: #{args}")
108
108
  @client.send(method_name, *args)
109
109
  end
110
110
 
@@ -124,12 +124,6 @@ module Multiwoven
124
124
  read_json(CATALOG_SPEC_PATH)
125
125
  end
126
126
 
127
- def tracking_message(success, failure)
128
- Multiwoven::Integrations::Protocol::TrackingMessage.new(
129
- success: success, failed: failure
130
- ).to_multiwoven_message
131
- end
132
-
133
127
  def log_debug(message)
134
128
  Multiwoven::Integrations::Service.logger.debug(message)
135
129
  end
@@ -197,6 +197,7 @@ module Multiwoven
197
197
  attribute :success, Types::Integer.default(0)
198
198
  attribute :failed, Types::Integer.default(0)
199
199
  attribute? :meta, Types::Hash
200
+ attribute? :logs, Types::Array.of(LogMessage)
200
201
 
201
202
  def to_multiwoven_message
202
203
  MultiwovenMessage.new(
@@ -2,7 +2,7 @@
2
2
 
3
3
  module Multiwoven
4
4
  module Integrations
5
- VERSION = "0.3.4"
5
+ VERSION = "0.4.0"
6
6
 
7
7
  ENABLED_SOURCES = %w[
8
8
  Snowflake
@@ -32,6 +32,7 @@ module Multiwoven
32
32
  Http
33
33
  Zendesk
34
34
  Iterable
35
+ MariaDB
35
36
  ].freeze
36
37
  end
37
38
  end
@@ -76,6 +76,7 @@ require_relative "integrations/destination/postgresql/client"
76
76
  require_relative "integrations/destination/http/client"
77
77
  require_relative "integrations/destination/zendesk/client"
78
78
  require_relative "integrations/destination/iterable/client"
79
+ require_relative "integrations/destination/maria_db/client"
79
80
 
80
81
  module Multiwoven
81
82
  module Integrations
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: multiwoven-integrations
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.4
4
+ version: 0.4.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Subin T P
@@ -522,6 +522,10 @@ files:
522
522
  - lib/multiwoven/integrations/destination/klaviyo/config/meta.json
523
523
  - lib/multiwoven/integrations/destination/klaviyo/config/spec.json
524
524
  - lib/multiwoven/integrations/destination/klaviyo/icon.svg
525
+ - lib/multiwoven/integrations/destination/maria_db/client.rb
526
+ - lib/multiwoven/integrations/destination/maria_db/config/meta.json
527
+ - lib/multiwoven/integrations/destination/maria_db/config/spec.json
528
+ - lib/multiwoven/integrations/destination/maria_db/icon.svg
525
529
  - lib/multiwoven/integrations/destination/postgresql/client.rb
526
530
  - lib/multiwoven/integrations/destination/postgresql/config/meta.json
527
531
  - lib/multiwoven/integrations/destination/postgresql/config/spec.json