multiwoven-integrations 0.3.4 → 0.4.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: cb9f117bf574599b93ccfd2238f1629166c473184837ec48a16a447650b1dc54
4
- data.tar.gz: efa39262c596d2b50bbd04d2d8a9d98889b708a9e2859e6ec3801b2c6eb79169
3
+ metadata.gz: 70c20cbce32a5325dde37e8cb8e1930905d3d9003300d9d552db1f50f63817b0
4
+ data.tar.gz: 54e03d9a4dd254d20fbee73ca9479b849beab979b5d9f7b76a708c34fa4881cb
5
5
  SHA512:
6
- metadata.gz: 959b53319cb4581fd8aeb2731891f967031a23096c42e1e855fa79802bf81e799a746ba62c1c08f20484c3a443418d2175e1d9218abd28939eceaf89b6f87964
7
- data.tar.gz: 22c706c8d47d9803cbdee5fa91a8f6862f7d0efb11c13466e2b0b0e07753a5e784e43443487a939a05f0cee0457bf310d12ce274e24ad322e5aa2cf8767c05ea
6
+ metadata.gz: f13156eac84b28e95904d9e32025f3e9943da758cfb14d69561ba1d69b40fd25c4eb9f1d97e91d34013eddf723d17ac579a35ab713a7272a6da78fa10d56c298
7
+ data.tar.gz: 35345fcd1d52d6237be64829092611ae69c629c13ca8964f38de7cf0fd4e133914c54265f87d3c9259dbf5948aa67ea4b017e31be5e2faecec3cd85d56493552
@@ -9,6 +9,12 @@ module Multiwoven
9
9
  raise "Not implemented"
10
10
  # return Protocol::TrackingMessage
11
11
  end
12
+
13
+ def tracking_message(success, failure, log_message_array)
14
+ Multiwoven::Integrations::Protocol::TrackingMessage.new(
15
+ success: success, failed: failure, logs: log_message_array
16
+ ).to_multiwoven_message
17
+ end
12
18
  end
13
19
  end
14
20
  end
@@ -53,6 +53,14 @@ module Multiwoven
53
53
  reporter&.report(exception, meta)
54
54
  end
55
55
 
56
+ def log_request_response(level, request, response)
57
+ Integrations::Protocol::LogMessage.new(
58
+ name: self.class.name,
59
+ level: level,
60
+ message: { request: request.to_s, response: response.to_s, level: level }.to_json
61
+ )
62
+ end
63
+
56
64
  def create_log_message(context, type, exception)
57
65
  Integrations::Protocol::LogMessage.new(
58
66
  name: context,
@@ -0,0 +1,115 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Multiwoven::Integrations::Destination
4
+ module MariaDB
5
+ include Multiwoven::Integrations::Core
6
+ class Client < DestinationConnector
7
+ def check_connection(connection_config)
8
+ connection_config = connection_config.with_indifferent_access
9
+ create_connection(connection_config)
10
+ ConnectionStatus.new(status: ConnectionStatusType["succeeded"]).to_multiwoven_message
11
+ rescue StandardError => e
12
+ ConnectionStatus.new(status: ConnectionStatusType["failed"], message: e.message).to_multiwoven_message
13
+ end
14
+
15
+ def discover(connection_config)
16
+ connection_config = connection_config.with_indifferent_access
17
+ query = "SELECT table_name, column_name, data_type, is_nullable
18
+ FROM information_schema.columns
19
+ WHERE table_schema = '#{connection_config[:database]}'
20
+ ORDER BY table_name, ordinal_position;"
21
+
22
+ db = create_connection(connection_config)
23
+ records = db.fetch(query) do |result|
24
+ result.map do |row|
25
+ row
26
+ end
27
+ end
28
+ catalog = Catalog.new(streams: create_streams(records))
29
+ catalog.to_multiwoven_message
30
+ rescue StandardError => e
31
+ handle_exception(
32
+ "MARIA:DB:DISCOVER:EXCEPTION",
33
+ "error",
34
+ e
35
+ )
36
+ end
37
+
38
+ def write(sync_config, records, action = "destination_insert")
39
+ connection_config = sync_config.destination.connection_specification.with_indifferent_access
40
+ table_name = sync_config.stream.name
41
+ primary_key = sync_config.model.primary_key
42
+ db = create_connection(connection_config)
43
+
44
+ write_success = 0
45
+ write_failure = 0
46
+
47
+ records.each do |record|
48
+ query = Multiwoven::Integrations::Core::QueryBuilder.perform(action, table_name, record, primary_key)
49
+ logger.debug("MARIA:DB:WRITE:QUERY query = #{query} sync_id = #{sync_config.sync_id} sync_run_id = #{sync_config.sync_run_id}")
50
+ begin
51
+ db.run(query)
52
+ write_success += 1
53
+ rescue StandardError => e
54
+ handle_exception(e, {
55
+ context: "MARIA:DB:RECORD:WRITE:EXCEPTION",
56
+ type: "error",
57
+ sync_id: sync_config.sync_id,
58
+ sync_run_id: sync_config.sync_run_id
59
+ })
60
+ write_failure += 1
61
+ end
62
+ end
63
+ tracking_message(write_success, write_failure)
64
+ rescue StandardError => e
65
+ handle_exception(e, {
66
+ context: "MARIA:DB:RECORD:WRITE:EXCEPTION",
67
+ type: "error",
68
+ sync_id: sync_config.sync_id,
69
+ sync_run_id: sync_config.sync_run_id
70
+ })
71
+ end
72
+
73
+ private
74
+
75
+ def create_connection(connection_config)
76
+ Sequel.connect(
77
+ adapter: "mysql2",
78
+ host: connection_config[:host],
79
+ port: connection_config[:port],
80
+ user: connection_config[:username],
81
+ password: connection_config[:password],
82
+ database: connection_config[:database]
83
+ )
84
+ end
85
+
86
+ def create_streams(records)
87
+ group_by_table(records).map do |_, r|
88
+ Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
89
+ end
90
+ end
91
+
92
+ def group_by_table(records)
93
+ result = {}
94
+ records.each_with_index do |entry, index|
95
+ table_name = entry[:table_name]
96
+ column_data = {
97
+ column_name: entry[:column_name],
98
+ data_type: entry[:data_type],
99
+ is_nullable: entry[:is_nullable] == "YES"
100
+ }
101
+ result[index] ||= {}
102
+ result[index][:tablename] = table_name
103
+ result[index][:columns] = [column_data]
104
+ end
105
+ result
106
+ end
107
+
108
+ def tracking_message(success, failure)
109
+ Multiwoven::Integrations::Protocol::TrackingMessage.new(
110
+ success: success, failed: failure
111
+ ).to_multiwoven_message
112
+ end
113
+ end
114
+ end
115
+ end
@@ -0,0 +1,15 @@
1
+ {
2
+ "data": {
3
+ "name": "MariaDB",
4
+ "title": "Maria DB",
5
+ "connector_type": "destination",
6
+ "category": "Database",
7
+ "documentation_url": "https://docs.squared.ai/guides/data-integration/destination/mariadb",
8
+ "github_issue_label": "destination-maria-db",
9
+ "icon": "icon.svg",
10
+ "license": "MIT",
11
+ "release_stage": "alpha",
12
+ "support_level": "community",
13
+ "tags": ["language:ruby", "multiwoven"]
14
+ }
15
+ }
@@ -0,0 +1,48 @@
1
+ {
2
+ "documentation_url": "https://docs.squared.ai/guides/data-integration/destination/mariadb",
3
+ "stream_type": "dynamic",
4
+ "connector_query_type": "raw_sql",
5
+ "connection_specification": {
6
+ "$schema": "http://json-schema.org/draft-07/schema#",
7
+ "title": "Maria DB",
8
+ "type": "object",
9
+ "required": ["host", "port", "username", "password", "database"],
10
+ "properties": {
11
+ "host": {
12
+ "description": "The hostname or IP address of the server where the MariaDB database is hosted.",
13
+ "examples": ["localhost"],
14
+ "type": "string",
15
+ "title": "Host",
16
+ "order": 0
17
+ },
18
+ "port": {
19
+ "description": "The port number on which the MariaDB server is listening for connections.",
20
+ "examples": ["3306"],
21
+ "type": "string",
22
+ "title": "Port",
23
+ "order": 1
24
+ },
25
+ "username": {
26
+ "description": "The username used to authenticate and connect to the MariaDB database.",
27
+ "examples": ["root"],
28
+ "type": "string",
29
+ "title": "Username",
30
+ "order": 2
31
+ },
32
+ "password": {
33
+ "description": "The password corresponding to the username used for authentication.",
34
+ "type": "string",
35
+ "multiwoven_secret": true,
36
+ "title": "Password",
37
+ "order": 3
38
+ },
39
+ "database": {
40
+ "description": "The name of the specific database within the MariaDB server to connect to.",
41
+ "examples": ["test"],
42
+ "type": "string",
43
+ "title": "Database",
44
+ "order": 4
45
+ }
46
+ }
47
+ }
48
+ }
@@ -0,0 +1,15 @@
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <svg xmlns="http://www.w3.org/2000/svg" id="Layer_1" data-name="Layer 1" viewBox="0 0 309.88 252.72">
3
+ <defs>
4
+ <style>.cls-1{fill:#003545;}</style>
5
+ </defs>
6
+ <title>MDB-VLogo_RGB</title>
7
+ <path class="cls-1" d="M61.74,214.91,73,258.46h-8.5L57,227.09,42.86,258.46H35.68L21.62,227.27,13.9,258.46H5.58l11.16-43.55H24.1l15.2,33.72,15.14-33.72Z" transform="translate(-5.58 -6.4)"></path>
8
+ <path class="cls-1" d="M105.11,231.74v-5.07h8.21v31.79h-8.21v-5.12c-2.17,3.68-6.63,5.79-12.06,5.79-11.35,0-17.68-8-17.68-17.2,0-8.87,6-16,16.47-16C97.81,226,102.76,228,105.11,231.74Zm-21,10.62c0,5.85,3.68,10.86,10.8,10.86,6.87,0,10.61-4.89,10.61-10.68s-3.86-10.74-11-10.74C87.68,231.8,84.12,236.74,84.12,242.36Z" transform="translate(-5.58 -6.4)"></path>
9
+ <path class="cls-1" d="M131.83,258.46h-8.2V226.67h8.2v7.12A12.58,12.58,0,0,1,143.29,226a14.84,14.84,0,0,1,5.13.84l-1.75,6a18,18,0,0,0-4.34-.6c-5.92,0-10.5,4.46-10.5,11Z" transform="translate(-5.58 -6.4)"></path>
10
+ <path class="cls-1" d="M152.52,218.28a4.17,4.17,0,0,1,4.4-4.28,4.33,4.33,0,0,1,4.47,4.41,4.15,4.15,0,0,1-4.47,4.22A4.22,4.22,0,0,1,152.52,218.28Zm.36,8.39h8.21V249.9c0,1.44.3,2.47,1.5,2.47a8.85,8.85,0,0,0,1.57-.18l1.27,6a14.36,14.36,0,0,1-5.43,1c-3.44,0-7.12-1-7.12-8.81Z" transform="translate(-5.58 -6.4)"></path>
11
+ <path class="cls-1" d="M197.76,231.74v-5.07H206v31.79h-8.21v-5.12c-2.17,3.68-6.63,5.79-12.06,5.79-11.34,0-17.68-8-17.68-17.2,0-8.87,6-16,16.47-16C190.46,226,195.41,228,197.76,231.74Zm-21,10.62c0,5.85,3.68,10.86,10.8,10.86,6.88,0,10.62-4.89,10.62-10.68s-3.87-10.74-11-10.74C180.33,231.8,176.77,236.74,176.77,242.36Z" transform="translate(-5.58 -6.4)"></path>
12
+ <path class="cls-1" d="M218.57,214.91h19.67c16.53,0,24.79,9.11,24.67,21.77.12,13.16-9,21.78-23.22,21.78H218.57Zm5.43,3.86v35.89h15c13.15,0,18.16-8.87,18.16-18.1,0-10.43-6.27-17.79-18.16-17.79Z" transform="translate(-5.58 -6.4)"></path>
13
+ <path class="cls-1" d="M296.45,258.46h-25V214.91H294c8.62,0,16.83,1.62,16.71,11.28,0,6.81-4.23,8.69-8.69,9.41,6.33.54,10.14,4.58,10.14,11.1C312.2,256.47,303.63,258.46,296.45,258.46Zm-1.87-24.55c8.63,0,10.56-3.32,10.56-7.54,0-6.34-3.86-7.78-10.56-7.78H276.66v15.32Zm.24,3.68H276.66v17.19H295.6c5.31,0,10.92-1.75,10.92-8.44C306.52,238.62,300.07,237.59,294.82,237.59Z" transform="translate(-5.58 -6.4)"></path>
14
+ <path class="cls-1" d="M314.08,7.35a4.18,4.18,0,0,0-2.84-.95c-2.83,0-6.49,1.92-8.46,2.95l-.78.4a26.86,26.86,0,0,1-10.57,2.66c-3.76.12-7,.34-11.22.78-25,2.57-36.15,21.73-46.89,40.26C227.47,63.53,221.43,74,213.15,82a54.4,54.4,0,0,1-5.45,4.63C199.13,93,188.37,97.55,180,100.77c-8.06,3.08-16.86,5.85-25.36,8.53-7.79,2.45-15.14,4.77-21.9,7.28-3.05,1.13-5.64,2-7.93,2.76-6.16,2-10.6,3.53-17.09,8-2.53,1.73-5.07,3.6-6.79,5a71.62,71.62,0,0,0-13.55,14.27A84.25,84.25,0,0,1,76,160.27c-1.37,1.34-3.8,2-7.44,2-4.26,0-9.43-.88-14.9-1.81-5.64-1-11.47-1.95-16.47-1.95-4.06,0-7.17.66-9.49,2,0,0-3.91,2.28-5.56,5.23l1.62.73a33.21,33.21,0,0,1,6.92,5,34.72,34.72,0,0,0,7.2,5.12A6.18,6.18,0,0,1,40.1,178c-.68,1-1.68,2.29-2.73,3.67-5.77,7.55-9.14,12.32-7.21,14.92a6.07,6.07,0,0,0,3,.68c12.58,0,19.34-3.27,27.89-7.41,2.48-1.2,5-2.43,8-3.7,5-2.17,10.38-5.63,16.09-9.29C92.61,172,100.42,167,108,164.59a62.3,62.3,0,0,1,19.23-2.7c8,0,16.42,1.07,24.54,2.11,6.05.78,12.32,1.58,18.47,1.95,2.39.14,4.6.21,6.75.21a78.21,78.21,0,0,0,8.61-.45l.69-.24c4.31-2.65,6.33-8.34,8.29-13.84,1.26-3.54,2.32-6.72,4-8.74a2.55,2.55,0,0,1,.32-.27.4.4,0,0,1,.49.08s0,.05,0,.16c-1,21.51-9.66,35.17-18.42,47.31l-5.85,6.27s8.19,0,12.85-1.8c17-5.08,29.83-16.28,39.17-34.14a145.7,145.7,0,0,0,6.17-14.09c.16-.4,1.63-1.14,1.49.93-.05.61-.09,1.29-.14,2h0c0,.42,0,.85-.08,1.28-.24,3-.95,9.34-.95,9.34l5.25-2.81c12.66-8,22.42-24.14,29.82-49.25,3.08-10.46,5.34-20.85,7.33-30,2.38-11,4.43-20.43,6.78-24.09,3.69-5.74,9.32-9.62,14.77-13.39.74-.51,1.49-1,2.22-1.54,6.85-4.81,13.66-10.36,15.16-20.71l0-.23C316.05,10.22,315.13,8.25,314.08,7.35Z" transform="translate(-5.58 -6.4)"></path>
15
+ </svg>
@@ -46,6 +46,7 @@ module Multiwoven::Integrations::Destination
46
46
  connection_config = sync_config.destination.connection_specification.with_indifferent_access
47
47
  table_name = sync_config.stream.name
48
48
  primary_key = sync_config.model.primary_key
49
+ log_message_array = []
49
50
  db = create_connection(connection_config)
50
51
 
51
52
  write_success = 0
@@ -55,8 +56,9 @@ module Multiwoven::Integrations::Destination
55
56
  query = Multiwoven::Integrations::Core::QueryBuilder.perform(action, table_name, record, primary_key)
56
57
  logger.debug("POSTGRESQL:WRITE:QUERY query = #{query} sync_id = #{sync_config.sync_id} sync_run_id = #{sync_config.sync_run_id}")
57
58
  begin
58
- db.exec(query)
59
+ response = db.exec(query)
59
60
  write_success += 1
61
+ log_message_array << log_request_response("info", query, response)
60
62
  rescue StandardError => e
61
63
  handle_exception(e, {
62
64
  context: "POSTGRESQL:RECORD:WRITE:EXCEPTION",
@@ -65,9 +67,10 @@ module Multiwoven::Integrations::Destination
65
67
  sync_run_id: sync_config.sync_run_id
66
68
  })
67
69
  write_failure += 1
70
+ log_message_array << log_request_response("error", query, e.message)
68
71
  end
69
72
  end
70
- tracking_message(write_success, write_failure)
73
+ tracking_message(write_success, write_failure, log_message_array)
71
74
  rescue StandardError => e
72
75
  handle_exception(e, {
73
76
  context: "POSTGRESQL:RECORD:WRITE:EXCEPTION",
@@ -119,12 +122,6 @@ module Multiwoven::Integrations::Destination
119
122
  }
120
123
  end
121
124
  end
122
-
123
- def tracking_message(success, failure)
124
- Multiwoven::Integrations::Protocol::TrackingMessage.new(
125
- success: success, failed: failure
126
- ).to_multiwoven_message
127
- end
128
125
  end
129
126
  end
130
127
  end
@@ -80,10 +80,14 @@ module Multiwoven
80
80
  write_success = 0
81
81
  write_failure = 0
82
82
  properties = stream.json_schema[:properties]
83
+ log_message_array = []
84
+
83
85
  records.each do |record_object|
84
86
  record = extract_data(record_object, properties)
85
- process_record(stream, record)
87
+ args = [stream.name, "Id", record]
88
+ response = send_data_to_salesforce(args)
86
89
  write_success += 1
90
+ log_message_array << log_request_response("info", args, response)
87
91
  rescue StandardError => e
88
92
  # TODO: add sync_id and sync run id to the logs
89
93
  handle_exception(e, {
@@ -93,18 +97,14 @@ module Multiwoven
93
97
  sync_run_id: @sync_config.sync_run_id
94
98
  })
95
99
  write_failure += 1
100
+ log_message_array << log_request_response("error", args, e.message)
96
101
  end
97
- tracking_message(write_success, write_failure)
98
- end
99
-
100
- def process_record(stream, record)
101
- send_data_to_salesforce(stream.name, record)
102
+ tracking_message(write_success, write_failure, log_message_array)
102
103
  end
103
104
 
104
- def send_data_to_salesforce(stream_name, record = {})
105
+ def send_data_to_salesforce(args)
105
106
  method_name = "upsert!"
106
- args = [stream_name, "Id", record]
107
- @logger.debug("sync_id: #{@sync_config.sync_id}, sync_run_id: #{@sync_config.sync_run_id}, record: #{record}")
107
+ @logger.debug("sync_id: #{@sync_config.sync_id}, sync_run_id: #{@sync_config.sync_run_id}, args: #{args}")
108
108
  @client.send(method_name, *args)
109
109
  end
110
110
 
@@ -124,12 +124,6 @@ module Multiwoven
124
124
  read_json(CATALOG_SPEC_PATH)
125
125
  end
126
126
 
127
- def tracking_message(success, failure)
128
- Multiwoven::Integrations::Protocol::TrackingMessage.new(
129
- success: success, failed: failure
130
- ).to_multiwoven_message
131
- end
132
-
133
127
  def log_debug(message)
134
128
  Multiwoven::Integrations::Service.logger.debug(message)
135
129
  end
@@ -197,6 +197,7 @@ module Multiwoven
197
197
  attribute :success, Types::Integer.default(0)
198
198
  attribute :failed, Types::Integer.default(0)
199
199
  attribute? :meta, Types::Hash
200
+ attribute? :logs, Types::Array.of(LogMessage)
200
201
 
201
202
  def to_multiwoven_message
202
203
  MultiwovenMessage.new(
@@ -2,7 +2,7 @@
2
2
 
3
3
  module Multiwoven
4
4
  module Integrations
5
- VERSION = "0.3.4"
5
+ VERSION = "0.4.0"
6
6
 
7
7
  ENABLED_SOURCES = %w[
8
8
  Snowflake
@@ -32,6 +32,7 @@ module Multiwoven
32
32
  Http
33
33
  Zendesk
34
34
  Iterable
35
+ MariaDB
35
36
  ].freeze
36
37
  end
37
38
  end
@@ -76,6 +76,7 @@ require_relative "integrations/destination/postgresql/client"
76
76
  require_relative "integrations/destination/http/client"
77
77
  require_relative "integrations/destination/zendesk/client"
78
78
  require_relative "integrations/destination/iterable/client"
79
+ require_relative "integrations/destination/maria_db/client"
79
80
 
80
81
  module Multiwoven
81
82
  module Integrations
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: multiwoven-integrations
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.4
4
+ version: 0.4.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Subin T P
@@ -522,6 +522,10 @@ files:
522
522
  - lib/multiwoven/integrations/destination/klaviyo/config/meta.json
523
523
  - lib/multiwoven/integrations/destination/klaviyo/config/spec.json
524
524
  - lib/multiwoven/integrations/destination/klaviyo/icon.svg
525
+ - lib/multiwoven/integrations/destination/maria_db/client.rb
526
+ - lib/multiwoven/integrations/destination/maria_db/config/meta.json
527
+ - lib/multiwoven/integrations/destination/maria_db/config/spec.json
528
+ - lib/multiwoven/integrations/destination/maria_db/icon.svg
525
529
  - lib/multiwoven/integrations/destination/postgresql/client.rb
526
530
  - lib/multiwoven/integrations/destination/postgresql/config/meta.json
527
531
  - lib/multiwoven/integrations/destination/postgresql/config/spec.json