multiwoven-integrations 0.11.5 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/multiwoven/integrations/destination/microsoft_sql/client.rb +137 -0
- data/lib/multiwoven/integrations/destination/microsoft_sql/config/meta.json +15 -0
- data/lib/multiwoven/integrations/destination/microsoft_sql/config/spec.json +68 -0
- data/lib/multiwoven/integrations/destination/microsoft_sql/icon.svg +22 -0
- data/lib/multiwoven/integrations/rollout.rb +2 -1
- data/lib/multiwoven/integrations/source/databrics_model/client.rb +1 -1
- data/lib/multiwoven/integrations.rb +1 -0
- data/multiwoven-integrations.gemspec +1 -0
- metadata +20 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 39b0d65bc581946d3576734695b267d4d9b32e8b973489d5d4d2cebfd0848443
|
|
4
|
+
data.tar.gz: 778a1d8f6d056e8769699c36849de4f96992a3e1646687d93fe8428c9e56f06f
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 44c7d134bdf48b7548658f91c4abbac9a7cd79caecdb1f75c628ef06cfc7e3562f6af62abfd2614661da4b90c3adf21b6c01b5514efda46ae88e46673e155386
|
|
7
|
+
data.tar.gz: fa3e918cc7b1aa524946bc59960638004a33b78baae8bc83cdf0abef6a0a90452211253d199637fc6d937c41b109871be6b70eb54b2d551946b65fc6222b4630
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "tiny_tds"
|
|
4
|
+
|
|
5
|
+
module Multiwoven::Integrations::Destination
|
|
6
|
+
module MicrosoftSql
|
|
7
|
+
include Multiwoven::Integrations::Core
|
|
8
|
+
class Client < DestinationConnector
|
|
9
|
+
def check_connection(connection_config)
|
|
10
|
+
connection_config = connection_config.with_indifferent_access
|
|
11
|
+
db_client = create_connection(connection_config)
|
|
12
|
+
|
|
13
|
+
if db_client.active?
|
|
14
|
+
ConnectionStatus.new(
|
|
15
|
+
status: ConnectionStatusType["succeeded"]
|
|
16
|
+
).to_multiwoven_message
|
|
17
|
+
else
|
|
18
|
+
ConnectionStatus.new(
|
|
19
|
+
status: ConnectionStatusType["failed"], message: "MS SQL connection not active"
|
|
20
|
+
).to_multiwoven_message
|
|
21
|
+
end
|
|
22
|
+
rescue TinyTds::Error => e
|
|
23
|
+
ConnectionStatus.new(
|
|
24
|
+
status: ConnectionStatusType["failed"], message: e.message
|
|
25
|
+
).to_multiwoven_message
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def discover(connection_config)
|
|
29
|
+
connection_config = connection_config.with_indifferent_access
|
|
30
|
+
query = "SELECT table_name, column_name, data_type, is_nullable
|
|
31
|
+
FROM information_schema.columns
|
|
32
|
+
WHERE table_schema = '#{connection_config[:schema]}' AND table_catalog = '#{connection_config[:database]}'
|
|
33
|
+
ORDER BY table_name, ordinal_position;"
|
|
34
|
+
|
|
35
|
+
db_client = create_connection(connection_config)
|
|
36
|
+
records = db_client.execute(query) do |result|
|
|
37
|
+
result.map do |row|
|
|
38
|
+
row
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
catalog = Catalog.new(streams: create_streams(records))
|
|
42
|
+
catalog.to_multiwoven_message
|
|
43
|
+
rescue StandardError => e
|
|
44
|
+
handle_exception(e, {
|
|
45
|
+
context: "MSSQL:DISCOVER:EXCEPTION",
|
|
46
|
+
type: "error"
|
|
47
|
+
})
|
|
48
|
+
ensure
|
|
49
|
+
db_client&.close
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def write(sync_config, records, action = "destination_insert")
|
|
53
|
+
connection_config = sync_config.destination.connection_specification.with_indifferent_access
|
|
54
|
+
table_name = sync_config.stream.name
|
|
55
|
+
primary_key = sync_config.model.primary_key
|
|
56
|
+
log_message_array = []
|
|
57
|
+
db = create_connection(connection_config)
|
|
58
|
+
|
|
59
|
+
write_success = 0
|
|
60
|
+
write_failure = 0
|
|
61
|
+
|
|
62
|
+
records.each do |record|
|
|
63
|
+
query = Multiwoven::Integrations::Core::QueryBuilder.perform(action, table_name, record, primary_key)
|
|
64
|
+
logger.debug("MSSQL:WRITE:QUERY query = #{query} sync_id = #{sync_config.sync_id} sync_run_id = #{sync_config.sync_run_id}")
|
|
65
|
+
begin
|
|
66
|
+
response = db.execute(query)
|
|
67
|
+
response.do
|
|
68
|
+
write_success += 1
|
|
69
|
+
log_message_array << log_request_response("info", query, response)
|
|
70
|
+
rescue StandardError => e
|
|
71
|
+
handle_exception(e, {
|
|
72
|
+
context: "MSSQL:RECORD:WRITE:EXCEPTION",
|
|
73
|
+
type: "error",
|
|
74
|
+
sync_id: sync_config.sync_id,
|
|
75
|
+
sync_run_id: sync_config.sync_run_id
|
|
76
|
+
})
|
|
77
|
+
write_failure += 1
|
|
78
|
+
log_message_array << log_request_response("error", query, e.message)
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
tracking_message(write_success, write_failure, log_message_array)
|
|
82
|
+
rescue StandardError => e
|
|
83
|
+
handle_exception(e, {
|
|
84
|
+
context: "MSSQL:RECORD:WRITE:EXCEPTION",
|
|
85
|
+
type: "error",
|
|
86
|
+
sync_id: sync_config.sync_id,
|
|
87
|
+
sync_run_id: sync_config.sync_run_id
|
|
88
|
+
})
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
private
|
|
92
|
+
|
|
93
|
+
def query(connection, query)
|
|
94
|
+
connection.exec(query) do |result|
|
|
95
|
+
result.map do |row|
|
|
96
|
+
RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def create_connection(connection_config)
|
|
102
|
+
raise "Unsupported Auth type" unless connection_config[:credentials][:auth_type] == "username/password"
|
|
103
|
+
|
|
104
|
+
TinyTds::Client.new(
|
|
105
|
+
username: connection_config[:credentials][:username],
|
|
106
|
+
password: connection_config[:credentials][:password],
|
|
107
|
+
host: connection_config[:host],
|
|
108
|
+
port: connection_config[:port],
|
|
109
|
+
database: connection_config[:database],
|
|
110
|
+
azure: true,
|
|
111
|
+
timeout: 3000
|
|
112
|
+
)
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def create_streams(records)
|
|
116
|
+
group_by_table(records).map do |r|
|
|
117
|
+
Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
def group_by_table(records)
|
|
122
|
+
records.group_by { |entry| entry["table_name"] }.map do |table_name, columns|
|
|
123
|
+
{
|
|
124
|
+
tablename: table_name,
|
|
125
|
+
columns: columns.map do |column|
|
|
126
|
+
{
|
|
127
|
+
column_name: column["column_name"],
|
|
128
|
+
type: column["data_type"],
|
|
129
|
+
optional: column["is_nullable"] == "YES"
|
|
130
|
+
}
|
|
131
|
+
end
|
|
132
|
+
}
|
|
133
|
+
end
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
end
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
{
|
|
2
|
+
"data": {
|
|
3
|
+
"name": "MicrosoftSql",
|
|
4
|
+
"title": "Microsoft SQL",
|
|
5
|
+
"connector_type": "destination",
|
|
6
|
+
"category": "Database",
|
|
7
|
+
"documentation_url": "https://docs.mutliwoven.com",
|
|
8
|
+
"github_issue_label": "destination-mssql",
|
|
9
|
+
"icon": "icon.svg",
|
|
10
|
+
"license": "MIT",
|
|
11
|
+
"release_stage": "alpha",
|
|
12
|
+
"support_level": "community",
|
|
13
|
+
"tags": ["language:ruby", "multiwoven"]
|
|
14
|
+
}
|
|
15
|
+
}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
{
|
|
2
|
+
"documentation_url": "https://docs.multiwoven.com/integrations/sources/mssql",
|
|
3
|
+
"stream_type": "dynamic",
|
|
4
|
+
"connection_specification": {
|
|
5
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
6
|
+
"title": "MicrosoftSql",
|
|
7
|
+
"type": "object",
|
|
8
|
+
"required": ["host", "port", "database", "schema"],
|
|
9
|
+
"properties": {
|
|
10
|
+
"credentials": {
|
|
11
|
+
"title": "",
|
|
12
|
+
"type": "object",
|
|
13
|
+
"required": ["auth_type", "username", "password"],
|
|
14
|
+
"properties": {
|
|
15
|
+
"auth_type": {
|
|
16
|
+
"type": "string",
|
|
17
|
+
"default": "username/password",
|
|
18
|
+
"order": 0,
|
|
19
|
+
"readOnly": true
|
|
20
|
+
},
|
|
21
|
+
"username": {
|
|
22
|
+
"description": "Username refers to your individual MS SQL login credentials. At a minimum, the user associated with these credentials must be granted read access to the data intended for synchronization.",
|
|
23
|
+
"examples": ["MSSQL_USER"],
|
|
24
|
+
"type": "string",
|
|
25
|
+
"title": "Username",
|
|
26
|
+
"order": 1
|
|
27
|
+
},
|
|
28
|
+
"password": {
|
|
29
|
+
"description": "This field requires the password associated with the user account specified in the preceding section.",
|
|
30
|
+
"type": "string",
|
|
31
|
+
"multiwoven_secret": true,
|
|
32
|
+
"title": "Password",
|
|
33
|
+
"order": 2
|
|
34
|
+
}
|
|
35
|
+
},
|
|
36
|
+
"order": 0
|
|
37
|
+
},
|
|
38
|
+
"host": {
|
|
39
|
+
"description": "The hostname or IP address of your MS SQL server.",
|
|
40
|
+
"examples": ["127.0.0.1"],
|
|
41
|
+
"type": "string",
|
|
42
|
+
"title": "Host",
|
|
43
|
+
"order": 1
|
|
44
|
+
},
|
|
45
|
+
"port": {
|
|
46
|
+
"description": "The port number for your MS SQL server, which defaults to 1433, may vary based on your configuration. ",
|
|
47
|
+
"examples": ["1433"],
|
|
48
|
+
"type": "string",
|
|
49
|
+
"title": "Port",
|
|
50
|
+
"order": 2
|
|
51
|
+
},
|
|
52
|
+
"database": {
|
|
53
|
+
"description": "The specific MS SQL database to connect to.",
|
|
54
|
+
"examples": ["MSSQL_DB"],
|
|
55
|
+
"type": "string",
|
|
56
|
+
"title": "Database",
|
|
57
|
+
"order": 3
|
|
58
|
+
},
|
|
59
|
+
"schema": {
|
|
60
|
+
"description": "The schema within the MS SQL database.",
|
|
61
|
+
"examples": ["dbo"],
|
|
62
|
+
"type": "string",
|
|
63
|
+
"title": "Schema",
|
|
64
|
+
"order": 4
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
module Multiwoven
|
|
4
4
|
module Integrations
|
|
5
|
-
VERSION = "0.
|
|
5
|
+
VERSION = "0.12.0"
|
|
6
6
|
|
|
7
7
|
ENABLED_SOURCES = %w[
|
|
8
8
|
Snowflake
|
|
@@ -40,6 +40,7 @@ module Multiwoven
|
|
|
40
40
|
DatabricksLakehouse
|
|
41
41
|
Oracle
|
|
42
42
|
MicrosoftExcel
|
|
43
|
+
MicrosoftSql
|
|
43
44
|
].freeze
|
|
44
45
|
end
|
|
45
46
|
end
|
|
@@ -71,7 +71,7 @@ module Multiwoven::Integrations::Source
|
|
|
71
71
|
|
|
72
72
|
def build_url(url, connection_config)
|
|
73
73
|
format(url, databricks_host: connection_config[:databricks_host],
|
|
74
|
-
endpoint_name: connection_config[:
|
|
74
|
+
endpoint_name: connection_config[:endpoint])
|
|
75
75
|
end
|
|
76
76
|
|
|
77
77
|
def send_request(url, token, payload)
|
|
@@ -89,6 +89,7 @@ require_relative "integrations/destination/maria_db/client"
|
|
|
89
89
|
require_relative "integrations/destination/databricks_lakehouse/client"
|
|
90
90
|
require_relative "integrations/destination/oracle_db/client"
|
|
91
91
|
require_relative "integrations/destination/microsoft_excel/client"
|
|
92
|
+
require_relative "integrations/destination/microsoft_sql/client"
|
|
92
93
|
|
|
93
94
|
module Multiwoven
|
|
94
95
|
module Integrations
|
|
@@ -63,6 +63,7 @@ Gem::Specification.new do |spec|
|
|
|
63
63
|
spec.add_runtime_dependency "sequel"
|
|
64
64
|
spec.add_runtime_dependency "slack-ruby-client"
|
|
65
65
|
spec.add_runtime_dependency "stripe"
|
|
66
|
+
spec.add_runtime_dependency "tiny_tds"
|
|
66
67
|
spec.add_runtime_dependency "zendesk_api"
|
|
67
68
|
|
|
68
69
|
spec.add_development_dependency "byebug"
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: multiwoven-integrations
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.12.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Subin T P
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2024-
|
|
11
|
+
date: 2024-10-04 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activesupport
|
|
@@ -430,6 +430,20 @@ dependencies:
|
|
|
430
430
|
- - ">="
|
|
431
431
|
- !ruby/object:Gem::Version
|
|
432
432
|
version: '0'
|
|
433
|
+
- !ruby/object:Gem::Dependency
|
|
434
|
+
name: tiny_tds
|
|
435
|
+
requirement: !ruby/object:Gem::Requirement
|
|
436
|
+
requirements:
|
|
437
|
+
- - ">="
|
|
438
|
+
- !ruby/object:Gem::Version
|
|
439
|
+
version: '0'
|
|
440
|
+
type: :runtime
|
|
441
|
+
prerelease: false
|
|
442
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
443
|
+
requirements:
|
|
444
|
+
- - ">="
|
|
445
|
+
- !ruby/object:Gem::Version
|
|
446
|
+
version: '0'
|
|
433
447
|
- !ruby/object:Gem::Dependency
|
|
434
448
|
name: zendesk_api
|
|
435
449
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -605,6 +619,10 @@ files:
|
|
|
605
619
|
- lib/multiwoven/integrations/destination/microsoft_excel/config/meta.json
|
|
606
620
|
- lib/multiwoven/integrations/destination/microsoft_excel/config/spec.json
|
|
607
621
|
- lib/multiwoven/integrations/destination/microsoft_excel/icon.svg
|
|
622
|
+
- lib/multiwoven/integrations/destination/microsoft_sql/client.rb
|
|
623
|
+
- lib/multiwoven/integrations/destination/microsoft_sql/config/meta.json
|
|
624
|
+
- lib/multiwoven/integrations/destination/microsoft_sql/config/spec.json
|
|
625
|
+
- lib/multiwoven/integrations/destination/microsoft_sql/icon.svg
|
|
608
626
|
- lib/multiwoven/integrations/destination/oracle_db/client.rb
|
|
609
627
|
- lib/multiwoven/integrations/destination/oracle_db/config/meta.json
|
|
610
628
|
- lib/multiwoven/integrations/destination/oracle_db/config/spec.json
|