multiwoven-integrations 0.11.5 → 0.12.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/multiwoven/integrations/destination/microsoft_sql/client.rb +137 -0
- data/lib/multiwoven/integrations/destination/microsoft_sql/config/meta.json +15 -0
- data/lib/multiwoven/integrations/destination/microsoft_sql/config/spec.json +68 -0
- data/lib/multiwoven/integrations/destination/microsoft_sql/icon.svg +22 -0
- data/lib/multiwoven/integrations/rollout.rb +2 -1
- data/lib/multiwoven/integrations/source/databrics_model/client.rb +1 -1
- data/lib/multiwoven/integrations.rb +1 -0
- data/multiwoven-integrations.gemspec +1 -0
- metadata +20 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 39b0d65bc581946d3576734695b267d4d9b32e8b973489d5d4d2cebfd0848443
|
|
4
|
+
data.tar.gz: 778a1d8f6d056e8769699c36849de4f96992a3e1646687d93fe8428c9e56f06f
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 44c7d134bdf48b7548658f91c4abbac9a7cd79caecdb1f75c628ef06cfc7e3562f6af62abfd2614661da4b90c3adf21b6c01b5514efda46ae88e46673e155386
|
|
7
|
+
data.tar.gz: fa3e918cc7b1aa524946bc59960638004a33b78baae8bc83cdf0abef6a0a90452211253d199637fc6d937c41b109871be6b70eb54b2d551946b65fc6222b4630
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "tiny_tds"
|
|
4
|
+
|
|
5
|
+
module Multiwoven::Integrations::Destination
|
|
6
|
+
module MicrosoftSql
|
|
7
|
+
include Multiwoven::Integrations::Core
|
|
8
|
+
class Client < DestinationConnector
|
|
9
|
+
def check_connection(connection_config)
|
|
10
|
+
connection_config = connection_config.with_indifferent_access
|
|
11
|
+
db_client = create_connection(connection_config)
|
|
12
|
+
|
|
13
|
+
if db_client.active?
|
|
14
|
+
ConnectionStatus.new(
|
|
15
|
+
status: ConnectionStatusType["succeeded"]
|
|
16
|
+
).to_multiwoven_message
|
|
17
|
+
else
|
|
18
|
+
ConnectionStatus.new(
|
|
19
|
+
status: ConnectionStatusType["failed"], message: "MS SQL connection not active"
|
|
20
|
+
).to_multiwoven_message
|
|
21
|
+
end
|
|
22
|
+
rescue TinyTds::Error => e
|
|
23
|
+
ConnectionStatus.new(
|
|
24
|
+
status: ConnectionStatusType["failed"], message: e.message
|
|
25
|
+
).to_multiwoven_message
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def discover(connection_config)
|
|
29
|
+
connection_config = connection_config.with_indifferent_access
|
|
30
|
+
query = "SELECT table_name, column_name, data_type, is_nullable
|
|
31
|
+
FROM information_schema.columns
|
|
32
|
+
WHERE table_schema = '#{connection_config[:schema]}' AND table_catalog = '#{connection_config[:database]}'
|
|
33
|
+
ORDER BY table_name, ordinal_position;"
|
|
34
|
+
|
|
35
|
+
db_client = create_connection(connection_config)
|
|
36
|
+
records = db_client.execute(query) do |result|
|
|
37
|
+
result.map do |row|
|
|
38
|
+
row
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
catalog = Catalog.new(streams: create_streams(records))
|
|
42
|
+
catalog.to_multiwoven_message
|
|
43
|
+
rescue StandardError => e
|
|
44
|
+
handle_exception(e, {
|
|
45
|
+
context: "MSSQL:DISCOVER:EXCEPTION",
|
|
46
|
+
type: "error"
|
|
47
|
+
})
|
|
48
|
+
ensure
|
|
49
|
+
db_client&.close
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def write(sync_config, records, action = "destination_insert")
|
|
53
|
+
connection_config = sync_config.destination.connection_specification.with_indifferent_access
|
|
54
|
+
table_name = sync_config.stream.name
|
|
55
|
+
primary_key = sync_config.model.primary_key
|
|
56
|
+
log_message_array = []
|
|
57
|
+
db = create_connection(connection_config)
|
|
58
|
+
|
|
59
|
+
write_success = 0
|
|
60
|
+
write_failure = 0
|
|
61
|
+
|
|
62
|
+
records.each do |record|
|
|
63
|
+
query = Multiwoven::Integrations::Core::QueryBuilder.perform(action, table_name, record, primary_key)
|
|
64
|
+
logger.debug("MSSQL:WRITE:QUERY query = #{query} sync_id = #{sync_config.sync_id} sync_run_id = #{sync_config.sync_run_id}")
|
|
65
|
+
begin
|
|
66
|
+
response = db.execute(query)
|
|
67
|
+
response.do
|
|
68
|
+
write_success += 1
|
|
69
|
+
log_message_array << log_request_response("info", query, response)
|
|
70
|
+
rescue StandardError => e
|
|
71
|
+
handle_exception(e, {
|
|
72
|
+
context: "MSSQL:RECORD:WRITE:EXCEPTION",
|
|
73
|
+
type: "error",
|
|
74
|
+
sync_id: sync_config.sync_id,
|
|
75
|
+
sync_run_id: sync_config.sync_run_id
|
|
76
|
+
})
|
|
77
|
+
write_failure += 1
|
|
78
|
+
log_message_array << log_request_response("error", query, e.message)
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
tracking_message(write_success, write_failure, log_message_array)
|
|
82
|
+
rescue StandardError => e
|
|
83
|
+
handle_exception(e, {
|
|
84
|
+
context: "MSSQL:RECORD:WRITE:EXCEPTION",
|
|
85
|
+
type: "error",
|
|
86
|
+
sync_id: sync_config.sync_id,
|
|
87
|
+
sync_run_id: sync_config.sync_run_id
|
|
88
|
+
})
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
private
|
|
92
|
+
|
|
93
|
+
def query(connection, query)
|
|
94
|
+
connection.exec(query) do |result|
|
|
95
|
+
result.map do |row|
|
|
96
|
+
RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def create_connection(connection_config)
|
|
102
|
+
raise "Unsupported Auth type" unless connection_config[:credentials][:auth_type] == "username/password"
|
|
103
|
+
|
|
104
|
+
TinyTds::Client.new(
|
|
105
|
+
username: connection_config[:credentials][:username],
|
|
106
|
+
password: connection_config[:credentials][:password],
|
|
107
|
+
host: connection_config[:host],
|
|
108
|
+
port: connection_config[:port],
|
|
109
|
+
database: connection_config[:database],
|
|
110
|
+
azure: true,
|
|
111
|
+
timeout: 3000
|
|
112
|
+
)
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def create_streams(records)
|
|
116
|
+
group_by_table(records).map do |r|
|
|
117
|
+
Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
def group_by_table(records)
|
|
122
|
+
records.group_by { |entry| entry["table_name"] }.map do |table_name, columns|
|
|
123
|
+
{
|
|
124
|
+
tablename: table_name,
|
|
125
|
+
columns: columns.map do |column|
|
|
126
|
+
{
|
|
127
|
+
column_name: column["column_name"],
|
|
128
|
+
type: column["data_type"],
|
|
129
|
+
optional: column["is_nullable"] == "YES"
|
|
130
|
+
}
|
|
131
|
+
end
|
|
132
|
+
}
|
|
133
|
+
end
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
end
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
{
|
|
2
|
+
"data": {
|
|
3
|
+
"name": "MicrosoftSql",
|
|
4
|
+
"title": "Microsoft SQL",
|
|
5
|
+
"connector_type": "destination",
|
|
6
|
+
"category": "Database",
|
|
7
|
+
"documentation_url": "https://docs.mutliwoven.com",
|
|
8
|
+
"github_issue_label": "destination-mssql",
|
|
9
|
+
"icon": "icon.svg",
|
|
10
|
+
"license": "MIT",
|
|
11
|
+
"release_stage": "alpha",
|
|
12
|
+
"support_level": "community",
|
|
13
|
+
"tags": ["language:ruby", "multiwoven"]
|
|
14
|
+
}
|
|
15
|
+
}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
{
|
|
2
|
+
"documentation_url": "https://docs.multiwoven.com/integrations/sources/mssql",
|
|
3
|
+
"stream_type": "dynamic",
|
|
4
|
+
"connection_specification": {
|
|
5
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
6
|
+
"title": "MicrosoftSql",
|
|
7
|
+
"type": "object",
|
|
8
|
+
"required": ["host", "port", "database", "schema"],
|
|
9
|
+
"properties": {
|
|
10
|
+
"credentials": {
|
|
11
|
+
"title": "",
|
|
12
|
+
"type": "object",
|
|
13
|
+
"required": ["auth_type", "username", "password"],
|
|
14
|
+
"properties": {
|
|
15
|
+
"auth_type": {
|
|
16
|
+
"type": "string",
|
|
17
|
+
"default": "username/password",
|
|
18
|
+
"order": 0,
|
|
19
|
+
"readOnly": true
|
|
20
|
+
},
|
|
21
|
+
"username": {
|
|
22
|
+
"description": "Username refers to your individual MS SQL login credentials. At a minimum, the user associated with these credentials must be granted read access to the data intended for synchronization.",
|
|
23
|
+
"examples": ["MSSQL_USER"],
|
|
24
|
+
"type": "string",
|
|
25
|
+
"title": "Username",
|
|
26
|
+
"order": 1
|
|
27
|
+
},
|
|
28
|
+
"password": {
|
|
29
|
+
"description": "This field requires the password associated with the user account specified in the preceding section.",
|
|
30
|
+
"type": "string",
|
|
31
|
+
"multiwoven_secret": true,
|
|
32
|
+
"title": "Password",
|
|
33
|
+
"order": 2
|
|
34
|
+
}
|
|
35
|
+
},
|
|
36
|
+
"order": 0
|
|
37
|
+
},
|
|
38
|
+
"host": {
|
|
39
|
+
"description": "The hostname or IP address of your MS SQL server.",
|
|
40
|
+
"examples": ["127.0.0.1"],
|
|
41
|
+
"type": "string",
|
|
42
|
+
"title": "Host",
|
|
43
|
+
"order": 1
|
|
44
|
+
},
|
|
45
|
+
"port": {
|
|
46
|
+
"description": "The port number for your MS SQL server, which defaults to 1433, may vary based on your configuration. ",
|
|
47
|
+
"examples": ["1433"],
|
|
48
|
+
"type": "string",
|
|
49
|
+
"title": "Port",
|
|
50
|
+
"order": 2
|
|
51
|
+
},
|
|
52
|
+
"database": {
|
|
53
|
+
"description": "The specific MS SQL database to connect to.",
|
|
54
|
+
"examples": ["MSSQL_DB"],
|
|
55
|
+
"type": "string",
|
|
56
|
+
"title": "Database",
|
|
57
|
+
"order": 3
|
|
58
|
+
},
|
|
59
|
+
"schema": {
|
|
60
|
+
"description": "The schema within the MS SQL database.",
|
|
61
|
+
"examples": ["dbo"],
|
|
62
|
+
"type": "string",
|
|
63
|
+
"title": "Schema",
|
|
64
|
+
"order": 4
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
module Multiwoven
|
|
4
4
|
module Integrations
|
|
5
|
-
VERSION = "0.
|
|
5
|
+
VERSION = "0.12.0"
|
|
6
6
|
|
|
7
7
|
ENABLED_SOURCES = %w[
|
|
8
8
|
Snowflake
|
|
@@ -40,6 +40,7 @@ module Multiwoven
|
|
|
40
40
|
DatabricksLakehouse
|
|
41
41
|
Oracle
|
|
42
42
|
MicrosoftExcel
|
|
43
|
+
MicrosoftSql
|
|
43
44
|
].freeze
|
|
44
45
|
end
|
|
45
46
|
end
|
|
@@ -71,7 +71,7 @@ module Multiwoven::Integrations::Source
|
|
|
71
71
|
|
|
72
72
|
def build_url(url, connection_config)
|
|
73
73
|
format(url, databricks_host: connection_config[:databricks_host],
|
|
74
|
-
endpoint_name: connection_config[:
|
|
74
|
+
endpoint_name: connection_config[:endpoint])
|
|
75
75
|
end
|
|
76
76
|
|
|
77
77
|
def send_request(url, token, payload)
|
|
@@ -89,6 +89,7 @@ require_relative "integrations/destination/maria_db/client"
|
|
|
89
89
|
require_relative "integrations/destination/databricks_lakehouse/client"
|
|
90
90
|
require_relative "integrations/destination/oracle_db/client"
|
|
91
91
|
require_relative "integrations/destination/microsoft_excel/client"
|
|
92
|
+
require_relative "integrations/destination/microsoft_sql/client"
|
|
92
93
|
|
|
93
94
|
module Multiwoven
|
|
94
95
|
module Integrations
|
|
@@ -63,6 +63,7 @@ Gem::Specification.new do |spec|
|
|
|
63
63
|
spec.add_runtime_dependency "sequel"
|
|
64
64
|
spec.add_runtime_dependency "slack-ruby-client"
|
|
65
65
|
spec.add_runtime_dependency "stripe"
|
|
66
|
+
spec.add_runtime_dependency "tiny_tds"
|
|
66
67
|
spec.add_runtime_dependency "zendesk_api"
|
|
67
68
|
|
|
68
69
|
spec.add_development_dependency "byebug"
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: multiwoven-integrations
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.12.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Subin T P
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2024-
|
|
11
|
+
date: 2024-10-04 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activesupport
|
|
@@ -430,6 +430,20 @@ dependencies:
|
|
|
430
430
|
- - ">="
|
|
431
431
|
- !ruby/object:Gem::Version
|
|
432
432
|
version: '0'
|
|
433
|
+
- !ruby/object:Gem::Dependency
|
|
434
|
+
name: tiny_tds
|
|
435
|
+
requirement: !ruby/object:Gem::Requirement
|
|
436
|
+
requirements:
|
|
437
|
+
- - ">="
|
|
438
|
+
- !ruby/object:Gem::Version
|
|
439
|
+
version: '0'
|
|
440
|
+
type: :runtime
|
|
441
|
+
prerelease: false
|
|
442
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
443
|
+
requirements:
|
|
444
|
+
- - ">="
|
|
445
|
+
- !ruby/object:Gem::Version
|
|
446
|
+
version: '0'
|
|
433
447
|
- !ruby/object:Gem::Dependency
|
|
434
448
|
name: zendesk_api
|
|
435
449
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -605,6 +619,10 @@ files:
|
|
|
605
619
|
- lib/multiwoven/integrations/destination/microsoft_excel/config/meta.json
|
|
606
620
|
- lib/multiwoven/integrations/destination/microsoft_excel/config/spec.json
|
|
607
621
|
- lib/multiwoven/integrations/destination/microsoft_excel/icon.svg
|
|
622
|
+
- lib/multiwoven/integrations/destination/microsoft_sql/client.rb
|
|
623
|
+
- lib/multiwoven/integrations/destination/microsoft_sql/config/meta.json
|
|
624
|
+
- lib/multiwoven/integrations/destination/microsoft_sql/config/spec.json
|
|
625
|
+
- lib/multiwoven/integrations/destination/microsoft_sql/icon.svg
|
|
608
626
|
- lib/multiwoven/integrations/destination/oracle_db/client.rb
|
|
609
627
|
- lib/multiwoven/integrations/destination/oracle_db/config/meta.json
|
|
610
628
|
- lib/multiwoven/integrations/destination/oracle_db/config/spec.json
|