multiwoven-integrations 0.41.12 → 0.43.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/multiwoven/integrations/rollout.rb +3 -1
- data/lib/multiwoven/integrations/source/mistral_ai/client.rb +19 -0
- data/lib/multiwoven/integrations/source/mistral_ai/config/catalog.json +6 -0
- data/lib/multiwoven/integrations/source/mistral_ai/config/meta.json +22 -0
- data/lib/multiwoven/integrations/source/mistral_ai/config/models.json +168 -0
- data/lib/multiwoven/integrations/source/mistral_ai/config/spec.json +71 -0
- data/lib/multiwoven/integrations/source/mistral_ai/icon.svg +1 -0
- data/lib/multiwoven/integrations/source/sql_server/client.rb +280 -0
- data/lib/multiwoven/integrations/source/sql_server/config/meta.json +16 -0
- data/lib/multiwoven/integrations/source/sql_server/config/spec.json +94 -0
- data/lib/multiwoven/integrations/source/sql_server/icon.svg +22 -0
- data/lib/multiwoven/integrations.rb +2 -0
- metadata +12 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 62e0ad2ac6fc880f4b88dd4a511312b240645d5d1134966659a16225a927848d
|
|
4
|
+
data.tar.gz: a7368698f3dd5a28894549b635c136b348d43feb25513a1a93f8333a6f9c5ae7
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: f0a58e7df26e7579cf5037d6b6605374695a07c6276279da1bd4b9582d7c9f797a001eb9ab7b52e41f402468c7e43c0958d1d1882f27f66aa92617e9a6bd8058
|
|
7
|
+
data.tar.gz: 8867babc43650be6e51e1a40c129089fa1b2ba46b561967b6d58bf251b3c48d3e42ef1d21ac46b5731d3a44012ffbd5edba298d76d621b3422db06f487b8eb5c
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
module Multiwoven
|
|
4
4
|
module Integrations
|
|
5
|
-
VERSION = "0.
|
|
5
|
+
VERSION = "0.43.0"
|
|
6
6
|
|
|
7
7
|
ENABLED_SOURCES = %w[
|
|
8
8
|
Snowflake
|
|
@@ -33,6 +33,7 @@ module Multiwoven
|
|
|
33
33
|
Xai
|
|
34
34
|
Groq
|
|
35
35
|
OpenRouter
|
|
36
|
+
MistralAi
|
|
36
37
|
IntuitQuickBooks
|
|
37
38
|
PineconeDB
|
|
38
39
|
Qdrant
|
|
@@ -44,6 +45,7 @@ module Multiwoven
|
|
|
44
45
|
OneDrive
|
|
45
46
|
MicrosoftDynamics
|
|
46
47
|
EpicFhir
|
|
48
|
+
SqlServer
|
|
47
49
|
].freeze
|
|
48
50
|
|
|
49
51
|
ENABLED_DESTINATIONS = %w[
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Multiwoven::Integrations::Source
|
|
4
|
+
module MistralAi
|
|
5
|
+
include Multiwoven::Integrations::Core
|
|
6
|
+
# Mistral exposes an OpenAI-compatible chat completions API, so read/stream
|
|
7
|
+
# handling matches GenericOpenAI.
|
|
8
|
+
#
|
|
9
|
+
# Model Hub stays curated in config/models.json: OpenRouter's mistralai/* ids
|
|
10
|
+
# (e.g. mistral-large) do not match Mistral API aliases (mistral-large-latest).
|
|
11
|
+
class Client < GenericOpenAI::Client
|
|
12
|
+
private
|
|
13
|
+
|
|
14
|
+
def log_context
|
|
15
|
+
"MISTRAL_AI"
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
end
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
{
|
|
2
|
+
"data": {
|
|
3
|
+
"name": "MistralAi",
|
|
4
|
+
"title": "Mistral AI",
|
|
5
|
+
"connector_type": "source",
|
|
6
|
+
"category": "AI Model",
|
|
7
|
+
"sub_category": "LLM",
|
|
8
|
+
"provider_type": "third_party",
|
|
9
|
+
"display_order": 170,
|
|
10
|
+
"description": "Mistral AI models via the OpenAI-compatible chat completions API. Catalog is curated to Mistral API model aliases.",
|
|
11
|
+
"documentation_url": "https://docs.mistral.ai/",
|
|
12
|
+
"github_issue_label": "source-mistral-ai",
|
|
13
|
+
"icon": "https://res.cloudinary.com/dspflukeu/image/upload/v1790199623/Multiwoven/connectors/mistral_ai/icon.svg",
|
|
14
|
+
"license": "MIT",
|
|
15
|
+
"release_stage": "alpha",
|
|
16
|
+
"support_level": "community",
|
|
17
|
+
"tags": [
|
|
18
|
+
"language:ruby",
|
|
19
|
+
"multiwoven"
|
|
20
|
+
]
|
|
21
|
+
}
|
|
22
|
+
}
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
{
|
|
2
|
+
"//models": "Curated for Mistral AI API aliases (https://docs.mistral.ai/getting-started/models). OpenRouter mistralai/* ids do not match (e.g. mistral-large vs mistral-large-latest). Do not set openrouter_slug.",
|
|
3
|
+
"models": [
|
|
4
|
+
{
|
|
5
|
+
"id": "mistral-large-latest",
|
|
6
|
+
"name": "Mistral Large 3",
|
|
7
|
+
"model_type": "llm",
|
|
8
|
+
"type": "completion",
|
|
9
|
+
"tasks": [
|
|
10
|
+
"Text Generation",
|
|
11
|
+
"Reasoning"
|
|
12
|
+
],
|
|
13
|
+
"context_window": 262144,
|
|
14
|
+
"max_output": 262144,
|
|
15
|
+
"pricing": {
|
|
16
|
+
"input": 0.5,
|
|
17
|
+
"output": 1.5,
|
|
18
|
+
"cached_read": 0.05,
|
|
19
|
+
"unit": "per_1m_tokens"
|
|
20
|
+
},
|
|
21
|
+
"capabilities": [
|
|
22
|
+
"Tool Use",
|
|
23
|
+
"JSON Object Mode",
|
|
24
|
+
"JSON Schema Mode",
|
|
25
|
+
"Image analysis"
|
|
26
|
+
]
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
"id": "mistral-medium-latest",
|
|
30
|
+
"name": "Mistral Medium 3.5",
|
|
31
|
+
"model_type": "llm",
|
|
32
|
+
"type": "completion",
|
|
33
|
+
"tasks": [
|
|
34
|
+
"Text Generation",
|
|
35
|
+
"Reasoning"
|
|
36
|
+
],
|
|
37
|
+
"context_window": 262144,
|
|
38
|
+
"max_output": 262144,
|
|
39
|
+
"pricing": {
|
|
40
|
+
"input": 1.5,
|
|
41
|
+
"output": 7.5,
|
|
42
|
+
"cached_read": 0.15,
|
|
43
|
+
"unit": "per_1m_tokens"
|
|
44
|
+
},
|
|
45
|
+
"capabilities": [
|
|
46
|
+
"Tool Use",
|
|
47
|
+
"JSON Object Mode",
|
|
48
|
+
"JSON Schema Mode",
|
|
49
|
+
"Image analysis"
|
|
50
|
+
]
|
|
51
|
+
},
|
|
52
|
+
{
|
|
53
|
+
"id": "mistral-small-latest",
|
|
54
|
+
"name": "Mistral Small 4",
|
|
55
|
+
"model_type": "llm",
|
|
56
|
+
"type": "completion",
|
|
57
|
+
"tasks": [
|
|
58
|
+
"Text Generation",
|
|
59
|
+
"Reasoning"
|
|
60
|
+
],
|
|
61
|
+
"context_window": 262144,
|
|
62
|
+
"max_output": 262144,
|
|
63
|
+
"pricing": {
|
|
64
|
+
"input": 0.15,
|
|
65
|
+
"output": 0.6,
|
|
66
|
+
"cached_read": 0.015,
|
|
67
|
+
"unit": "per_1m_tokens"
|
|
68
|
+
},
|
|
69
|
+
"capabilities": [
|
|
70
|
+
"Tool Use",
|
|
71
|
+
"JSON Object Mode",
|
|
72
|
+
"JSON Schema Mode",
|
|
73
|
+
"Image analysis"
|
|
74
|
+
]
|
|
75
|
+
},
|
|
76
|
+
{
|
|
77
|
+
"id": "ministral-14b-latest",
|
|
78
|
+
"name": "Ministral 3 14B",
|
|
79
|
+
"model_type": "llm",
|
|
80
|
+
"type": "completion",
|
|
81
|
+
"tasks": [
|
|
82
|
+
"Text Generation"
|
|
83
|
+
],
|
|
84
|
+
"context_window": 262144,
|
|
85
|
+
"max_output": 262144,
|
|
86
|
+
"pricing": {
|
|
87
|
+
"input": 0.2,
|
|
88
|
+
"output": 0.2,
|
|
89
|
+
"cached_read": 0.02,
|
|
90
|
+
"unit": "per_1m_tokens"
|
|
91
|
+
},
|
|
92
|
+
"capabilities": [
|
|
93
|
+
"Tool Use",
|
|
94
|
+
"JSON Object Mode",
|
|
95
|
+
"JSON Schema Mode",
|
|
96
|
+
"Image analysis"
|
|
97
|
+
]
|
|
98
|
+
},
|
|
99
|
+
{
|
|
100
|
+
"id": "ministral-8b-latest",
|
|
101
|
+
"name": "Ministral 3 8B",
|
|
102
|
+
"model_type": "llm",
|
|
103
|
+
"type": "completion",
|
|
104
|
+
"tasks": [
|
|
105
|
+
"Text Generation"
|
|
106
|
+
],
|
|
107
|
+
"context_window": 262144,
|
|
108
|
+
"max_output": 262144,
|
|
109
|
+
"pricing": {
|
|
110
|
+
"input": 0.15,
|
|
111
|
+
"output": 0.15,
|
|
112
|
+
"cached_read": 0.015,
|
|
113
|
+
"unit": "per_1m_tokens"
|
|
114
|
+
},
|
|
115
|
+
"capabilities": [
|
|
116
|
+
"Tool Use",
|
|
117
|
+
"JSON Object Mode",
|
|
118
|
+
"JSON Schema Mode",
|
|
119
|
+
"Image analysis"
|
|
120
|
+
]
|
|
121
|
+
},
|
|
122
|
+
{
|
|
123
|
+
"id": "ministral-3b-latest",
|
|
124
|
+
"name": "Ministral 3 3B",
|
|
125
|
+
"model_type": "llm",
|
|
126
|
+
"type": "completion",
|
|
127
|
+
"tasks": [
|
|
128
|
+
"Text Generation"
|
|
129
|
+
],
|
|
130
|
+
"context_window": 262144,
|
|
131
|
+
"max_output": 262144,
|
|
132
|
+
"pricing": {
|
|
133
|
+
"input": 0.1,
|
|
134
|
+
"output": 0.1,
|
|
135
|
+
"cached_read": 0.01,
|
|
136
|
+
"unit": "per_1m_tokens"
|
|
137
|
+
},
|
|
138
|
+
"capabilities": [
|
|
139
|
+
"Tool Use",
|
|
140
|
+
"JSON Object Mode",
|
|
141
|
+
"JSON Schema Mode",
|
|
142
|
+
"Image analysis"
|
|
143
|
+
]
|
|
144
|
+
},
|
|
145
|
+
{
|
|
146
|
+
"id": "codestral-latest",
|
|
147
|
+
"name": "Codestral",
|
|
148
|
+
"model_type": "llm",
|
|
149
|
+
"type": "completion",
|
|
150
|
+
"tasks": [
|
|
151
|
+
"Code Generation"
|
|
152
|
+
],
|
|
153
|
+
"context_window": 131072,
|
|
154
|
+
"max_output": 131072,
|
|
155
|
+
"pricing": {
|
|
156
|
+
"input": 0.3,
|
|
157
|
+
"output": 0.9,
|
|
158
|
+
"cached_read": 0.03,
|
|
159
|
+
"unit": "per_1m_tokens"
|
|
160
|
+
},
|
|
161
|
+
"capabilities": [
|
|
162
|
+
"Tool Use",
|
|
163
|
+
"JSON Object Mode",
|
|
164
|
+
"JSON Schema Mode"
|
|
165
|
+
]
|
|
166
|
+
}
|
|
167
|
+
]
|
|
168
|
+
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
{
|
|
2
|
+
"documentation_url": "https://docs.mistral.ai/api/endpoint/chat",
|
|
3
|
+
"stream_type": "user_defined",
|
|
4
|
+
"connector_query_type": "ai_ml",
|
|
5
|
+
"connection_specification": {
|
|
6
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
7
|
+
"title": "MistralAi",
|
|
8
|
+
"type": "object",
|
|
9
|
+
"x-dynamic-input-path": "messages.0.content",
|
|
10
|
+
"x-output-path": "choices.0.message.content",
|
|
11
|
+
"required": [
|
|
12
|
+
"url",
|
|
13
|
+
"api_key",
|
|
14
|
+
"request_format",
|
|
15
|
+
"response_format"
|
|
16
|
+
],
|
|
17
|
+
"properties": {
|
|
18
|
+
"url": {
|
|
19
|
+
"type": "string",
|
|
20
|
+
"title": "Endpoint URL",
|
|
21
|
+
"description": "The Mistral OpenAI-compatible chat completions endpoint. For data residency, you can use the regional endpoints https://api.eu.mistral.ai/v1/chat/completions (EU) or https://api.us.mistral.ai/v1/chat/completions (US).",
|
|
22
|
+
"default": "https://api.mistral.ai/v1/chat/completions",
|
|
23
|
+
"order": 0
|
|
24
|
+
},
|
|
25
|
+
"api_key": {
|
|
26
|
+
"type": "string",
|
|
27
|
+
"multiwoven_secret": true,
|
|
28
|
+
"title": "API Key",
|
|
29
|
+
"description": "Your Mistral API key.",
|
|
30
|
+
"order": 1
|
|
31
|
+
},
|
|
32
|
+
"is_stream": {
|
|
33
|
+
"type": "boolean",
|
|
34
|
+
"title": "Enable streaming",
|
|
35
|
+
"description": "Enables data streaming when supported by the model. When true, responses are processed in chunks for immediate delivery. Default is false.",
|
|
36
|
+
"default": false,
|
|
37
|
+
"order": 2
|
|
38
|
+
},
|
|
39
|
+
"config": {
|
|
40
|
+
"title": "",
|
|
41
|
+
"type": "object",
|
|
42
|
+
"properties": {
|
|
43
|
+
"timeout": {
|
|
44
|
+
"type": "string",
|
|
45
|
+
"default": "30",
|
|
46
|
+
"title": "HTTP Timeout",
|
|
47
|
+
"description": "The maximum time, in seconds, to wait for a response from the server before the request is canceled.",
|
|
48
|
+
"order": 0
|
|
49
|
+
}
|
|
50
|
+
},
|
|
51
|
+
"order": 3
|
|
52
|
+
},
|
|
53
|
+
"request_format": {
|
|
54
|
+
"title": "Request Format",
|
|
55
|
+
"description": "Sample Request Format",
|
|
56
|
+
"type": "string",
|
|
57
|
+
"default": "{\"model\":\"mistral-small-latest\", \"messages\":[{\"role\": \"user\", \"content\": \"Hi.\"}], \"stream\": false}",
|
|
58
|
+
"x-request-format": true,
|
|
59
|
+
"order": 4
|
|
60
|
+
},
|
|
61
|
+
"response_format": {
|
|
62
|
+
"title": "Response Format",
|
|
63
|
+
"description": "Sample Response Format",
|
|
64
|
+
"type": "string",
|
|
65
|
+
"default": "{\"choices\":[{\"index\": 0, \"message\":{\"role\": \"assistant\", \"content\": \"Hello!\"}, \"finish_reason\": \"stop\"}]}",
|
|
66
|
+
"x-response-format": true,
|
|
67
|
+
"order": 5
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
<svg height="1em" style="flex:none;line-height:1" viewBox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><title>Mistral</title><path d="M3.428 3.4h3.429v3.428H3.428V3.4zm13.714 0h3.43v3.428h-3.43V3.4z" fill="gold"></path><path d="M3.428 6.828h6.857v3.429H3.429V6.828zm10.286 0h6.857v3.429h-6.857V6.828z" fill="#FFAF00"></path><path d="M3.428 10.258h17.144v3.428H3.428v-3.428z" fill="#FF8205"></path><path d="M3.428 13.686h3.429v3.428H3.428v-3.428zm6.858 0h3.429v3.428h-3.429v-3.428zm6.856 0h3.43v3.428h-3.43v-3.428z" fill="#FA500F"></path><path d="M0 17.114h10.286v3.429H0v-3.429zm13.714 0H24v3.429H13.714v-3.429z" fill="#E10500"></path></svg>
|
|
@@ -0,0 +1,280 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "tiny_tds"
|
|
4
|
+
|
|
5
|
+
module Multiwoven::Integrations::Source
|
|
6
|
+
module SqlServer
|
|
7
|
+
include Multiwoven::Integrations::Core
|
|
8
|
+
class Client < SourceConnector
|
|
9
|
+
def check_connection(connection_config)
|
|
10
|
+
connection_config = connection_config.with_indifferent_access
|
|
11
|
+
db = create_connection(connection_config)
|
|
12
|
+
ConnectionStatus.new(
|
|
13
|
+
status: ConnectionStatusType["succeeded"]
|
|
14
|
+
).to_multiwoven_message
|
|
15
|
+
rescue TinyTds::Error => e
|
|
16
|
+
ConnectionStatus.new(
|
|
17
|
+
status: ConnectionStatusType["failed"],
|
|
18
|
+
message: e.message
|
|
19
|
+
).to_multiwoven_message
|
|
20
|
+
ensure
|
|
21
|
+
db&.close
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def discover(connection_config)
|
|
25
|
+
connection_config = connection_config.with_indifferent_access
|
|
26
|
+
schema = connection_config[:schema].presence || "dbo"
|
|
27
|
+
query = "SELECT table_name, column_name, data_type, is_nullable
|
|
28
|
+
FROM information_schema.columns
|
|
29
|
+
WHERE table_schema = '#{escape(schema)}'
|
|
30
|
+
ORDER BY table_name, ordinal_position;"
|
|
31
|
+
|
|
32
|
+
db = create_connection(connection_config)
|
|
33
|
+
records = db.execute(query).map do |row|
|
|
34
|
+
row.transform_keys { |key| key.to_s.downcase }
|
|
35
|
+
end
|
|
36
|
+
catalog = Catalog.new(streams: create_streams(records))
|
|
37
|
+
catalog.to_multiwoven_message
|
|
38
|
+
rescue StandardError => e
|
|
39
|
+
handle_exception(e, {
|
|
40
|
+
context: "SQLSERVER:DISCOVER:EXCEPTION",
|
|
41
|
+
type: "error"
|
|
42
|
+
})
|
|
43
|
+
ensure
|
|
44
|
+
db&.close
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def read(sync_config)
|
|
48
|
+
connection_config = sync_config.source.connection_specification.with_indifferent_access
|
|
49
|
+
@pagination_primary_key = sync_config.model.primary_key
|
|
50
|
+
query = sync_config.model.query
|
|
51
|
+
query = batched_query(query, sync_config.limit, sync_config.offset) unless sync_config.limit.nil? && sync_config.offset.nil?
|
|
52
|
+
|
|
53
|
+
db = create_connection(connection_config)
|
|
54
|
+
|
|
55
|
+
query(db, query)
|
|
56
|
+
rescue StandardError => e
|
|
57
|
+
handle_exception(e, {
|
|
58
|
+
context: "SQLSERVER:READ:EXCEPTION",
|
|
59
|
+
type: "error",
|
|
60
|
+
sync_id: sync_config.sync_id,
|
|
61
|
+
sync_run_id: sync_config.sync_run_id
|
|
62
|
+
})
|
|
63
|
+
ensure
|
|
64
|
+
@pagination_primary_key = nil
|
|
65
|
+
db&.close
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def search(vector_search_config)
|
|
69
|
+
connection_config = vector_search_config.source.connection_specification.with_indifferent_access
|
|
70
|
+
query = vector_search_config[:vector]
|
|
71
|
+
limit = vector_search_config[:limit]
|
|
72
|
+
query = batched_query(query, limit, 0) unless limit.nil?
|
|
73
|
+
|
|
74
|
+
db = create_connection(connection_config)
|
|
75
|
+
query(db, query)
|
|
76
|
+
rescue StandardError => e
|
|
77
|
+
handle_exception(e, {
|
|
78
|
+
context: "SQLSERVER:SEARCH:EXCEPTION",
|
|
79
|
+
type: "error"
|
|
80
|
+
})
|
|
81
|
+
ensure
|
|
82
|
+
db&.close
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
private
|
|
86
|
+
|
|
87
|
+
def query(connection, sql)
|
|
88
|
+
connection.execute(reformat_query(sql)).map do |row|
|
|
89
|
+
RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# SQL Server does not support LIMIT/OFFSET; use OFFSET/FETCH NEXT.
|
|
94
|
+
def batched_query(sql_query, limit, offset)
|
|
95
|
+
offset = offset.to_i
|
|
96
|
+
limit = limit&.to_i
|
|
97
|
+
raise ArgumentError, "Offset and limit must be non-negative" if offset.negative? || (!limit.nil? && limit.negative?)
|
|
98
|
+
|
|
99
|
+
sql_query = strip_trailing_terminator(sql_query)
|
|
100
|
+
raise ArgumentError, "Query already contains a LIMIT clause" if clause_outside_literals?(sql_query, /\bLIMIT\s+\d+\b/i)
|
|
101
|
+
raise ArgumentError, "Query already contains an OFFSET clause" if clause_outside_literals?(sql_query, /\bOFFSET\s+\d+\b/i)
|
|
102
|
+
|
|
103
|
+
apply_pagination(sql_query, limit, offset)
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
# Convert Postgres/MySQL-style LIMIT/OFFSET (from upstream, including query_source) into SQL Server pagination.
|
|
107
|
+
def reformat_query(sql_query)
|
|
108
|
+
sql_query = strip_trailing_terminator(sql_query)
|
|
109
|
+
|
|
110
|
+
return sql_query if clause_outside_literals?(sql_query, /\bOFFSET\s+\d+\s+ROWS\b/i)
|
|
111
|
+
|
|
112
|
+
limit = nil
|
|
113
|
+
offset = nil
|
|
114
|
+
|
|
115
|
+
normalized = with_masked_literals(sql_query) do |masked|
|
|
116
|
+
if (match = masked.match(/\bLIMIT\s+(\d+)\b/i))
|
|
117
|
+
limit = match[1].to_i
|
|
118
|
+
masked = masked.sub(/\bLIMIT\s+\d+\b/i, "")
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
if (match = masked.match(/\bOFFSET\s+(\d+)\b(?!\s+ROWS)/i))
|
|
122
|
+
offset = match[1].to_i
|
|
123
|
+
masked = masked.sub(/\bOFFSET\s+\d+\b(?!\s+ROWS)/i, "")
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
masked.gsub(/[^\S\n]+/, " ").strip
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
return normalized if limit.nil? && offset.nil?
|
|
130
|
+
|
|
131
|
+
apply_pagination(normalized, limit, offset || 0)
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
def apply_pagination(sql_query, limit, offset)
|
|
135
|
+
raise ArgumentError, "Limit must be at least 1" if !limit.nil? && limit.to_i < 1
|
|
136
|
+
|
|
137
|
+
keys = normalize_primary_keys(@pagination_primary_key)
|
|
138
|
+
has_order = clause_outside_literals?(sql_query, /\bORDER\s+BY\b/i)
|
|
139
|
+
|
|
140
|
+
# query_source appends LIMIT without ORDER BY or primary_key metadata.
|
|
141
|
+
# TOP does not require ORDER BY; OFFSET/FETCH does.
|
|
142
|
+
return apply_top(sql_query, limit) if offset.to_i.zero? && !limit.nil? && !has_order && keys.empty?
|
|
143
|
+
|
|
144
|
+
sql_query = ensure_stable_order(sql_query, keys: keys, has_order: has_order)
|
|
145
|
+
|
|
146
|
+
clause = "OFFSET #{offset.to_i} ROWS"
|
|
147
|
+
clause = "#{clause} FETCH NEXT #{limit.to_i} ROWS ONLY" unless limit.nil?
|
|
148
|
+
|
|
149
|
+
"#{sql_query} #{clause}"
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
def apply_top(sql_query, limit)
|
|
153
|
+
with_masked_literals(sql_query) do |masked|
|
|
154
|
+
raise ArgumentError, "Query already contains TOP" if masked.match?(/\A\s*SELECT\s+(?:(?:DISTINCT|ALL)\s+)?TOP\b/i)
|
|
155
|
+
|
|
156
|
+
pattern = /\A\s*SELECT\s+((?:DISTINCT|ALL)\s+)?/i
|
|
157
|
+
raise ArgumentError, "Cannot apply TOP to this query; add ORDER BY" unless masked.match?(pattern)
|
|
158
|
+
|
|
159
|
+
masked.sub(pattern) { "SELECT #{Regexp.last_match(1)}TOP #{limit.to_i} " }
|
|
160
|
+
end
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
def ensure_stable_order(sql_query, keys:, has_order:)
|
|
164
|
+
return append_order_tie_breakers(sql_query, keys) if has_order
|
|
165
|
+
|
|
166
|
+
if keys.empty?
|
|
167
|
+
raise ArgumentError,
|
|
168
|
+
"Paginated SQL Server queries require an ORDER BY with a unique key, or a model primary_key"
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
"#{sql_query} ORDER BY #{order_by_list(keys)}"
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
def append_order_tie_breakers(sql_query, keys)
|
|
175
|
+
missing = keys.reject { |key| order_by_includes_key?(sql_query, key) }
|
|
176
|
+
return sql_query if missing.empty?
|
|
177
|
+
|
|
178
|
+
"#{sql_query}, #{order_by_list(missing)}"
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
def order_by_includes_key?(sql_query, key)
|
|
182
|
+
included = false
|
|
183
|
+
with_masked_literals(sql_query) do |masked|
|
|
184
|
+
order_clause = masked[/\bORDER\s+BY\b(.+)\z/im, 1]
|
|
185
|
+
if order_clause
|
|
186
|
+
quoted = Regexp.escape(quote_identifier(key))
|
|
187
|
+
bare = Regexp.escape(key.to_s)
|
|
188
|
+
included = order_clause.match?(/#{quoted}|\b#{bare}\b/i)
|
|
189
|
+
end
|
|
190
|
+
masked
|
|
191
|
+
end
|
|
192
|
+
included
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
def normalize_primary_keys(primary_key)
|
|
196
|
+
Array(primary_key).flat_map { |key| key.to_s.split(",") }.map(&:strip).reject(&:empty?).uniq
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
def order_by_list(keys)
|
|
200
|
+
keys.map { |key| quote_identifier(key) }.join(", ")
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
def quote_identifier(name)
|
|
204
|
+
"[#{name.to_s.gsub("]", "]]")}]"
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
def strip_trailing_terminator(sql_query)
|
|
208
|
+
with_masked_literals(sql_query.to_s.strip) do |masked|
|
|
209
|
+
masked = masked.rstrip
|
|
210
|
+
masked = masked.chomp(";") while masked.end_with?(";")
|
|
211
|
+
masked
|
|
212
|
+
end
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
def clause_outside_literals?(sql_query, pattern)
|
|
216
|
+
matched = false
|
|
217
|
+
with_masked_literals(sql_query) do |masked|
|
|
218
|
+
matched = masked.match?(pattern)
|
|
219
|
+
masked
|
|
220
|
+
end
|
|
221
|
+
matched
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
# Mask single-quoted SQL literals (including N'...' and escaped '') so rewrites
|
|
225
|
+
# do not touch values like 'a;b' or 'LIMIT 1'.
|
|
226
|
+
def with_masked_literals(sql_query)
|
|
227
|
+
literals = []
|
|
228
|
+
masked = sql_query.to_s.gsub(/(?:N)?'(?:''|[^'])*'/i) do |literal|
|
|
229
|
+
literals << literal
|
|
230
|
+
"__SQL_LITERAL_#{literals.length - 1}__"
|
|
231
|
+
end
|
|
232
|
+
|
|
233
|
+
result = yield(masked)
|
|
234
|
+
literals.each_with_index do |literal, index|
|
|
235
|
+
result = result.gsub("__SQL_LITERAL_#{index}__", literal)
|
|
236
|
+
end
|
|
237
|
+
result
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
def create_connection(connection_config)
|
|
241
|
+
raise "Unsupported Auth type" unless connection_config[:credentials][:auth_type] == "username/password"
|
|
242
|
+
|
|
243
|
+
TinyTds::Client.new(
|
|
244
|
+
username: connection_config[:credentials][:username],
|
|
245
|
+
password: connection_config[:credentials][:password],
|
|
246
|
+
host: connection_config[:host],
|
|
247
|
+
port: connection_config[:port].presence || 1433,
|
|
248
|
+
database: connection_config[:database],
|
|
249
|
+
timeout: 10,
|
|
250
|
+
azure: connection_config[:azure].presence || false
|
|
251
|
+
)
|
|
252
|
+
end
|
|
253
|
+
|
|
254
|
+
def create_streams(records)
|
|
255
|
+
group_by_table(records).map do |r|
|
|
256
|
+
Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["fetch"], json_schema: convert_to_json_schema(r[:columns]))
|
|
257
|
+
end
|
|
258
|
+
end
|
|
259
|
+
|
|
260
|
+
def group_by_table(records)
|
|
261
|
+
records.group_by { |entry| entry["table_name"] }.map do |table_name, columns|
|
|
262
|
+
{
|
|
263
|
+
tablename: table_name,
|
|
264
|
+
columns: columns.map do |column|
|
|
265
|
+
{
|
|
266
|
+
column_name: column["column_name"],
|
|
267
|
+
type: column["data_type"],
|
|
268
|
+
optional: column["is_nullable"] == "YES"
|
|
269
|
+
}
|
|
270
|
+
end
|
|
271
|
+
}
|
|
272
|
+
end
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
def escape(value)
|
|
276
|
+
value.to_s.gsub("'", "''")
|
|
277
|
+
end
|
|
278
|
+
end
|
|
279
|
+
end
|
|
280
|
+
end
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
{
|
|
2
|
+
"data": {
|
|
3
|
+
"name": "SqlServer",
|
|
4
|
+
"title": "SQL Server",
|
|
5
|
+
"connector_type": "source",
|
|
6
|
+
"category": "Data Warehouse",
|
|
7
|
+
"sub_category": "Relational Database",
|
|
8
|
+
"documentation_url": "https://docs.squared.ai/guides/sources/data-sources/sql-server",
|
|
9
|
+
"github_issue_label": "source-sql-server",
|
|
10
|
+
"icon": "https://res.cloudinary.com/dspflukeu/image/upload/v1790358609/Multiwoven/connectors/sql_server/icon.svg",
|
|
11
|
+
"license": "MIT",
|
|
12
|
+
"release_stage": "alpha",
|
|
13
|
+
"support_level": "community",
|
|
14
|
+
"tags": ["language:ruby", "multiwoven"]
|
|
15
|
+
}
|
|
16
|
+
}
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
{
|
|
2
|
+
"documentation_url": "https://docs.squared.ai/guides/sources/data-sources/sql-server",
|
|
3
|
+
"stream_type": "dynamic",
|
|
4
|
+
"connector_query_type": "raw_sql",
|
|
5
|
+
"connection_specification": {
|
|
6
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
7
|
+
"title": "SqlServer",
|
|
8
|
+
"type": "object",
|
|
9
|
+
"required": ["data_type", "credentials", "host", "database", "schema"],
|
|
10
|
+
"properties": {
|
|
11
|
+
"data_type": {
|
|
12
|
+
"description": "Type of data in the database",
|
|
13
|
+
"type": "string",
|
|
14
|
+
"title": "Data Format Type",
|
|
15
|
+
"oneOf": [
|
|
16
|
+
{
|
|
17
|
+
"const": "structured",
|
|
18
|
+
"title": "Tables & Records (Structured)"
|
|
19
|
+
},
|
|
20
|
+
{
|
|
21
|
+
"const": "vector",
|
|
22
|
+
"title": "Tables & Records containing Vector fields"
|
|
23
|
+
}
|
|
24
|
+
],
|
|
25
|
+
"default": "structured",
|
|
26
|
+
"order": 0
|
|
27
|
+
},
|
|
28
|
+
"credentials": {
|
|
29
|
+
"title": "",
|
|
30
|
+
"type": "object",
|
|
31
|
+
"required": ["auth_type", "username", "password"],
|
|
32
|
+
"properties": {
|
|
33
|
+
"auth_type": {
|
|
34
|
+
"type": "string",
|
|
35
|
+
"default": "username/password",
|
|
36
|
+
"order": 0,
|
|
37
|
+
"readOnly": true
|
|
38
|
+
},
|
|
39
|
+
"username": {
|
|
40
|
+
"description": "Username refers to your individual SQL Server login credentials. At a minimum, the user associated with these credentials must be granted read access to the data intended for synchronization.",
|
|
41
|
+
"examples": ["SQLSERVER_USER"],
|
|
42
|
+
"type": "string",
|
|
43
|
+
"title": "Username",
|
|
44
|
+
"order": 1
|
|
45
|
+
},
|
|
46
|
+
"password": {
|
|
47
|
+
"description": "This field requires the password associated with the SQL Server login specified in the preceding section.",
|
|
48
|
+
"type": "string",
|
|
49
|
+
"multiwoven_secret": true,
|
|
50
|
+
"title": "Password",
|
|
51
|
+
"order": 2
|
|
52
|
+
}
|
|
53
|
+
},
|
|
54
|
+
"order": 1
|
|
55
|
+
},
|
|
56
|
+
"host": {
|
|
57
|
+
"description": "The hostname or IP address of your SQL Server instance.",
|
|
58
|
+
"examples": ["127.0.0.1"],
|
|
59
|
+
"type": "string",
|
|
60
|
+
"title": "Host",
|
|
61
|
+
"order": 2
|
|
62
|
+
},
|
|
63
|
+
"port": {
|
|
64
|
+
"description": "The port number for your SQL Server instance, which defaults to 1433 and may vary based on your configuration.",
|
|
65
|
+
"examples": ["1433"],
|
|
66
|
+
"type": "string",
|
|
67
|
+
"title": "Port",
|
|
68
|
+
"order": 3
|
|
69
|
+
},
|
|
70
|
+
"database": {
|
|
71
|
+
"description": "The specific SQL Server database to connect to.",
|
|
72
|
+
"examples": ["SQLSERVER_DB"],
|
|
73
|
+
"type": "string",
|
|
74
|
+
"title": "Database",
|
|
75
|
+
"order": 4
|
|
76
|
+
},
|
|
77
|
+
"schema": {
|
|
78
|
+
"description": "The schema within the SQL Server database, typically 'dbo'.",
|
|
79
|
+
"examples": ["dbo"],
|
|
80
|
+
"default": "dbo",
|
|
81
|
+
"type": "string",
|
|
82
|
+
"title": "Schema",
|
|
83
|
+
"order": 5
|
|
84
|
+
},
|
|
85
|
+
"azure": {
|
|
86
|
+
"description": "Enable when connecting to Azure SQL Database (required for Azure encryption/login behaviour).",
|
|
87
|
+
"type": "boolean",
|
|
88
|
+
"title": "Azure SQL",
|
|
89
|
+
"default": false,
|
|
90
|
+
"order": 6
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
}
|
|
@@ -101,6 +101,7 @@ require_relative "integrations/source/azure_open_ai/client"
|
|
|
101
101
|
require_relative "integrations/source/xai/client"
|
|
102
102
|
require_relative "integrations/source/groq/client"
|
|
103
103
|
require_relative "integrations/source/open_router/client"
|
|
104
|
+
require_relative "integrations/source/mistral_ai/client"
|
|
104
105
|
require_relative "integrations/source/intuit_quick_books/client"
|
|
105
106
|
require_relative "integrations/source/pinecone_db/client"
|
|
106
107
|
require_relative "integrations/source/qdrant/client"
|
|
@@ -112,6 +113,7 @@ require_relative "integrations/source/aisquared/client"
|
|
|
112
113
|
require_relative "integrations/source/one_drive/client"
|
|
113
114
|
require_relative "integrations/source/microsoft_dynamics/client"
|
|
114
115
|
require_relative "integrations/source/epic_fhir/client"
|
|
116
|
+
require_relative "integrations/source/sql_server/client"
|
|
115
117
|
|
|
116
118
|
# Destination
|
|
117
119
|
require_relative "integrations/destination/klaviyo/client"
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: multiwoven-integrations
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.43.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Subin T P
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-09-
|
|
11
|
+
date: 2026-09-25 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activesupport
|
|
@@ -851,6 +851,12 @@ files:
|
|
|
851
851
|
- lib/multiwoven/integrations/source/microsoft_dynamics/config/meta.json
|
|
852
852
|
- lib/multiwoven/integrations/source/microsoft_dynamics/config/spec.json
|
|
853
853
|
- lib/multiwoven/integrations/source/microsoft_dynamics/icon.svg
|
|
854
|
+
- lib/multiwoven/integrations/source/mistral_ai/client.rb
|
|
855
|
+
- lib/multiwoven/integrations/source/mistral_ai/config/catalog.json
|
|
856
|
+
- lib/multiwoven/integrations/source/mistral_ai/config/meta.json
|
|
857
|
+
- lib/multiwoven/integrations/source/mistral_ai/config/models.json
|
|
858
|
+
- lib/multiwoven/integrations/source/mistral_ai/config/spec.json
|
|
859
|
+
- lib/multiwoven/integrations/source/mistral_ai/icon.svg
|
|
854
860
|
- lib/multiwoven/integrations/source/odoo/client.rb
|
|
855
861
|
- lib/multiwoven/integrations/source/odoo/config/meta.json
|
|
856
862
|
- lib/multiwoven/integrations/source/odoo/config/spec.json
|
|
@@ -907,6 +913,10 @@ files:
|
|
|
907
913
|
- lib/multiwoven/integrations/source/snowflake/config/meta.json
|
|
908
914
|
- lib/multiwoven/integrations/source/snowflake/config/spec.json
|
|
909
915
|
- lib/multiwoven/integrations/source/snowflake/icon.svg
|
|
916
|
+
- lib/multiwoven/integrations/source/sql_server/client.rb
|
|
917
|
+
- lib/multiwoven/integrations/source/sql_server/config/meta.json
|
|
918
|
+
- lib/multiwoven/integrations/source/sql_server/config/spec.json
|
|
919
|
+
- lib/multiwoven/integrations/source/sql_server/icon.svg
|
|
910
920
|
- lib/multiwoven/integrations/source/watsonx_ai/client.rb
|
|
911
921
|
- lib/multiwoven/integrations/source/watsonx_ai/config/catalog.json
|
|
912
922
|
- lib/multiwoven/integrations/source/watsonx_ai/config/meta.json
|