multiwoven-integrations 0.25.0 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/multiwoven/integrations/destination/qdrant/client.rb +184 -0
- data/lib/multiwoven/integrations/destination/qdrant/config/meta.json +15 -0
- data/lib/multiwoven/integrations/destination/qdrant/config/spec.json +23 -0
- data/lib/multiwoven/integrations/destination/qdrant/icon.svg +1 -0
- data/lib/multiwoven/integrations/rollout.rb +2 -1
- data/lib/multiwoven/integrations.rb +1 -0
- metadata +6 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: d9a97f8e29b1319b13c9828b58874c459aa1775834471212e067b50c37c89f45
|
|
4
|
+
data.tar.gz: 4cca3dd890240b8d831a7ef100ed99e6961c142bb4202631f2022d655692e13c
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: f4e91d96513b9b53a274b15a831d811f3e23e8d0b4958ed24a6914ca77d3b961c7389a7a2cdaae60546906e0abe68e772a678ad618f13e7b50ae9fd388379e32
|
|
7
|
+
data.tar.gz: 480de5e8ea3d1f6cf93457f4d7717ab8ae4cbae939d704cbd7a3403412c8c452b6e877b40fe91ac33baa1fad71ca21cfc4623fceb62fa7e5c46628dfca83d7e4
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Multiwoven::Integrations::Destination
|
|
4
|
+
module Qdrant
|
|
5
|
+
include Multiwoven::Integrations::Core
|
|
6
|
+
class Client < DestinationConnector
|
|
7
|
+
def check_connection(connection_config)
|
|
8
|
+
connection_config = connection_config.with_indifferent_access
|
|
9
|
+
api_url = connection_config[:api_url]
|
|
10
|
+
api_key = connection_config[:api_key]
|
|
11
|
+
|
|
12
|
+
response = Multiwoven::Integrations::Core::HttpClient.request(
|
|
13
|
+
api_url,
|
|
14
|
+
HTTP_GET,
|
|
15
|
+
headers: auth_headers(api_key)
|
|
16
|
+
)
|
|
17
|
+
if success?(response)
|
|
18
|
+
success_status
|
|
19
|
+
else
|
|
20
|
+
failure_status(nil)
|
|
21
|
+
end
|
|
22
|
+
rescue StandardError => e
|
|
23
|
+
handle_exception(e, {
|
|
24
|
+
context: "QDRANT:CHECK_CONNECTION:EXCEPTION",
|
|
25
|
+
type: "error"
|
|
26
|
+
})
|
|
27
|
+
failure_status(e)
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def discover(connection_config = nil)
|
|
31
|
+
connection_config = connection_config.with_indifferent_access
|
|
32
|
+
@api_url = connection_config[:api_url]
|
|
33
|
+
@api_key = connection_config[:api_key]
|
|
34
|
+
|
|
35
|
+
response = Multiwoven::Integrations::Core::HttpClient.request(
|
|
36
|
+
"#{@api_url}/collections",
|
|
37
|
+
HTTP_GET,
|
|
38
|
+
headers: auth_headers(@api_key)
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
data = JSON.parse(response.body)
|
|
42
|
+
catalog = build_catalog(data)
|
|
43
|
+
catalog.to_multiwoven_message
|
|
44
|
+
rescue StandardError => e
|
|
45
|
+
handle_exception(e, {
|
|
46
|
+
context: "QDRANT:DISCOVER:EXCEPTION",
|
|
47
|
+
type: "error"
|
|
48
|
+
})
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def write(sync_config, records, _action = "upsert")
|
|
52
|
+
connection_config = sync_config.destination.connection_specification.with_indifferent_access
|
|
53
|
+
collection_name = sync_config.stream.name
|
|
54
|
+
primary_key = sync_config.model.primary_key
|
|
55
|
+
log_message_array = []
|
|
56
|
+
|
|
57
|
+
api_url = connection_config[:api_url]
|
|
58
|
+
api_key = connection_config[:api_key]
|
|
59
|
+
|
|
60
|
+
write_success = 0
|
|
61
|
+
write_failure = 0
|
|
62
|
+
records.each do |record|
|
|
63
|
+
points = []
|
|
64
|
+
points.push({
|
|
65
|
+
id: record[primary_key],
|
|
66
|
+
vector: JSON.parse(record["vector"]),
|
|
67
|
+
payload: record["payload"]
|
|
68
|
+
})
|
|
69
|
+
begin
|
|
70
|
+
response = upsert_points(api_url, api_key, collection_name, { points: points })
|
|
71
|
+
if success?(response)
|
|
72
|
+
write_success += 1
|
|
73
|
+
log_message_array << log_request_response("info", { points: points }, JSON.parse(response.body))
|
|
74
|
+
else
|
|
75
|
+
# write_failure could be duplicated if JSON.parse errors.
|
|
76
|
+
write_failure += 1
|
|
77
|
+
log_message_array << log_request_response("error", { points: points }, JSON.parse(response.body))
|
|
78
|
+
end
|
|
79
|
+
rescue StandardError => e
|
|
80
|
+
handle_exception(e, {
|
|
81
|
+
context: "QDRANT:RECORD:WRITE:EXCEPTION",
|
|
82
|
+
type: "error",
|
|
83
|
+
sync_id: sync_config.sync_id,
|
|
84
|
+
sync_run_id: sync_config.sync_run_id
|
|
85
|
+
})
|
|
86
|
+
write_failure += 1
|
|
87
|
+
log_message_array << log_request_response("error", { points: points }, e.message)
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
tracking_message(write_success, write_failure, log_message_array)
|
|
91
|
+
rescue StandardError => e
|
|
92
|
+
handle_exception(e, {
|
|
93
|
+
context: "QDRANT:RECORD:WRITE:EXCEPTION",
|
|
94
|
+
type: "error",
|
|
95
|
+
sync_id: sync_config.sync_id,
|
|
96
|
+
sync_run_id: sync_config.sync_run_id
|
|
97
|
+
})
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
private
|
|
101
|
+
|
|
102
|
+
def upsert_points(api_url, api_key, collection_name, payload)
|
|
103
|
+
Multiwoven::Integrations::Core::HttpClient.request(
|
|
104
|
+
api_url + "/collections/#{collection_name}/points",
|
|
105
|
+
HTTP_PUT,
|
|
106
|
+
payload: payload,
|
|
107
|
+
headers: auth_headers(api_key)
|
|
108
|
+
)
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
def build_catalog(data)
|
|
112
|
+
streams = data["result"]["collections"].map { |collection| build_stream(collection) }
|
|
113
|
+
Multiwoven::Integrations::Protocol::Catalog.new(
|
|
114
|
+
streams: streams,
|
|
115
|
+
request_rate_limit: 60,
|
|
116
|
+
request_rate_limit_unit: "minute",
|
|
117
|
+
request_rate_concurrency: 10
|
|
118
|
+
)
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
def build_stream(collection)
|
|
122
|
+
response = Multiwoven::Integrations::Core::HttpClient.request(
|
|
123
|
+
"#{@api_url}/collections/#{collection["name"]}",
|
|
124
|
+
HTTP_GET,
|
|
125
|
+
headers: auth_headers(@api_key)
|
|
126
|
+
)
|
|
127
|
+
|
|
128
|
+
payload = { "type" => "object", "properties" => {} }
|
|
129
|
+
if success?(response)
|
|
130
|
+
data = JSON.parse(response.body)
|
|
131
|
+
payload_schema = data["result"]["payload_schema"]
|
|
132
|
+
payload_schema.each { |key, value| payload["properties"][key] = map_qdrant_types(value) } unless payload_schema.empty?
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
Multiwoven::Integrations::Protocol::Stream.new(
|
|
136
|
+
name: collection["name"],
|
|
137
|
+
action: "update",
|
|
138
|
+
method: HTTP_PUT,
|
|
139
|
+
supported_sync_modes: %w[incremental],
|
|
140
|
+
json_schema: {
|
|
141
|
+
"type" => "object",
|
|
142
|
+
"required" => %w[id vector payload],
|
|
143
|
+
"properties" => {
|
|
144
|
+
"id" => {
|
|
145
|
+
"type" => "string"
|
|
146
|
+
},
|
|
147
|
+
"payload" => payload,
|
|
148
|
+
"vector" => {
|
|
149
|
+
"type" => "vector"
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
)
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
def map_qdrant_types(value)
|
|
157
|
+
case value["data_type"]
|
|
158
|
+
when "integer"
|
|
159
|
+
{ "type" => "integer" }
|
|
160
|
+
when "float"
|
|
161
|
+
{ "type" => "number" }
|
|
162
|
+
when "bool"
|
|
163
|
+
{ "type" => "boolean" }
|
|
164
|
+
when "geo"
|
|
165
|
+
{
|
|
166
|
+
"type" => "object",
|
|
167
|
+
"required" => %w[lon lat],
|
|
168
|
+
"properties" => {
|
|
169
|
+
"lon" => {
|
|
170
|
+
"type" => "number"
|
|
171
|
+
},
|
|
172
|
+
"lat" => {
|
|
173
|
+
"type" => "number"
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
else
|
|
178
|
+
# datetime, keyword, text, uuid
|
|
179
|
+
{ "type" => "string" }
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
end
|
|
183
|
+
end
|
|
184
|
+
end
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
{
|
|
2
|
+
"data": {
|
|
3
|
+
"name": "Qdrant",
|
|
4
|
+
"title": "Qdrant",
|
|
5
|
+
"connector_type": "destination",
|
|
6
|
+
"category": "Database",
|
|
7
|
+
"documentation_url": "https://docs.squared.ai/guides/destinations/retl-destinations/database/qdrant",
|
|
8
|
+
"github_issue_label": "destination-qdrant",
|
|
9
|
+
"icon": "icon.svg",
|
|
10
|
+
"license": "MIT",
|
|
11
|
+
"release_stage": "alpha",
|
|
12
|
+
"support_level": "community",
|
|
13
|
+
"tags": ["language:ruby", "multiwoven"]
|
|
14
|
+
}
|
|
15
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
{
|
|
2
|
+
"documentation_url": "https://docs.squared.ai/guides/destinations/retl-destinations/database/qdrant",
|
|
3
|
+
"stream_type": "dynamic",
|
|
4
|
+
"connection_specification": {
|
|
5
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
6
|
+
"title": "Qdrant",
|
|
7
|
+
"type": "object",
|
|
8
|
+
"required": ["api_url", "api_key"],
|
|
9
|
+
"properties": {
|
|
10
|
+
"api_url": {
|
|
11
|
+
"type": "string",
|
|
12
|
+
"title": "API Url",
|
|
13
|
+
"order": 0
|
|
14
|
+
},
|
|
15
|
+
"api_key": {
|
|
16
|
+
"type": "string",
|
|
17
|
+
"multiwoven_secret": true,
|
|
18
|
+
"title": "API Key",
|
|
19
|
+
"order": 1
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 451.84 152.4" width="451.84" height="152.4" fill="none"><g fill="#dc244c"><path d="M212.69 116c0 5.523-4.477 10.001-10 10.001h-6.836v-7.808h-.244c-.732 1.057-1.708 2.155-2.928 3.293-1.139 1.058-2.521 2.034-4.148 2.929-1.545.894-3.294 1.626-5.246 2.196a20.015 20.015 0 0 1-5.856.853c-4.392 0-8.377-.732-11.956-2.196-3.579-1.545-6.669-3.66-9.272-6.344-2.521-2.765-4.473-6.018-5.856-9.759-1.383-3.742-2.074-7.849-2.074-12.322 0-4.148.61-8.093 1.83-11.835 1.301-3.822 3.091-7.198 5.368-10.126a26.176 26.176 0 0 1 8.54-6.954c3.335-1.789 7.117-2.683 11.346-2.683 3.823 0 7.361.61 10.614 1.83 3.335 1.138 6.059 3.13 8.174 5.978h.244V43.769c0-5.523 4.477-10 10-10h8.3zm-16.836-19.646c0-4.473-1.301-8.092-3.904-10.858-2.521-2.765-6.1-4.148-10.736-4.148s-8.255 1.383-10.858 4.148c-2.521 2.766-3.782 6.385-3.782 10.858 0 4.474 1.261 8.093 3.782 10.858 2.603 2.766 6.222 4.149 10.858 4.149s8.215-1.383 10.736-4.149c2.603-2.765 3.904-6.384 3.904-10.858zM224.53 76.708c0-5.522 4.477-10 10-10h8.3v9.516h.244c1.952-3.66 4.27-6.384 6.954-8.174 2.684-1.87 6.059-2.805 10.126-2.805 1.057 0 2.114.04 3.172.121 1.057.082 2.033.245 2.928.489v16.713a22.4 22.4 0 0 0-3.904-.854 19.895 19.895 0 0 0-3.904-.367c-3.498 0-6.263.489-8.296 1.465-2.034.975-3.62 2.358-4.758 4.147-1.058 1.708-1.749 3.782-2.074 6.222-.326 2.441-.488 5.124-.488 8.052V116c0 5.524-4.477 10.002-10 10.002h-8.3zM310.64 118.56h-.244c-2.033 3.172-4.758 5.449-8.174 6.832-3.334 1.382-6.872 2.073-10.614 2.073-2.765 0-5.449-.406-8.052-1.219-2.521-.732-4.758-1.871-6.71-3.416-1.952-1.546-3.497-3.457-4.636-5.735-1.138-2.277-1.708-4.92-1.708-7.929 0-3.416.61-6.304 1.83-8.662 1.302-2.359 3.01-4.311 5.124-5.856 2.196-1.546 4.677-2.725 7.442-3.538a50.076 50.076 0 0 1 8.54-1.953c3.01-.406 5.978-.65 8.906-.732 3.01-.081 5.775-.121 8.296-.121 0-3.254-1.179-5.816-3.538-7.687-2.277-1.952-5.002-2.928-8.174-2.928-3.009 0-5.774.651-8.296 1.952-2.44 1.221-4.636 2.929-6.588 5.124l-9.76-10.004c3.416-3.171 7.402-5.53 11.956-7.075a41.766 41.766 0 0 1 14.152-2.44c5.368 0 9.76.69 13.176 2.073 3.498 1.302 6.263 3.254 8.296 5.856 2.115 2.603 3.579 5.816 4.392 9.638.814 3.742 1.22 8.093 1.22 13.054v20.135c0 5.522-4.477 10-10 10h-6.836zm-4.514-18.545c-1.382 0-3.131.082-5.246.244a24.865 24.865 0 0 0-5.978.976c-1.87.57-3.497 1.424-4.88 2.562-1.301 1.139-1.952 2.725-1.952 4.759 0 2.196.936 3.822 2.806 4.879 1.871 1.058 3.823 1.586 5.856 1.586 1.79 0 3.498-.244 5.124-.732 1.708-.488 3.213-1.179 4.514-2.074a9.45 9.45 0 0 0 3.05-3.416c.814-1.382 1.22-3.009 1.22-4.879v-3.905zM340.18 76.708c0-5.522 4.477-10 10-10h7.568v8.052h.244c.569-1.138 1.382-2.277 2.44-3.416 1.057-1.138 2.318-2.155 3.782-3.05s3.131-1.626 5.002-2.195c1.87-.57 3.904-.854 6.1-.854 4.636 0 8.377.732 11.224 2.196 2.846 1.382 5.042 3.334 6.588 5.855 1.626 2.522 2.724 5.49 3.294 8.906.569 3.416.854 7.117.854 11.103V116c0 5.523-4.477 10.001-10 10.001h-8.3V96.964a58.2 58.2 0 0 0-.244-5.246c-.082-1.87-.448-3.578-1.098-5.123a8.154 8.154 0 0 0-2.806-3.783c-1.22-.976-3.01-1.464-5.368-1.464-2.359 0-4.27.448-5.734 1.342a8.629 8.629 0 0 0-3.416 3.416c-.732 1.383-1.22 2.969-1.464 4.758a41.535 41.535 0 0 0-.366 5.612V116c0 5.523-4.477 10.001-10 10.001h-8.3zM451.84 71.348c0 5.522-4.477 10-10 10h-6.104v19.764c0 1.627.082 3.132.244 4.514.163 1.301.529 2.44 1.098 3.415.57.977 1.424 1.75 2.562 2.32 1.22.487 2.806.73 4.758.73.976 0 2.237-.08 3.782-.243 1.627-.245 2.847-.732 3.66-1.464v8.725c0 3.915-2.452 7.557-6.344 7.989a58.528 58.528 0 0 1-6.466.366c-3.09 0-5.937-.325-8.54-.976-2.602-.65-4.88-1.667-6.832-3.05-1.952-1.463-3.497-3.334-4.636-5.612-1.057-2.277-1.586-5.042-1.586-8.295V81.347h-11.712v-4.64c0-5.522 4.478-9.999 10-9.999h1.712V59.14c0-5.523 4.478-10 10-10h8.3v17.568h16.104z"/></g><g fill-rule="evenodd" clip-rule="evenodd"><path fill="#24386c" d="m103.79 140.09-3.039-83.784-5.503-22.089 36.734 3.889v101.35l-22.44 12.95z"/><path fill="#7589be" d="m131.98 38.1-22.44 12.96-46.308-10.158L9.029 62.971-.001 38.1l32.99-19.05 33-19.05 32.99 19.05z"/><path fill="#b2bfe8" d="m0 38.1 22.44 12.96 13.008 38.686 43.921 35.142L65.991 152.4l-33-19.051L0 114.299v-76.2"/><path fill="#24386c" d="m80.868 104.56-14.877 21.932v25.91l21.11-12.18 10.877-16.242"/><path fill="#7589be" d="M66 100.59 44.881 64.025l4.549-12.119 17.293-8.384L87.1 64.026z"/><path fill="#b2bfe8" d="m44.881 64.022 21.11 12.18v24.38l-19.524.84-11.81-15.08 10.224-22.32"/><path fill="#24386c" d="m65.991 76.2 21.11-12.179 14.367 23.922-17.386 14.365-18.091-1.727z"/><path fill="#dc244c" d="m87.101 140.22 22.44 12.181V51.061l-21.78-12.57-21.77-12.57-21.78 12.57-21.77 12.57v50.289l21.77 12.57 21.78 12.571 21.11-12.191zm0-51.83-21.11 12.19-21.11-12.19V64.02l21.11-12.19 21.11 12.19v24.37"/></g><path fill="url(#a)" d="M66 126.5v-25.914L45 88.5v25.871z"/><defs><linearGradient id="a" x1="62.128" x2="41.202" y1="105.54" y2="105.54" gradientUnits="userSpaceOnUse"><stop offset="0" stop-color="#FF3364"/><stop offset="1" stop-color="#C91540" stop-opacity="0"/></linearGradient></defs></svg>
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
module Multiwoven
|
|
4
4
|
module Integrations
|
|
5
|
-
VERSION = "0.
|
|
5
|
+
VERSION = "0.26.0"
|
|
6
6
|
|
|
7
7
|
ENABLED_SOURCES = %w[
|
|
8
8
|
Snowflake
|
|
@@ -53,6 +53,7 @@ module Multiwoven
|
|
|
53
53
|
AISDataStore
|
|
54
54
|
AmazonS3
|
|
55
55
|
MicrosoftDynamics
|
|
56
|
+
Qdrant
|
|
56
57
|
PineconeDB
|
|
57
58
|
].freeze
|
|
58
59
|
end
|
|
@@ -108,6 +108,7 @@ require_relative "integrations/destination/mailchimp/client"
|
|
|
108
108
|
require_relative "integrations/destination/ais_data_store/client"
|
|
109
109
|
require_relative "integrations/destination/amazon_s3/client"
|
|
110
110
|
require_relative "integrations/destination/microsoft_dynamics/client"
|
|
111
|
+
require_relative "integrations/destination/qdrant/client"
|
|
111
112
|
require_relative "integrations/destination/pinecone_db/client"
|
|
112
113
|
|
|
113
114
|
module Multiwoven
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: multiwoven-integrations
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.26.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Subin T P
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2025-05-
|
|
11
|
+
date: 2025-05-19 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activesupport
|
|
@@ -671,6 +671,10 @@ files:
|
|
|
671
671
|
- lib/multiwoven/integrations/destination/postgresql/config/meta.json
|
|
672
672
|
- lib/multiwoven/integrations/destination/postgresql/config/spec.json
|
|
673
673
|
- lib/multiwoven/integrations/destination/postgresql/icon.svg
|
|
674
|
+
- lib/multiwoven/integrations/destination/qdrant/client.rb
|
|
675
|
+
- lib/multiwoven/integrations/destination/qdrant/config/meta.json
|
|
676
|
+
- lib/multiwoven/integrations/destination/qdrant/config/spec.json
|
|
677
|
+
- lib/multiwoven/integrations/destination/qdrant/icon.svg
|
|
674
678
|
- lib/multiwoven/integrations/destination/salesforce_consumer_goods_cloud/client.rb
|
|
675
679
|
- lib/multiwoven/integrations/destination/salesforce_consumer_goods_cloud/config/catalog.json
|
|
676
680
|
- lib/multiwoven/integrations/destination/salesforce_consumer_goods_cloud/config/meta.json
|