multiwoven-integrations 0.27.0 → 0.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/multiwoven/integrations/core/vector_source_connector.rb +14 -0
- data/lib/multiwoven/integrations/protocol/protocol.rb +7 -1
- data/lib/multiwoven/integrations/rollout.rb +2 -1
- data/lib/multiwoven/integrations/source/pinecone_db/client.rb +73 -0
- data/lib/multiwoven/integrations/source/pinecone_db/config/catalog.json +6 -0
- data/lib/multiwoven/integrations/source/pinecone_db/config/meta.json +15 -0
- data/lib/multiwoven/integrations/source/pinecone_db/config/spec.json +34 -0
- data/lib/multiwoven/integrations/source/pinecone_db/icon.svg +2 -0
- data/lib/multiwoven/integrations.rb +2 -0
- metadata +8 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: acdbc7603a2a3ec00b87ba9dfab76702391d030c78ade57444ff0ff5d219a464
|
|
4
|
+
data.tar.gz: b51c2cbddb3597d1053d08af43dce594b86ab3a308fbd37093167ed99bc2bad7
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: b964f7f98033dc64b70f2080bc9e0006d28c6a9fc2b83ee48cfa7c0ec72ecfc79afa3f641202eaa2749bcd12c1adcfb98d9062ae1133872506d76deb6918ce91
|
|
7
|
+
data.tar.gz: 34921ea2abcf6053149983cc23c5d106f041e21e7b592cdf34e4c45ffb218012e47ae48a27dcddaa9f0b2408723caf5ca423ff9381c9c526a801ad0787a2cc55
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Multiwoven
|
|
4
|
+
module Integrations::Core
|
|
5
|
+
class VectorSourceConnector < SourceConnector
|
|
6
|
+
# This needs to be implemented
|
|
7
|
+
# for all vector database sources
|
|
8
|
+
# that will be used in RAG workflows
|
|
9
|
+
def search(_vector_search_config)
|
|
10
|
+
raise "Not implemented"
|
|
11
|
+
end
|
|
12
|
+
end
|
|
13
|
+
end
|
|
14
|
+
end
|
|
@@ -10,7 +10,7 @@ module Multiwoven
|
|
|
10
10
|
SyncStatus = Types::String.enum("started", "running", "complete", "incomplete")
|
|
11
11
|
DestinationSyncMode = Types::String.enum("insert", "upsert")
|
|
12
12
|
ConnectorType = Types::String.enum("source", "destination")
|
|
13
|
-
ConnectorQueryType = Types::String.enum("raw_sql", "soql", "ai_ml")
|
|
13
|
+
ConnectorQueryType = Types::String.enum("raw_sql", "soql", "ai_ml", "vector_search")
|
|
14
14
|
ModelQueryType = Types::String.enum("raw_sql", "dbt", "soql", "table_selector", "ai_ml", "dynamic_sql", "unstructured", "vector_search")
|
|
15
15
|
ConnectionStatusType = Types::String.enum("succeeded", "failed")
|
|
16
16
|
StreamType = Types::String.enum("static", "dynamic", "user_defined")
|
|
@@ -179,6 +179,12 @@ module Multiwoven
|
|
|
179
179
|
attribute :sync_id, Types::String.default("unknown")
|
|
180
180
|
end
|
|
181
181
|
|
|
182
|
+
class VectorConfig < ProtocolModel
|
|
183
|
+
attribute :source, Connector
|
|
184
|
+
attribute :vector, Types::Array.of(Types::Float)
|
|
185
|
+
attribute :limit, Types::Integer.default(1)
|
|
186
|
+
end
|
|
187
|
+
|
|
182
188
|
class ControlMessage < ProtocolModel
|
|
183
189
|
attribute :type, ControlMessageType
|
|
184
190
|
attribute :emitted_at, Types::Integer
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
module Multiwoven
|
|
4
4
|
module Integrations
|
|
5
|
-
VERSION = "0.
|
|
5
|
+
VERSION = "0.28.0"
|
|
6
6
|
|
|
7
7
|
ENABLED_SOURCES = %w[
|
|
8
8
|
Snowflake
|
|
@@ -28,6 +28,7 @@ module Multiwoven
|
|
|
28
28
|
AwsBedrockModel
|
|
29
29
|
GenericOpenAI
|
|
30
30
|
IntuitQuickBooks
|
|
31
|
+
PineconeDB
|
|
31
32
|
].freeze
|
|
32
33
|
|
|
33
34
|
ENABLED_DESTINATIONS = %w[
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Multiwoven::Integrations::Source
|
|
4
|
+
module PineconeDB
|
|
5
|
+
include Multiwoven::Integrations::Core
|
|
6
|
+
class Client < VectorSourceConnector
|
|
7
|
+
def check_connection(connection_config)
|
|
8
|
+
connection_config = connection_config.with_indifferent_access
|
|
9
|
+
pinecone = create_connection(connection_config)
|
|
10
|
+
result = pinecone.describe_index(@index_name)
|
|
11
|
+
if success?(result)
|
|
12
|
+
success_status
|
|
13
|
+
else
|
|
14
|
+
failure_status(nil)
|
|
15
|
+
end
|
|
16
|
+
rescue StandardError => e
|
|
17
|
+
handle_exception(e, { context: "PINECONE:CHECK_CONNECTION:EXCEPTION", type: "error" })
|
|
18
|
+
failure_status(e)
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def discover(_connection_config = nil)
|
|
22
|
+
catalog_json = read_json(CATALOG_SPEC_PATH)
|
|
23
|
+
catalog = build_catalog(catalog_json)
|
|
24
|
+
catalog.to_multiwoven_message
|
|
25
|
+
rescue StandardError => e
|
|
26
|
+
handle_exception(e, {
|
|
27
|
+
context: "PINECONE:DISCOVER:EXCEPTION",
|
|
28
|
+
type: "error"
|
|
29
|
+
})
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def search(vector_search_config)
|
|
33
|
+
connection_config = vector_search_config.source.connection_specification
|
|
34
|
+
connection_config = connection_config.with_indifferent_access
|
|
35
|
+
connection = create_connection(connection_config)
|
|
36
|
+
pinecone_index = connection.index(@index_name)
|
|
37
|
+
response = pinecone_index.query(vector: vector_search_config[:vector],
|
|
38
|
+
namespace: @namespace,
|
|
39
|
+
top_k: vector_search_config[:limit],
|
|
40
|
+
include_values: true,
|
|
41
|
+
include_metadata: true)
|
|
42
|
+
result = JSON.parse(response.body).with_indifferent_access
|
|
43
|
+
records = result["matches"]
|
|
44
|
+
records.map do |row|
|
|
45
|
+
RecordMessage.new(data: row, emitted_at: Time.now.to_i).to_multiwoven_message
|
|
46
|
+
end
|
|
47
|
+
rescue StandardError => e
|
|
48
|
+
handle_exception(e, {
|
|
49
|
+
context: "PINECONE:SEARCH:EXCEPTION",
|
|
50
|
+
type: "error"
|
|
51
|
+
})
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
private
|
|
55
|
+
|
|
56
|
+
def create_connection(connection_config)
|
|
57
|
+
initialize_client(connection_config)
|
|
58
|
+
Pinecone.configure do |config|
|
|
59
|
+
config.api_key = @api_key
|
|
60
|
+
config.environment = @region
|
|
61
|
+
end
|
|
62
|
+
Pinecone::Client.new
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def initialize_client(connection_config)
|
|
66
|
+
@api_key = connection_config["api_key"]
|
|
67
|
+
@region = connection_config["region"]
|
|
68
|
+
@index_name = connection_config["index_name"]
|
|
69
|
+
@namespace = connection_config["namespace"]
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
end
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
{
|
|
2
|
+
"data": {
|
|
3
|
+
"name": "PineconeDB",
|
|
4
|
+
"title": "Pinecone DB",
|
|
5
|
+
"connector_type": "source",
|
|
6
|
+
"category": "Database",
|
|
7
|
+
"documentation_url": "https://docs.squared.ai/guides/source/data-sources/pinecone_db",
|
|
8
|
+
"github_issue_label": "source-pinecone-db",
|
|
9
|
+
"icon": "icon.svg",
|
|
10
|
+
"license": "MIT",
|
|
11
|
+
"release_stage": "alpha",
|
|
12
|
+
"support_level": "community",
|
|
13
|
+
"tags": ["language:ruby", "multiwoven"]
|
|
14
|
+
}
|
|
15
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
{
|
|
2
|
+
"documentation_url": "https://docs.squared.ai/guides/source/data-sources/pinecone_db",
|
|
3
|
+
"stream_type": "dynamic",
|
|
4
|
+
"connector_query_type": "vector_search",
|
|
5
|
+
"connection_specification": {
|
|
6
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
7
|
+
"title": "Pinecone DB",
|
|
8
|
+
"type": "object",
|
|
9
|
+
"required": ["api_key", "region", "index_name", "namespace"],
|
|
10
|
+
"properties": {
|
|
11
|
+
"api_key": {
|
|
12
|
+
"type": "string",
|
|
13
|
+
"multiwoven_secret": true,
|
|
14
|
+
"title": "API Key",
|
|
15
|
+
"order": 0
|
|
16
|
+
},
|
|
17
|
+
"region": {
|
|
18
|
+
"type": "string",
|
|
19
|
+
"title": "Region",
|
|
20
|
+
"order": 1
|
|
21
|
+
},
|
|
22
|
+
"index_name": {
|
|
23
|
+
"type": "string",
|
|
24
|
+
"title": "Index Name",
|
|
25
|
+
"order": 2
|
|
26
|
+
},
|
|
27
|
+
"namespace": {
|
|
28
|
+
"type": "string",
|
|
29
|
+
"title": "Index Name",
|
|
30
|
+
"order": 3
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
}
|
|
@@ -0,0 +1,2 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="utf-8"?><!-- Uploaded to: SVG Repo, www.svgrepo.com, Generator: SVG Repo Mixer Tools -->
|
|
2
|
+
<svg fill="#000000" width="800px" height="800px" viewBox="0 0 24 24" role="img" xmlns="http://www.w3.org/2000/svg"><title>Dynamics 365 icon</title><path d="M4.59 7.41l4.94 3.54L4.59 24zm0-7.41v6.36l9.53 5.29 4.59-3.52zm0 24l14.82-8.47v-6.7Z"/></svg>
|
|
@@ -60,6 +60,7 @@ require_relative "integrations/core/http_client"
|
|
|
60
60
|
require_relative "integrations/core/streaming_http_client"
|
|
61
61
|
require_relative "integrations/core/query_builder"
|
|
62
62
|
require_relative "integrations/core/unstructured_source_connector"
|
|
63
|
+
require_relative "integrations/core/vector_source_connector"
|
|
63
64
|
|
|
64
65
|
# Source
|
|
65
66
|
require_relative "integrations/source/snowflake/client"
|
|
@@ -85,6 +86,7 @@ require_relative "integrations/source/anthropic/client"
|
|
|
85
86
|
require_relative "integrations/source/aws_bedrock_model/client"
|
|
86
87
|
require_relative "integrations/source/generic_open_ai/client"
|
|
87
88
|
require_relative "integrations/source/intuit_quick_books/client"
|
|
89
|
+
require_relative "integrations/source/pinecone_db/client"
|
|
88
90
|
|
|
89
91
|
# Destination
|
|
90
92
|
require_relative "integrations/destination/klaviyo/client"
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: multiwoven-integrations
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.28.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Subin T P
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2025-
|
|
11
|
+
date: 2025-06-12 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activesupport
|
|
@@ -587,6 +587,7 @@ files:
|
|
|
587
587
|
- lib/multiwoven/integrations/core/streaming_http_client.rb
|
|
588
588
|
- lib/multiwoven/integrations/core/unstructured_source_connector.rb
|
|
589
589
|
- lib/multiwoven/integrations/core/utils.rb
|
|
590
|
+
- lib/multiwoven/integrations/core/vector_source_connector.rb
|
|
590
591
|
- lib/multiwoven/integrations/destination/airtable/client.rb
|
|
591
592
|
- lib/multiwoven/integrations/destination/airtable/config/catalog.json
|
|
592
593
|
- lib/multiwoven/integrations/destination/airtable/config/meta.json
|
|
@@ -783,6 +784,11 @@ files:
|
|
|
783
784
|
- lib/multiwoven/integrations/source/oracle_db/config/meta.json
|
|
784
785
|
- lib/multiwoven/integrations/source/oracle_db/config/spec.json
|
|
785
786
|
- lib/multiwoven/integrations/source/oracle_db/icon.svg
|
|
787
|
+
- lib/multiwoven/integrations/source/pinecone_db/client.rb
|
|
788
|
+
- lib/multiwoven/integrations/source/pinecone_db/config/catalog.json
|
|
789
|
+
- lib/multiwoven/integrations/source/pinecone_db/config/meta.json
|
|
790
|
+
- lib/multiwoven/integrations/source/pinecone_db/config/spec.json
|
|
791
|
+
- lib/multiwoven/integrations/source/pinecone_db/icon.svg
|
|
786
792
|
- lib/multiwoven/integrations/source/postgresql/client.rb
|
|
787
793
|
- lib/multiwoven/integrations/source/postgresql/config/meta.json
|
|
788
794
|
- lib/multiwoven/integrations/source/postgresql/config/spec.json
|