multiwoven-integrations 0.23.2 → 0.24.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/multiwoven/integrations/destination/pinecone_db/client.rb +154 -0
- data/lib/multiwoven/integrations/destination/pinecone_db/config/meta.json +15 -0
- data/lib/multiwoven/integrations/destination/pinecone_db/config/spec.json +45 -0
- data/lib/multiwoven/integrations/destination/pinecone_db/icon.svg +1 -0
- data/lib/multiwoven/integrations/rollout.rb +2 -1
- data/lib/multiwoven/integrations/source/amazon_s3/config/spec.json +3 -3
- data/lib/multiwoven/integrations/source/postgresql/config/spec.json +9 -3
- data/lib/multiwoven/integrations.rb +2 -0
- metadata +6 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: ad60cb551af1e7779f94369ba8696ca3057b197a4eeff5593fab09b78c12441a
|
|
4
|
+
data.tar.gz: 42097886868c03c1a5f3454c07b487565e7c16df906810aca5528141a3de8584
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 23f17dd48e50c6cff237592f8d43b2ac5eafc120b89482ff346d0c0fee7451b21a460b37a3e2ac81a6092c5cea0ee55bce8febfdad69e1516deaf7c88a81bc59
|
|
7
|
+
data.tar.gz: 51441d50a9bc79fec30d1ee98c108b27fbb9f2b36fe8199de919d266a111682162873d8da53eddd37244b2b3d51b8ccb2ef143eae97e118e6d8a6beb58a14b54
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Multiwoven::Integrations::Destination
|
|
4
|
+
module PineconeDB
|
|
5
|
+
include Multiwoven::Integrations::Core
|
|
6
|
+
PINECONE_OBJECTS = [
|
|
7
|
+
{ column_name: "id", data_type: "string", is_nullable: false },
|
|
8
|
+
{ column_name: "value", data_type: "vector", is_nullable: false },
|
|
9
|
+
{ column_name: "meta_data", data_type: "string", is_nullable: false }
|
|
10
|
+
].freeze
|
|
11
|
+
class Client < DestinationConnector
|
|
12
|
+
def check_connection(connection_config)
|
|
13
|
+
connection_config = connection_config.with_indifferent_access
|
|
14
|
+
create_connection(connection_config)
|
|
15
|
+
result = @pinecone.describe_index(@index_name)
|
|
16
|
+
if result
|
|
17
|
+
success_status
|
|
18
|
+
else
|
|
19
|
+
failure_status(nil)
|
|
20
|
+
end
|
|
21
|
+
rescue StandardError => e
|
|
22
|
+
handle_exception(e, { context: "PINECONE:CHECK_CONNECTION:EXCEPTION", type: "error" })
|
|
23
|
+
failure_status(e)
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def discover(connection_config)
|
|
27
|
+
connection_config = connection_config.with_indifferent_access
|
|
28
|
+
create_connection(connection_config)
|
|
29
|
+
pinecone_index = @pinecone.index(@index_name)
|
|
30
|
+
response = pinecone_index.describe_index_stats
|
|
31
|
+
results = JSON.parse(response.body)
|
|
32
|
+
records = results["namespaces"].keys
|
|
33
|
+
catalog = Catalog.new(streams: create_streams(records))
|
|
34
|
+
catalog.to_multiwoven_message
|
|
35
|
+
rescue StandardError => e
|
|
36
|
+
handle_exception(e, {
|
|
37
|
+
context: "PINECONE:DISCOVER:EXCEPTION",
|
|
38
|
+
type: "error"
|
|
39
|
+
})
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def write(sync_config, records, _action = "upsert")
|
|
43
|
+
@sync_config = sync_config
|
|
44
|
+
connection_config = sync_config.destination.connection_specification.with_indifferent_access
|
|
45
|
+
create_connection(connection_config)
|
|
46
|
+
process_records(records, sync_config.stream)
|
|
47
|
+
rescue StandardError => e
|
|
48
|
+
handle_exception(e, {
|
|
49
|
+
context: "PINECONE:WRITE:EXCEPTION",
|
|
50
|
+
type: "error",
|
|
51
|
+
sync_id: @sync_config.sync_id,
|
|
52
|
+
sync_run_id: @sync_config.sync_run_id
|
|
53
|
+
})
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
private
|
|
57
|
+
|
|
58
|
+
def create_connection(connection_config)
|
|
59
|
+
initialize_client(connection_config)
|
|
60
|
+
Pinecone.configure do |config|
|
|
61
|
+
config.api_key = @api_key
|
|
62
|
+
config.environment = @region
|
|
63
|
+
end
|
|
64
|
+
@pinecone = Pinecone::Client.new
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def initialize_client(connection_config)
|
|
68
|
+
@api_key = connection_config["api_key"]
|
|
69
|
+
@region = connection_config["region"]
|
|
70
|
+
@index_name = connection_config["index_name"]
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def process_records(records, stream)
|
|
74
|
+
log_message_array = []
|
|
75
|
+
write_success = 0
|
|
76
|
+
write_failure = 0
|
|
77
|
+
properties = stream.json_schema[:properties]
|
|
78
|
+
|
|
79
|
+
records.each do |record_object|
|
|
80
|
+
record = extract_data(record_object, properties)
|
|
81
|
+
@namespace = stream.name
|
|
82
|
+
args = [@index_name, @namespace, record]
|
|
83
|
+
begin
|
|
84
|
+
pinecone_index = @pinecone.index(@index_name)
|
|
85
|
+
response = send_to_pinecone(pinecone_index, record)
|
|
86
|
+
if success?(response)
|
|
87
|
+
write_success += 1
|
|
88
|
+
else
|
|
89
|
+
write_failure += 1
|
|
90
|
+
end
|
|
91
|
+
log_message_array << log_request_response("info", args, response)
|
|
92
|
+
rescue StandardError => e
|
|
93
|
+
handle_exception(e, {
|
|
94
|
+
context: "PINECONE:WRITE:EXCEPTION",
|
|
95
|
+
type: "error",
|
|
96
|
+
sync_id: @sync_config.sync_id,
|
|
97
|
+
sync_run_id: @sync_config.sync_run_id
|
|
98
|
+
})
|
|
99
|
+
write_failure += 1
|
|
100
|
+
log_message_array << log_request_response("error", args, e.message)
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
tracking_message(write_success, write_failure, log_message_array)
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def parse_meta_data(vector_meta_data)
|
|
107
|
+
return {} if vector_meta_data.nil?
|
|
108
|
+
|
|
109
|
+
metadata = vector_meta_data.to_s
|
|
110
|
+
metadata = metadata.gsub(/([{,]\s*)([A-Za-z_]\w*)(\s*:)/, '\1"\2"\3')
|
|
111
|
+
metadata = metadata.gsub(/:\s*([A-Za-z_]\w*)/, ': "\1"')
|
|
112
|
+
|
|
113
|
+
JSON.parse(metadata)
|
|
114
|
+
rescue JSON::ParserError
|
|
115
|
+
{}
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def send_to_pinecone(pinecone_index, record)
|
|
119
|
+
meta_data = parse_meta_data(record[:meta_data])
|
|
120
|
+
pinecone_index.upsert(
|
|
121
|
+
namespace: @namespace,
|
|
122
|
+
vectors: [
|
|
123
|
+
{
|
|
124
|
+
id: record[:id].to_s,
|
|
125
|
+
values: record[:value],
|
|
126
|
+
metadata: meta_data
|
|
127
|
+
}
|
|
128
|
+
]
|
|
129
|
+
)
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def create_streams(records)
|
|
133
|
+
group_by_table(records).map do |r|
|
|
134
|
+
Multiwoven::Integrations::Protocol::Stream.new(name: r[:tablename], action: StreamAction["create"], json_schema: convert_to_json_schema(r[:columns]))
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
def group_by_table(records)
|
|
139
|
+
records.map do |table_name|
|
|
140
|
+
{
|
|
141
|
+
tablename: table_name,
|
|
142
|
+
columns: PINECONE_OBJECTS.map do |column|
|
|
143
|
+
{
|
|
144
|
+
column_name: column[:column_name],
|
|
145
|
+
type: column[:data_type],
|
|
146
|
+
optional: column[:is_nullable]
|
|
147
|
+
}
|
|
148
|
+
end
|
|
149
|
+
}
|
|
150
|
+
end
|
|
151
|
+
end
|
|
152
|
+
end
|
|
153
|
+
end
|
|
154
|
+
end
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
{
|
|
2
|
+
"data": {
|
|
3
|
+
"name": "PineconeDB",
|
|
4
|
+
"title": "Pinecone DB",
|
|
5
|
+
"connector_type": "destination",
|
|
6
|
+
"category": "Database",
|
|
7
|
+
"documentation_url": "https://docs.squared.ai/guides/data-integration/destination/pinecone_db",
|
|
8
|
+
"github_issue_label": "destination-pinecone-db",
|
|
9
|
+
"icon": "icon.svg",
|
|
10
|
+
"license": "MIT",
|
|
11
|
+
"release_stage": "alpha",
|
|
12
|
+
"support_level": "community",
|
|
13
|
+
"tags": ["language:ruby", "multiwoven"]
|
|
14
|
+
}
|
|
15
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
{
|
|
2
|
+
"documentation_url": "https://docs.multiwoven.com/integrations/destination/pinecone_db",
|
|
3
|
+
"stream_type": "dynamic",
|
|
4
|
+
"connector_query_type": "raw_sql",
|
|
5
|
+
"connection_specification": {
|
|
6
|
+
"$schema": "http://json-schema.org/draft-07/schema#",
|
|
7
|
+
"title": "Pinecone DB",
|
|
8
|
+
"type": "object",
|
|
9
|
+
"required": ["api_key", "region", "index_name", "model", "namespace"],
|
|
10
|
+
"properties": {
|
|
11
|
+
"api_key": {
|
|
12
|
+
"type": "string",
|
|
13
|
+
"multiwoven_secret": true,
|
|
14
|
+
"title": "API Key",
|
|
15
|
+
"description": "Your secret Pinecone API key used to authenticate requests.",
|
|
16
|
+
"order": 0
|
|
17
|
+
},
|
|
18
|
+
"region": {
|
|
19
|
+
"type": "string",
|
|
20
|
+
"title": "Region",
|
|
21
|
+
"description": "The Pinecone region where your index is hosted (e.g., 'us-east-1').",
|
|
22
|
+
"order": 1
|
|
23
|
+
},
|
|
24
|
+
"index_name": {
|
|
25
|
+
"type": "string",
|
|
26
|
+
"title": "Index Name",
|
|
27
|
+
"description": "The name of the Pinecone index where vectors will be written.",
|
|
28
|
+
"order": 2
|
|
29
|
+
},
|
|
30
|
+
"model": {
|
|
31
|
+
"type": "string",
|
|
32
|
+
"title": "Model",
|
|
33
|
+
"enum": ["multilingual-e5-large", "llama-text-embed-v2", "pinecone-sparse-english-v0"],
|
|
34
|
+
"description": "The embedding model used to convert text into vector representations.",
|
|
35
|
+
"order": 3
|
|
36
|
+
},
|
|
37
|
+
"namespace": {
|
|
38
|
+
"type": "string",
|
|
39
|
+
"title": "Namespace",
|
|
40
|
+
"description": "The namespace within the index for logical separation of data.",
|
|
41
|
+
"order": 4
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
<svg width="256" height="287.319" viewBox="0 0 256 287.319" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" preserveAspectRatio="xMidYMid"><title>Pinecone</title><g><path d="M108.633615,254.43629 C117.713862,254.43629 125.074857,261.797284 125.074857,270.877532 C125.074857,279.957779 117.713862,287.318774 108.633615,287.318774 C99.5533677,287.318774 92.1923728,279.957779 92.1923728,270.877532 C92.1923728,261.797284 99.5533677,254.43629 108.633615,254.43629 Z M199.849665,224.438339 L216.09705,229.252379 L203.199913,272.780219 C202.072982,276.58361 198.458049,279.095992 194.500389,278.826397 L190.516677,278.552973 L190.419263,278.633409 L149.02918,275.728903 L150.180842,258.822508 L177.989056,260.709686 L159.783784,234.447622 L173.709616,224.792379 L191.938895,251.08702 L199.849665,224.438339 Z M23.0126771,194.347476 L39.9158866,195.544979 L37.935897,223.348728 L64.1501315,205.120082 L73.8271476,219.030793 L47.578736,237.278394 L74.3707554,245.173037 L69.5818063,261.427835 L25.8485266,248.543243 C22.0304448,247.418369 19.5101155,243.787479 19.7913963,239.817092 L23.0126771,194.347476 Z M132.151306,170.671396 L162.658679,207.503468 L148.909247,218.891886 L130.753266,196.972134 L124.866941,230.673893 L107.280249,227.599613 L113.172232,193.845272 L88.7296311,208.256891 L79.6674587,192.874434 L120.745504,168.674377 C124.522104,166.449492 129.355297,167.295726 132.151306,170.671396 Z M217.504528,145.960198 L232.744017,137.668804 L254.94482,178.473633 C256.889641,182.048192 256.088221,186.494171 253.017682,189.164674 L249.876622,191.878375 L217.826246,219.77131 L206.441034,206.680621 L227.988588,187.934494 L195.893546,182.152609 L198.972402,165.078949 L231.044844,170.857793 L217.504528,145.960198 Z M37.7821805,103.299272 L49.2622123,116.306888 L28.0106317,135.050179 L60.1668233,140.664193 L57.1863573,157.755303 L24.9947229,152.136967 L38.822104,177.134576 L23.6411026,185.532577 L1.08439616,144.756992 C-0.885025494,141.196884 -0.115545265,136.746375 2.93488097,134.054184 L37.7821805,103.299272 Z M146.476311,89.8796828 L176.88045,126.612847 L163.1271,137.996532 L144.975445,116.067101 L139.08912,149.778947 L121.502428,146.704666 L127.374238,113.081452 L103.025237,127.354817 L93.9976317,111.952048 L131.398812,90.0233663 L131.435631,89.880899 L131.600545,89.9023265 L135.085833,87.870141 C138.861877,85.6569913 143.68556,86.5079996 146.476311,89.8796828 Z M185.655786,71.8143168 L192.305535,55.7902703 L235.318239,73.6399229 C239.072486,75.1978811 241.2415,79.1537636 240.536356,83.1568091 L239.820231,87.1385839 L232.47517,128.919545 L215.389188,125.909819 L220.312646,97.9413879 L191.776157,113.7129 L183.390302,98.5251862 L211.981072,82.7408038 L185.655786,71.8143168 Z M103.71696,40.2373824 L104.456513,57.5706533 L76.0432671,58.785006 L97.4730368,83.2749086 L84.4165529,94.6993319 L62.9507932,70.1728358 L57.949673,98.1737132 L40.8716575,95.1191088 L49.0561498,49.3603563 C49.771444,45.3612115 53.1664633,42.3942036 57.2253811,42.2210231 L61.246149,42.0411642 L61.3363168,41.9758 L103.71696,40.2373824 Z M161.838155,3.27194826 L192.104824,40.2369789 L178.291207,51.5474574 L160.327329,29.6043227 L154.268381,63.2715157 L136.697231,60.1096121 L142.766468,26.3665075 L118.24002,40.7062765 L109.232678,25.2916494 L150.427675,1.21987397 C154.218286,-0.995121237 159.056796,-0.124957814 161.838155,3.27194826 Z" fill="#201D1E"/></g></svg>
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
module Multiwoven
|
|
4
4
|
module Integrations
|
|
5
|
-
VERSION = "0.
|
|
5
|
+
VERSION = "0.24.1"
|
|
6
6
|
|
|
7
7
|
ENABLED_SOURCES = %w[
|
|
8
8
|
Snowflake
|
|
@@ -52,6 +52,7 @@ module Multiwoven
|
|
|
52
52
|
AISDataStore
|
|
53
53
|
AmazonS3
|
|
54
54
|
MicrosoftDynamics
|
|
55
|
+
PineconeDB
|
|
55
56
|
].freeze
|
|
56
57
|
end
|
|
57
58
|
end
|
|
@@ -36,9 +36,9 @@
|
|
|
36
36
|
"description": "Type of data in the files",
|
|
37
37
|
"type": "string",
|
|
38
38
|
"title": "Data Format Type",
|
|
39
|
-
"
|
|
40
|
-
"structured",
|
|
41
|
-
"unstructured"
|
|
39
|
+
"oneOf": [
|
|
40
|
+
{ "const": "structured", "title": "Tables & Records (Structured)" },
|
|
41
|
+
{ "const": "unstructured", "title": "Documents & Files (Unstructured)" }
|
|
42
42
|
],
|
|
43
43
|
"default": "structured",
|
|
44
44
|
"order": 9
|
|
@@ -12,9 +12,15 @@
|
|
|
12
12
|
"description": "Type of data in the database",
|
|
13
13
|
"type": "string",
|
|
14
14
|
"title": "Data Format Type",
|
|
15
|
-
"
|
|
16
|
-
|
|
17
|
-
|
|
15
|
+
"oneOf": [
|
|
16
|
+
{
|
|
17
|
+
"const": "structured",
|
|
18
|
+
"title": "Tables & Records (Structured)"
|
|
19
|
+
},
|
|
20
|
+
{
|
|
21
|
+
"const": "vector",
|
|
22
|
+
"title": "Tables & Records containing Vector fields"
|
|
23
|
+
}
|
|
18
24
|
],
|
|
19
25
|
"default": "structured",
|
|
20
26
|
"order": 0
|
|
@@ -38,6 +38,7 @@ require "google/cloud/ai_platform/v1"
|
|
|
38
38
|
require "grpc"
|
|
39
39
|
require "MailchimpMarketing"
|
|
40
40
|
require "aws-sdk-bedrockruntime"
|
|
41
|
+
require "pinecone"
|
|
41
42
|
|
|
42
43
|
# Service
|
|
43
44
|
require_relative "integrations/config"
|
|
@@ -106,6 +107,7 @@ require_relative "integrations/destination/mailchimp/client"
|
|
|
106
107
|
require_relative "integrations/destination/ais_data_store/client"
|
|
107
108
|
require_relative "integrations/destination/amazon_s3/client"
|
|
108
109
|
require_relative "integrations/destination/microsoft_dynamics/client"
|
|
110
|
+
require_relative "integrations/destination/pinecone_db/client"
|
|
109
111
|
|
|
110
112
|
module Multiwoven
|
|
111
113
|
module Integrations
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: multiwoven-integrations
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.24.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Subin T P
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2025-
|
|
11
|
+
date: 2025-05-12 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activesupport
|
|
@@ -663,6 +663,10 @@ files:
|
|
|
663
663
|
- lib/multiwoven/integrations/destination/oracle_db/config/meta.json
|
|
664
664
|
- lib/multiwoven/integrations/destination/oracle_db/config/spec.json
|
|
665
665
|
- lib/multiwoven/integrations/destination/oracle_db/icon.svg
|
|
666
|
+
- lib/multiwoven/integrations/destination/pinecone_db/client.rb
|
|
667
|
+
- lib/multiwoven/integrations/destination/pinecone_db/config/meta.json
|
|
668
|
+
- lib/multiwoven/integrations/destination/pinecone_db/config/spec.json
|
|
669
|
+
- lib/multiwoven/integrations/destination/pinecone_db/icon.svg
|
|
666
670
|
- lib/multiwoven/integrations/destination/postgresql/client.rb
|
|
667
671
|
- lib/multiwoven/integrations/destination/postgresql/config/meta.json
|
|
668
672
|
- lib/multiwoven/integrations/destination/postgresql/config/spec.json
|