vector_amp 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +34 -2
- data/lib/vector_amp/dataset.rb +10 -1
- data/lib/vector_amp/datasets.rb +36 -2
- data/lib/vector_amp/ingestion.rb +98 -0
- data/lib/vector_amp/metadata_schema.rb +20 -0
- data/lib/vector_amp/source.rb +150 -1
- data/lib/vector_amp/version.rb +1 -1
- data/lib/vector_amp.rb +1 -0
- metadata +3 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 8275404bc4db2f1270091d20315634d273fa140417503bdfea605c9d3baf0c5b
|
|
4
|
+
data.tar.gz: 426ae6dd92df94fd947d92ace32cbae23af4b0b0b69cf6a7eb2e9daf5e380257
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 585008c3ef2c83a95d8e55956ac4e11a6b006f05e3aa8d3af3d5932967edf53ce3f47af88612dd9e56e443eea327b4de30cdd6b82e5732d0d93e15a67edfc7ea
|
|
7
|
+
data.tar.gz: ab6d8685a761322203f41278049106953c0c3a12a3e3b12cd844cc2c7c3d89745027d3df9c1745ac41b656708a84b742eb199156dcf6e8e038c795ba7a5c5a5f
|
data/README.md
CHANGED
|
@@ -153,6 +153,20 @@ client.org_secrets.put_openai_api_key(
|
|
|
153
153
|
)
|
|
154
154
|
```
|
|
155
155
|
|
|
156
|
+
Declare typed metadata at creation, then merge fields or replace the complete schema:
|
|
157
|
+
|
|
158
|
+
```ruby
|
|
159
|
+
schema = [
|
|
160
|
+
VectorAmp::MetadataSchema.field(:category, VectorAmp::MetadataSchema::STRING),
|
|
161
|
+
VectorAmp::MetadataSchema.field(:price, VectorAmp::MetadataSchema::F32)
|
|
162
|
+
]
|
|
163
|
+
dataset = client.datasets.create(name: "products", metadata_schema: schema)
|
|
164
|
+
dataset.patch_metadata_schema([VectorAmp::MetadataSchema.field(:inventory, VectorAmp::MetadataSchema::U32)])
|
|
165
|
+
dataset.replace_metadata_schema(schema) # [] removes all declared fields
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
Canonical types are `STRING`, `U32`, `I32`, `I64`, `F32`, and `F64`.
|
|
169
|
+
|
|
156
170
|
### Source documents
|
|
157
171
|
|
|
158
172
|
Dataset document listing is cursor-based: pass `next_cursor` into the next request and do not assume offsets or totals. Downloads return retained original bytes and follow API/storage redirects.
|
|
@@ -183,7 +197,8 @@ client.ingestion.list_sources(limit: 50, offset: 0)
|
|
|
183
197
|
client.ingestion.get_source("source-uuid")
|
|
184
198
|
|
|
185
199
|
# Typed builders cover the supported source_type values:
|
|
186
|
-
# "web", "s3", "gcs", "gdrive", "file_upload", "jira",
|
|
200
|
+
# "web", "s3", "gcs", "gdrive", "file_upload", "jira", "confluence",
|
|
201
|
+
# "github", and "gitlab".
|
|
187
202
|
source = client.sources.create_web(
|
|
188
203
|
start_urls: ["https://docs.example.com"],
|
|
189
204
|
max_depth: 1,
|
|
@@ -215,6 +230,20 @@ gdrive_source = client.sources.create_google_drive(
|
|
|
215
230
|
folder_ids: ["google-drive-folder-id"]
|
|
216
231
|
)
|
|
217
232
|
|
|
233
|
+
# GitHub reads through the VectorAmp GitHub App, so the SDK never takes a token.
|
|
234
|
+
# Install the app from the Sources page and pass the installation id it reports.
|
|
235
|
+
github_source = client.sources.create_github(
|
|
236
|
+
installation_id: 42,
|
|
237
|
+
repositories: ["acme/api", "acme/web"]
|
|
238
|
+
)
|
|
239
|
+
|
|
240
|
+
gitlab_source = client.sources.create_gitlab(
|
|
241
|
+
projects: ["mygroup/myproject"], # and/or groups: ["mygroup"]
|
|
242
|
+
auth_mode: "token", # defaults to "oauth"
|
|
243
|
+
access_token: ENV["GITLAB_TOKEN"],
|
|
244
|
+
gitlab_url: "https://gitlab.com" # override for self-managed
|
|
245
|
+
)
|
|
246
|
+
|
|
218
247
|
file_source = client.sources.create_file_upload
|
|
219
248
|
|
|
220
249
|
# You can still pass name: when you want control. If omitted, the SDK chooses
|
|
@@ -360,6 +389,8 @@ equivalent `datasetObj.<m>(...)` on a returned `VectorAmp::Dataset`.
|
|
|
360
389
|
| Method | Required | Optional (defaults) |
|
|
361
390
|
|---|---|---|
|
|
362
391
|
| `create(name:)` | `name` | `dim` (inferred), `embedding` (vectoramp/VectorAmp-Embedding-4B), `metric` ("cosine"), `hybrid`, `filters`, `metadata_schema`, `tuning`, `metadata` |
|
|
392
|
+
| `patch_metadata_schema(id, schema)` | `id`, `schema` | — |
|
|
393
|
+
| `replace_metadata_schema(id, schema)` | `id`, `schema` | — |
|
|
363
394
|
| `list` | — | `limit` (50), `offset` (0) |
|
|
364
395
|
| `get(id)` | `id` | — |
|
|
365
396
|
| `delete(id)` | `id` | — |
|
|
@@ -390,7 +421,8 @@ Dataset-object helpers: `search`, `insert`, `add_texts`, `embed`, `delete`, `sta
|
|
|
390
421
|
|
|
391
422
|
`create(source)`, `create_web(start_urls:)`, `create_s3(bucket:)`, `create_gcs(bucket:)`,
|
|
392
423
|
`create_google_drive(folder_ids:|file_ids:)`, `create_jira(cloud_id:)`,
|
|
393
|
-
`create_confluence(cloud_id:|base_url:)`, `
|
|
424
|
+
`create_confluence(cloud_id:|base_url:)`, `create_github(installation_id:, repositories:)`,
|
|
425
|
+
`create_gitlab(groups:|projects:)`, `create_file_upload`, `list_sources`, `get_source(id)`.
|
|
394
426
|
Jobs: `start_job(source_id:, dataset_id:)`, `list_jobs`, `get_job(id)`, `retry_job(id)`,
|
|
395
427
|
`ingest_files(dataset_id:, paths:)`.
|
|
396
428
|
|
data/lib/vector_amp/dataset.rb
CHANGED
|
@@ -95,13 +95,22 @@ module VectorAmp
|
|
|
95
95
|
service.delete(id)
|
|
96
96
|
end
|
|
97
97
|
|
|
98
|
+
# Add or update typed metadata fields, retaining omitted fields.
|
|
99
|
+
def patch_metadata_schema(schema)
|
|
100
|
+
service.patch_metadata_schema(id, schema)
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# Replace the complete typed metadata schema.
|
|
104
|
+
def replace_metadata_schema(schema)
|
|
105
|
+
service.replace_metadata_schema(id, schema)
|
|
106
|
+
end
|
|
107
|
+
|
|
98
108
|
# Fetch stats for this dataset.
|
|
99
109
|
# @return [Hash] dataset statistics.
|
|
100
110
|
def stats
|
|
101
111
|
service.stats(id)
|
|
102
112
|
end
|
|
103
113
|
|
|
104
|
-
|
|
105
114
|
# List retained source documents for this dataset using cursor pagination.
|
|
106
115
|
# @param limit [Integer, nil] maximum documents to return.
|
|
107
116
|
# @param cursor [String, nil] cursor from a previous response's `next_cursor`.
|
data/lib/vector_amp/datasets.rb
CHANGED
|
@@ -59,7 +59,7 @@ module VectorAmp
|
|
|
59
59
|
# @param metric [String] distance metric; defaults to `cosine`.
|
|
60
60
|
# @param hybrid [Boolean, nil] enable hybrid (dense + sparse) search; sends `hybrid: true`.
|
|
61
61
|
# @param filters [Hash, nil] optional filter schema/config.
|
|
62
|
-
# @param metadata_schema [Hash, nil] optional metadata schema.
|
|
62
|
+
# @param metadata_schema [Array<Hash>, Hash, nil] optional metadata schema.
|
|
63
63
|
# @param tuning [Hash, nil] optional SABLE tuning parameters.
|
|
64
64
|
# @param metadata [Hash, nil] optional dataset metadata.
|
|
65
65
|
# @return [Dataset] created dataset.
|
|
@@ -85,7 +85,7 @@ module VectorAmp
|
|
|
85
85
|
index_type: "sable",
|
|
86
86
|
hybrid: hybrid,
|
|
87
87
|
filters: filters,
|
|
88
|
-
|
|
88
|
+
schema: normalize_metadata_schema(metadata_schema),
|
|
89
89
|
tuning: tuning,
|
|
90
90
|
metadata: metadata
|
|
91
91
|
)
|
|
@@ -115,6 +115,21 @@ module VectorAmp
|
|
|
115
115
|
create(name: name, embedding: Embedding.openai(size, secret_ref: secret_ref), **options)
|
|
116
116
|
end
|
|
117
117
|
|
|
118
|
+
# Add or update typed metadata fields, retaining omitted fields.
|
|
119
|
+
# @param dataset_id [String] dataset id.
|
|
120
|
+
# @param schema [Array<Hash>] fields with `name` and canonical `type`.
|
|
121
|
+
# @return [Dataset] updated dataset.
|
|
122
|
+
def patch_metadata_schema(dataset_id, schema)
|
|
123
|
+
update_metadata_schema(dataset_id, schema, "merge")
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
# Replace the complete typed metadata schema.
|
|
127
|
+
# @param dataset_id [String] dataset id.
|
|
128
|
+
# @param schema [Array<Hash>] complete field list; may be empty.
|
|
129
|
+
# @return [Dataset] updated dataset.
|
|
130
|
+
def replace_metadata_schema(dataset_id, schema)
|
|
131
|
+
update_metadata_schema(dataset_id, schema, "replace")
|
|
132
|
+
end
|
|
118
133
|
|
|
119
134
|
# List retained source documents for a dataset using cursor pagination.
|
|
120
135
|
# @param dataset_id [String] dataset id.
|
|
@@ -268,6 +283,25 @@ module VectorAmp
|
|
|
268
283
|
end
|
|
269
284
|
end
|
|
270
285
|
|
|
286
|
+
def normalize_metadata_schema(schema)
|
|
287
|
+
return schema unless schema.is_a?(Hash)
|
|
288
|
+
|
|
289
|
+
schema.map do |name, config|
|
|
290
|
+
raise ArgumentError, "metadata schema field #{name.inspect} must be a Hash" unless config.is_a?(Hash)
|
|
291
|
+
|
|
292
|
+
{ name: name.to_s }.merge(config)
|
|
293
|
+
end
|
|
294
|
+
end
|
|
295
|
+
|
|
296
|
+
def update_metadata_schema(dataset_id, schema, mode)
|
|
297
|
+
response = @transport.request(
|
|
298
|
+
:patch,
|
|
299
|
+
"/datasets/#{dataset_id}/schema",
|
|
300
|
+
body: { schema: schema, mode: mode }
|
|
301
|
+
)
|
|
302
|
+
wrap_dataset(response)
|
|
303
|
+
end
|
|
304
|
+
|
|
271
305
|
def wrap_dataset(data)
|
|
272
306
|
Dataset.new(data, service: self, client: @client)
|
|
273
307
|
end
|
data/lib/vector_amp/ingestion.rb
CHANGED
|
@@ -224,6 +224,102 @@ module VectorAmp
|
|
|
224
224
|
))
|
|
225
225
|
end
|
|
226
226
|
|
|
227
|
+
# Create a GitHub source backed by the VectorAmp GitHub App.
|
|
228
|
+
#
|
|
229
|
+
# Install the VectorAmp GitHub App from the Sources page in the app first and
|
|
230
|
+
# pass the resulting installation id here; the SDK never handles a GitHub token.
|
|
231
|
+
# @param installation_id [Integer] required GitHub App installation id.
|
|
232
|
+
# @param repositories [String, Array<String>] required `owner/repo` full names.
|
|
233
|
+
# @param name [String, nil] defaults to `github-<owner>-<repo>` from the first repository.
|
|
234
|
+
# @param ref_mode [String, nil] `active` (server default), `default`, or `explicit`.
|
|
235
|
+
# @param refs [String, Array<String>, nil] explicit branch names for `ref_mode: "explicit"`.
|
|
236
|
+
# @param excluded_refs [String, Array<String>, nil] branch names to skip.
|
|
237
|
+
# @param active_branch_days [Integer, nil] activity window in days (1-90); server default 7.
|
|
238
|
+
# @param include_pull_requests [Boolean, nil] ingest pull requests; server default true.
|
|
239
|
+
# @param include_review_threads [Boolean, nil] ingest review discussions; server default true.
|
|
240
|
+
# @param include_direct_commits [Boolean, nil] ingest commits outside a pull request; server default true.
|
|
241
|
+
# @param include_globs [String, Array<String>, nil] path globs to include; server default `**/*`.
|
|
242
|
+
# @param exclude_globs [String, Array<String>, nil] path globs to skip.
|
|
243
|
+
# @param max_file_size_bytes [Integer, nil] per-file ceiling; server default 1_000_000.
|
|
244
|
+
# @param description [String, nil] optional description.
|
|
245
|
+
# @param metadata [Hash, nil] optional metadata.
|
|
246
|
+
# @param config [Hash] additional GitHub-source config forwarded to the API.
|
|
247
|
+
# @return [Hash] created source response.
|
|
248
|
+
def create_github(installation_id:, repositories:, name: nil, ref_mode: nil, refs: nil, excluded_refs: nil,
|
|
249
|
+
active_branch_days: nil, include_pull_requests: nil, include_review_threads: nil,
|
|
250
|
+
include_direct_commits: nil, include_globs: nil, exclude_globs: nil,
|
|
251
|
+
max_file_size_bytes: nil, description: nil, metadata: nil, **config)
|
|
252
|
+
create_source(GitHubSource.new(
|
|
253
|
+
installation_id: installation_id,
|
|
254
|
+
repositories: repositories,
|
|
255
|
+
name: name,
|
|
256
|
+
ref_mode: ref_mode,
|
|
257
|
+
refs: refs,
|
|
258
|
+
excluded_refs: excluded_refs,
|
|
259
|
+
active_branch_days: active_branch_days,
|
|
260
|
+
include_pull_requests: include_pull_requests,
|
|
261
|
+
include_review_threads: include_review_threads,
|
|
262
|
+
include_direct_commits: include_direct_commits,
|
|
263
|
+
include_globs: include_globs,
|
|
264
|
+
exclude_globs: exclude_globs,
|
|
265
|
+
max_file_size_bytes: max_file_size_bytes,
|
|
266
|
+
description: description,
|
|
267
|
+
metadata: metadata,
|
|
268
|
+
**config
|
|
269
|
+
))
|
|
270
|
+
end
|
|
271
|
+
|
|
272
|
+
# Create a GitLab source for gitlab.com or a self-managed instance.
|
|
273
|
+
# @param groups [String, Array<String>, nil] group paths; required unless projects is given.
|
|
274
|
+
# @param projects [String, Array<String>, nil] project paths with namespace; required unless groups is given.
|
|
275
|
+
# @param name [String, nil] defaults to `gitlab-<path>` from the first project or group.
|
|
276
|
+
# @param auth_mode [String] `oauth` (default) or `token`.
|
|
277
|
+
# @param gitlab_url [String] instance base URL; defaults to `https://gitlab.com`.
|
|
278
|
+
# @param access_token [String, nil] personal or group access token for `auth_mode: "token"`.
|
|
279
|
+
# @param connection_id [String, nil] optional stored OAuth connection id used instead of a token.
|
|
280
|
+
# @param ref_mode [String, nil] `active` (server default), `default`, or `explicit`.
|
|
281
|
+
# @param refs [String, Array<String>, nil] explicit branch names for `ref_mode: "explicit"`.
|
|
282
|
+
# @param excluded_refs [String, Array<String>, nil] branch names to skip.
|
|
283
|
+
# @param active_branch_days [Integer, nil] activity window in days (1-90); server default 7.
|
|
284
|
+
# @param include_merge_requests [Boolean, nil] ingest merge requests; server default true.
|
|
285
|
+
# @param include_review_threads [Boolean, nil] ingest review discussions; server default true.
|
|
286
|
+
# @param include_direct_commits [Boolean, nil] ingest commits outside a merge request; server default true.
|
|
287
|
+
# @param include_globs [String, Array<String>, nil] path globs to include; server default `**/*`.
|
|
288
|
+
# @param exclude_globs [String, Array<String>, nil] path globs to skip.
|
|
289
|
+
# @param max_file_size_bytes [Integer, nil] per-file ceiling; server default 1_000_000.
|
|
290
|
+
# @param description [String, nil] optional description.
|
|
291
|
+
# @param metadata [Hash, nil] optional metadata.
|
|
292
|
+
# @param config [Hash] additional GitLab-source config forwarded to the API.
|
|
293
|
+
# @return [Hash] created source response.
|
|
294
|
+
def create_gitlab(groups: nil, projects: nil, name: nil, auth_mode: "oauth", gitlab_url: "https://gitlab.com",
|
|
295
|
+
access_token: nil, connection_id: nil, ref_mode: nil, refs: nil, excluded_refs: nil,
|
|
296
|
+
active_branch_days: nil, include_merge_requests: nil, include_review_threads: nil,
|
|
297
|
+
include_direct_commits: nil, include_globs: nil, exclude_globs: nil,
|
|
298
|
+
max_file_size_bytes: nil, description: nil, metadata: nil, **config)
|
|
299
|
+
create_source(GitLabSource.new(
|
|
300
|
+
groups: groups,
|
|
301
|
+
projects: projects,
|
|
302
|
+
name: name,
|
|
303
|
+
auth_mode: auth_mode,
|
|
304
|
+
gitlab_url: gitlab_url,
|
|
305
|
+
access_token: access_token,
|
|
306
|
+
connection_id: connection_id,
|
|
307
|
+
ref_mode: ref_mode,
|
|
308
|
+
refs: refs,
|
|
309
|
+
excluded_refs: excluded_refs,
|
|
310
|
+
active_branch_days: active_branch_days,
|
|
311
|
+
include_merge_requests: include_merge_requests,
|
|
312
|
+
include_review_threads: include_review_threads,
|
|
313
|
+
include_direct_commits: include_direct_commits,
|
|
314
|
+
include_globs: include_globs,
|
|
315
|
+
exclude_globs: exclude_globs,
|
|
316
|
+
max_file_size_bytes: max_file_size_bytes,
|
|
317
|
+
description: description,
|
|
318
|
+
metadata: metadata,
|
|
319
|
+
**config
|
|
320
|
+
))
|
|
321
|
+
end
|
|
322
|
+
|
|
227
323
|
# Create a file-upload source.
|
|
228
324
|
# @param name [String, nil] defaults to timestamped `ruby-sdk-file-upload-YYYYmmddHHMMSS`.
|
|
229
325
|
# @param description [String, nil] optional description.
|
|
@@ -386,6 +482,8 @@ module VectorAmp
|
|
|
386
482
|
when "jira" then SourceNames.jira(project_keys: config[:project_keys] || config["project_keys"], cloud_id: config[:cloud_id] || config["cloud_id"])
|
|
387
483
|
when "confluence" then SourceNames.confluence(spaces: config[:spaces] || config["spaces"], cloud_id: config[:cloud_id] || config["cloud_id"], base_url: config[:base_url] || config["base_url"])
|
|
388
484
|
when "gdrive" then SourceNames.google_drive(folder_ids: config[:folder_ids] || config["folder_ids"], file_ids: config[:file_ids] || config["file_ids"])
|
|
485
|
+
when "github" then SourceNames.github(repositories: config[:repositories] || config["repositories"])
|
|
486
|
+
when "gitlab" then SourceNames.gitlab(projects: config[:projects] || config["projects"], groups: config[:groups] || config["groups"])
|
|
389
487
|
end
|
|
390
488
|
end
|
|
391
489
|
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module VectorAmp
|
|
4
|
+
# Helpers and canonical scalar types for dataset metadata schemas.
|
|
5
|
+
module MetadataSchema
|
|
6
|
+
STRING = "string"
|
|
7
|
+
U32 = "u32"
|
|
8
|
+
I32 = "i32"
|
|
9
|
+
I64 = "i64"
|
|
10
|
+
F32 = "f32"
|
|
11
|
+
F64 = "f64"
|
|
12
|
+
TYPES = [STRING, U32, I32, I64, F32, F64].freeze
|
|
13
|
+
|
|
14
|
+
def self.field(name, type)
|
|
15
|
+
raise ArgumentError, "unsupported metadata schema type: #{type.inspect}" unless TYPES.include?(type)
|
|
16
|
+
|
|
17
|
+
{ name: name.to_s, type: type }
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
end
|
data/lib/vector_amp/source.rb
CHANGED
|
@@ -10,7 +10,7 @@ module VectorAmp
|
|
|
10
10
|
# create a source or to `dataset.ingest_source(source)` once they include an id
|
|
11
11
|
# returned by the API.
|
|
12
12
|
class Source
|
|
13
|
-
SUPPORTED_SOURCE_TYPES = %w[s3 web gcs gdrive file_upload jira confluence].freeze
|
|
13
|
+
SUPPORTED_SOURCE_TYPES = %w[s3 web gcs gdrive file_upload jira confluence github gitlab].freeze
|
|
14
14
|
|
|
15
15
|
# @return [String, nil] API source id when returned by the API.
|
|
16
16
|
# @return [String] source type (`s3`, `web`, `gdrive`, or `file_upload`).
|
|
@@ -164,6 +164,28 @@ module VectorAmp
|
|
|
164
164
|
nil
|
|
165
165
|
end
|
|
166
166
|
|
|
167
|
+
# @param repositories [String, Array<String>, nil] `owner/repo` full names.
|
|
168
|
+
# @return [String] `github-<owner>-<repo>` from the first repository, or `github-source`.
|
|
169
|
+
def github(repositories: nil)
|
|
170
|
+
repository = Array(repositories).first
|
|
171
|
+
repository ? "github-#{slugify(repository)}" : "github-source"
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
# @param projects [String, Array<String>, nil] project paths with namespace.
|
|
175
|
+
# @param groups [String, Array<String>, nil] group paths.
|
|
176
|
+
# @return [String] `gitlab-<path>` from the first project or group, or `gitlab-source`.
|
|
177
|
+
def gitlab(projects: nil, groups: nil)
|
|
178
|
+
path = Array(projects).first || Array(groups).first
|
|
179
|
+
path ? "gitlab-#{slugify(path)}" : "gitlab-source"
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
# Lowercase a repository/group path into a dash-separated name fragment.
|
|
183
|
+
# @param value [String] path such as `owner/repo` or `group/subgroup/project`.
|
|
184
|
+
# @return [String]
|
|
185
|
+
def slugify(value)
|
|
186
|
+
value.to_s.downcase.gsub(/[^a-z0-9]+/, "-").gsub(/\A-+|-+\z/, "")
|
|
187
|
+
end
|
|
188
|
+
|
|
167
189
|
# @param folder_ids [String, Array<String>, nil] folder ids.
|
|
168
190
|
# @param file_ids [String, Array<String>, nil] file ids.
|
|
169
191
|
# @return [String] `google-drive-<first id>` or `google-drive-source`.
|
|
@@ -360,6 +382,133 @@ module VectorAmp
|
|
|
360
382
|
end
|
|
361
383
|
end
|
|
362
384
|
|
|
385
|
+
# GitHub ingestion source backed by the VectorAmp GitHub App.
|
|
386
|
+
#
|
|
387
|
+
# Repositories, active branches, pull requests, and review discussions are read
|
|
388
|
+
# through a GitHub App installation, so no token is passed to the SDK. Install
|
|
389
|
+
# the VectorAmp GitHub App from the Sources page in the app first; the
|
|
390
|
+
# installation id shown there is what this class needs.
|
|
391
|
+
class GitHubSource < Source
|
|
392
|
+
# @param installation_id [Integer] required GitHub App installation id; must be positive.
|
|
393
|
+
# @param repositories [String, Array<String>] required `owner/repo` full names.
|
|
394
|
+
# @param name [String, nil] defaults to `github-<owner>-<repo>` from the first repository.
|
|
395
|
+
# @param ref_mode [String, nil] `active` (server default), `default`, or `explicit`.
|
|
396
|
+
# @param refs [String, Array<String>, nil] explicit branch names for `ref_mode: "explicit"`.
|
|
397
|
+
# @param excluded_refs [String, Array<String>, nil] branch names to skip.
|
|
398
|
+
# @param active_branch_days [Integer, nil] activity window in days (1-90); server default 7.
|
|
399
|
+
# @param include_pull_requests [Boolean, nil] ingest pull requests; server default true.
|
|
400
|
+
# @param include_review_threads [Boolean, nil] ingest review discussions; server default true.
|
|
401
|
+
# @param include_direct_commits [Boolean, nil] ingest commits outside a pull request; server default true.
|
|
402
|
+
# @param include_globs [String, Array<String>, nil] path globs to include; server default `**/*`.
|
|
403
|
+
# @param exclude_globs [String, Array<String>, nil] path globs to skip.
|
|
404
|
+
# @param max_file_size_bytes [Integer, nil] per-file ceiling (1-25_000_000); server default 1_000_000.
|
|
405
|
+
# @param description [String, nil] optional description.
|
|
406
|
+
# @param metadata [Hash, nil] optional metadata.
|
|
407
|
+
# @param id [String, nil] optional API source id.
|
|
408
|
+
# @param config [Hash] additional GitHub-source config forwarded to the API.
|
|
409
|
+
# @return [GitHubSource]
|
|
410
|
+
def initialize(installation_id:, repositories:, name: nil, ref_mode: nil, refs: nil, excluded_refs: nil,
|
|
411
|
+
active_branch_days: nil, include_pull_requests: nil, include_review_threads: nil,
|
|
412
|
+
include_direct_commits: nil, include_globs: nil, exclude_globs: nil, max_file_size_bytes: nil,
|
|
413
|
+
description: nil, metadata: nil, id: nil, **config)
|
|
414
|
+
unless installation_id.is_a?(Integer) && installation_id.positive?
|
|
415
|
+
raise ArgumentError, "installation_id must be a positive Integer"
|
|
416
|
+
end
|
|
417
|
+
|
|
418
|
+
repository_list = Array(repositories)
|
|
419
|
+
raise ArgumentError, "repositories must not be empty" if repository_list.empty?
|
|
420
|
+
|
|
421
|
+
super(
|
|
422
|
+
id: id,
|
|
423
|
+
source_type: "github",
|
|
424
|
+
name: name || SourceNames.github(repositories: repository_list),
|
|
425
|
+
description: description,
|
|
426
|
+
metadata: metadata,
|
|
427
|
+
config: Utils.compact_hash(config.merge(
|
|
428
|
+
installation_id: installation_id,
|
|
429
|
+
repositories: repository_list,
|
|
430
|
+
ref_mode: ref_mode,
|
|
431
|
+
refs: refs.nil? ? nil : Array(refs),
|
|
432
|
+
excluded_refs: excluded_refs.nil? ? nil : Array(excluded_refs),
|
|
433
|
+
active_branch_days: active_branch_days,
|
|
434
|
+
include_pull_requests: include_pull_requests,
|
|
435
|
+
include_review_threads: include_review_threads,
|
|
436
|
+
include_direct_commits: include_direct_commits,
|
|
437
|
+
include_globs: include_globs.nil? ? nil : Array(include_globs),
|
|
438
|
+
exclude_globs: exclude_globs.nil? ? nil : Array(exclude_globs),
|
|
439
|
+
max_file_size_bytes: max_file_size_bytes
|
|
440
|
+
))
|
|
441
|
+
)
|
|
442
|
+
end
|
|
443
|
+
end
|
|
444
|
+
|
|
445
|
+
# GitLab ingestion source for gitlab.com or a self-managed instance.
|
|
446
|
+
#
|
|
447
|
+
# Authenticates with an access token (`auth_mode: "token"` plus `access_token`)
|
|
448
|
+
# or a stored OAuth connection (`connection_id`). At least one group or project
|
|
449
|
+
# is required.
|
|
450
|
+
class GitLabSource < Source
|
|
451
|
+
# @param groups [String, Array<String>, nil] group paths; required unless projects is given.
|
|
452
|
+
# @param projects [String, Array<String>, nil] project paths with namespace; required unless groups is given.
|
|
453
|
+
# @param name [String, nil] defaults to `gitlab-<path>` from the first project or group.
|
|
454
|
+
# @param auth_mode [String] `oauth` (default) or `token`.
|
|
455
|
+
# @param gitlab_url [String] instance base URL; defaults to `https://gitlab.com`.
|
|
456
|
+
# @param access_token [String, nil] personal or group access token for `auth_mode: "token"`.
|
|
457
|
+
# @param connection_id [String, nil] optional stored OAuth connection id used instead of a token.
|
|
458
|
+
# @param ref_mode [String, nil] `active` (server default), `default`, or `explicit`.
|
|
459
|
+
# @param refs [String, Array<String>, nil] explicit branch names for `ref_mode: "explicit"`.
|
|
460
|
+
# @param excluded_refs [String, Array<String>, nil] branch names to skip.
|
|
461
|
+
# @param active_branch_days [Integer, nil] activity window in days (1-90); server default 7.
|
|
462
|
+
# @param include_merge_requests [Boolean, nil] ingest merge requests; server default true.
|
|
463
|
+
# @param include_review_threads [Boolean, nil] ingest review discussions; server default true.
|
|
464
|
+
# @param include_direct_commits [Boolean, nil] ingest commits outside a merge request; server default true.
|
|
465
|
+
# @param include_globs [String, Array<String>, nil] path globs to include; server default `**/*`.
|
|
466
|
+
# @param exclude_globs [String, Array<String>, nil] path globs to skip.
|
|
467
|
+
# @param max_file_size_bytes [Integer, nil] per-file ceiling (1-25_000_000); server default 1_000_000.
|
|
468
|
+
# @param description [String, nil] optional description.
|
|
469
|
+
# @param metadata [Hash, nil] optional metadata.
|
|
470
|
+
# @param id [String, nil] optional API source id.
|
|
471
|
+
# @param config [Hash] additional GitLab-source config forwarded to the API.
|
|
472
|
+
# @return [GitLabSource]
|
|
473
|
+
def initialize(groups: nil, projects: nil, name: nil, auth_mode: "oauth", gitlab_url: "https://gitlab.com",
|
|
474
|
+
access_token: nil, connection_id: nil, ref_mode: nil, refs: nil, excluded_refs: nil,
|
|
475
|
+
active_branch_days: nil, include_merge_requests: nil, include_review_threads: nil,
|
|
476
|
+
include_direct_commits: nil, include_globs: nil, exclude_globs: nil, max_file_size_bytes: nil,
|
|
477
|
+
description: nil, metadata: nil, id: nil, **config)
|
|
478
|
+
group_list = Array(groups)
|
|
479
|
+
project_list = Array(projects)
|
|
480
|
+
if group_list.empty? && project_list.empty?
|
|
481
|
+
raise ArgumentError, "groups or projects is required"
|
|
482
|
+
end
|
|
483
|
+
|
|
484
|
+
super(
|
|
485
|
+
id: id,
|
|
486
|
+
source_type: "gitlab",
|
|
487
|
+
name: name || SourceNames.gitlab(projects: project_list, groups: group_list),
|
|
488
|
+
description: description,
|
|
489
|
+
metadata: metadata,
|
|
490
|
+
config: Utils.compact_hash(config.merge(
|
|
491
|
+
auth_mode: auth_mode,
|
|
492
|
+
gitlab_url: gitlab_url,
|
|
493
|
+
groups: group_list.empty? ? nil : group_list,
|
|
494
|
+
projects: project_list.empty? ? nil : project_list,
|
|
495
|
+
access_token: access_token,
|
|
496
|
+
connection_id: connection_id,
|
|
497
|
+
ref_mode: ref_mode,
|
|
498
|
+
refs: refs.nil? ? nil : Array(refs),
|
|
499
|
+
excluded_refs: excluded_refs.nil? ? nil : Array(excluded_refs),
|
|
500
|
+
active_branch_days: active_branch_days,
|
|
501
|
+
include_merge_requests: include_merge_requests,
|
|
502
|
+
include_review_threads: include_review_threads,
|
|
503
|
+
include_direct_commits: include_direct_commits,
|
|
504
|
+
include_globs: include_globs.nil? ? nil : Array(include_globs),
|
|
505
|
+
exclude_globs: exclude_globs.nil? ? nil : Array(exclude_globs),
|
|
506
|
+
max_file_size_bytes: max_file_size_bytes
|
|
507
|
+
))
|
|
508
|
+
)
|
|
509
|
+
end
|
|
510
|
+
end
|
|
511
|
+
|
|
363
512
|
# Escape hatch for API-compatible source types/configs not yet modeled by the SDK.
|
|
364
513
|
class GenericSource < Source
|
|
365
514
|
end
|
data/lib/vector_amp/version.rb
CHANGED
data/lib/vector_amp.rb
CHANGED
|
@@ -5,6 +5,7 @@ require_relative "vector_amp/client"
|
|
|
5
5
|
require_relative "vector_amp/embedding"
|
|
6
6
|
require_relative "vector_amp/error"
|
|
7
7
|
require_relative "vector_amp/org_secrets"
|
|
8
|
+
require_relative "vector_amp/metadata_schema"
|
|
8
9
|
require_relative "vector_amp/source"
|
|
9
10
|
|
|
10
11
|
module VectorAmp
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: vector_amp
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.4.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- VectorAmp
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-
|
|
11
|
+
date: 2026-08-20 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: minitest
|
|
@@ -114,6 +114,7 @@ files:
|
|
|
114
114
|
- lib/vector_amp/error.rb
|
|
115
115
|
- lib/vector_amp/ingestion.rb
|
|
116
116
|
- lib/vector_amp/intelligence.rb
|
|
117
|
+
- lib/vector_amp/metadata_schema.rb
|
|
117
118
|
- lib/vector_amp/org_secrets.rb
|
|
118
119
|
- lib/vector_amp/schedules.rb
|
|
119
120
|
- lib/vector_amp/source.rb
|