@agent-native/core 0.111.3 → 0.112.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/corpus/README.md +2 -2
- package/corpus/core/CHANGELOG.md +18 -0
- package/corpus/core/package.json +3 -1
- package/corpus/core/src/cli/code-agent-executor.ts +17 -5
- package/corpus/core/src/cli/code-agent-runs.ts +43 -2
- package/corpus/core/src/cli/migration-codemod.ts +59 -9
- package/corpus/core/src/cli/multi-frontier-runs.ts +841 -0
- package/corpus/core/src/coding-tools/index.ts +27 -8
- package/corpus/core/src/embeddings/index.ts +233 -0
- package/corpus/core/src/index.ts +2 -0
- package/corpus/core/src/search/index.ts +413 -0
- package/corpus/templates/brain/.agents/skills/brain/RUNBOOK.md +48 -5
- package/corpus/templates/brain/.agents/skills/brain/SKILL.md +46 -14
- package/corpus/templates/brain/README.md +34 -3
- package/corpus/templates/brain/actions/approve-proposal.ts +2 -0
- package/corpus/templates/brain/actions/claim-distillation.ts +24 -27
- package/corpus/templates/brain/actions/create-source.ts +1 -1
- package/corpus/templates/brain/actions/get-knowledge.ts +6 -2
- package/corpus/templates/brain/actions/get-pilot-report.ts +44 -8
- package/corpus/templates/brain/actions/import-capture.ts +21 -9
- package/corpus/templates/brain/actions/import-transcript.ts +42 -14
- package/corpus/templates/brain/actions/list-captures.ts +10 -1
- package/corpus/templates/brain/actions/list-distillation-queue.ts +39 -14
- package/corpus/templates/brain/actions/list-projects.ts +51 -0
- package/corpus/templates/brain/actions/list-proposals.ts +80 -4
- package/corpus/templates/brain/actions/list-sources.ts +17 -3
- package/corpus/templates/brain/actions/manage-project.ts +118 -0
- package/corpus/templates/brain/actions/mark-capture-distilled.ts +68 -20
- package/corpus/templates/brain/actions/navigate.ts +1 -0
- package/corpus/templates/brain/actions/reject-proposal.ts +6 -1
- package/corpus/templates/brain/actions/resanitize-captures.ts +159 -33
- package/corpus/templates/brain/actions/review-proposal.ts +2 -0
- package/corpus/templates/brain/actions/search-everything.ts +10 -1
- package/corpus/templates/brain/actions/search-knowledge.ts +1 -1
- package/corpus/templates/brain/actions/set-settings.ts +36 -1
- package/corpus/templates/brain/actions/update-proposal.ts +2 -0
- package/corpus/templates/brain/app/hooks/use-distillation-bridge.ts +18 -9
- package/corpus/templates/brain/app/hooks/use-navigation-state.ts +3 -0
- package/corpus/templates/brain/app/i18n-data.ts +731 -2
- package/corpus/templates/brain/app/lib/brain.ts +33 -1
- package/corpus/templates/brain/app/routes/ops.tsx +47 -0
- package/corpus/templates/brain/app/routes/review.tsx +41 -2
- package/corpus/templates/brain/app/routes/search.tsx +33 -2
- package/corpus/templates/brain/app/routes/settings.tsx +158 -0
- package/corpus/templates/brain/app/routes/sources.tsx +31 -0
- package/corpus/templates/brain/changelog/2026-07-19-added-private-source-safe-semantic-search.md +6 -0
- package/corpus/templates/brain/jobs/process-ingest-queue.ts +270 -36
- package/corpus/templates/brain/package.json +1 -0
- package/corpus/templates/brain/server/db/index.ts +18 -0
- package/corpus/templates/brain/server/db/schema.ts +197 -1
- package/corpus/templates/brain/server/jobs/distillation-queue.ts +12 -5
- package/corpus/templates/brain/server/lib/audiences.ts +708 -0
- package/corpus/templates/brain/server/lib/brain-health.ts +251 -38
- package/corpus/templates/brain/server/lib/brain.ts +877 -50
- package/corpus/templates/brain/server/lib/capture-sanitization.ts +217 -53
- package/corpus/templates/brain/server/lib/connectors.ts +556 -70
- package/corpus/templates/brain/server/lib/demo.ts +17 -4
- package/corpus/templates/brain/server/lib/hybrid-search.ts +363 -0
- package/corpus/templates/brain/server/lib/ingest-queue.ts +104 -0
- package/corpus/templates/brain/server/lib/privacy-readiness.ts +24 -0
- package/corpus/templates/brain/server/lib/search-index-contracts.ts +72 -0
- package/corpus/templates/brain/server/lib/search-index.ts +714 -0
- package/corpus/templates/brain/server/lib/search.ts +120 -4
- package/corpus/templates/brain/server/lib/sensitivity-policy.ts +174 -0
- package/corpus/templates/brain/server/lib/slack-events.ts +281 -0
- package/corpus/templates/brain/server/onboarding.ts +29 -0
- package/corpus/templates/brain/server/plugins/db.ts +204 -0
- package/corpus/templates/brain/server/register-secrets.ts +46 -0
- package/corpus/templates/brain/server/routes/api/_agent-native/brain/ingest.post.ts +36 -16
- package/corpus/templates/brain/server/routes/api/_agent-native/brain/slack-events.post.ts +28 -0
- package/corpus/templates/brain/shared/types.ts +10 -0
- package/dist/cli/code-agent-executor.d.ts +8 -9
- package/dist/cli/code-agent-executor.d.ts.map +1 -1
- package/dist/cli/code-agent-executor.js +9 -3
- package/dist/cli/code-agent-executor.js.map +1 -1
- package/dist/cli/code-agent-runs.d.ts +5 -0
- package/dist/cli/code-agent-runs.d.ts.map +1 -1
- package/dist/cli/code-agent-runs.js +33 -2
- package/dist/cli/code-agent-runs.js.map +1 -1
- package/dist/cli/migration-codemod.d.ts.map +1 -1
- package/dist/cli/migration-codemod.js +48 -10
- package/dist/cli/migration-codemod.js.map +1 -1
- package/dist/cli/multi-frontier-runs.d.ts +131 -0
- package/dist/cli/multi-frontier-runs.d.ts.map +1 -0
- package/dist/cli/multi-frontier-runs.js +529 -0
- package/dist/cli/multi-frontier-runs.js.map +1 -0
- package/dist/coding-tools/index.d.ts +1 -0
- package/dist/coding-tools/index.d.ts.map +1 -1
- package/dist/coding-tools/index.js +25 -7
- package/dist/coding-tools/index.js.map +1 -1
- package/dist/collab/awareness.d.ts +2 -2
- package/dist/collab/awareness.d.ts.map +1 -1
- package/dist/embeddings/index.d.ts +24 -0
- package/dist/embeddings/index.d.ts.map +1 -0
- package/dist/embeddings/index.js +172 -0
- package/dist/embeddings/index.js.map +1 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/notifications/routes.d.ts +3 -3
- package/dist/progress/routes.d.ts +1 -1
- package/dist/resources/handlers.d.ts +1 -1
- package/dist/search/index.d.ts +88 -0
- package/dist/search/index.d.ts.map +1 -0
- package/dist/search/index.js +267 -0
- package/dist/search/index.js.map +1 -0
- package/dist/server/agent-engine-api-key-route.d.ts +1 -1
- package/package.json +3 -1
|
@@ -451,7 +451,11 @@ export async function runCodingCommand(
|
|
|
451
451
|
command: string,
|
|
452
452
|
cwd: string,
|
|
453
453
|
timeoutMs: number,
|
|
454
|
-
options: {
|
|
454
|
+
options: {
|
|
455
|
+
stdin?: string;
|
|
456
|
+
onChunk?: BashOutputChunkCallback;
|
|
457
|
+
signal?: AbortSignal;
|
|
458
|
+
} = {},
|
|
455
459
|
): Promise<CodingCommandResult> {
|
|
456
460
|
const child = spawn(command, {
|
|
457
461
|
cwd,
|
|
@@ -463,9 +467,10 @@ export async function runCodingCommand(
|
|
|
463
467
|
let stderr = "";
|
|
464
468
|
let timedOut = false;
|
|
465
469
|
const startMs = Date.now();
|
|
470
|
+
const abort = () => child.kill("SIGTERM");
|
|
466
471
|
const timer = setTimeout(() => {
|
|
467
472
|
timedOut = true;
|
|
468
|
-
|
|
473
|
+
abort();
|
|
469
474
|
}, timeoutMs);
|
|
470
475
|
child.stdout?.on("data", (chunk: Buffer) => {
|
|
471
476
|
const text = chunk.toString();
|
|
@@ -479,12 +484,26 @@ export async function runCodingCommand(
|
|
|
479
484
|
});
|
|
480
485
|
if (options.stdin) child.stdin?.end(options.stdin);
|
|
481
486
|
else child.stdin?.end();
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
}
|
|
486
|
-
|
|
487
|
-
|
|
487
|
+
if (options.signal) {
|
|
488
|
+
if (options.signal.aborted) abort();
|
|
489
|
+
else options.signal.addEventListener("abort", abort, { once: true });
|
|
490
|
+
}
|
|
491
|
+
try {
|
|
492
|
+
const code = await new Promise<number | null>((resolve, reject) => {
|
|
493
|
+
child.once("error", reject);
|
|
494
|
+
child.once("close", resolve);
|
|
495
|
+
});
|
|
496
|
+
return {
|
|
497
|
+
code,
|
|
498
|
+
stdout,
|
|
499
|
+
stderr,
|
|
500
|
+
timedOut,
|
|
501
|
+
durationMs: Date.now() - startMs,
|
|
502
|
+
};
|
|
503
|
+
} finally {
|
|
504
|
+
clearTimeout(timer);
|
|
505
|
+
options.signal?.removeEventListener("abort", abort);
|
|
506
|
+
}
|
|
488
507
|
}
|
|
489
508
|
|
|
490
509
|
/**
|
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
import { readBoundedResponseBytes } from "../ingestion/index.js";
|
|
2
|
+
import { resolveSecret } from "../server/credential-provider.js";
|
|
3
|
+
|
|
4
|
+
export type EmbeddingInputPurpose = "query" | "document";
|
|
5
|
+
export interface EmbeddingImageInput {
|
|
6
|
+
mimeType: "image/png" | "image/jpeg" | "image/webp" | "image/gif";
|
|
7
|
+
base64: string;
|
|
8
|
+
}
|
|
9
|
+
export interface MultimodalEmbeddingInput {
|
|
10
|
+
text?: string;
|
|
11
|
+
images?: EmbeddingImageInput[];
|
|
12
|
+
}
|
|
13
|
+
export interface EmbeddingFamily {
|
|
14
|
+
id: string;
|
|
15
|
+
provider: "gemini" | "cohere" | "voyage" | (string & {});
|
|
16
|
+
model: string;
|
|
17
|
+
version: string;
|
|
18
|
+
dimensions: number;
|
|
19
|
+
supportedImageMimeTypes?: readonly EmbeddingImageInput["mimeType"][];
|
|
20
|
+
embed(
|
|
21
|
+
inputs: readonly MultimodalEmbeddingInput[],
|
|
22
|
+
purpose: EmbeddingInputPurpose,
|
|
23
|
+
): Promise<number[][]>;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
const DEFAULT_DIMENSIONS = 1024;
|
|
27
|
+
function dataUrl(image: EmbeddingImageInput) {
|
|
28
|
+
return `data:${image.mimeType};base64,${image.base64}`;
|
|
29
|
+
}
|
|
30
|
+
function normalizedInput(input: MultimodalEmbeddingInput) {
|
|
31
|
+
const text = input.text?.trim();
|
|
32
|
+
const images = input.images ?? [];
|
|
33
|
+
if (!text && !images.length)
|
|
34
|
+
throw new Error("Embedding input needs text, an image, or both.");
|
|
35
|
+
return { text, images };
|
|
36
|
+
}
|
|
37
|
+
async function postJson(
|
|
38
|
+
url: string,
|
|
39
|
+
headers: Record<string, string>,
|
|
40
|
+
body: unknown,
|
|
41
|
+
providerModel: string,
|
|
42
|
+
): Promise<Record<string, unknown>> {
|
|
43
|
+
const controller = new AbortController();
|
|
44
|
+
const timeout = setTimeout(() => controller.abort(), 30_000);
|
|
45
|
+
try {
|
|
46
|
+
const response = await fetch(url, {
|
|
47
|
+
method: "POST",
|
|
48
|
+
headers: { "Content-Type": "application/json", ...headers },
|
|
49
|
+
body: JSON.stringify(body),
|
|
50
|
+
signal: controller.signal,
|
|
51
|
+
});
|
|
52
|
+
if (!response.ok)
|
|
53
|
+
throw new Error(
|
|
54
|
+
`Embedding provider ${providerModel} failed with status ${response.status}.`,
|
|
55
|
+
);
|
|
56
|
+
const bytes = await readBoundedResponseBytes(response, 1_000_000);
|
|
57
|
+
return JSON.parse(new TextDecoder().decode(bytes)) as Record<
|
|
58
|
+
string,
|
|
59
|
+
unknown
|
|
60
|
+
>;
|
|
61
|
+
} catch (error) {
|
|
62
|
+
if (controller.signal.aborted)
|
|
63
|
+
throw new Error(`Embedding provider ${providerModel} timed out.`);
|
|
64
|
+
throw error;
|
|
65
|
+
} finally {
|
|
66
|
+
clearTimeout(timeout);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
function numberVectors(value: unknown): number[][] {
|
|
70
|
+
if (!Array.isArray(value))
|
|
71
|
+
throw new Error("Embedding response was malformed.");
|
|
72
|
+
return value.map((vector) => {
|
|
73
|
+
if (
|
|
74
|
+
!Array.isArray(vector) ||
|
|
75
|
+
vector.some((entry) => !Number.isFinite(entry))
|
|
76
|
+
)
|
|
77
|
+
throw new Error("Embedding response contained an invalid vector.");
|
|
78
|
+
return vector.map(Number);
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export function createGeminiEmbeddingFamily(
|
|
83
|
+
apiKey: string,
|
|
84
|
+
dimensions = DEFAULT_DIMENSIONS,
|
|
85
|
+
): EmbeddingFamily {
|
|
86
|
+
const model = "gemini-embedding-2";
|
|
87
|
+
return {
|
|
88
|
+
id: `gemini:${model}:${dimensions}`,
|
|
89
|
+
provider: "gemini",
|
|
90
|
+
model,
|
|
91
|
+
version: "stable-2026-04",
|
|
92
|
+
dimensions,
|
|
93
|
+
supportedImageMimeTypes: ["image/png", "image/jpeg"],
|
|
94
|
+
async embed(inputs, purpose) {
|
|
95
|
+
const vectors: number[][] = [];
|
|
96
|
+
for (const raw of inputs) {
|
|
97
|
+
const input = normalizedInput(raw);
|
|
98
|
+
const instruction =
|
|
99
|
+
purpose === "query"
|
|
100
|
+
? "task: search result | query:"
|
|
101
|
+
: "title: none | text:";
|
|
102
|
+
const result = await postJson(
|
|
103
|
+
`https://generativelanguage.googleapis.com/v1beta/models/${model}:embedContent`,
|
|
104
|
+
{ "x-goog-api-key": apiKey },
|
|
105
|
+
{
|
|
106
|
+
content: {
|
|
107
|
+
parts: [
|
|
108
|
+
{ text: `${instruction} ${input.text ?? ""}`.trim() },
|
|
109
|
+
...input.images.map((image) => ({
|
|
110
|
+
inlineData: { mimeType: image.mimeType, data: image.base64 },
|
|
111
|
+
})),
|
|
112
|
+
],
|
|
113
|
+
},
|
|
114
|
+
output_dimensionality: dimensions,
|
|
115
|
+
},
|
|
116
|
+
`gemini/${model}`,
|
|
117
|
+
);
|
|
118
|
+
vectors.push(
|
|
119
|
+
...numberVectors([
|
|
120
|
+
(result.embedding as { values?: unknown } | undefined)?.values,
|
|
121
|
+
]),
|
|
122
|
+
);
|
|
123
|
+
}
|
|
124
|
+
return vectors;
|
|
125
|
+
},
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
export function createCohereEmbeddingFamily(
|
|
129
|
+
apiKey: string,
|
|
130
|
+
dimensions = DEFAULT_DIMENSIONS,
|
|
131
|
+
): EmbeddingFamily {
|
|
132
|
+
const model = "embed-v4.0";
|
|
133
|
+
return {
|
|
134
|
+
id: `cohere:${model}:${dimensions}`,
|
|
135
|
+
provider: "cohere",
|
|
136
|
+
model,
|
|
137
|
+
version: "v4.0",
|
|
138
|
+
dimensions,
|
|
139
|
+
supportedImageMimeTypes: [
|
|
140
|
+
"image/png",
|
|
141
|
+
"image/jpeg",
|
|
142
|
+
"image/webp",
|
|
143
|
+
"image/gif",
|
|
144
|
+
],
|
|
145
|
+
async embed(inputs, purpose) {
|
|
146
|
+
const result = await postJson(
|
|
147
|
+
"https://api.cohere.com/v2/embed",
|
|
148
|
+
{ Authorization: `Bearer ${apiKey}` },
|
|
149
|
+
{
|
|
150
|
+
model,
|
|
151
|
+
inputs: inputs.map((raw) => {
|
|
152
|
+
const input = normalizedInput(raw);
|
|
153
|
+
return {
|
|
154
|
+
content: [
|
|
155
|
+
...(input.text ? [{ type: "text", text: input.text }] : []),
|
|
156
|
+
...input.images.map((image) => ({
|
|
157
|
+
type: "image_url",
|
|
158
|
+
image_url: { url: dataUrl(image) },
|
|
159
|
+
})),
|
|
160
|
+
],
|
|
161
|
+
};
|
|
162
|
+
}),
|
|
163
|
+
input_type: purpose === "query" ? "search_query" : "search_document",
|
|
164
|
+
embedding_types: ["float"],
|
|
165
|
+
output_dimension: dimensions,
|
|
166
|
+
},
|
|
167
|
+
`cohere/${model}`,
|
|
168
|
+
);
|
|
169
|
+
const embeddings = result.embeddings as
|
|
170
|
+
| { float?: unknown; float_?: unknown }
|
|
171
|
+
| undefined;
|
|
172
|
+
return numberVectors(embeddings?.float ?? embeddings?.float_);
|
|
173
|
+
},
|
|
174
|
+
};
|
|
175
|
+
}
|
|
176
|
+
export function createVoyageEmbeddingFamily(apiKey: string): EmbeddingFamily {
|
|
177
|
+
const model = "voyage-multimodal-3.5";
|
|
178
|
+
return {
|
|
179
|
+
id: `voyage:${model}:1024`,
|
|
180
|
+
provider: "voyage",
|
|
181
|
+
model,
|
|
182
|
+
version: "3.5",
|
|
183
|
+
dimensions: 1024,
|
|
184
|
+
supportedImageMimeTypes: [
|
|
185
|
+
"image/png",
|
|
186
|
+
"image/jpeg",
|
|
187
|
+
"image/webp",
|
|
188
|
+
"image/gif",
|
|
189
|
+
],
|
|
190
|
+
async embed(inputs, purpose) {
|
|
191
|
+
const result = await postJson(
|
|
192
|
+
"https://api.voyageai.com/v1/multimodalembeddings",
|
|
193
|
+
{ Authorization: `Bearer ${apiKey}` },
|
|
194
|
+
{
|
|
195
|
+
model,
|
|
196
|
+
inputs: inputs.map((raw) => {
|
|
197
|
+
const input = normalizedInput(raw);
|
|
198
|
+
return {
|
|
199
|
+
content: [
|
|
200
|
+
...(input.text ? [{ type: "text", text: input.text }] : []),
|
|
201
|
+
...input.images.map((image) => ({
|
|
202
|
+
type: "image_base64",
|
|
203
|
+
image_base64: dataUrl(image),
|
|
204
|
+
})),
|
|
205
|
+
],
|
|
206
|
+
};
|
|
207
|
+
}),
|
|
208
|
+
input_type: purpose,
|
|
209
|
+
truncation: true,
|
|
210
|
+
},
|
|
211
|
+
`voyage/${model}`,
|
|
212
|
+
);
|
|
213
|
+
return numberVectors(result.embeddings);
|
|
214
|
+
},
|
|
215
|
+
};
|
|
216
|
+
}
|
|
217
|
+
export async function availableEmbeddingFamilies(): Promise<EmbeddingFamily[]> {
|
|
218
|
+
const [gemini, cohere, voyage] = await Promise.all([
|
|
219
|
+
resolveSecret("GEMINI_API_KEY").catch(() => null),
|
|
220
|
+
resolveSecret("COHERE_API_KEY").catch(() => null),
|
|
221
|
+
resolveSecret("VOYAGE_API_KEY").catch(() => null),
|
|
222
|
+
]);
|
|
223
|
+
return [
|
|
224
|
+
...(gemini ? [createGeminiEmbeddingFamily(gemini)] : []),
|
|
225
|
+
...(cohere ? [createCohereEmbeddingFamily(cohere)] : []),
|
|
226
|
+
...(voyage ? [createVoyageEmbeddingFamily(voyage)] : []),
|
|
227
|
+
];
|
|
228
|
+
}
|
|
229
|
+
export function defaultEmbeddingFamily(
|
|
230
|
+
families: readonly EmbeddingFamily[],
|
|
231
|
+
): EmbeddingFamily | null {
|
|
232
|
+
return families.length === 1 ? (families[0] ?? null) : null;
|
|
233
|
+
}
|
package/corpus/core/src/index.ts
CHANGED
|
@@ -399,6 +399,8 @@ export * from "./review/index.js";
|
|
|
399
399
|
export * from "./org-team/index.js";
|
|
400
400
|
export * from "./navigation/index.js";
|
|
401
401
|
export * from "./command-navigation/index.js";
|
|
402
|
+
export * from "./search/index.js";
|
|
403
|
+
export * from "./embeddings/index.js";
|
|
402
404
|
export {
|
|
403
405
|
registerWorkspaceConnectionOnboardingStep,
|
|
404
406
|
type RegisterWorkspaceConnectionOnboardingStepOptions,
|
|
@@ -0,0 +1,413 @@
|
|
|
1
|
+
import type { DbExec } from "../db/index.js";
|
|
2
|
+
import { isPostgres } from "../db/index.js";
|
|
3
|
+
|
|
4
|
+
export const DEFAULT_SEARCH_NAMESPACE = "creative_context";
|
|
5
|
+
export const PGVECTOR_REQUIRED_MESSAGE =
|
|
6
|
+
"Vector search requires Postgres with the pgvector extension in the configured DATABASE_URL database.";
|
|
7
|
+
|
|
8
|
+
export interface SearchNamespaceIdentifiers {
|
|
9
|
+
namespace: string;
|
|
10
|
+
ftsTable: string;
|
|
11
|
+
ftsIndex: string;
|
|
12
|
+
ftsAudienceIndex: string;
|
|
13
|
+
vectorTable: string;
|
|
14
|
+
vectorIndex: string;
|
|
15
|
+
vectorAudienceIndex: string;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export interface PgVectorHit {
|
|
19
|
+
vectorKey: string;
|
|
20
|
+
embeddingSetId: string;
|
|
21
|
+
score: number;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export interface PostgresFtsHit {
|
|
25
|
+
chunkId: string;
|
|
26
|
+
itemVersionId: string;
|
|
27
|
+
score: number;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export interface RankedCandidate<T> {
|
|
31
|
+
key: string;
|
|
32
|
+
value: T;
|
|
33
|
+
score: number;
|
|
34
|
+
reason?: string;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export interface FusedCandidate<T> extends RankedCandidate<T> {
|
|
38
|
+
laneRanks: Record<string, number>;
|
|
39
|
+
reasons: string[];
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
type PgVectorOptions =
|
|
43
|
+
| boolean
|
|
44
|
+
| {
|
|
45
|
+
namespace?: string;
|
|
46
|
+
postgres?: boolean;
|
|
47
|
+
/** The caller has already provisioned this namespace and dimension. */
|
|
48
|
+
indexInitialized?: boolean;
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
function pgVectorOptions(options: PgVectorOptions | undefined) {
|
|
52
|
+
return typeof options === "boolean" ? { postgres: options } : (options ?? {});
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export function searchNamespaceIdentifiers(
|
|
56
|
+
namespace = DEFAULT_SEARCH_NAMESPACE,
|
|
57
|
+
dimensions?: number,
|
|
58
|
+
): SearchNamespaceIdentifiers {
|
|
59
|
+
if (!/^[a-z][a-z0-9_]{0,62}$/.test(namespace)) {
|
|
60
|
+
throw new Error(
|
|
61
|
+
"Search namespace must start with a lowercase letter and contain only lowercase letters, numbers, and underscores.",
|
|
62
|
+
);
|
|
63
|
+
}
|
|
64
|
+
const suffix =
|
|
65
|
+
dimensions === undefined ? "" : `_${checkedDimensions(dimensions)}`;
|
|
66
|
+
return {
|
|
67
|
+
namespace,
|
|
68
|
+
ftsTable: `${namespace}_search_documents`,
|
|
69
|
+
ftsIndex: `${namespace}_search_documents_gin`,
|
|
70
|
+
ftsAudienceIndex: `${namespace}_search_documents_audience_gin`,
|
|
71
|
+
vectorTable: `${namespace}_vectors${suffix}`,
|
|
72
|
+
vectorIndex: `${namespace}_vectors${suffix}_hnsw`,
|
|
73
|
+
vectorAudienceIndex: `${namespace}_vectors${suffix}_audience_gin`,
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function checkedDimensions(value: number): number {
|
|
78
|
+
if (!Number.isInteger(value) || value < 1 || value > 16_000) {
|
|
79
|
+
throw new Error("Embedding dimensions must be an integer from 1 to 16000.");
|
|
80
|
+
}
|
|
81
|
+
return value;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function vectorLiteral(vector: readonly number[], dimensions: number): string {
|
|
85
|
+
if (vector.length !== dimensions) {
|
|
86
|
+
throw new Error(
|
|
87
|
+
`Embedding has ${vector.length} values; expected ${dimensions}.`,
|
|
88
|
+
);
|
|
89
|
+
}
|
|
90
|
+
for (const value of vector) {
|
|
91
|
+
if (!Number.isFinite(value)) {
|
|
92
|
+
throw new Error("Embedding vectors may contain only finite numbers.");
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
return `[${vector.join(",")}]`;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function audienceClause(
|
|
99
|
+
audienceIds: readonly string[] | undefined,
|
|
100
|
+
column = "audience_ids",
|
|
101
|
+
): { sql: string; args: string[] } {
|
|
102
|
+
if (!audienceIds?.length) return { sql: "", args: [] };
|
|
103
|
+
return {
|
|
104
|
+
sql: ` AND ${column} && ARRAY[${audienceIds.map(() => "?").join(", ")}]::TEXT[]`,
|
|
105
|
+
args: [...audienceIds],
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
export function assertPgVectorAvailable(postgres = isPostgres()): void {
|
|
110
|
+
if (!postgres) throw new Error(PGVECTOR_REQUIRED_MESSAGE);
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
export async function ensurePgVectorIndex(
|
|
114
|
+
db: DbExec,
|
|
115
|
+
dimensions: number,
|
|
116
|
+
options?: PgVectorOptions,
|
|
117
|
+
): Promise<void> {
|
|
118
|
+
const resolved = pgVectorOptions(options);
|
|
119
|
+
const names = searchNamespaceIdentifiers(resolved.namespace, dimensions);
|
|
120
|
+
assertPgVectorAvailable(resolved.postgres);
|
|
121
|
+
try {
|
|
122
|
+
await db.execute("CREATE EXTENSION IF NOT EXISTS vector");
|
|
123
|
+
await db.execute(`
|
|
124
|
+
CREATE TABLE IF NOT EXISTS ${names.vectorTable} (
|
|
125
|
+
vector_key TEXT PRIMARY KEY,
|
|
126
|
+
embedding_set_id TEXT NOT NULL,
|
|
127
|
+
audience_ids TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[],
|
|
128
|
+
embedding vector(${checkedDimensions(dimensions)}) NOT NULL,
|
|
129
|
+
updated_at TEXT NOT NULL
|
|
130
|
+
)
|
|
131
|
+
`);
|
|
132
|
+
await db.execute(
|
|
133
|
+
`ALTER TABLE ${names.vectorTable} ADD COLUMN IF NOT EXISTS audience_ids TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[]`,
|
|
134
|
+
);
|
|
135
|
+
await db.execute(
|
|
136
|
+
`CREATE INDEX IF NOT EXISTS ${names.vectorIndex} ON ${names.vectorTable} USING hnsw (embedding vector_cosine_ops)`,
|
|
137
|
+
);
|
|
138
|
+
await db.execute(
|
|
139
|
+
`CREATE INDEX IF NOT EXISTS ${names.vectorAudienceIndex} ON ${names.vectorTable} USING GIN (audience_ids)`,
|
|
140
|
+
);
|
|
141
|
+
} catch (error) {
|
|
142
|
+
throw new Error(PGVECTOR_REQUIRED_MESSAGE, { cause: error });
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
export async function upsertPgVector(
|
|
147
|
+
db: DbExec,
|
|
148
|
+
input: {
|
|
149
|
+
vectorKey: string;
|
|
150
|
+
embeddingSetId: string;
|
|
151
|
+
dimensions: number;
|
|
152
|
+
vector: readonly number[];
|
|
153
|
+
audienceIds?: readonly string[];
|
|
154
|
+
updatedAt?: string;
|
|
155
|
+
},
|
|
156
|
+
options?: PgVectorOptions,
|
|
157
|
+
): Promise<void> {
|
|
158
|
+
const resolved = pgVectorOptions(options);
|
|
159
|
+
const names = searchNamespaceIdentifiers(
|
|
160
|
+
resolved.namespace,
|
|
161
|
+
input.dimensions,
|
|
162
|
+
);
|
|
163
|
+
assertPgVectorAvailable(resolved.postgres);
|
|
164
|
+
if (!resolved.indexInitialized) {
|
|
165
|
+
await ensurePgVectorIndex(db, input.dimensions, resolved);
|
|
166
|
+
}
|
|
167
|
+
await db.execute({
|
|
168
|
+
sql: `
|
|
169
|
+
INSERT INTO ${names.vectorTable} (vector_key, embedding_set_id, audience_ids, embedding, updated_at)
|
|
170
|
+
VALUES (?, ?, ARRAY[${(input.audienceIds ?? []).map(() => "?").join(", ")}]::TEXT[], ?::vector, ?)
|
|
171
|
+
ON CONFLICT (vector_key) DO UPDATE SET
|
|
172
|
+
embedding_set_id = EXCLUDED.embedding_set_id,
|
|
173
|
+
audience_ids = EXCLUDED.audience_ids,
|
|
174
|
+
embedding = EXCLUDED.embedding,
|
|
175
|
+
updated_at = EXCLUDED.updated_at
|
|
176
|
+
`,
|
|
177
|
+
args: [
|
|
178
|
+
input.vectorKey,
|
|
179
|
+
input.embeddingSetId,
|
|
180
|
+
...(input.audienceIds ?? []),
|
|
181
|
+
vectorLiteral(input.vector, input.dimensions),
|
|
182
|
+
input.updatedAt ?? new Date().toISOString(),
|
|
183
|
+
],
|
|
184
|
+
});
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
export async function deletePgVectors(
|
|
188
|
+
db: DbExec,
|
|
189
|
+
input: {
|
|
190
|
+
dimensions: number;
|
|
191
|
+
vectorKeys: readonly string[];
|
|
192
|
+
namespace?: string;
|
|
193
|
+
},
|
|
194
|
+
postgres = isPostgres(),
|
|
195
|
+
): Promise<number> {
|
|
196
|
+
const names = searchNamespaceIdentifiers(input.namespace, input.dimensions);
|
|
197
|
+
assertPgVectorAvailable(postgres);
|
|
198
|
+
if (!input.vectorKeys.length) return 0;
|
|
199
|
+
const result = await db.execute({
|
|
200
|
+
sql: `DELETE FROM ${names.vectorTable} WHERE vector_key IN (${input.vectorKeys.map(() => "?").join(", ")})`,
|
|
201
|
+
args: [...input.vectorKeys],
|
|
202
|
+
});
|
|
203
|
+
return result.rowsAffected;
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
export async function queryPgVectorIndex(
|
|
207
|
+
db: DbExec,
|
|
208
|
+
input: {
|
|
209
|
+
embeddingSetId: string;
|
|
210
|
+
dimensions: number;
|
|
211
|
+
vector: readonly number[];
|
|
212
|
+
limit?: number;
|
|
213
|
+
allowedVectorKeys?: readonly string[];
|
|
214
|
+
allowedAudienceIds?: readonly string[];
|
|
215
|
+
namespace?: string;
|
|
216
|
+
},
|
|
217
|
+
postgres = isPostgres(),
|
|
218
|
+
): Promise<PgVectorHit[]> {
|
|
219
|
+
const names = searchNamespaceIdentifiers(input.namespace, input.dimensions);
|
|
220
|
+
assertPgVectorAvailable(postgres);
|
|
221
|
+
if (
|
|
222
|
+
input.allowedVectorKeys?.length === 0 ||
|
|
223
|
+
input.allowedAudienceIds?.length === 0
|
|
224
|
+
)
|
|
225
|
+
return [];
|
|
226
|
+
const limit = Math.max(1, Math.min(200, Math.floor(input.limit ?? 40)));
|
|
227
|
+
const literal = vectorLiteral(input.vector, input.dimensions);
|
|
228
|
+
const keySql = input.allowedVectorKeys?.length
|
|
229
|
+
? ` AND vector_key IN (${input.allowedVectorKeys.map(() => "?").join(", ")})`
|
|
230
|
+
: "";
|
|
231
|
+
const audiences = audienceClause(input.allowedAudienceIds);
|
|
232
|
+
try {
|
|
233
|
+
await db.execute("SET hnsw.iterative_scan = strict_order");
|
|
234
|
+
} catch {}
|
|
235
|
+
const result = await db.execute({
|
|
236
|
+
sql: `
|
|
237
|
+
SELECT vector_key, embedding_set_id, 1 - (embedding <=> ?::vector) AS score
|
|
238
|
+
FROM ${names.vectorTable}
|
|
239
|
+
WHERE embedding_set_id = ?${keySql}${audiences.sql}
|
|
240
|
+
ORDER BY embedding <=> ?::vector
|
|
241
|
+
LIMIT ?
|
|
242
|
+
`,
|
|
243
|
+
args: [
|
|
244
|
+
literal,
|
|
245
|
+
input.embeddingSetId,
|
|
246
|
+
...(input.allowedVectorKeys ?? []),
|
|
247
|
+
...audiences.args,
|
|
248
|
+
literal,
|
|
249
|
+
limit,
|
|
250
|
+
],
|
|
251
|
+
});
|
|
252
|
+
return result.rows.map((row) => ({
|
|
253
|
+
vectorKey: String(row.vector_key),
|
|
254
|
+
embeddingSetId: String(row.embedding_set_id),
|
|
255
|
+
score: Number(row.score),
|
|
256
|
+
}));
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
export async function ensurePostgresFts(
|
|
260
|
+
db: DbExec,
|
|
261
|
+
namespace = DEFAULT_SEARCH_NAMESPACE,
|
|
262
|
+
): Promise<boolean> {
|
|
263
|
+
const names = searchNamespaceIdentifiers(namespace);
|
|
264
|
+
if (!isPostgres()) return false;
|
|
265
|
+
await db.execute(`
|
|
266
|
+
CREATE TABLE IF NOT EXISTS ${names.ftsTable} (
|
|
267
|
+
chunk_id TEXT PRIMARY KEY,
|
|
268
|
+
item_version_id TEXT NOT NULL,
|
|
269
|
+
audience_ids TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[],
|
|
270
|
+
document TSVECTOR NOT NULL,
|
|
271
|
+
updated_at TEXT NOT NULL
|
|
272
|
+
)
|
|
273
|
+
`);
|
|
274
|
+
await db.execute(
|
|
275
|
+
`ALTER TABLE ${names.ftsTable} ADD COLUMN IF NOT EXISTS audience_ids TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[]`,
|
|
276
|
+
);
|
|
277
|
+
await db.execute(
|
|
278
|
+
`CREATE INDEX IF NOT EXISTS ${names.ftsIndex} ON ${names.ftsTable} USING GIN (document)`,
|
|
279
|
+
);
|
|
280
|
+
await db.execute(
|
|
281
|
+
`CREATE INDEX IF NOT EXISTS ${names.ftsAudienceIndex} ON ${names.ftsTable} USING GIN (audience_ids)`,
|
|
282
|
+
);
|
|
283
|
+
return true;
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
export async function upsertPostgresFtsDocument(
|
|
287
|
+
db: DbExec,
|
|
288
|
+
input: {
|
|
289
|
+
chunkId: string;
|
|
290
|
+
itemVersionId: string;
|
|
291
|
+
title: string;
|
|
292
|
+
summary?: string | null;
|
|
293
|
+
body: string;
|
|
294
|
+
audienceIds?: readonly string[];
|
|
295
|
+
updatedAt?: string;
|
|
296
|
+
namespace?: string;
|
|
297
|
+
},
|
|
298
|
+
): Promise<boolean> {
|
|
299
|
+
const namespace = input.namespace ?? DEFAULT_SEARCH_NAMESPACE;
|
|
300
|
+
if (!(await ensurePostgresFts(db, namespace))) return false;
|
|
301
|
+
const names = searchNamespaceIdentifiers(namespace);
|
|
302
|
+
await db.execute({
|
|
303
|
+
sql: `
|
|
304
|
+
INSERT INTO ${names.ftsTable} (chunk_id, item_version_id, audience_ids, document, updated_at)
|
|
305
|
+
VALUES (?, ?, ARRAY[${(input.audienceIds ?? []).map(() => "?").join(", ")}]::TEXT[], setweight(to_tsvector('simple', ?), 'A') || setweight(to_tsvector('simple', ?), 'B') || setweight(to_tsvector('simple', ?), 'C'), ?)
|
|
306
|
+
ON CONFLICT (chunk_id) DO UPDATE SET item_version_id = EXCLUDED.item_version_id, audience_ids = EXCLUDED.audience_ids, document = EXCLUDED.document, updated_at = EXCLUDED.updated_at
|
|
307
|
+
`,
|
|
308
|
+
args: [
|
|
309
|
+
input.chunkId,
|
|
310
|
+
input.itemVersionId,
|
|
311
|
+
...(input.audienceIds ?? []),
|
|
312
|
+
input.title,
|
|
313
|
+
input.summary ?? "",
|
|
314
|
+
input.body,
|
|
315
|
+
input.updatedAt ?? new Date().toISOString(),
|
|
316
|
+
],
|
|
317
|
+
});
|
|
318
|
+
return true;
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
export async function queryPostgresFts(
|
|
322
|
+
db: DbExec,
|
|
323
|
+
input: {
|
|
324
|
+
query: string;
|
|
325
|
+
allowedChunkIds?: readonly string[];
|
|
326
|
+
allowedAudienceIds?: readonly string[];
|
|
327
|
+
limit?: number;
|
|
328
|
+
namespace?: string;
|
|
329
|
+
},
|
|
330
|
+
): Promise<PostgresFtsHit[]> {
|
|
331
|
+
const namespace = input.namespace ?? DEFAULT_SEARCH_NAMESPACE;
|
|
332
|
+
const names = searchNamespaceIdentifiers(namespace);
|
|
333
|
+
if (
|
|
334
|
+
!isPostgres() ||
|
|
335
|
+
input.allowedChunkIds?.length === 0 ||
|
|
336
|
+
input.allowedAudienceIds?.length === 0
|
|
337
|
+
)
|
|
338
|
+
return [];
|
|
339
|
+
const keys = input.allowedChunkIds?.length
|
|
340
|
+
? `chunk_id IN (${input.allowedChunkIds.map(() => "?").join(", ")}) AND`
|
|
341
|
+
: "";
|
|
342
|
+
const audiences = audienceClause(input.allowedAudienceIds);
|
|
343
|
+
const limit = Math.max(1, Math.min(200, Math.floor(input.limit ?? 40)));
|
|
344
|
+
try {
|
|
345
|
+
const result = await db.execute({
|
|
346
|
+
sql: `SELECT chunk_id, item_version_id, ts_rank_cd(document, websearch_to_tsquery('simple', ?)) AS score FROM ${names.ftsTable} WHERE ${keys} document @@ websearch_to_tsquery('simple', ?)${audiences.sql} ORDER BY score DESC, chunk_id ASC LIMIT ?`,
|
|
347
|
+
args: [
|
|
348
|
+
input.query,
|
|
349
|
+
...(input.allowedChunkIds ?? []),
|
|
350
|
+
input.query,
|
|
351
|
+
...audiences.args,
|
|
352
|
+
limit,
|
|
353
|
+
],
|
|
354
|
+
});
|
|
355
|
+
return result.rows.map((row) => ({
|
|
356
|
+
chunkId: String(row.chunk_id),
|
|
357
|
+
itemVersionId: String(row.item_version_id),
|
|
358
|
+
score: Number(row.score),
|
|
359
|
+
}));
|
|
360
|
+
} catch (error) {
|
|
361
|
+
// A read can race the first write for a tenant-specific namespace. Writers
|
|
362
|
+
// provision it; an absent lane simply contributes no candidates.
|
|
363
|
+
if (isMissingPostgresRelation(error)) return [];
|
|
364
|
+
throw error;
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
function isMissingPostgresRelation(error: unknown): boolean {
|
|
369
|
+
if (!error || typeof error !== "object") return false;
|
|
370
|
+
return (error as { code?: unknown }).code === "42P01";
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
export async function deletePostgresFtsDocuments(
|
|
374
|
+
db: DbExec,
|
|
375
|
+
chunkIds: readonly string[],
|
|
376
|
+
namespace = DEFAULT_SEARCH_NAMESPACE,
|
|
377
|
+
): Promise<number> {
|
|
378
|
+
const names = searchNamespaceIdentifiers(namespace);
|
|
379
|
+
if (!isPostgres() || !chunkIds.length) return 0;
|
|
380
|
+
await ensurePostgresFts(db, namespace);
|
|
381
|
+
const result = await db.execute({
|
|
382
|
+
sql: `DELETE FROM ${names.ftsTable} WHERE chunk_id IN (${chunkIds.map(() => "?").join(", ")})`,
|
|
383
|
+
args: [...chunkIds],
|
|
384
|
+
});
|
|
385
|
+
return result.rowsAffected;
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
export function reciprocalRankFusion<T>(
|
|
389
|
+
lanes: Readonly<Record<string, readonly RankedCandidate<T>[]>>,
|
|
390
|
+
options: { rankConstant?: number; limit?: number } = {},
|
|
391
|
+
): FusedCandidate<T>[] {
|
|
392
|
+
const rankConstant = Math.max(1, options.rankConstant ?? 60);
|
|
393
|
+
const fused = new Map<string, FusedCandidate<T>>();
|
|
394
|
+
for (const [lane, candidates] of Object.entries(lanes)) {
|
|
395
|
+
candidates.forEach((candidate, index) => {
|
|
396
|
+
const rank = index + 1;
|
|
397
|
+
const current = fused.get(candidate.key) ?? {
|
|
398
|
+
...candidate,
|
|
399
|
+
score: 0,
|
|
400
|
+
laneRanks: {},
|
|
401
|
+
reasons: [],
|
|
402
|
+
};
|
|
403
|
+
current.score += 1 / (rankConstant + rank);
|
|
404
|
+
current.laneRanks[lane] = rank;
|
|
405
|
+
if (candidate.reason && !current.reasons.includes(candidate.reason))
|
|
406
|
+
current.reasons.push(candidate.reason);
|
|
407
|
+
fused.set(candidate.key, current);
|
|
408
|
+
});
|
|
409
|
+
}
|
|
410
|
+
return [...fused.values()]
|
|
411
|
+
.sort((a, b) => b.score - a.score || a.key.localeCompare(b.key))
|
|
412
|
+
.slice(0, Math.max(1, options.limit ?? 40));
|
|
413
|
+
}
|