@cogenta/agents 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assist/chat.d.ts +69 -0
- package/dist/assist/chat.d.ts.map +1 -0
- package/dist/assist/chat.js +108 -0
- package/dist/assist/chat.js.map +1 -0
- package/dist/assist/classify.d.ts +101 -0
- package/dist/assist/classify.d.ts.map +1 -0
- package/dist/assist/classify.js +233 -0
- package/dist/assist/classify.js.map +1 -0
- package/dist/assist/faq.d.ts +61 -0
- package/dist/assist/faq.d.ts.map +1 -0
- package/dist/assist/faq.js +136 -0
- package/dist/assist/faq.js.map +1 -0
- package/dist/assist/images.d.ts +45 -0
- package/dist/assist/images.d.ts.map +1 -0
- package/dist/assist/images.js +79 -0
- package/dist/assist/images.js.map +1 -0
- package/dist/assist/runtime.d.ts +58 -0
- package/dist/assist/runtime.d.ts.map +1 -0
- package/dist/assist/runtime.js +84 -0
- package/dist/assist/runtime.js.map +1 -0
- package/dist/assist/suggestion.d.ts +20 -0
- package/dist/assist/suggestion.d.ts.map +1 -0
- package/dist/assist/suggestion.js +28 -0
- package/dist/assist/suggestion.js.map +1 -0
- package/dist/assist/toolset.d.ts +76 -0
- package/dist/assist/toolset.d.ts.map +1 -0
- package/dist/assist/toolset.js +72 -0
- package/dist/assist/toolset.js.map +1 -0
- package/dist/assist/writing.d.ts +74 -0
- package/dist/assist/writing.d.ts.map +1 -0
- package/dist/assist/writing.js +429 -0
- package/dist/assist/writing.js.map +1 -0
- package/dist/documents/docx.d.ts +6 -0
- package/dist/documents/docx.d.ts.map +1 -0
- package/dist/documents/docx.js +169 -0
- package/dist/documents/docx.js.map +1 -0
- package/dist/documents/extract-text.d.ts +35 -0
- package/dist/documents/extract-text.d.ts.map +1 -0
- package/dist/documents/extract-text.js +134 -0
- package/dist/documents/extract-text.js.map +1 -0
- package/dist/documents/extract-tool.d.ts +40 -0
- package/dist/documents/extract-tool.d.ts.map +1 -0
- package/dist/documents/extract-tool.js +65 -0
- package/dist/documents/extract-tool.js.map +1 -0
- package/dist/documents/limits.d.ts +23 -0
- package/dist/documents/limits.d.ts.map +1 -0
- package/dist/documents/limits.js +23 -0
- package/dist/documents/limits.js.map +1 -0
- package/dist/documents/pdf.d.ts +26 -0
- package/dist/documents/pdf.d.ts.map +1 -0
- package/dist/documents/pdf.js +478 -0
- package/dist/documents/pdf.js.map +1 -0
- package/dist/documents/zip.d.ts +18 -0
- package/dist/documents/zip.d.ts.map +1 -0
- package/dist/documents/zip.js +113 -0
- package/dist/documents/zip.js.map +1 -0
- package/dist/index.d.ts +50 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +25 -0
- package/dist/index.js.map +1 -1
- package/dist/providers/image/openai.d.ts +28 -0
- package/dist/providers/image/openai.d.ts.map +1 -0
- package/dist/providers/image/openai.js +80 -0
- package/dist/providers/image/openai.js.map +1 -0
- package/dist/providers/image/registry.d.ts +30 -0
- package/dist/providers/image/registry.d.ts.map +1 -0
- package/dist/providers/image/registry.js +36 -0
- package/dist/providers/image/registry.js.map +1 -0
- package/dist/providers/image/stability.d.ts +39 -0
- package/dist/providers/image/stability.d.ts.map +1 -0
- package/dist/providers/image/stability.js +88 -0
- package/dist/providers/image/stability.js.map +1 -0
- package/dist/providers/image/types.d.ts +50 -0
- package/dist/providers/image/types.d.ts.map +1 -0
- package/dist/providers/image/types.js +33 -0
- package/dist/providers/image/types.js.map +1 -0
- package/dist/rag/semantic/search.d.ts +84 -0
- package/dist/rag/semantic/search.d.ts.map +1 -0
- package/dist/rag/semantic/search.js +113 -0
- package/dist/rag/semantic/search.js.map +1 -0
- package/dist/rag/vector/file.d.ts +23 -0
- package/dist/rag/vector/file.d.ts.map +1 -0
- package/dist/rag/vector/file.js +170 -0
- package/dist/rag/vector/file.js.map +1 -0
- package/dist/rag/vector/index.d.ts +24 -0
- package/dist/rag/vector/index.d.ts.map +1 -0
- package/dist/rag/vector/index.js +27 -0
- package/dist/rag/vector/index.js.map +1 -0
- package/dist/rag/vector/memory.d.ts +16 -0
- package/dist/rag/vector/memory.d.ts.map +1 -0
- package/dist/rag/vector/memory.js +107 -0
- package/dist/rag/vector/memory.js.map +1 -0
- package/dist/rag/vector/pgvector.d.ts +34 -0
- package/dist/rag/vector/pgvector.d.ts.map +1 -0
- package/dist/rag/vector/pgvector.js +273 -0
- package/dist/rag/vector/pgvector.js.map +1 -0
- package/dist/rag/vector/types.d.ts +113 -0
- package/dist/rag/vector/types.d.ts.map +1 -0
- package/dist/rag/vector/types.js +51 -0
- package/dist/rag/vector/types.js.map +1 -0
- package/dist/site-plan/analyse-brief.d.ts +23 -0
- package/dist/site-plan/analyse-brief.d.ts.map +1 -0
- package/dist/site-plan/analyse-brief.js +207 -0
- package/dist/site-plan/analyse-brief.js.map +1 -0
- package/dist/site-plan/approval.d.ts +80 -0
- package/dist/site-plan/approval.d.ts.map +1 -0
- package/dist/site-plan/approval.js +201 -0
- package/dist/site-plan/approval.js.map +1 -0
- package/dist/site-plan/constraints.d.ts +45 -0
- package/dist/site-plan/constraints.d.ts.map +1 -0
- package/dist/site-plan/constraints.js +213 -0
- package/dist/site-plan/constraints.js.map +1 -0
- package/dist/site-plan/content-model.d.ts +22 -0
- package/dist/site-plan/content-model.d.ts.map +1 -0
- package/dist/site-plan/content-model.js +385 -0
- package/dist/site-plan/content-model.js.map +1 -0
- package/dist/site-plan/demo-content.d.ts +27 -0
- package/dist/site-plan/demo-content.d.ts.map +1 -0
- package/dist/site-plan/demo-content.js +170 -0
- package/dist/site-plan/demo-content.js.map +1 -0
- package/dist/site-plan/draft-store.d.ts +37 -0
- package/dist/site-plan/draft-store.d.ts.map +1 -0
- package/dist/site-plan/draft-store.js +117 -0
- package/dist/site-plan/draft-store.js.map +1 -0
- package/dist/site-plan/enforce.d.ts +17 -0
- package/dist/site-plan/enforce.d.ts.map +1 -0
- package/dist/site-plan/enforce.js +120 -0
- package/dist/site-plan/enforce.js.map +1 -0
- package/dist/site-plan/json.d.ts +9 -0
- package/dist/site-plan/json.d.ts.map +1 -0
- package/dist/site-plan/json.js +29 -0
- package/dist/site-plan/json.js.map +1 -0
- package/dist/site-plan/propose-plan.d.ts +43 -0
- package/dist/site-plan/propose-plan.d.ts.map +1 -0
- package/dist/site-plan/propose-plan.js +65 -0
- package/dist/site-plan/propose-plan.js.map +1 -0
- package/dist/site-plan/skin-candidates.d.ts +61 -0
- package/dist/site-plan/skin-candidates.d.ts.map +1 -0
- package/dist/site-plan/skin-candidates.js +99 -0
- package/dist/site-plan/skin-candidates.js.map +1 -0
- package/dist/site-plan/types.d.ts +93 -0
- package/dist/site-plan/types.d.ts.map +1 -0
- package/dist/site-plan/types.js +2 -0
- package/dist/site-plan/types.js.map +1 -0
- package/package.json +4 -3
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* L18 task 4 — the image provider abstraction.
|
|
3
|
+
*
|
|
4
|
+
* Shaped exactly like `ProviderClient` (L4) and for the same stated reason:
|
|
5
|
+
* "un fournisseur d'image (driver, comme les fournisseurs LLM existants —
|
|
6
|
+
* plusieurs implémentations, jamais un seul fournisseur en dur)". Vendors
|
|
7
|
+
* disagree on nearly everything above the wire — OpenAI returns
|
|
8
|
+
* `data[].b64_json`, Stability returns `artifacts[].base64`, sizes are a
|
|
9
|
+
* `"1024x1024"` string in one and a width/height pair in the other — and this
|
|
10
|
+
* interface is the one place that difference is allowed to live.
|
|
11
|
+
*
|
|
12
|
+
* Unlike cache or storage there is **no degraded tier**, and there deliberately
|
|
13
|
+
* is no `Driver`/registry-with-fallback here: there is no local, service-free
|
|
14
|
+
* way to draw a picture from a sentence, so the honest degradation is that the
|
|
15
|
+
* feature is absent, not that it is slower. `createImageProviderRegistry`
|
|
16
|
+
* returns an empty registry for a site that configured none, and the tool is
|
|
17
|
+
* never built.
|
|
18
|
+
*/
|
|
19
|
+
/** Deliberately a small closed set: every vendor supports these three shapes. */
|
|
20
|
+
export const IMAGE_SIZES = ['square', 'landscape', 'portrait'];
|
|
21
|
+
const MAX_COUNT = 4;
|
|
22
|
+
export function clampCount(count) {
|
|
23
|
+
if (count === undefined)
|
|
24
|
+
return 1;
|
|
25
|
+
return Math.min(Math.max(Math.trunc(count), 1), MAX_COUNT);
|
|
26
|
+
}
|
|
27
|
+
/** Pixel dimensions for the three named shapes. One table, so two adapters cannot disagree. */
|
|
28
|
+
export const IMAGE_DIMENSIONS = Object.freeze({
|
|
29
|
+
square: { width: 1024, height: 1024 },
|
|
30
|
+
landscape: { width: 1536, height: 640 },
|
|
31
|
+
portrait: { width: 640, height: 1536 },
|
|
32
|
+
});
|
|
33
|
+
//# sourceMappingURL=types.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/providers/image/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,iFAAiF;AACjF,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,QAAQ,EAAE,WAAW,EAAE,UAAU,CAAU,CAAA;AAgCvE,MAAM,SAAS,GAAG,CAAC,CAAA;AAEnB,MAAM,UAAU,UAAU,CAAC,KAAyB;IAClD,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,CAAC,CAAA;IACjC,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,SAAS,CAAC,CAAA;AAC5D,CAAC;AAED,+FAA+F;AAC/F,MAAM,CAAC,MAAM,gBAAgB,GAEzB,MAAM,CAAC,MAAM,CAAC;IAChB,MAAM,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE;IACrC,SAAS,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE;IACvC,QAAQ,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE;CACvC,CAAC,CAAA"}
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import type { EmbeddingProvider } from '../embeddings/types.js';
|
|
2
|
+
import type { VectorStore } from '../vector/types.js';
|
|
3
|
+
/**
|
|
4
|
+
* The slice of `@cogenta/schema`'s `SearchDriver` this module calls, declared
|
|
5
|
+
* structurally rather than imported — the same reasoning `ContentServiceLike`
|
|
6
|
+
* documents: the dependency arrow from `@cogenta/agents` to the content engine
|
|
7
|
+
* points one way only, and it is not this way.
|
|
8
|
+
*
|
|
9
|
+
* A real `SearchDriver` satisfies it as-is.
|
|
10
|
+
*/
|
|
11
|
+
export interface FullTextSearchLike {
|
|
12
|
+
search(query: {
|
|
13
|
+
readonly text: string;
|
|
14
|
+
readonly locale: string;
|
|
15
|
+
readonly status?: string;
|
|
16
|
+
readonly collections?: readonly string[];
|
|
17
|
+
readonly limit?: number;
|
|
18
|
+
}): Promise<{
|
|
19
|
+
readonly hits: readonly {
|
|
20
|
+
readonly id: string;
|
|
21
|
+
readonly collection: string;
|
|
22
|
+
readonly locale: string;
|
|
23
|
+
readonly status: string;
|
|
24
|
+
readonly title: string;
|
|
25
|
+
readonly score: number;
|
|
26
|
+
}[];
|
|
27
|
+
}>;
|
|
28
|
+
}
|
|
29
|
+
export interface SemanticHit {
|
|
30
|
+
readonly id: string;
|
|
31
|
+
readonly collection: string;
|
|
32
|
+
readonly locale: string;
|
|
33
|
+
readonly status: string;
|
|
34
|
+
readonly title: string;
|
|
35
|
+
/** The chunk that matched, when the hit came from the vector side. Absent for a full-text-only hit. */
|
|
36
|
+
readonly excerpt?: string;
|
|
37
|
+
/** Fused RRF score — a rank signal, never a distance or a probability. */
|
|
38
|
+
readonly score: number;
|
|
39
|
+
/** Which halves of the hybrid produced this hit. Shown in the admin, and the thing that makes "why is this here?" answerable. */
|
|
40
|
+
readonly matched: readonly ('semantic' | 'full-text')[];
|
|
41
|
+
}
|
|
42
|
+
export interface SemanticSearchQuery {
|
|
43
|
+
readonly text: string;
|
|
44
|
+
readonly locale: string;
|
|
45
|
+
readonly siteId: string;
|
|
46
|
+
readonly status?: string;
|
|
47
|
+
/** Already narrowed to what the actor may read — this module never decides that. */
|
|
48
|
+
readonly collections: readonly string[];
|
|
49
|
+
readonly limit?: number;
|
|
50
|
+
}
|
|
51
|
+
export interface SemanticSearch {
|
|
52
|
+
/** True when a vector store and an embedding provider are both wired in. */
|
|
53
|
+
readonly available: boolean;
|
|
54
|
+
search(query: SemanticSearchQuery): Promise<readonly SemanticHit[]>;
|
|
55
|
+
}
|
|
56
|
+
export interface SemanticSearchOptions {
|
|
57
|
+
readonly store: VectorStore;
|
|
58
|
+
readonly embeddings: EmbeddingProvider;
|
|
59
|
+
/**
|
|
60
|
+
* L10's full-text index. Optional, and its absence is not an error: the
|
|
61
|
+
* hybrid degrades to the vector half alone rather than refusing. The reverse
|
|
62
|
+
* — no vector half — is handled by not building this object at all.
|
|
63
|
+
*/
|
|
64
|
+
readonly fullText?: FullTextSearchLike;
|
|
65
|
+
/** How many candidates each half contributes before fusion. */
|
|
66
|
+
readonly candidates?: number;
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Semantic search, fused with the full-text search L10 wired up — **beside it,
|
|
70
|
+
* never instead of it**, which is what L18's dependency line asks for in as many
|
|
71
|
+
* words.
|
|
72
|
+
*
|
|
73
|
+
* Fusion is RRF over the two ranked lists, reusing L4's `reciprocalRankFusion`
|
|
74
|
+
* unchanged. Fusing ranks rather than scores is not a stylistic choice: a
|
|
75
|
+
* `ts_rank`, an InnoDB fulltext score and a cosine similarity live on three
|
|
76
|
+
* incomparable scales, and averaging them produces a number that means nothing.
|
|
77
|
+
*
|
|
78
|
+
* The known failure of pure vector search — exact-keyword queries, a part
|
|
79
|
+
* number, a person's name — is exactly what the full-text half covers, and the
|
|
80
|
+
* architecture document says so at line 190. Neither half is allowed to be the
|
|
81
|
+
* only one.
|
|
82
|
+
*/
|
|
83
|
+
export declare function createSemanticSearch(options: SemanticSearchOptions): SemanticSearch;
|
|
84
|
+
//# sourceMappingURL=search.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../../src/rag/semantic/search.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAA;AAG/D,OAAO,KAAK,EAAe,WAAW,EAAE,MAAM,oBAAoB,CAAA;AAElE;;;;;;;GAOG;AACH,MAAM,WAAW,kBAAkB;IACjC,MAAM,CAAC,KAAK,EAAE;QACZ,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;QACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;QACvB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;QACxB,QAAQ,CAAC,WAAW,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;QACxC,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KACxB,GAAG,OAAO,CAAC;QACV,QAAQ,CAAC,IAAI,EAAE,SAAS;YACtB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;YACnB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;YAC3B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;YACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;YACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;YACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;SACvB,EAAE,CAAA;KACJ,CAAC,CAAA;CACH;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,uGAAuG;IACvG,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAA;IACzB,0EAA0E;IAC1E,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,iIAAiI;IACjI,QAAQ,CAAC,OAAO,EAAE,SAAS,CAAC,UAAU,GAAG,WAAW,CAAC,EAAE,CAAA;CACxD;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;IACxB,oFAAoF;IACpF,QAAQ,CAAC,WAAW,EAAE,SAAS,MAAM,EAAE,CAAA;IACvC,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAA;CACxB;AAED,MAAM,WAAW,cAAc;IAC7B,4EAA4E;IAC5E,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAA;IAC3B,MAAM,CAAC,KAAK,EAAE,mBAAmB,GAAG,OAAO,CAAC,SAAS,WAAW,EAAE,CAAC,CAAA;CACpE;AAED,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAA;IAC3B,QAAQ,CAAC,UAAU,EAAE,iBAAiB,CAAA;IACtC;;;;OAIG;IACH,QAAQ,CAAC,QAAQ,CAAC,EAAE,kBAAkB,CAAA;IACtC,+DAA+D;IAC/D,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAA;CAC7B;AASD;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,qBAAqB,GAAG,cAAc,CAmGnF"}
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import { reciprocalRankFusion } from '../index/rrf.js';
|
|
2
|
+
const DEFAULT_LIMIT = 10;
|
|
3
|
+
const DEFAULT_CANDIDATES = 30;
|
|
4
|
+
function keyOf(collection, id) {
|
|
5
|
+
return `${collection}:${id}`;
|
|
6
|
+
}
|
|
7
|
+
/**
|
|
8
|
+
* Semantic search, fused with the full-text search L10 wired up — **beside it,
|
|
9
|
+
* never instead of it**, which is what L18's dependency line asks for in as many
|
|
10
|
+
* words.
|
|
11
|
+
*
|
|
12
|
+
* Fusion is RRF over the two ranked lists, reusing L4's `reciprocalRankFusion`
|
|
13
|
+
* unchanged. Fusing ranks rather than scores is not a stylistic choice: a
|
|
14
|
+
* `ts_rank`, an InnoDB fulltext score and a cosine similarity live on three
|
|
15
|
+
* incomparable scales, and averaging them produces a number that means nothing.
|
|
16
|
+
*
|
|
17
|
+
* The known failure of pure vector search — exact-keyword queries, a part
|
|
18
|
+
* number, a person's name — is exactly what the full-text half covers, and the
|
|
19
|
+
* architecture document says so at line 190. Neither half is allowed to be the
|
|
20
|
+
* only one.
|
|
21
|
+
*/
|
|
22
|
+
export function createSemanticSearch(options) {
|
|
23
|
+
const candidates = options.candidates ?? DEFAULT_CANDIDATES;
|
|
24
|
+
async function vectorHalf(query) {
|
|
25
|
+
const [vector] = await options.embeddings.embed([query.text]);
|
|
26
|
+
const byKey = new Map();
|
|
27
|
+
if (vector === undefined)
|
|
28
|
+
return { ranked: [], byKey };
|
|
29
|
+
const matches = await options.store.search(vector, {
|
|
30
|
+
limit: candidates,
|
|
31
|
+
filter: {
|
|
32
|
+
siteId: query.siteId,
|
|
33
|
+
collections: query.collections,
|
|
34
|
+
locales: [query.locale],
|
|
35
|
+
...(query.status === undefined ? {} : { statuses: [query.status] }),
|
|
36
|
+
},
|
|
37
|
+
});
|
|
38
|
+
const ranked = [];
|
|
39
|
+
for (const match of matches) {
|
|
40
|
+
const key = keyOf(match.record.collection, match.record.entryId);
|
|
41
|
+
// Several chunks of one entry can match. The entry appears once, at its
|
|
42
|
+
// best chunk's rank — otherwise a long article crowds out every other
|
|
43
|
+
// result purely by having more chunks.
|
|
44
|
+
if (byKey.has(key))
|
|
45
|
+
continue;
|
|
46
|
+
byKey.set(key, match);
|
|
47
|
+
ranked.push({ id: key, score: match.score });
|
|
48
|
+
}
|
|
49
|
+
return { ranked, byKey };
|
|
50
|
+
}
|
|
51
|
+
async function textHalf(query) {
|
|
52
|
+
const byKey = new Map();
|
|
53
|
+
if (options.fullText === undefined)
|
|
54
|
+
return { ranked: [], byKey };
|
|
55
|
+
const results = await options.fullText.search({
|
|
56
|
+
text: query.text,
|
|
57
|
+
locale: query.locale,
|
|
58
|
+
collections: query.collections,
|
|
59
|
+
limit: candidates,
|
|
60
|
+
...(query.status === undefined ? {} : { status: query.status }),
|
|
61
|
+
});
|
|
62
|
+
const ranked = [];
|
|
63
|
+
for (const hit of results.hits) {
|
|
64
|
+
const key = keyOf(hit.collection, hit.id);
|
|
65
|
+
if (byKey.has(key))
|
|
66
|
+
continue;
|
|
67
|
+
byKey.set(key, { title: hit.title, status: hit.status });
|
|
68
|
+
ranked.push({ id: key, score: hit.score });
|
|
69
|
+
}
|
|
70
|
+
return { ranked, byKey };
|
|
71
|
+
}
|
|
72
|
+
return {
|
|
73
|
+
available: true,
|
|
74
|
+
async search(query) {
|
|
75
|
+
if (query.collections.length === 0)
|
|
76
|
+
return [];
|
|
77
|
+
// Both halves run against the *already narrowed* collection list, so a
|
|
78
|
+
// collection the actor may not read is invisible to ranking rather than
|
|
79
|
+
// filtered out afterwards — the same ordering `createMemoryRagIndex`
|
|
80
|
+
// enforces, and for the same reason.
|
|
81
|
+
const [vector, text] = await Promise.all([vectorHalf(query), textHalf(query)]);
|
|
82
|
+
const fused = reciprocalRankFusion([vector.ranked, text.ranked]);
|
|
83
|
+
const limit = query.limit ?? DEFAULT_LIMIT;
|
|
84
|
+
const hits = [];
|
|
85
|
+
for (const { id: key, score } of fused) {
|
|
86
|
+
const match = vector.byKey.get(key);
|
|
87
|
+
const textHit = text.byKey.get(key);
|
|
88
|
+
const matched = [];
|
|
89
|
+
if (match !== undefined)
|
|
90
|
+
matched.push('semantic');
|
|
91
|
+
if (textHit !== undefined)
|
|
92
|
+
matched.push('full-text');
|
|
93
|
+
const collection = key.slice(0, key.indexOf(':'));
|
|
94
|
+
const entryId = key.slice(key.indexOf(':') + 1);
|
|
95
|
+
const title = textHit?.title ?? match?.record.chunk.text.slice(0, 80) ?? entryId;
|
|
96
|
+
hits.push({
|
|
97
|
+
id: entryId,
|
|
98
|
+
collection,
|
|
99
|
+
locale: query.locale,
|
|
100
|
+
status: textHit?.status ?? match?.record.status ?? 'published',
|
|
101
|
+
title,
|
|
102
|
+
...(match === undefined ? {} : { excerpt: match.record.chunk.text }),
|
|
103
|
+
score,
|
|
104
|
+
matched,
|
|
105
|
+
});
|
|
106
|
+
if (hits.length >= limit)
|
|
107
|
+
break;
|
|
108
|
+
}
|
|
109
|
+
return hits;
|
|
110
|
+
},
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
//# sourceMappingURL=search.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"search.js","sourceRoot":"","sources":["../../../src/rag/semantic/search.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAA;AA0EtD,MAAM,aAAa,GAAG,EAAE,CAAA;AACxB,MAAM,kBAAkB,GAAG,EAAE,CAAA;AAE7B,SAAS,KAAK,CAAC,UAAkB,EAAE,EAAU;IAC3C,OAAO,GAAG,UAAU,IAAI,EAAE,EAAE,CAAA;AAC9B,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,oBAAoB,CAAC,OAA8B;IACjE,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,kBAAkB,CAAA;IAE3D,KAAK,UAAU,UAAU,CACvB,KAA0B;QAE1B,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAA;QAC7D,MAAM,KAAK,GAAG,IAAI,GAAG,EAAuB,CAAA;QAC5C,IAAI,MAAM,KAAK,SAAS;YAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,CAAA;QAEtD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE;YACjD,KAAK,EAAE,UAAU;YACjB,MAAM,EAAE;gBACN,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,WAAW,EAAE,KAAK,CAAC,WAAW;gBAC9B,OAAO,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC;gBACvB,GAAG,CAAC,KAAK,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;aACpE;SACF,CAAC,CAAA;QAEF,MAAM,MAAM,GAAe,EAAE,CAAA;QAC7B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;YAChE,wEAAwE;YACxE,sEAAsE;YACtE,uCAAuC;YACvC,IAAI,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAQ;YAC5B,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;YACrB,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAA;QAC9C,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAA;IAC1B,CAAC;IAED,KAAK,UAAU,QAAQ,CAAC,KAA0B;QAIhD,MAAM,KAAK,GAAG,IAAI,GAAG,EAA+D,CAAA;QACpF,IAAI,OAAO,CAAC,QAAQ,KAAK,SAAS;YAAE,OAAO,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,CAAA;QAEhE,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;YAC5C,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,KAAK,EAAE,UAAU;YACjB,GAAG,CAAC,KAAK,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC;SAChE,CAAC,CAAA;QAEF,MAAM,MAAM,GAAe,EAAE,CAAA;QAC7B,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YAC/B,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,EAAE,CAAC,CAAA;YACzC,IAAI,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAQ;YAC5B,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAA;YACxD,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAA;QAC5C,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAA;IAC1B,CAAC;IAED,OAAO;QACL,SAAS,EAAE,IAAI;QAEf,KAAK,CAAC,MAAM,CAAC,KAAK;YAChB,IAAI,KAAK,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,EAAE,CAAA;YAE7C,uEAAuE;YACvE,wEAAwE;YACxE,qEAAqE;YACrE,qCAAqC;YACrC,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;YAC9E,MAAM,KAAK,GAAG,oBAAoB,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAA;YAEhE,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,aAAa,CAAA;YAC1C,MAAM,IAAI,GAAkB,EAAE,CAAA;YAC9B,KAAK,MAAM,EAAE,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,KAAK,EAAE,CAAC;gBACvC,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;gBACnC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;gBACnC,MAAM,OAAO,GAAiC,EAAE,CAAA;gBAChD,IAAI,KAAK,KAAK,SAAS;oBAAE,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;gBACjD,IAAI,OAAO,KAAK,SAAS;oBAAE,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;gBAEpD,MAAM,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAA;gBACjD,MAAM,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAA;gBAC/C,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,OAAO,CAAA;gBAEhF,IAAI,CAAC,IAAI,CAAC;oBACR,EAAE,EAAE,OAAO;oBACX,UAAU;oBACV,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,MAAM,EAAE,OAAO,EAAE,MAAM,IAAI,KAAK,EAAE,MAAM,CAAC,MAAM,IAAI,WAAW;oBAC9D,KAAK;oBACL,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;oBACpE,KAAK;oBACL,OAAO;iBACR,CAAC,CAAA;gBACF,IAAI,IAAI,CAAC,MAAM,IAAI,KAAK;oBAAE,MAAK;YACjC,CAAC;YACD,OAAO,IAAI,CAAA;QACb,CAAC;KACF,CAAA;AACH,CAAC"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { type Driver } from '@cogenta/core';
|
|
2
|
+
import type { VectorConfig, VectorStore } from './types.js';
|
|
3
|
+
export interface FileVectorOptions {
|
|
4
|
+
readonly dimensions: number;
|
|
5
|
+
readonly path?: string;
|
|
6
|
+
}
|
|
7
|
+
/**
|
|
8
|
+
* Vectors that survive a restart, with no service to install.
|
|
9
|
+
*
|
|
10
|
+
* Ranking is delegated to the memory store — the whole file is the working set,
|
|
11
|
+
* and re-ranking it in memory is exactly what the degraded row of
|
|
12
|
+
* `docs/02-architecture.md` describes. What this driver adds over `memory` is
|
|
13
|
+
* only durability, and that is deliberately the *only* difference: two drivers
|
|
14
|
+
* that also ranked differently would make the shared contract suite meaningless.
|
|
15
|
+
*
|
|
16
|
+
* JSONL, rewritten whole on every mutation through a temp file and a rename.
|
|
17
|
+
* That is O(n) per write and openly so: this driver is for the site that does
|
|
18
|
+
* not run Postgres, where n is thousands of chunks. A site past that point
|
|
19
|
+
* belongs on `pgvector`, and its health report says which one is running.
|
|
20
|
+
*/
|
|
21
|
+
export declare function createFileVectorStore(options: FileVectorOptions): Promise<VectorStore>;
|
|
22
|
+
export declare function fileVectorDriver(): Driver<VectorStore, VectorConfig>;
|
|
23
|
+
//# sourceMappingURL=file.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"file.d.ts","sourceRoot":"","sources":["../../../src/rag/vector/file.ts"],"names":[],"mappings":"AAGA,OAAO,EAAgB,KAAK,MAAM,EAAqB,MAAM,eAAe,CAAA;AAE5E,OAAO,KAAK,EACV,YAAY,EAIZ,WAAW,EACZ,MAAM,YAAY,CAAA;AAsBnB,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CACvB;AAyCD;;;;;;;;;;;;;GAaG;AACH,wBAAsB,qBAAqB,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,WAAW,CAAC,CA2E5F;AAED,wBAAgB,gBAAgB,IAAI,MAAM,CAAC,WAAW,EAAE,YAAY,CAAC,CAoCpE"}
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import process from 'node:process';
|
|
4
|
+
import { CogentaError } from '@cogenta/core';
|
|
5
|
+
import { createMemoryVectorStore } from './memory.js';
|
|
6
|
+
const DEFAULT_PATH = './.cogenta/vectors';
|
|
7
|
+
const FILE_NAME = 'index.jsonl';
|
|
8
|
+
/** Transient on Windows while another handle is still open on the target; clears within milliseconds. */
|
|
9
|
+
const RETRYABLE = new Set(['EPERM', 'EACCES', 'EBUSY']);
|
|
10
|
+
const RETRY_DELAYS_MS = [5, 10, 20, 40, 80, 160];
|
|
11
|
+
async function withWindowsRetry(operation) {
|
|
12
|
+
for (const [attempt, delay] of RETRY_DELAYS_MS.entries()) {
|
|
13
|
+
try {
|
|
14
|
+
await operation();
|
|
15
|
+
return;
|
|
16
|
+
}
|
|
17
|
+
catch (error) {
|
|
18
|
+
const code = error.code ?? '';
|
|
19
|
+
if (!RETRYABLE.has(code) || attempt === RETRY_DELAYS_MS.length - 1)
|
|
20
|
+
throw error;
|
|
21
|
+
await new Promise((resolve) => setTimeout(resolve, delay * (0.5 + Math.random())));
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
function storeFailed(message, hint, cause) {
|
|
26
|
+
return new CogentaError({
|
|
27
|
+
code: 'VECTOR_STORE_FAILED',
|
|
28
|
+
message,
|
|
29
|
+
hint,
|
|
30
|
+
...(cause === undefined ? {} : { cause }),
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
async function readRecords(file) {
|
|
34
|
+
let text;
|
|
35
|
+
try {
|
|
36
|
+
text = await readFile(file, 'utf8');
|
|
37
|
+
}
|
|
38
|
+
catch (error) {
|
|
39
|
+
if (error.code === 'ENOENT')
|
|
40
|
+
return [];
|
|
41
|
+
throw storeFailed(`The vector index at ${file} could not be read.`, 'Check the file permissions, or delete the file to re-index from scratch.', error);
|
|
42
|
+
}
|
|
43
|
+
const records = [];
|
|
44
|
+
for (const line of text.split('\n')) {
|
|
45
|
+
const trimmed = line.trim();
|
|
46
|
+
if (trimmed.length === 0)
|
|
47
|
+
continue;
|
|
48
|
+
try {
|
|
49
|
+
records.push(JSON.parse(trimmed));
|
|
50
|
+
}
|
|
51
|
+
catch (error) {
|
|
52
|
+
throw storeFailed(`The vector index at ${file} contains a line that is not JSON.`, 'Delete the file to re-index from scratch — it holds only derived embeddings, never content of record.', error);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
return records;
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Vectors that survive a restart, with no service to install.
|
|
59
|
+
*
|
|
60
|
+
* Ranking is delegated to the memory store — the whole file is the working set,
|
|
61
|
+
* and re-ranking it in memory is exactly what the degraded row of
|
|
62
|
+
* `docs/02-architecture.md` describes. What this driver adds over `memory` is
|
|
63
|
+
* only durability, and that is deliberately the *only* difference: two drivers
|
|
64
|
+
* that also ranked differently would make the shared contract suite meaningless.
|
|
65
|
+
*
|
|
66
|
+
* JSONL, rewritten whole on every mutation through a temp file and a rename.
|
|
67
|
+
* That is O(n) per write and openly so: this driver is for the site that does
|
|
68
|
+
* not run Postgres, where n is thousands of chunks. A site past that point
|
|
69
|
+
* belongs on `pgvector`, and its health report says which one is running.
|
|
70
|
+
*/
|
|
71
|
+
export async function createFileVectorStore(options) {
|
|
72
|
+
const directory = options.path ?? DEFAULT_PATH;
|
|
73
|
+
const file = join(directory, FILE_NAME);
|
|
74
|
+
const inner = createMemoryVectorStore({ dimensions: options.dimensions });
|
|
75
|
+
await mkdir(directory, { recursive: true });
|
|
76
|
+
/** The store interface has no enumeration, so durability needs its own mirror of what was written. */
|
|
77
|
+
const mirror = new Map();
|
|
78
|
+
const loaded = await readRecords(file);
|
|
79
|
+
// A file written under another model's dimensions is refused here, loudly,
|
|
80
|
+
// rather than silently ranking two vector spaces against each other.
|
|
81
|
+
await inner.upsert(loaded);
|
|
82
|
+
for (const record of loaded)
|
|
83
|
+
mirror.set(record.chunk.id, record);
|
|
84
|
+
async function persist() {
|
|
85
|
+
const body = [...mirror.values()].map((record) => JSON.stringify(record)).join('\n');
|
|
86
|
+
const temporary = `${file}.${process.pid}.tmp`;
|
|
87
|
+
await writeFile(temporary, body.length === 0 ? '' : `${body}\n`, 'utf8');
|
|
88
|
+
try {
|
|
89
|
+
await withWindowsRetry(() => rename(temporary, file));
|
|
90
|
+
}
|
|
91
|
+
catch (error) {
|
|
92
|
+
await rm(temporary, { force: true });
|
|
93
|
+
throw storeFailed(`The vector index at ${file} could not be written.`, 'Check that the directory is writable, or point vector.path somewhere that is.', error);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
return {
|
|
97
|
+
dimensions: options.dimensions,
|
|
98
|
+
async upsert(records) {
|
|
99
|
+
// Validation first (the memory store refuses a wrong dimension), so a
|
|
100
|
+
// rejected batch never reaches the mirror and never reaches the disk.
|
|
101
|
+
await inner.upsert(records);
|
|
102
|
+
for (const record of records)
|
|
103
|
+
mirror.set(record.chunk.id, record);
|
|
104
|
+
await persist();
|
|
105
|
+
},
|
|
106
|
+
async remove(chunkIds) {
|
|
107
|
+
await inner.remove(chunkIds);
|
|
108
|
+
for (const id of chunkIds)
|
|
109
|
+
mirror.delete(id);
|
|
110
|
+
await persist();
|
|
111
|
+
},
|
|
112
|
+
async removeEntries(scope) {
|
|
113
|
+
await inner.removeEntries(scope);
|
|
114
|
+
const entries = new Set(scope.entryIds);
|
|
115
|
+
for (const [id, record] of mirror) {
|
|
116
|
+
if (record.siteId === scope.siteId &&
|
|
117
|
+
record.collection === scope.collection &&
|
|
118
|
+
entries.has(record.entryId)) {
|
|
119
|
+
mirror.delete(id);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
await persist();
|
|
123
|
+
},
|
|
124
|
+
search: (queryVector, searchOptions) => inner.search(queryVector, searchOptions),
|
|
125
|
+
count: (filter) => inner.count(filter),
|
|
126
|
+
async clear() {
|
|
127
|
+
await inner.clear();
|
|
128
|
+
mirror.clear();
|
|
129
|
+
await persist();
|
|
130
|
+
},
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
export function fileVectorDriver() {
|
|
134
|
+
let store = null;
|
|
135
|
+
let path = DEFAULT_PATH;
|
|
136
|
+
return {
|
|
137
|
+
name: 'file',
|
|
138
|
+
tier: 'degraded',
|
|
139
|
+
available: async (config) => {
|
|
140
|
+
try {
|
|
141
|
+
await mkdir(config.path ?? DEFAULT_PATH, { recursive: true });
|
|
142
|
+
return true;
|
|
143
|
+
}
|
|
144
|
+
catch {
|
|
145
|
+
// A read-only filesystem is a real answer, not a crash: the registry
|
|
146
|
+
// falls through to `memory` and the site still starts.
|
|
147
|
+
return false;
|
|
148
|
+
}
|
|
149
|
+
},
|
|
150
|
+
init: async (config) => {
|
|
151
|
+
path = config.path ?? DEFAULT_PATH;
|
|
152
|
+
store = await createFileVectorStore({
|
|
153
|
+
dimensions: config.dimensions,
|
|
154
|
+
...(config.path === undefined ? {} : { path: config.path }),
|
|
155
|
+
});
|
|
156
|
+
return store;
|
|
157
|
+
},
|
|
158
|
+
dispose: async () => {
|
|
159
|
+
store = null;
|
|
160
|
+
},
|
|
161
|
+
health: async () => ({
|
|
162
|
+
status: 'degraded',
|
|
163
|
+
driver: 'file',
|
|
164
|
+
tier: 'degraded',
|
|
165
|
+
message: `Embeddings are kept on disk in ${path} and survive a restart; ranking is exact, over the whole index.`,
|
|
166
|
+
...(store === null ? {} : { details: { records: await store.count() } }),
|
|
167
|
+
}),
|
|
168
|
+
};
|
|
169
|
+
}
|
|
170
|
+
//# sourceMappingURL=file.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"file.js","sourceRoot":"","sources":["../../../src/rag/vector/file.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACzE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAChC,OAAO,OAAO,MAAM,cAAc,CAAA;AAClC,OAAO,EAAE,YAAY,EAAkC,MAAM,eAAe,CAAA;AAC5E,OAAO,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAA;AASrD,MAAM,YAAY,GAAG,oBAAoB,CAAA;AACzC,MAAM,SAAS,GAAG,aAAa,CAAA;AAE/B,yGAAyG;AACzG,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAA;AACvD,MAAM,eAAe,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,GAAG,CAAC,CAAA;AAEhD,KAAK,UAAU,gBAAgB,CAAC,SAA8B;IAC5D,KAAK,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,eAAe,CAAC,OAAO,EAAE,EAAE,CAAC;QACzD,IAAI,CAAC;YACH,MAAM,SAAS,EAAE,CAAA;YACjB,OAAM;QACR,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,GAAI,KAA+B,CAAC,IAAI,IAAI,EAAE,CAAA;YACxD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,OAAO,KAAK,eAAe,CAAC,MAAM,GAAG,CAAC;gBAAE,MAAM,KAAK,CAAA;YAC/E,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAA;QACpF,CAAC;IACH,CAAC;AACH,CAAC;AAOD,SAAS,WAAW,CAAC,OAAe,EAAE,IAAY,EAAE,KAAe;IACjE,OAAO,IAAI,YAAY,CAAC;QACtB,IAAI,EAAE,qBAAqB;QAC3B,OAAO;QACP,IAAI;QACJ,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC;KAC1C,CAAC,CAAA;AACJ,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,IAAY;IACrC,IAAI,IAAY,CAAA;IAChB,IAAI,CAAC;QACH,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IACrC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,EAAE,CAAA;QACjE,MAAM,WAAW,CACf,uBAAuB,IAAI,qBAAqB,EAChD,0EAA0E,EAC1E,KAAK,CACN,CAAA;IACH,CAAC;IAED,MAAM,OAAO,GAAmB,EAAE,CAAA;IAClC,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACpC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;QAC3B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,SAAQ;QAClC,IAAI,CAAC;YACH,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAiB,CAAC,CAAA;QACnD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,WAAW,CACf,uBAAuB,IAAI,oCAAoC,EAC/D,uGAAuG,EACvG,KAAK,CACN,CAAA;QACH,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAA;AAChB,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,OAA0B;IACpE,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,IAAI,YAAY,CAAA;IAC9C,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,CAAA;IACvC,MAAM,KAAK,GAAG,uBAAuB,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,CAAA;IAEzE,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAE3C,sGAAsG;IACtG,MAAM,MAAM,GAAG,IAAI,GAAG,EAAwB,CAAA;IAE9C,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,CAAA;IACtC,2EAA2E;IAC3E,qEAAqE;IACrE,MAAM,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;IAC1B,KAAK,MAAM,MAAM,IAAI,MAAM;QAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAA;IAEhE,KAAK,UAAU,OAAO;QACpB,MAAM,IAAI,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpF,MAAM,SAAS,GAAG,GAAG,IAAI,IAAI,OAAO,CAAC,GAAG,MAAM,CAAA;QAC9C,MAAM,SAAS,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,EAAE,MAAM,CAAC,CAAA;QACxE,IAAI,CAAC;YACH,MAAM,gBAAgB,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,CAAA;QACvD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,EAAE,CAAC,SAAS,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;YACpC,MAAM,WAAW,CACf,uBAAuB,IAAI,wBAAwB,EACnD,+EAA+E,EAC/E,KAAK,CACN,CAAA;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,UAAU,EAAE,OAAO,CAAC,UAAU;QAE9B,KAAK,CAAC,MAAM,CAAC,OAAO;YAClB,sEAAsE;YACtE,sEAAsE;YACtE,MAAM,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;YAC3B,KAAK,MAAM,MAAM,IAAI,OAAO;gBAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAA;YACjE,MAAM,OAAO,EAAE,CAAA;QACjB,CAAC;QAED,KAAK,CAAC,MAAM,CAAC,QAAQ;YACnB,MAAM,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;YAC5B,KAAK,MAAM,EAAE,IAAI,QAAQ;gBAAE,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;YAC5C,MAAM,OAAO,EAAE,CAAA;QACjB,CAAC;QAED,KAAK,CAAC,aAAa,CAAC,KAAK;YACvB,MAAM,KAAK,CAAC,aAAa,CAAC,KAAK,CAAC,CAAA;YAChC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;YACvC,KAAK,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;gBAClC,IACE,MAAM,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM;oBAC9B,MAAM,CAAC,UAAU,KAAK,KAAK,CAAC,UAAU;oBACtC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,EAC3B,CAAC;oBACD,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;gBACnB,CAAC;YACH,CAAC;YACD,MAAM,OAAO,EAAE,CAAA;QACjB,CAAC;QAED,MAAM,EAAE,CAAC,WAAW,EAAE,aAAmC,EAAE,EAAE,CAC3D,KAAK,CAAC,MAAM,CAAC,WAAW,EAAE,aAAa,CAAC;QAE1C,KAAK,EAAE,CAAC,MAAqB,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC;QAErD,KAAK,CAAC,KAAK;YACT,MAAM,KAAK,CAAC,KAAK,EAAE,CAAA;YACnB,MAAM,CAAC,KAAK,EAAE,CAAA;YACd,MAAM,OAAO,EAAE,CAAA;QACjB,CAAC;KACF,CAAA;AACH,CAAC;AAED,MAAM,UAAU,gBAAgB;IAC9B,IAAI,KAAK,GAAuB,IAAI,CAAA;IACpC,IAAI,IAAI,GAAG,YAAY,CAAA;IAEvB,OAAO;QACL,IAAI,EAAE,MAAM;QACZ,IAAI,EAAE,UAAU;QAChB,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;YAC1B,IAAI,CAAC;gBACH,MAAM,KAAK,CAAC,MAAM,CAAC,IAAI,IAAI,YAAY,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;gBAC7D,OAAO,IAAI,CAAA;YACb,CAAC;YAAC,MAAM,CAAC;gBACP,qEAAqE;gBACrE,uDAAuD;gBACvD,OAAO,KAAK,CAAA;YACd,CAAC;QACH,CAAC;QACD,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;YACrB,IAAI,GAAG,MAAM,CAAC,IAAI,IAAI,YAAY,CAAA;YAClC,KAAK,GAAG,MAAM,qBAAqB,CAAC;gBAClC,UAAU,EAAE,MAAM,CAAC,UAAU;gBAC7B,GAAG,CAAC,MAAM,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC;aAC5D,CAAC,CAAA;YACF,OAAO,KAAK,CAAA;QACd,CAAC;QACD,OAAO,EAAE,KAAK,IAAI,EAAE;YAClB,KAAK,GAAG,IAAI,CAAA;QACd,CAAC;QACD,MAAM,EAAE,KAAK,IAA2B,EAAE,CAAC,CAAC;YAC1C,MAAM,EAAE,UAAU;YAClB,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,UAAU;YAChB,OAAO,EAAE,kCAAkC,IAAI,iEAAiE;YAChH,GAAG,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,OAAO,EAAE,MAAM,KAAK,CAAC,KAAK,EAAE,EAAE,EAAE,CAAC;SACzE,CAAC;KACH,CAAA;AACH,CAAC"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { type DatabaseHandle, type DriverRegistry, type Logger } from '@cogenta/core';
|
|
2
|
+
import type { VectorConfig, VectorStore } from './types.js';
|
|
3
|
+
export type { FileVectorOptions } from './file.js';
|
|
4
|
+
export { createFileVectorStore, fileVectorDriver } from './file.js';
|
|
5
|
+
export { createMemoryVectorStore, memoryVectorDriver } from './memory.js';
|
|
6
|
+
export type { PgVectorDriverOptions, PgVectorOptions } from './pgvector.js';
|
|
7
|
+
export { createPgVectorStore, pgVectorDriver, vectorLiteral } from './pgvector.js';
|
|
8
|
+
export type { VectorConfig, VectorFilter, VectorMatch, VectorRecord, VectorScope, VectorSearchOptions, VectorStore, } from './types.js';
|
|
9
|
+
export { matchesFilter, VECTOR_DEFAULTS } from './types.js';
|
|
10
|
+
export interface VectorRegistryOptions {
|
|
11
|
+
/** The site database, lent to `pgvector`. Absent means only the two service-free drivers can run. */
|
|
12
|
+
readonly db?: DatabaseHandle;
|
|
13
|
+
readonly logger?: Logger;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* The vector drivers Cogenta ships, in tier order.
|
|
17
|
+
*
|
|
18
|
+
* `file` before `memory` among the degraded pair, for the same reason the cache
|
|
19
|
+
* registry orders them that way: both need no service, and the one that survives
|
|
20
|
+
* a restart is worth more than the one that does not. A site with no Postgres
|
|
21
|
+
* still gets working semantic search — that is R1, not a nicety.
|
|
22
|
+
*/
|
|
23
|
+
export declare function createVectorRegistry(options?: VectorRegistryOptions): DriverRegistry<VectorStore, VectorConfig>;
|
|
24
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/rag/vector/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,KAAK,cAAc,EACnB,KAAK,cAAc,EACnB,KAAK,MAAM,EACZ,MAAM,eAAe,CAAA;AAItB,OAAO,KAAK,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AAE3D,YAAY,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAA;AAClD,OAAO,EAAE,qBAAqB,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAA;AACnE,OAAO,EAAE,uBAAuB,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAA;AACzE,YAAY,EAAE,qBAAqB,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;AAC3E,OAAO,EAAE,mBAAmB,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAClF,YAAY,EACV,YAAY,EACZ,YAAY,EACZ,WAAW,EACX,YAAY,EACZ,WAAW,EACX,mBAAmB,EACnB,WAAW,GACZ,MAAM,YAAY,CAAA;AACnB,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AAE3D,MAAM,WAAW,qBAAqB;IACpC,qGAAqG;IACrG,QAAQ,CAAC,EAAE,CAAC,EAAE,cAAc,CAAA;IAC5B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CACzB;AAED;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAClC,OAAO,GAAE,qBAA0B,GAClC,cAAc,CAAC,WAAW,EAAE,YAAY,CAAC,CAW3C"}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { createDriverRegistry, } from '@cogenta/core';
|
|
2
|
+
import { fileVectorDriver } from './file.js';
|
|
3
|
+
import { memoryVectorDriver } from './memory.js';
|
|
4
|
+
import { pgVectorDriver } from './pgvector.js';
|
|
5
|
+
export { createFileVectorStore, fileVectorDriver } from './file.js';
|
|
6
|
+
export { createMemoryVectorStore, memoryVectorDriver } from './memory.js';
|
|
7
|
+
export { createPgVectorStore, pgVectorDriver, vectorLiteral } from './pgvector.js';
|
|
8
|
+
export { matchesFilter, VECTOR_DEFAULTS } from './types.js';
|
|
9
|
+
/**
|
|
10
|
+
* The vector drivers Cogenta ships, in tier order.
|
|
11
|
+
*
|
|
12
|
+
* `file` before `memory` among the degraded pair, for the same reason the cache
|
|
13
|
+
* registry orders them that way: both need no service, and the one that survives
|
|
14
|
+
* a restart is worth more than the one that does not. A site with no Postgres
|
|
15
|
+
* still gets working semantic search — that is R1, not a nicety.
|
|
16
|
+
*/
|
|
17
|
+
export function createVectorRegistry(options = {}) {
|
|
18
|
+
const registry = createDriverRegistry({
|
|
19
|
+
need: 'vector',
|
|
20
|
+
...(options.logger === undefined ? {} : { logger: options.logger }),
|
|
21
|
+
});
|
|
22
|
+
registry.register(pgVectorDriver(options.db === undefined ? {} : { db: options.db }));
|
|
23
|
+
registry.register(fileVectorDriver());
|
|
24
|
+
registry.register(memoryVectorDriver());
|
|
25
|
+
return registry;
|
|
26
|
+
}
|
|
27
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/rag/vector/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,oBAAoB,GAIrB,MAAM,eAAe,CAAA;AACtB,OAAO,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAA;AAC5C,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAA;AAChD,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAA;AAI9C,OAAO,EAAE,qBAAqB,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAA;AACnE,OAAO,EAAE,uBAAuB,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAA;AAEzE,OAAO,EAAE,mBAAmB,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAUlF,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AAQ3D;;;;;;;GAOG;AACH,MAAM,UAAU,oBAAoB,CAClC,OAAO,GAA0B,EAAE;IAEnC,MAAM,QAAQ,GAAG,oBAAoB,CAA4B;QAC/D,IAAI,EAAE,QAAQ;QACd,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;KACpE,CAAC,CAAA;IAEF,QAAQ,CAAC,QAAQ,CAAC,cAAc,CAAC,OAAO,CAAC,EAAE,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC,CAAA;IACrF,QAAQ,CAAC,QAAQ,CAAC,gBAAgB,EAAE,CAAC,CAAA;IACrC,QAAQ,CAAC,QAAQ,CAAC,kBAAkB,EAAE,CAAC,CAAA;IAEvC,OAAO,QAAQ,CAAA;AACjB,CAAC"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { type Driver } from '@cogenta/core';
|
|
2
|
+
import { type VectorConfig, type VectorStore } from './types.js';
|
|
3
|
+
/**
|
|
4
|
+
* The always-available tier (R1): every process has memory, nothing to install,
|
|
5
|
+
* nothing to configure. Exact cosine over every record in scope — which is what
|
|
6
|
+
* `docs/02-architecture.md` names as the degraded row for this need, and it is
|
|
7
|
+
* exact rather than approximate, so it is the *reference* answer the persistent
|
|
8
|
+
* drivers are checked against by the shared contract suite.
|
|
9
|
+
*
|
|
10
|
+
* Scoring is `vectorRank` from L4, unchanged and un-duplicated.
|
|
11
|
+
*/
|
|
12
|
+
export declare function createMemoryVectorStore(options: {
|
|
13
|
+
readonly dimensions: number;
|
|
14
|
+
}): VectorStore;
|
|
15
|
+
export declare function memoryVectorDriver(): Driver<VectorStore, VectorConfig>;
|
|
16
|
+
//# sourceMappingURL=memory.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"memory.d.ts","sourceRoot":"","sources":["../../../src/rag/vector/memory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,KAAK,MAAM,EAAqB,MAAM,eAAe,CAAA;AAE5E,OAAO,EAGL,KAAK,YAAY,EAKjB,KAAK,WAAW,EACjB,MAAM,YAAY,CAAA;AAEnB;;;;;;;;GAQG;AACH,wBAAgB,uBAAuB,CAAC,OAAO,EAAE;IAAE,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;CAAE,GAAG,WAAW,CA8E7F;AAED,wBAAgB,kBAAkB,IAAI,MAAM,CAAC,WAAW,EAAE,YAAY,CAAC,CA0BtE"}
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import { CogentaError } from '@cogenta/core';
|
|
2
|
+
import { vectorRank } from '../index/vector-rank.js';
|
|
3
|
+
import { matchesFilter, VECTOR_DEFAULTS, } from './types.js';
|
|
4
|
+
/**
|
|
5
|
+
* The always-available tier (R1): every process has memory, nothing to install,
|
|
6
|
+
* nothing to configure. Exact cosine over every record in scope — which is what
|
|
7
|
+
* `docs/02-architecture.md` names as the degraded row for this need, and it is
|
|
8
|
+
* exact rather than approximate, so it is the *reference* answer the persistent
|
|
9
|
+
* drivers are checked against by the shared contract suite.
|
|
10
|
+
*
|
|
11
|
+
* Scoring is `vectorRank` from L4, unchanged and un-duplicated.
|
|
12
|
+
*/
|
|
13
|
+
export function createMemoryVectorStore(options) {
|
|
14
|
+
const { dimensions } = options;
|
|
15
|
+
const records = new Map();
|
|
16
|
+
function assertDimensions(record) {
|
|
17
|
+
if (record.vector.length === dimensions)
|
|
18
|
+
return;
|
|
19
|
+
throw new CogentaError({
|
|
20
|
+
code: 'VECTOR_DIMENSION_MISMATCH',
|
|
21
|
+
message: `This store holds ${dimensions}-dimension vectors and was handed one of ${record.vector.length}.`,
|
|
22
|
+
hint: 'Changing embedding model means a new, parallel index — never mixing two models in one store. Re-index from scratch after a model change.',
|
|
23
|
+
details: { expected: dimensions, received: record.vector.length, chunkId: record.chunk.id },
|
|
24
|
+
});
|
|
25
|
+
}
|
|
26
|
+
return {
|
|
27
|
+
dimensions,
|
|
28
|
+
async upsert(incoming) {
|
|
29
|
+
// Validated up front, all of them, before a single write lands: a batch
|
|
30
|
+
// that fails half-way through would leave the index in a state no caller
|
|
31
|
+
// can reason about.
|
|
32
|
+
for (const record of incoming)
|
|
33
|
+
assertDimensions(record);
|
|
34
|
+
for (const record of incoming)
|
|
35
|
+
records.set(record.chunk.id, record);
|
|
36
|
+
},
|
|
37
|
+
async remove(chunkIds) {
|
|
38
|
+
for (const id of chunkIds)
|
|
39
|
+
records.delete(id);
|
|
40
|
+
},
|
|
41
|
+
async removeEntries(scope) {
|
|
42
|
+
const entries = new Set(scope.entryIds);
|
|
43
|
+
for (const [id, record] of records) {
|
|
44
|
+
if (record.siteId === scope.siteId &&
|
|
45
|
+
record.collection === scope.collection &&
|
|
46
|
+
entries.has(record.entryId)) {
|
|
47
|
+
records.delete(id);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
},
|
|
51
|
+
async search(queryVector, searchOptions) {
|
|
52
|
+
const scoped = [...records.values()].filter((record) => matchesFilter(record, searchOptions?.filter));
|
|
53
|
+
const byId = new Map(scoped.map((record) => [record.chunk.id, record]));
|
|
54
|
+
const ranked = vectorRank(scoped.map((record) => ({
|
|
55
|
+
chunk: record.chunk,
|
|
56
|
+
siteId: record.siteId,
|
|
57
|
+
vector: record.vector,
|
|
58
|
+
})), queryVector);
|
|
59
|
+
const limit = searchOptions?.limit ?? VECTOR_DEFAULTS.limit;
|
|
60
|
+
const minScore = searchOptions?.minScore;
|
|
61
|
+
const matches = [];
|
|
62
|
+
for (const { id, score } of ranked) {
|
|
63
|
+
if (minScore !== undefined && score < minScore)
|
|
64
|
+
break;
|
|
65
|
+
const record = byId.get(id);
|
|
66
|
+
if (record === undefined)
|
|
67
|
+
continue;
|
|
68
|
+
matches.push({ record, score });
|
|
69
|
+
if (matches.length >= limit)
|
|
70
|
+
break;
|
|
71
|
+
}
|
|
72
|
+
return matches;
|
|
73
|
+
},
|
|
74
|
+
async count(filter) {
|
|
75
|
+
return [...records.values()].filter((record) => matchesFilter(record, filter)).length;
|
|
76
|
+
},
|
|
77
|
+
async clear() {
|
|
78
|
+
records.clear();
|
|
79
|
+
},
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
export function memoryVectorDriver() {
|
|
83
|
+
let store = null;
|
|
84
|
+
return {
|
|
85
|
+
name: 'memory',
|
|
86
|
+
tier: 'degraded',
|
|
87
|
+
// Memory is the one thing that is always there. Saying so plainly is what
|
|
88
|
+
// makes it a usable last resort rather than one more thing that can refuse.
|
|
89
|
+
available: async () => true,
|
|
90
|
+
init: async (config) => {
|
|
91
|
+
store = createMemoryVectorStore({ dimensions: config.dimensions });
|
|
92
|
+
return store;
|
|
93
|
+
},
|
|
94
|
+
dispose: async () => {
|
|
95
|
+
await store?.clear();
|
|
96
|
+
store = null;
|
|
97
|
+
},
|
|
98
|
+
health: async () => ({
|
|
99
|
+
status: 'degraded',
|
|
100
|
+
driver: 'memory',
|
|
101
|
+
tier: 'degraded',
|
|
102
|
+
message: 'Embeddings are kept in memory and are lost on restart; the site re-indexes on the next ingest.',
|
|
103
|
+
...(store === null ? {} : { details: { records: await store.count() } }),
|
|
104
|
+
}),
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
//# sourceMappingURL=memory.js.map
|