@adonis-agora/agent 0.17.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +133 -0
- package/README.md +210 -0
- package/dist/providers/agent_provider.d.ts +6 -4
- package/dist/providers/agent_provider.d.ts.map +1 -1
- package/dist/providers/agent_provider.js +23 -16
- package/dist/providers/agent_provider.js.map +1 -1
- package/dist/src/agent-deps-factory.d.ts +6 -0
- package/dist/src/agent-deps-factory.d.ts.map +1 -1
- package/dist/src/agent-deps-factory.js +5 -1
- package/dist/src/agent-deps-factory.js.map +1 -1
- package/dist/src/agent-deps.d.ts +6 -1
- package/dist/src/agent-deps.d.ts.map +1 -1
- package/dist/src/agent-deps.js.map +1 -1
- package/dist/src/agent-loop.d.ts +11 -2
- package/dist/src/agent-loop.d.ts.map +1 -1
- package/dist/src/agent-loop.js +69 -2
- package/dist/src/agent-loop.js.map +1 -1
- package/dist/src/data/tenant-scope.d.ts +16 -1
- package/dist/src/data/tenant-scope.d.ts.map +1 -1
- package/dist/src/data/tenant-scope.js +50 -7
- package/dist/src/data/tenant-scope.js.map +1 -1
- package/dist/src/define_config.d.ts +25 -9
- package/dist/src/define_config.d.ts.map +1 -1
- package/dist/src/define_config.js.map +1 -1
- package/dist/src/governance-gate.d.ts +24 -8
- package/dist/src/governance-gate.d.ts.map +1 -1
- package/dist/src/governance-gate.js +14 -3
- package/dist/src/governance-gate.js.map +1 -1
- package/dist/src/index.d.ts +2 -1
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +5 -1
- package/dist/src/index.js.map +1 -1
- package/dist/src/rag/chunk.d.ts +35 -0
- package/dist/src/rag/chunk.d.ts.map +1 -1
- package/dist/src/rag/chunk.js +60 -0
- package/dist/src/rag/chunk.js.map +1 -1
- package/dist/src/rag/index.d.ts +1 -1
- package/dist/src/rag/index.d.ts.map +1 -1
- package/dist/src/rag/index.js +1 -1
- package/dist/src/rag/index.js.map +1 -1
- package/dist/src/rag/memory-vector-store.d.ts +20 -1
- package/dist/src/rag/memory-vector-store.d.ts.map +1 -1
- package/dist/src/rag/memory-vector-store.js +56 -1
- package/dist/src/rag/memory-vector-store.js.map +1 -1
- package/dist/src/rag/pg-vector-store.d.ts +32 -1
- package/dist/src/rag/pg-vector-store.d.ts.map +1 -1
- package/dist/src/rag/pg-vector-store.js +69 -0
- package/dist/src/rag/pg-vector-store.js.map +1 -1
- package/dist/src/rag/qdrant-store.d.ts +81 -2
- package/dist/src/rag/qdrant-store.d.ts.map +1 -1
- package/dist/src/rag/qdrant-store.js +148 -1
- package/dist/src/rag/qdrant-store.js.map +1 -1
- package/dist/src/rag/vector-store.d.ts +144 -0
- package/dist/src/rag/vector-store.d.ts.map +1 -1
- package/dist/src/rag/vector-store.js +73 -0
- package/dist/src/rag/vector-store.js.map +1 -1
- package/package.json +14 -9
|
@@ -1,11 +1,16 @@
|
|
|
1
1
|
import type { HttpContext } from '@adonisjs/core/http';
|
|
2
2
|
import type { Actor } from './types.js';
|
|
3
3
|
/**
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
4
|
+
* Authorization gate for the cross-actor `/agent/governance/*` read routes. It runs AFTER the actor
|
|
5
|
+
* is resolved (so the caller is already authenticated), and decides whether THIS actor may read the
|
|
6
|
+
* platform-wide governance read-model (every actor's spend/usage/threads/approvals). Return `false`
|
|
7
|
+
* to deny (the route replies `403`).
|
|
8
|
+
*
|
|
9
|
+
* The gate is what makes those routes exist: omit it and the provider does NOT mount
|
|
10
|
+
* `/agent/governance/*` at all (every route answers `404`). Set it to mount them gated, or set
|
|
11
|
+
* `governanceAuthorize: () => true` to deliberately restore the old behaviour of letting ANY
|
|
12
|
+
* authenticated actor read them. `GET /agent/approvals/mine` is unaffected — it stays mounted and
|
|
13
|
+
* scoped to the calling actor's own pending approvals.
|
|
9
14
|
*
|
|
10
15
|
* Mirrors `@adonis-agora/agent-dashboard`'s `authorize` hook so the JSON routes and the console SPA
|
|
11
16
|
* that reads them can be gated with the SAME predicate (typically "actor is ADMIN").
|
|
@@ -19,13 +24,24 @@ export interface GovernanceGateVerdict {
|
|
|
19
24
|
error?: string;
|
|
20
25
|
}
|
|
21
26
|
/**
|
|
22
|
-
* Decide whether an already-resolved actor
|
|
23
|
-
* the provider's governance gate, extracted so it can be unit tested without booting an app.
|
|
27
|
+
* Decide whether an already-resolved actor holds cross-actor governance privilege — the router-free
|
|
28
|
+
* core of the provider's governance gate, extracted so it can be unit tested without booting an app.
|
|
24
29
|
*
|
|
25
|
-
* - No `authorize` configured → `{ ok: true }` (
|
|
30
|
+
* - No `authorize` configured → `{ ok: true }` (see the caveat below — the provider never gets here).
|
|
26
31
|
* - `authorize` returns truthy → `{ ok: true }`.
|
|
27
32
|
* - `authorize` returns falsy → `{ ok: false, status: 403 }`.
|
|
28
33
|
* - `authorize` throws → `{ ok: false, status: 403 }` (fail-closed; the error message is surfaced).
|
|
34
|
+
*
|
|
35
|
+
* The `undefined` branch is a pure-function convenience, NOT a claim that governance is readable
|
|
36
|
+
* without a gate. Neither provider caller can reach it: `#resolveGovernanceActor` backs the
|
|
37
|
+
* `/agent/governance/*` routes, which are NOT mounted at all when no gate is configured (they answer
|
|
38
|
+
* `404`), so it always passes one; and `#assertOwner` — which reuses this predicate as the "may act
|
|
39
|
+
* across actors" seam for the per-actor run/thread routes — short-circuits to "not privileged" when
|
|
40
|
+
* no gate is configured, leaving ownership strict, rather than calling in. To mount the governance
|
|
41
|
+
* routes, set `governanceAuthorize` (typically an ADMIN check); to deliberately restore the old
|
|
42
|
+
* behaviour of letting ANY authenticated actor read them, set `governanceAuthorize: () => true`.
|
|
43
|
+
* Either way `GET /agent/approvals/mine` is unaffected — it stays mounted and scoped to the calling
|
|
44
|
+
* actor's own pending approvals.
|
|
29
45
|
*/
|
|
30
46
|
export declare function evaluateGovernanceGate(actor: Actor, ctx: HttpContext, authorize?: AgentGovernanceAuthorize): Promise<GovernanceGateVerdict>;
|
|
31
47
|
//# sourceMappingURL=governance-gate.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"governance-gate.d.ts","sourceRoot":"","sources":["../../src/governance-gate.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAExC
|
|
1
|
+
{"version":3,"file":"governance-gate.d.ts","sourceRoot":"","sources":["../../src/governance-gate.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAExC;;;;;;;;;;;;;;GAcG;AACH,MAAM,MAAM,wBAAwB,GAAG,CACrC,KAAK,EAAE,KAAK,EACZ,GAAG,EAAE,WAAW,KACb,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;AAEhC,oGAAoG;AACpG,MAAM,WAAW,qBAAqB;IACpC,EAAE,EAAE,OAAO,CAAC;IACZ,qGAAqG;IACrG,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAsB,sBAAsB,CAC1C,KAAK,EAAE,KAAK,EACZ,GAAG,EAAE,WAAW,EAChB,SAAS,CAAC,EAAE,wBAAwB,GACnC,OAAO,CAAC,qBAAqB,CAAC,CAYhC"}
|
|
@@ -1,11 +1,22 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Decide whether an already-resolved actor
|
|
3
|
-
* the provider's governance gate, extracted so it can be unit tested without booting an app.
|
|
2
|
+
* Decide whether an already-resolved actor holds cross-actor governance privilege — the router-free
|
|
3
|
+
* core of the provider's governance gate, extracted so it can be unit tested without booting an app.
|
|
4
4
|
*
|
|
5
|
-
* - No `authorize` configured → `{ ok: true }` (
|
|
5
|
+
* - No `authorize` configured → `{ ok: true }` (see the caveat below — the provider never gets here).
|
|
6
6
|
* - `authorize` returns truthy → `{ ok: true }`.
|
|
7
7
|
* - `authorize` returns falsy → `{ ok: false, status: 403 }`.
|
|
8
8
|
* - `authorize` throws → `{ ok: false, status: 403 }` (fail-closed; the error message is surfaced).
|
|
9
|
+
*
|
|
10
|
+
* The `undefined` branch is a pure-function convenience, NOT a claim that governance is readable
|
|
11
|
+
* without a gate. Neither provider caller can reach it: `#resolveGovernanceActor` backs the
|
|
12
|
+
* `/agent/governance/*` routes, which are NOT mounted at all when no gate is configured (they answer
|
|
13
|
+
* `404`), so it always passes one; and `#assertOwner` — which reuses this predicate as the "may act
|
|
14
|
+
* across actors" seam for the per-actor run/thread routes — short-circuits to "not privileged" when
|
|
15
|
+
* no gate is configured, leaving ownership strict, rather than calling in. To mount the governance
|
|
16
|
+
* routes, set `governanceAuthorize` (typically an ADMIN check); to deliberately restore the old
|
|
17
|
+
* behaviour of letting ANY authenticated actor read them, set `governanceAuthorize: () => true`.
|
|
18
|
+
* Either way `GET /agent/approvals/mine` is unaffected — it stays mounted and scoped to the calling
|
|
19
|
+
* actor's own pending approvals.
|
|
9
20
|
*/
|
|
10
21
|
export async function evaluateGovernanceGate(actor, ctx, authorize) {
|
|
11
22
|
if (authorize === undefined)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"governance-gate.js","sourceRoot":"","sources":["../../src/governance-gate.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"governance-gate.js","sourceRoot":"","sources":["../../src/governance-gate.ts"],"names":[],"mappings":"AA+BA;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,KAAY,EACZ,GAAgB,EAChB,SAAoC;IAEpC,IAAI,SAAS,KAAK,SAAS;QAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;IAC9D,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAC5C,OAAO,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC;IAC9F,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO;YACL,EAAE,EAAE,KAAK;YACT,MAAM,EAAE,GAAG;YACX,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW;SAC5D,CAAC;IACJ,CAAC;AACH,CAAC"}
|
package/dist/src/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export declare const VERSION = "0.
|
|
1
|
+
export declare const VERSION = "0.18.0";
|
|
2
2
|
export * from './types.js';
|
|
3
3
|
export * from './spi/tool.js';
|
|
4
4
|
export * from './base-tool.js';
|
|
@@ -62,4 +62,5 @@ export { LucidPricingStore } from './stores/lucid-pricing.js';
|
|
|
62
62
|
export { LucidGovernanceQueries } from './stores/lucid-governance-queries.js';
|
|
63
63
|
export { dataTool, SqlValidator, SqlValidationError, GroupTableAccessPolicy, TenantScopeRewriter, injectLimit, loadSqlParser, } from './data/index.js';
|
|
64
64
|
export type { DataToolConfig, DataToolResult, QueryRunner, SqlValidationResult, TableAccessPolicy, GroupTableAccessConfig, TenantScopeConfig, SqlParserLike, } from './data/index.js';
|
|
65
|
+
export { configure } from '../configure.js';
|
|
65
66
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/src/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,OAAO,WAAW,CAAC;AAEhC,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC;AAC9B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,yBAAyB,CAAC;AACxC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,sBAAsB,CAAC;AACrC,cAAc,uBAAuB,CAAC;AACtC,cAAc,sBAAsB,CAAC;AACrC,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AACtC,cAAc,yBAAyB,CAAC;AACxC,cAAc,0BAA0B,CAAC;AACzC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,6BAA6B,CAAC;AAC5C,cAAc,6BAA6B,CAAC;AAC5C,cAAc,oBAAoB,CAAC;AACnC,cAAc,mBAAmB,CAAC;AAClC,cAAc,gBAAgB,CAAC;AAC/B,OAAO,EACL,WAAW,EACX,eAAe,EACf,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,uBAAuB,CAAC;AAC/B,YAAY,EACV,gBAAgB,EAChB,kBAAkB,EAClB,YAAY,EACZ,WAAW,EACX,YAAY,GACb,MAAM,uBAAuB,CAAC;AAC/B,YAAY,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AACjE,cAAc,sBAAsB,CAAC;AACrC,cAAc,eAAe,CAAC;AAC9B,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EACL,YAAY,EACZ,kBAAkB,EAClB,kBAAkB,EAClB,iBAAiB,EACjB,qBAAqB,GACtB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,YAAY,EACZ,kBAAkB,EAClB,KAAK,aAAa,EAClB,KAAK,cAAc,GACpB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,wBAAwB,EACxB,oBAAoB,EACpB,qCAAqC,EACrC,uCAAuC,GACxC,MAAM,iBAAiB,CAAC;AACzB,YAAY,EACV,yBAAyB,EACzB,yBAAyB,EACzB,+BAA+B,GAChC,MAAM,iBAAiB,CAAC;AACzB,cAAc,kBAAkB,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAGtC,OAAO,EACL,YAAY,EACZ,MAAM,EACN,MAAM,EACN,aAAa,EACb,iBAAiB,EACjB,UAAU,EACV,gBAAgB,EAChB,UAAU,EACV,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAC3D,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,YAAY,EAAE,wBAAwB,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAC5F,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACnD,YAAY,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AACvD,YAAY,EACV,WAAW,EACX,mBAAmB,EACnB,YAAY,EACZ,WAAW,EACX,gBAAgB,EAChB,oBAAoB,EACpB,qBAAqB,EACrB,0BAA0B,EAC1B,YAAY,EACZ,YAAY,EACZ,WAAW,EACX,cAAc,EACd,cAAc,EACd,kBAAkB,EAClB,wBAAwB,EACxB,wBAAwB,EACxB,qBAAqB,EACrB,gBAAgB,EAChB,gBAAgB,EAChB,qBAAqB,EACrB,uBAAuB,EACvB,gBAAgB,EAChB,wBAAwB,EACxB,wBAAwB,EACxB,YAAY,EACZ,YAAY,EACZ,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,MAAM,EACN,UAAU,EACV,cAAc,EACd,uBAAuB,EACvB,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACV,aAAa,EACb,UAAU,EACV,SAAS,EACT,cAAc,EACd,qBAAqB,GACtB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EACL,aAAa,EACb,kBAAkB,EAClB,sBAAsB,EACtB,uBAAuB,GACxB,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACvE,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAC;AACpG,YAAY,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAC;AACtE,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACzC,YAAY,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAC;AAChE,OAAO,EACL,oBAAoB,EACpB,KAAK,2BAA2B,GACjC,MAAM,8BAA8B,CAAC;AACtC,YAAY,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,EACL,yBAAyB,EACzB,mBAAmB,EACnB,iBAAiB,EACjB,oBAAoB,GACrB,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AACpE,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,YAAY,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AACxD,YAAY,EAAE,WAAW,IAAI,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAC3E,OAAO,EAAE,iBAAiB,EAAE,MAAM,kCAAkC,CAAC;AACrE,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,YAAY,EACV,sBAAsB,EACtB,iBAAiB,EACjB,eAAe,EACf,qBAAqB,EACrB,sBAAsB,GACvB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACL,qBAAqB,EACrB,iBAAiB,EACjB,iBAAiB,EACjB,YAAY,GACb,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAC5D,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,MAAM,sCAAsC,CAAC;AAG9E,OAAO,EACL,QAAQ,EACR,YAAY,EACZ,kBAAkB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,WAAW,EACX,aAAa,GACd,MAAM,iBAAiB,CAAC;AACzB,YAAY,EACV,cAAc,EACd,cAAc,EACd,WAAW,EACX,mBAAmB,EACnB,iBAAiB,EACjB,sBAAsB,EACtB,iBAAiB,EACjB,aAAa,GACd,MAAM,iBAAiB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,OAAO,WAAW,CAAC;AAEhC,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC;AAC9B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,yBAAyB,CAAC;AACxC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,sBAAsB,CAAC;AACrC,cAAc,uBAAuB,CAAC;AACtC,cAAc,sBAAsB,CAAC;AACrC,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AACtC,cAAc,yBAAyB,CAAC;AACxC,cAAc,0BAA0B,CAAC;AACzC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,6BAA6B,CAAC;AAC5C,cAAc,6BAA6B,CAAC;AAC5C,cAAc,oBAAoB,CAAC;AACnC,cAAc,mBAAmB,CAAC;AAClC,cAAc,gBAAgB,CAAC;AAC/B,OAAO,EACL,WAAW,EACX,eAAe,EACf,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,uBAAuB,CAAC;AAC/B,YAAY,EACV,gBAAgB,EAChB,kBAAkB,EAClB,YAAY,EACZ,WAAW,EACX,YAAY,GACb,MAAM,uBAAuB,CAAC;AAC/B,YAAY,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AACjE,cAAc,sBAAsB,CAAC;AACrC,cAAc,eAAe,CAAC;AAC9B,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EACL,YAAY,EACZ,kBAAkB,EAClB,kBAAkB,EAClB,iBAAiB,EACjB,qBAAqB,GACtB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,YAAY,EACZ,kBAAkB,EAClB,KAAK,aAAa,EAClB,KAAK,cAAc,GACpB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,wBAAwB,EACxB,oBAAoB,EACpB,qCAAqC,EACrC,uCAAuC,GACxC,MAAM,iBAAiB,CAAC;AACzB,YAAY,EACV,yBAAyB,EACzB,yBAAyB,EACzB,+BAA+B,GAChC,MAAM,iBAAiB,CAAC;AACzB,cAAc,kBAAkB,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAGtC,OAAO,EACL,YAAY,EACZ,MAAM,EACN,MAAM,EACN,aAAa,EACb,iBAAiB,EACjB,UAAU,EACV,gBAAgB,EAChB,UAAU,EACV,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAC3D,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,YAAY,EAAE,wBAAwB,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAC5F,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACnD,YAAY,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AACvD,YAAY,EACV,WAAW,EACX,mBAAmB,EACnB,YAAY,EACZ,WAAW,EACX,gBAAgB,EAChB,oBAAoB,EACpB,qBAAqB,EACrB,0BAA0B,EAC1B,YAAY,EACZ,YAAY,EACZ,WAAW,EACX,cAAc,EACd,cAAc,EACd,kBAAkB,EAClB,wBAAwB,EACxB,wBAAwB,EACxB,qBAAqB,EACrB,gBAAgB,EAChB,gBAAgB,EAChB,qBAAqB,EACrB,uBAAuB,EACvB,gBAAgB,EAChB,wBAAwB,EACxB,wBAAwB,EACxB,YAAY,EACZ,YAAY,EACZ,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,MAAM,EACN,UAAU,EACV,cAAc,EACd,uBAAuB,EACvB,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACV,aAAa,EACb,UAAU,EACV,SAAS,EACT,cAAc,EACd,qBAAqB,GACtB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EACL,aAAa,EACb,kBAAkB,EAClB,sBAAsB,EACtB,uBAAuB,GACxB,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACvE,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAC;AACpG,YAAY,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAC;AACtE,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACzC,YAAY,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAC;AAChE,OAAO,EACL,oBAAoB,EACpB,KAAK,2BAA2B,GACjC,MAAM,8BAA8B,CAAC;AACtC,YAAY,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,EACL,yBAAyB,EACzB,mBAAmB,EACnB,iBAAiB,EACjB,oBAAoB,GACrB,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AACpE,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,YAAY,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AACxD,YAAY,EAAE,WAAW,IAAI,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAC3E,OAAO,EAAE,iBAAiB,EAAE,MAAM,kCAAkC,CAAC;AACrE,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,YAAY,EACV,sBAAsB,EACtB,iBAAiB,EACjB,eAAe,EACf,qBAAqB,EACrB,sBAAsB,GACvB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACL,qBAAqB,EACrB,iBAAiB,EACjB,iBAAiB,EACjB,YAAY,GACb,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAC5D,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,MAAM,sCAAsC,CAAC;AAG9E,OAAO,EACL,QAAQ,EACR,YAAY,EACZ,kBAAkB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,WAAW,EACX,aAAa,GACd,MAAM,iBAAiB,CAAC;AACzB,YAAY,EACV,cAAc,EACd,cAAc,EACd,WAAW,EACX,mBAAmB,EACnB,iBAAiB,EACjB,sBAAsB,EACtB,iBAAiB,EACjB,aAAa,GACd,MAAM,iBAAiB,CAAC;AAKzB,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC"}
|
package/dist/src/index.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export const VERSION = '0.
|
|
1
|
+
export const VERSION = '0.18.0';
|
|
2
2
|
export * from './types.js';
|
|
3
3
|
export * from './spi/tool.js';
|
|
4
4
|
export * from './base-tool.js';
|
|
@@ -48,4 +48,8 @@ export { LucidPricingStore } from './stores/lucid-pricing.js';
|
|
|
48
48
|
export { LucidGovernanceQueries } from './stores/lucid-governance-queries.js';
|
|
49
49
|
// ── Data satellite: governed read-only SQL tool ──────────────────────────────
|
|
50
50
|
export { dataTool, SqlValidator, SqlValidationError, GroupTableAccessPolicy, TenantScopeRewriter, injectLimit, loadSqlParser, } from './data/index.js';
|
|
51
|
+
// Re-export the configure hook from the package root so `node ace configure` finds it.
|
|
52
|
+
// AdonisJS imports the package MAIN and reads `configure` off the module namespace —
|
|
53
|
+
// the `./configure` subpath alone is never consulted.
|
|
54
|
+
export { configure } from '../configure.js';
|
|
51
55
|
//# sourceMappingURL=index.js.map
|
package/dist/src/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,OAAO,GAAG,QAAQ,CAAC;AAEhC,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC;AAC9B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,yBAAyB,CAAC;AACxC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,sBAAsB,CAAC;AACrC,cAAc,uBAAuB,CAAC;AACtC,cAAc,sBAAsB,CAAC;AACrC,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AACtC,cAAc,yBAAyB,CAAC;AACxC,cAAc,0BAA0B,CAAC;AACzC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,6BAA6B,CAAC;AAC5C,cAAc,6BAA6B,CAAC;AAC5C,cAAc,oBAAoB,CAAC;AACnC,cAAc,mBAAmB,CAAC;AAClC,cAAc,gBAAgB,CAAC;AAC/B,OAAO,EACL,WAAW,EACX,eAAe,EACf,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,uBAAuB,CAAC;AAS/B,cAAc,sBAAsB,CAAC;AACrC,cAAc,eAAe,CAAC;AAC9B,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EACL,YAAY,EACZ,kBAAkB,EAClB,kBAAkB,EAClB,iBAAiB,EACjB,qBAAqB,GACtB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,YAAY,EACZ,kBAAkB,GAGnB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,wBAAwB,EACxB,oBAAoB,EACpB,qCAAqC,EACrC,uCAAuC,GACxC,MAAM,iBAAiB,CAAC;AAMzB,cAAc,kBAAkB,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAEtC,gFAAgF;AAChF,OAAO,EACL,YAAY,EACZ,MAAM,EACN,MAAM,EACN,aAAa,EACb,iBAAiB,EACjB,UAAU,EACV,gBAAgB,EAChB,UAAU,EACV,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAC3D,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAE9D,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAgCnD,OAAO,EACL,MAAM,EACN,UAAU,EACV,cAAc,EACd,uBAAuB,EACvB,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,kBAAkB,CAAC;AAQ1B,OAAO,EACL,aAAa,EACb,kBAAkB,EAClB,sBAAsB,EACtB,uBAAuB,GACxB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAC;AAEpG,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAEzC,OAAO,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAC;AAChE,OAAO,EACL,oBAAoB,GAErB,MAAM,8BAA8B,CAAC;AAEtC,OAAO,EACL,yBAAyB,EACzB,mBAAmB,EACnB,iBAAiB,EACjB,oBAAoB,GACrB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAElD,OAAO,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AAExD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kCAAkC,CAAC;AACrE,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAQpD,OAAO,EACL,qBAAqB,EACrB,iBAAiB,EACjB,iBAAiB,EACjB,YAAY,GACb,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAC5D,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,MAAM,sCAAsC,CAAC;AAE9E,gFAAgF;AAChF,OAAO,EACL,QAAQ,EACR,YAAY,EACZ,kBAAkB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,WAAW,EACX,aAAa,GACd,MAAM,iBAAiB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,OAAO,GAAG,QAAQ,CAAC;AAEhC,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC;AAC9B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,yBAAyB,CAAC;AACxC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,sBAAsB,CAAC;AACrC,cAAc,uBAAuB,CAAC;AACtC,cAAc,sBAAsB,CAAC;AACrC,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AACtC,cAAc,yBAAyB,CAAC;AACxC,cAAc,0BAA0B,CAAC;AACzC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,6BAA6B,CAAC;AAC5C,cAAc,6BAA6B,CAAC;AAC5C,cAAc,oBAAoB,CAAC;AACnC,cAAc,mBAAmB,CAAC;AAClC,cAAc,gBAAgB,CAAC;AAC/B,OAAO,EACL,WAAW,EACX,eAAe,EACf,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,uBAAuB,CAAC;AAS/B,cAAc,sBAAsB,CAAC;AACrC,cAAc,eAAe,CAAC;AAC9B,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EACL,YAAY,EACZ,kBAAkB,EAClB,kBAAkB,EAClB,iBAAiB,EACjB,qBAAqB,GACtB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,YAAY,EACZ,kBAAkB,GAGnB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,wBAAwB,EACxB,oBAAoB,EACpB,qCAAqC,EACrC,uCAAuC,GACxC,MAAM,iBAAiB,CAAC;AAMzB,cAAc,kBAAkB,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAEtC,gFAAgF;AAChF,OAAO,EACL,YAAY,EACZ,MAAM,EACN,MAAM,EACN,aAAa,EACb,iBAAiB,EACjB,UAAU,EACV,gBAAgB,EAChB,UAAU,EACV,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAC3D,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAE9D,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAgCnD,OAAO,EACL,MAAM,EACN,UAAU,EACV,cAAc,EACd,uBAAuB,EACvB,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,kBAAkB,CAAC;AAQ1B,OAAO,EACL,aAAa,EACb,kBAAkB,EAClB,sBAAsB,EACtB,uBAAuB,GACxB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAC;AAEpG,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAEzC,OAAO,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAC;AAChE,OAAO,EACL,oBAAoB,GAErB,MAAM,8BAA8B,CAAC;AAEtC,OAAO,EACL,yBAAyB,EACzB,mBAAmB,EACnB,iBAAiB,EACjB,oBAAoB,GACrB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAElD,OAAO,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AAExD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kCAAkC,CAAC;AACrE,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAQpD,OAAO,EACL,qBAAqB,EACrB,iBAAiB,EACjB,iBAAiB,EACjB,YAAY,GACb,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAC5D,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,MAAM,sCAAsC,CAAC;AAE9E,gFAAgF;AAChF,OAAO,EACL,QAAQ,EACR,YAAY,EACZ,kBAAkB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,WAAW,EACX,aAAa,GACd,MAAM,iBAAiB,CAAC;AAYzB,uFAAuF;AACvF,qFAAqF;AACrF,sDAAsD;AACtD,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC"}
|
package/dist/src/rag/chunk.d.ts
CHANGED
|
@@ -3,12 +3,47 @@ export interface ChunkOptions {
|
|
|
3
3
|
chunkSize?: number;
|
|
4
4
|
/** Characters of overlap carried from the end of one chunk into the next. Default 100. */
|
|
5
5
|
overlap?: number;
|
|
6
|
+
/**
|
|
7
|
+
* The record delimiter, when the text has boundaries that *mean* something. Pass it and the chunker
|
|
8
|
+
* cuts ONLY on this string — never inside a record.
|
|
9
|
+
*
|
|
10
|
+
* Prose has no real boundaries, so the default heuristic (paragraph → sentence → word, in the back
|
|
11
|
+
* half of the window) is the right guess. Machine-shaped text is the opposite case: a spreadsheet
|
|
12
|
+
* flattened to one field-labelled record per line, a log, a JSONL export. There the boundaries are
|
|
13
|
+
* known exactly, and a heuristic cut lands mid-record — stranding the half that carries the row
|
|
14
|
+
* identifier in a different chunk from the half that carries the value, so neither chunk can answer
|
|
15
|
+
* a question about that row. `separator: '\n'` says "the lines are the records".
|
|
16
|
+
*
|
|
17
|
+
* Two consequences worth knowing before you pass it, both by design:
|
|
18
|
+
*
|
|
19
|
+
* 1. **`chunkSize` becomes a target, not a cap.** A single record longer than `chunkSize` is emitted
|
|
20
|
+
* whole, as its own over-size chunk. It is NOT split, and it does NOT fall back to the prose
|
|
21
|
+
* heuristic — a mid-record cut is the exact failure `separator` exists to prevent, so silently
|
|
22
|
+
* performing one to respect a size limit would defeat the feature. An over-size chunk is visible
|
|
23
|
+
* (you can see the record, and shorten it or raise `chunkSize`); a mangled record is not. It never
|
|
24
|
+
* drags its neighbours over the limit either: it always chunks alone.
|
|
25
|
+
* 2. **`overlap` is spent on whole records.** It stops being a character count and becomes a
|
|
26
|
+
* character *budget*, filled with entire trailing records of the previous chunk — as many as fit,
|
|
27
|
+
* never a partial one, and never so many that a chunk fails to advance. Half a record is not
|
|
28
|
+
* useful context; it is the same mid-record cut by another name. `overlap: 0` carries nothing,
|
|
29
|
+
* which is usually what self-contained records want.
|
|
30
|
+
*
|
|
31
|
+
* Empty or whitespace-only records are dropped. An empty string is treated as "not passed" (it would
|
|
32
|
+
* otherwise split every character), leaving the prose path. Omit it and NOTHING changes: the prose
|
|
33
|
+
* path below is untouched, byte-for-byte, so no existing corpus is re-chunked (and so none has to be
|
|
34
|
+
* re-embedded).
|
|
35
|
+
*/
|
|
36
|
+
separator?: string;
|
|
6
37
|
}
|
|
7
38
|
/**
|
|
8
39
|
* Splits text into overlapping chunks for embedding. Greedy up to `chunkSize`, but prefers to break on
|
|
9
40
|
* a paragraph → sentence → word boundary in the back half of the window (so a chunk rarely cuts
|
|
10
41
|
* mid-sentence), then carries `overlap` characters into the next chunk to preserve context across the
|
|
11
42
|
* seam. Pure and deterministic.
|
|
43
|
+
*
|
|
44
|
+
* Pass {@link ChunkOptions.separator} for text whose record boundaries are known rather than guessed;
|
|
45
|
+
* that path cuts only on the separator, and its two deviations (`chunkSize` becomes a target,
|
|
46
|
+
* `overlap` is spent on whole records) are documented there.
|
|
12
47
|
*/
|
|
13
48
|
export declare function chunkText(text: string, options?: ChunkOptions): string[];
|
|
14
49
|
//# sourceMappingURL=chunk.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chunk.d.ts","sourceRoot":"","sources":["../../../src/rag/chunk.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,YAAY;IAC3B,oDAAoD;IACpD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0FAA0F;IAC1F,OAAO,CAAC,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"chunk.d.ts","sourceRoot":"","sources":["../../../src/rag/chunk.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,YAAY;IAC3B,oDAAoD;IACpD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0FAA0F;IAC1F,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA6BG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;;;;GASG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,YAAiB,GAAG,MAAM,EAAE,CA2C5E"}
|
package/dist/src/rag/chunk.js
CHANGED
|
@@ -3,10 +3,18 @@
|
|
|
3
3
|
* a paragraph → sentence → word boundary in the back half of the window (so a chunk rarely cuts
|
|
4
4
|
* mid-sentence), then carries `overlap` characters into the next chunk to preserve context across the
|
|
5
5
|
* seam. Pure and deterministic.
|
|
6
|
+
*
|
|
7
|
+
* Pass {@link ChunkOptions.separator} for text whose record boundaries are known rather than guessed;
|
|
8
|
+
* that path cuts only on the separator, and its two deviations (`chunkSize` becomes a target,
|
|
9
|
+
* `overlap` is spent on whole records) are documented there.
|
|
6
10
|
*/
|
|
7
11
|
export function chunkText(text, options = {}) {
|
|
8
12
|
const chunkSize = options.chunkSize ?? 800;
|
|
9
13
|
const overlap = options.overlap ?? 100;
|
|
14
|
+
const separator = options.separator;
|
|
15
|
+
if (separator !== undefined && separator.length > 0) {
|
|
16
|
+
return chunkRecords(text, separator, chunkSize, overlap);
|
|
17
|
+
}
|
|
10
18
|
const normalized = text.trim();
|
|
11
19
|
if (normalized.length === 0) {
|
|
12
20
|
return [];
|
|
@@ -39,4 +47,56 @@ export function chunkText(text, options = {}) {
|
|
|
39
47
|
}
|
|
40
48
|
return chunks;
|
|
41
49
|
}
|
|
50
|
+
/**
|
|
51
|
+
* The {@link ChunkOptions.separator} path: pack WHOLE records into chunks. Records are the atoms here —
|
|
52
|
+
* no chunk ever contains a fragment of one — so `chunkSize` is honoured by taking fewer records rather
|
|
53
|
+
* than by cutting one, and `overlap` is honoured by repeating trailing records rather than by repeating
|
|
54
|
+
* characters. Pure and deterministic; see {@link ChunkOptions.separator} for the two trades.
|
|
55
|
+
*/
|
|
56
|
+
function chunkRecords(text, separator, chunkSize, overlap) {
|
|
57
|
+
const records = text
|
|
58
|
+
.split(separator)
|
|
59
|
+
.map((record) => record.trim())
|
|
60
|
+
.filter((record) => record.length > 0);
|
|
61
|
+
if (records.length === 0) {
|
|
62
|
+
return [];
|
|
63
|
+
}
|
|
64
|
+
const chunks = [];
|
|
65
|
+
let start = 0;
|
|
66
|
+
while (start < records.length) {
|
|
67
|
+
// Pack forward while the joined length fits. The FIRST record is taken unconditionally: that is
|
|
68
|
+
// what emits an over-long record whole instead of cutting it (trade 1).
|
|
69
|
+
let end = start;
|
|
70
|
+
let length = 0;
|
|
71
|
+
while (end < records.length) {
|
|
72
|
+
const record = records[end] ?? '';
|
|
73
|
+
const cost = end === start ? record.length : separator.length + record.length;
|
|
74
|
+
if (end > start && length + cost > chunkSize) {
|
|
75
|
+
break;
|
|
76
|
+
}
|
|
77
|
+
length += cost;
|
|
78
|
+
end += 1;
|
|
79
|
+
}
|
|
80
|
+
chunks.push(records.slice(start, end).join(separator));
|
|
81
|
+
if (end >= records.length) {
|
|
82
|
+
break;
|
|
83
|
+
}
|
|
84
|
+
// Overlap as a whole-record budget: walk back over trailing records while they fit in `overlap`.
|
|
85
|
+
// `back > start + 1` keeps at least one record consumed by this chunk, so `start` strictly
|
|
86
|
+
// increases — an over-long record (which always chunks alone) is therefore never carried forward,
|
|
87
|
+
// and no `overlap`, however large, can stall the loop.
|
|
88
|
+
let back = end;
|
|
89
|
+
let carried = 0;
|
|
90
|
+
while (back > start + 1) {
|
|
91
|
+
const cost = (records[back - 1] ?? '').length + separator.length;
|
|
92
|
+
if (carried + cost > overlap) {
|
|
93
|
+
break;
|
|
94
|
+
}
|
|
95
|
+
carried += cost;
|
|
96
|
+
back -= 1;
|
|
97
|
+
}
|
|
98
|
+
start = back;
|
|
99
|
+
}
|
|
100
|
+
return chunks;
|
|
101
|
+
}
|
|
42
102
|
//# sourceMappingURL=chunk.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chunk.js","sourceRoot":"","sources":["../../../src/rag/chunk.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"chunk.js","sourceRoot":"","sources":["../../../src/rag/chunk.ts"],"names":[],"mappings":"AAsCA;;;;;;;;;GASG;AACH,MAAM,UAAU,SAAS,CAAC,IAAY,EAAE,UAAwB,EAAE;IAChE,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,GAAG,CAAC;IAC3C,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,GAAG,CAAC;IACvC,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;IACpC,IAAI,SAAS,KAAK,SAAS,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpD,OAAO,YAAY,CAAC,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IAC3D,CAAC;IACD,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC/B,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,IAAI,UAAU,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;QACnC,OAAO,CAAC,UAAU,CAAC,CAAC;IACtB,CAAC;IAED,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,OAAO,KAAK,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC;QACjC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,SAAS,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;QAC3D,IAAI,OAAO,GAAG,GAAG,CAAC;QAClB,IAAI,GAAG,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC;YAC5B,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;YAC5C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CACvB,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,EAC1B,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,EACxB,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,CACxB,CAAC;YACF,uEAAuE;YACvE,IAAI,QAAQ,GAAG,SAAS,GAAG,CAAC,EAAE,CAAC;gBAC7B,OAAO,GAAG,KAAK,GAAG,QAAQ,GAAG,CAAC,CAAC;YACjC,CAAC;QACH,CAAC;QACD,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;QACtD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;QACD,IAAI,OAAO,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;YACjC,MAAM;QACR,CAAC;QACD,yFAAyF;QACzF,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,OAAO,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;IACjD,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;GAKG;AACH,SAAS,YAAY,CACnB,IAAY,EACZ,SAAiB,EACjB,SAAiB,EACjB,OAAe;IAEf,MAAM,OAAO,GAAG,IAAI;SACjB,KAAK,CAAC,SAAS,CAAC;SAChB,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;SAC9B,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACzC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,OAAO,KAAK,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;QAC9B,gGAAgG;QAChG,wEAAwE;QACxE,IAAI,GAAG,GAAG,KAAK,CAAC;QAChB,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,OAAO,GAAG,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;YAC5B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;YAC9E,IAAI,GAAG,GAAG,KAAK,IAAI,MAAM,GAAG,IAAI,GAAG,SAAS,EAAE,CAAC;gBAC7C,MAAM;YACR,CAAC;YACD,MAAM,IAAI,IAAI,CAAC;YACf,GAAG,IAAI,CAAC,CAAC;QACX,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;QACvD,IAAI,GAAG,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM;QACR,CAAC;QACD,iGAAiG;QACjG,2FAA2F;QAC3F,kGAAkG;QAClG,uDAAuD;QACvD,IAAI,IAAI,GAAG,GAAG,CAAC;QACf,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,OAAO,IAAI,GAAG,KAAK,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,GAAG,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC;YACjE,IAAI,OAAO,GAAG,IAAI,GAAG,OAAO,EAAE,CAAC;gBAC7B,MAAM;YACR,CAAC;YACD,OAAO,IAAI,IAAI,CAAC;YAChB,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;QACD,KAAK,GAAG,IAAI,CAAC;IACf,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
package/dist/src/rag/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export { chunkText, type ChunkOptions } from './chunk.js';
|
|
2
|
-
export { documentIdOf, type IndexedDocument, type VectorRecord, type VectorSearchOptions, type VectorStore, } from './vector-store.js';
|
|
2
|
+
export { applyMetadataPatch, assertRemovalFilter, documentIdOf, effectivePatchKeys, filterDeniesAll, UnsafeRemovalError, type IndexedDocument, type MetadataPatch, type VectorRecord, type VectorSearchOptions, type VectorStore, } from './vector-store.js';
|
|
3
3
|
export { matchesFilter } from './filter.js';
|
|
4
4
|
export { MemoryVectorStore, cosineSimilarity } from './memory-vector-store.js';
|
|
5
5
|
export { EmbeddingRetriever } from './embedding-retriever.js';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/rag/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,KAAK,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1D,OAAO,EACL,YAAY,EACZ,KAAK,eAAe,EACpB,KAAK,YAAY,EACjB,KAAK,mBAAmB,EACxB,KAAK,WAAW,GACjB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAC/E,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EACL,kBAAkB,EAClB,KAAK,yBAAyB,GAC/B,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,gBAAgB,EAAE,KAAK,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AACxF,OAAO,EAAE,eAAe,EAAE,KAAK,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AACrF,OAAO,EACL,aAAa,EACb,iBAAiB,EACjB,eAAe,EACf,KAAK,cAAc,EACnB,KAAK,eAAe,EACpB,KAAK,oBAAoB,GAC1B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,cAAc,EACd,YAAY,EACZ,eAAe,EACf,KAAK,WAAW,EAChB,KAAK,mBAAmB,EACxB,KAAK,cAAc,EACnB,KAAK,aAAa,GACnB,MAAM,aAAa,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/rag/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,KAAK,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1D,OAAO,EACL,kBAAkB,EAClB,mBAAmB,EACnB,YAAY,EACZ,kBAAkB,EAClB,eAAe,EACf,kBAAkB,EAClB,KAAK,eAAe,EACpB,KAAK,aAAa,EAClB,KAAK,YAAY,EACjB,KAAK,mBAAmB,EACxB,KAAK,WAAW,GACjB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAC/E,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EACL,kBAAkB,EAClB,KAAK,yBAAyB,GAC/B,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,gBAAgB,EAAE,KAAK,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AACxF,OAAO,EAAE,eAAe,EAAE,KAAK,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AACrF,OAAO,EACL,aAAa,EACb,iBAAiB,EACjB,eAAe,EACf,KAAK,cAAc,EACnB,KAAK,eAAe,EACpB,KAAK,oBAAoB,GAC1B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,cAAc,EACd,YAAY,EACZ,eAAe,EACf,KAAK,WAAW,EAChB,KAAK,mBAAmB,EACxB,KAAK,cAAc,EACnB,KAAK,aAAa,GACnB,MAAM,aAAa,CAAC"}
|
package/dist/src/rag/index.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export { chunkText } from './chunk.js';
|
|
2
|
-
export { documentIdOf, } from './vector-store.js';
|
|
2
|
+
export { applyMetadataPatch, assertRemovalFilter, documentIdOf, effectivePatchKeys, filterDeniesAll, UnsafeRemovalError, } from './vector-store.js';
|
|
3
3
|
export { matchesFilter } from './filter.js';
|
|
4
4
|
export { MemoryVectorStore, cosineSimilarity } from './memory-vector-store.js';
|
|
5
5
|
export { EmbeddingRetriever } from './embedding-retriever.js';
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/rag/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAqB,MAAM,YAAY,CAAC;AAC1D,OAAO,EACL,YAAY,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/rag/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAqB,MAAM,YAAY,CAAC;AAC1D,OAAO,EACL,kBAAkB,EAClB,mBAAmB,EACnB,YAAY,EACZ,kBAAkB,EAClB,eAAe,EACf,kBAAkB,GAMnB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAC/E,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EACL,kBAAkB,GAEnB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,gBAAgB,EAAgC,MAAM,wBAAwB,CAAC;AACxF,OAAO,EAAE,eAAe,EAA+B,MAAM,uBAAuB,CAAC;AACrF,OAAO,EACL,aAAa,EACb,iBAAiB,EACjB,eAAe,GAIhB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,cAAc,EACd,YAAY,EACZ,eAAe,GAKhB,MAAM,aAAa,CAAC"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { Passage } from '../spi/retriever.js';
|
|
2
|
-
import { type IndexedDocument, type VectorRecord, type VectorSearchOptions, type VectorStore } from './vector-store.js';
|
|
2
|
+
import { type IndexedDocument, type MetadataPatch, type VectorRecord, type VectorSearchOptions, type VectorStore } from './vector-store.js';
|
|
3
3
|
/**
|
|
4
4
|
* An in-process {@link VectorStore} — cosine similarity over a Map, no infra. The reference adapter for
|
|
5
5
|
* tests and small/embedded corpora; a pgvector/Lucid-backed store for production scale is deferred.
|
|
@@ -8,7 +8,26 @@ export declare class MemoryVectorStore implements VectorStore {
|
|
|
8
8
|
private readonly records;
|
|
9
9
|
upsert(records: VectorRecord[]): Promise<void>;
|
|
10
10
|
remove(documentId: string): Promise<void>;
|
|
11
|
+
/**
|
|
12
|
+
* {@link VectorStore.updateMetadata} — rewrite every chunk's metadata for one document, touching
|
|
13
|
+
* neither `text` nor `embedding`. Both are carried across by the spread, so the stored `embedding`
|
|
14
|
+
* stays the **same array instance** it was upserted with: nothing here can perturb a vector even in
|
|
15
|
+
* principle. Merge semantics come from the shared {@link applyMetadataPatch}, so this adapter cannot
|
|
16
|
+
* drift from the other two.
|
|
17
|
+
*/
|
|
18
|
+
updateMetadata(documentId: string, patch: MetadataPatch): Promise<number>;
|
|
11
19
|
listDocuments(filter?: Record<string, unknown>): Promise<IndexedDocument[]>;
|
|
20
|
+
/**
|
|
21
|
+
* {@link VectorStore.listDocumentIds} — the ids `listDocuments` would report, without building the
|
|
22
|
+
* per-document metadata entries. Filtering goes through the same {@link matchesFilter} `search` uses.
|
|
23
|
+
*/
|
|
24
|
+
listDocumentIds(filter?: Record<string, unknown>): Promise<string[]>;
|
|
25
|
+
/**
|
|
26
|
+
* {@link VectorStore.removeWhere} — delete every chunk the same filter would let `search` reach.
|
|
27
|
+
* Parity is by construction: the predicate below is the identical {@link matchesFilter} call, on the
|
|
28
|
+
* identical argument, as the one in {@link MemoryVectorStore.search}.
|
|
29
|
+
*/
|
|
30
|
+
removeWhere(filter: Record<string, unknown>): Promise<number>;
|
|
12
31
|
search(embedding: number[], options: VectorSearchOptions): Promise<Passage[]>;
|
|
13
32
|
}
|
|
14
33
|
/** Cosine similarity of two vectors; `0` when either has zero magnitude. Pure. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"memory-vector-store.d.ts","sourceRoot":"","sources":["../../../src/rag/memory-vector-store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AAEnD,OAAO,EACL,KAAK,eAAe,EACpB,KAAK,YAAY,EACjB,KAAK,mBAAmB,EACxB,KAAK,WAAW,
|
|
1
|
+
{"version":3,"file":"memory-vector-store.d.ts","sourceRoot":"","sources":["../../../src/rag/memory-vector-store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AAEnD,OAAO,EACL,KAAK,eAAe,EACpB,KAAK,aAAa,EAClB,KAAK,YAAY,EACjB,KAAK,mBAAmB,EACxB,KAAK,WAAW,EAMjB,MAAM,mBAAmB,CAAC;AAE3B;;;GAGG;AACH,qBAAa,iBAAkB,YAAW,WAAW;IACnD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmC;IAErD,MAAM,CAAC,OAAO,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAM9C,MAAM,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAQ/C;;;;;;OAMG;IACG,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC;IAezE,aAAa,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAiBjF;;;OAGG;IACG,eAAe,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAW1E;;;;OAIG;IACG,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IAgB7D,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;CAsBpF;AAED,kFAAkF;AAClF,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,CAcjE"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { matchesFilter } from './filter.js';
|
|
2
|
-
import { documentIdOf, } from './vector-store.js';
|
|
2
|
+
import { applyMetadataPatch, assertRemovalFilter, documentIdOf, effectivePatchKeys, filterDeniesAll, } from './vector-store.js';
|
|
3
3
|
/**
|
|
4
4
|
* An in-process {@link VectorStore} — cosine similarity over a Map, no infra. The reference adapter for
|
|
5
5
|
* tests and small/embedded corpora; a pgvector/Lucid-backed store for production scale is deferred.
|
|
@@ -18,6 +18,27 @@ export class MemoryVectorStore {
|
|
|
18
18
|
}
|
|
19
19
|
}
|
|
20
20
|
}
|
|
21
|
+
/**
|
|
22
|
+
* {@link VectorStore.updateMetadata} — rewrite every chunk's metadata for one document, touching
|
|
23
|
+
* neither `text` nor `embedding`. Both are carried across by the spread, so the stored `embedding`
|
|
24
|
+
* stays the **same array instance** it was upserted with: nothing here can perturb a vector even in
|
|
25
|
+
* principle. Merge semantics come from the shared {@link applyMetadataPatch}, so this adapter cannot
|
|
26
|
+
* drift from the other two.
|
|
27
|
+
*/
|
|
28
|
+
async updateMetadata(documentId, patch) {
|
|
29
|
+
if (effectivePatchKeys(patch).length === 0) {
|
|
30
|
+
return 0;
|
|
31
|
+
}
|
|
32
|
+
let written = 0;
|
|
33
|
+
for (const [id, record] of this.records) {
|
|
34
|
+
if (documentIdOf(id) !== documentId) {
|
|
35
|
+
continue;
|
|
36
|
+
}
|
|
37
|
+
this.records.set(id, { ...record, metadata: applyMetadataPatch(record.metadata, patch) });
|
|
38
|
+
written += 1;
|
|
39
|
+
}
|
|
40
|
+
return written;
|
|
41
|
+
}
|
|
21
42
|
async listDocuments(filter) {
|
|
22
43
|
const documents = new Map();
|
|
23
44
|
for (const record of this.records.values()) {
|
|
@@ -34,6 +55,40 @@ export class MemoryVectorStore {
|
|
|
34
55
|
}
|
|
35
56
|
return [...documents.values()];
|
|
36
57
|
}
|
|
58
|
+
/**
|
|
59
|
+
* {@link VectorStore.listDocumentIds} — the ids `listDocuments` would report, without building the
|
|
60
|
+
* per-document metadata entries. Filtering goes through the same {@link matchesFilter} `search` uses.
|
|
61
|
+
*/
|
|
62
|
+
async listDocumentIds(filter) {
|
|
63
|
+
const ids = new Set();
|
|
64
|
+
for (const record of this.records.values()) {
|
|
65
|
+
if (filter !== undefined && !matchesFilter(record.metadata, filter)) {
|
|
66
|
+
continue;
|
|
67
|
+
}
|
|
68
|
+
ids.add(documentIdOf(record.id));
|
|
69
|
+
}
|
|
70
|
+
return [...ids];
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* {@link VectorStore.removeWhere} — delete every chunk the same filter would let `search` reach.
|
|
74
|
+
* Parity is by construction: the predicate below is the identical {@link matchesFilter} call, on the
|
|
75
|
+
* identical argument, as the one in {@link MemoryVectorStore.search}.
|
|
76
|
+
*/
|
|
77
|
+
async removeWhere(filter) {
|
|
78
|
+
assertRemovalFilter(filter);
|
|
79
|
+
if (filterDeniesAll(filter)) {
|
|
80
|
+
return 0;
|
|
81
|
+
}
|
|
82
|
+
let removed = 0;
|
|
83
|
+
for (const [id, record] of [...this.records]) {
|
|
84
|
+
if (!matchesFilter(record.metadata, filter)) {
|
|
85
|
+
continue;
|
|
86
|
+
}
|
|
87
|
+
this.records.delete(id);
|
|
88
|
+
removed += 1;
|
|
89
|
+
}
|
|
90
|
+
return removed;
|
|
91
|
+
}
|
|
37
92
|
async search(embedding, options) {
|
|
38
93
|
const scored = [];
|
|
39
94
|
for (const record of this.records.values()) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"memory-vector-store.js","sourceRoot":"","sources":["../../../src/rag/memory-vector-store.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,
|
|
1
|
+
{"version":3,"file":"memory-vector-store.js","sourceRoot":"","sources":["../../../src/rag/memory-vector-store.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAML,kBAAkB,EAClB,mBAAmB,EACnB,YAAY,EACZ,kBAAkB,EAClB,eAAe,GAChB,MAAM,mBAAmB,CAAC;AAE3B;;;GAGG;AACH,MAAM,OAAO,iBAAiB;IACX,OAAO,GAAG,IAAI,GAAG,EAAwB,CAAC;IAE3D,KAAK,CAAC,MAAM,CAAC,OAAuB;QAClC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,UAAkB;QAC7B,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;YACrC,IAAI,YAAY,CAAC,EAAE,CAAC,KAAK,UAAU,EAAE,CAAC;gBACpC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,cAAc,CAAC,UAAkB,EAAE,KAAoB;QAC3D,IAAI,kBAAkB,CAAC,KAAK,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3C,OAAO,CAAC,CAAC;QACX,CAAC;QACD,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACxC,IAAI,YAAY,CAAC,EAAE,CAAC,KAAK,UAAU,EAAE,CAAC;gBACpC,SAAS;YACX,CAAC;YACD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,GAAG,MAAM,EAAE,QAAQ,EAAE,kBAAkB,CAAC,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC;YAC1F,OAAO,IAAI,CAAC,CAAC;QACf,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,MAAgC;QAClD,MAAM,SAAS,GAAG,IAAI,GAAG,EAA2B,CAAC;QACrD,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YAC3C,IAAI,MAAM,KAAK,SAAS,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC;gBACpE,SAAS;YACX,CAAC;YACD,MAAM,EAAE,GAAG,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACnC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;gBACvB,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE;oBAChB,EAAE;oBACF,GAAG,CAAC,MAAM,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBACxE,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,OAAO,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;IACjC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,eAAe,CAAC,MAAgC;QACpD,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;QAC9B,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YAC3C,IAAI,MAAM,KAAK,SAAS,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC;gBACpE,SAAS;YACX,CAAC;YACD,GAAG,CAAC,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QACnC,CAAC;QACD,OAAO,CAAC,GAAG,GAAG,CAAC,CAAC;IAClB,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,WAAW,CAAC,MAA+B;QAC/C,mBAAmB,CAAC,MAAM,CAAC,CAAC;QAC5B,IAAI,eAAe,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5B,OAAO,CAAC,CAAC;QACX,CAAC;QACD,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC7C,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC;gBAC5C,SAAS;YACX,CAAC;YACD,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACxB,OAAO,IAAI,CAAC,CAAC;QACf,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,SAAmB,EAAE,OAA4B;QAC5D,MAAM,MAAM,GAAc,EAAE,CAAC;QAC7B,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YAC3C,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBACpF,SAAS;YACX,CAAC;YACD,MAAM,KAAK,GAAG,gBAAgB,CAAC,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;YAC5D,2FAA2F;YAC3F,4DAA4D;YAC5D,IAAI,OAAO,CAAC,QAAQ,KAAK,SAAS,IAAI,KAAK,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;gBAC/D,SAAS;YACX,CAAC;YACD,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,MAAM,CAAC,EAAE;gBACb,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,KAAK;gBACL,GAAG,CAAC,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACjE,GAAG,CAAC,MAAM,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACxE,CAAC,CAAC;QACL,CAAC;QACD,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IACzE,CAAC;CACF;AAED,kFAAkF;AAClF,MAAM,UAAU,gBAAgB,CAAC,CAAW,EAAE,CAAW;IACvD,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;IAC5C,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QAC/C,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC7B,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC7B,GAAG,IAAI,MAAM,GAAG,MAAM,CAAC;QACvB,KAAK,IAAI,MAAM,GAAG,MAAM,CAAC;QACzB,KAAK,IAAI,MAAM,GAAG,MAAM,CAAC;IAC3B,CAAC;IACD,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACxD,OAAO,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,WAAW,CAAC;AACnD,CAAC"}
|
|
@@ -2,7 +2,7 @@ import type { EmbeddingProvider } from '../spi/embedding-provider.js';
|
|
|
2
2
|
import type { Passage } from '../spi/retriever.js';
|
|
3
3
|
import type { LucidDatabaseLike } from '../stores/lucid.js';
|
|
4
4
|
import { EmbeddingRetriever } from './embedding-retriever.js';
|
|
5
|
-
import type { IndexedDocument, VectorRecord, VectorSearchOptions, VectorStore } from './vector-store.js';
|
|
5
|
+
import type { IndexedDocument, MetadataPatch, VectorRecord, VectorSearchOptions, VectorStore } from './vector-store.js';
|
|
6
6
|
/**
|
|
7
7
|
* The similarity metric — which pgvector distance operator ranks results (and which index opclass the
|
|
8
8
|
* migration should build). `cosine` (the default) is scale-invariant and the right choice for most
|
|
@@ -71,7 +71,38 @@ export declare class PgVectorStore implements VectorStore {
|
|
|
71
71
|
schemaStatements(): string[];
|
|
72
72
|
upsert(records: VectorRecord[]): Promise<void>;
|
|
73
73
|
remove(documentId: string): Promise<void>;
|
|
74
|
+
/**
|
|
75
|
+
* {@link VectorStore.updateMetadata} — one `UPDATE` for the whole document, with the
|
|
76
|
+
* {@link import('./vector-store.js').MetadataPatch} merge done **in Postgres** rather than read back
|
|
77
|
+
* into JS: `(COALESCE(metadata, '{}') || <set>) - <removed keys>`. `||` is jsonb concatenation, which
|
|
78
|
+
* at the top level is exactly a shallow merge (right-hand keys win, wholesale); `jsonb - text[]`
|
|
79
|
+
* removes keys, which is how a `null` in the patch deletes. So there is no read-modify-write and no
|
|
80
|
+
* lost-update window between a `SELECT` and an `UPDATE`.
|
|
81
|
+
*
|
|
82
|
+
* The embedding is untouched **structurally**: `SET` names only the metadata column, so no statement
|
|
83
|
+
* this method can emit is capable of writing `embedding` or `text`. The row is not re-inserted, so
|
|
84
|
+
* pgvector never re-parses a vector literal either.
|
|
85
|
+
*
|
|
86
|
+
* `RETURNING` the id column yields the chunk count without a second query.
|
|
87
|
+
*/
|
|
88
|
+
updateMetadata(documentId: string, patch: MetadataPatch): Promise<number>;
|
|
74
89
|
listDocuments(filter?: Record<string, unknown>): Promise<IndexedDocument[]>;
|
|
90
|
+
/**
|
|
91
|
+
* {@link VectorStore.listDocumentIds} — `SELECT DISTINCT` of the document-id expression. Unlike
|
|
92
|
+
* {@link PgVectorStore.listDocuments} it never selects the `jsonb` metadata column, so Postgres does not
|
|
93
|
+
* transfer it and {@link parseMetadata} is never called: on a large corpus that is most of the cost.
|
|
94
|
+
* The `WHERE` comes from the same {@link buildMetadataWhere} `search` uses.
|
|
95
|
+
*/
|
|
96
|
+
listDocumentIds(filter?: Record<string, unknown>): Promise<string[]>;
|
|
97
|
+
/**
|
|
98
|
+
* {@link VectorStore.removeWhere} — one `DELETE … RETURNING`, so the chunk count comes back from the
|
|
99
|
+
* same statement that does the deleting: no count-then-delete, no race between the two, one round trip.
|
|
100
|
+
*
|
|
101
|
+
* Filter parity with {@link PgVectorStore.search} is by construction — the `WHERE` fragment is produced
|
|
102
|
+
* by the identical {@link buildMetadataWhere} call, so an empty-array value emits the same `false`
|
|
103
|
+
* clause here as it does there, and a scalar the same `@> ?::jsonb` containment.
|
|
104
|
+
*/
|
|
105
|
+
removeWhere(filter: Record<string, unknown>): Promise<number>;
|
|
75
106
|
search(embedding: number[], options: VectorSearchOptions): Promise<Passage[]>;
|
|
76
107
|
}
|
|
77
108
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"pg-vector-store.d.ts","sourceRoot":"","sources":["../../../src/rag/pg-vector-store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AACtE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;
|
|
1
|
+
{"version":3,"file":"pg-vector-store.d.ts","sourceRoot":"","sources":["../../../src/rag/pg-vector-store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AACtE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAE9D,OAAO,KAAK,EACV,eAAe,EACf,aAAa,EACb,YAAY,EACZ,mBAAmB,EACnB,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAE3B;;;;GAIG;AACH,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,IAAI,GAAG,OAAO,CAAC;AAEvD;;;;;GAKG;AACH,MAAM,WAAW,eAAe;IAC9B,6EAA6E;IAC7E,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,yCAAyC;IACzC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,2DAA2D;IAC3D,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,8DAA8D;IAC9D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,yDAAyD;IACzD,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,oBAAoB;IACnC,sFAAsF;IACtF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,oGAAoG;IACpG,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,+DAA+D;IAC/D,MAAM,CAAC,EAAE,cAAc,CAAC;IACxB,2DAA2D;IAC3D,OAAO,CAAC,EAAE,eAAe,CAAC;CAC3B;AAsDD,yGAAyG;AACzG,wBAAgB,eAAe,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,MAAM,CAE3D;AAyFD;;;;;;;;;;;;;;GAcG;AACH,qBAAa,aAAc,YAAW,WAAW;IAQ7C,OAAO,CAAC,QAAQ,CAAC,EAAE;IAPrB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAa;IACpC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAiB;IAC5C,OAAO,CAAC,QAAQ,CAAC,GAAG,CAA4B;gBAG7B,EAAE,EAAE,iBAAiB,EACtC,OAAO,GAAE,oBAAyB;IAgBpC;;;OAGG;IACG,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAMnC,uGAAuG;IACvG,gBAAgB,IAAI,MAAM,EAAE;IAgBtB,MAAM,CAAC,OAAO,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAsB9C,MAAM,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAM/C;;;;;;;;;;;;;OAaG;IACG,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC;IAyBzE,aAAa,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAoBjF;;;;;OAKG;IACG,eAAe,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAa1E;;;;;;;OAOG;IACG,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IAa7D,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;CA+CpF;AAED;;;;GAIG;AACH,qBAAa,iBAAkB,SAAQ,kBAAkB;gBAE3C,QAAQ,EAAE,iBAAiB,EAAE,KAAK,EAAE,aAAa;CAG9D"}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { EmbeddingRetriever } from './embedding-retriever.js';
|
|
2
|
+
import { assertRemovalFilter, effectivePatchKeys, filterDeniesAll } from './vector-store.js';
|
|
2
3
|
const METRICS = {
|
|
3
4
|
cosine: {
|
|
4
5
|
operator: '<=>',
|
|
@@ -203,6 +204,42 @@ export class PgVectorStore {
|
|
|
203
204
|
documentId,
|
|
204
205
|
]);
|
|
205
206
|
}
|
|
207
|
+
/**
|
|
208
|
+
* {@link VectorStore.updateMetadata} — one `UPDATE` for the whole document, with the
|
|
209
|
+
* {@link import('./vector-store.js').MetadataPatch} merge done **in Postgres** rather than read back
|
|
210
|
+
* into JS: `(COALESCE(metadata, '{}') || <set>) - <removed keys>`. `||` is jsonb concatenation, which
|
|
211
|
+
* at the top level is exactly a shallow merge (right-hand keys win, wholesale); `jsonb - text[]`
|
|
212
|
+
* removes keys, which is how a `null` in the patch deletes. So there is no read-modify-write and no
|
|
213
|
+
* lost-update window between a `SELECT` and an `UPDATE`.
|
|
214
|
+
*
|
|
215
|
+
* The embedding is untouched **structurally**: `SET` names only the metadata column, so no statement
|
|
216
|
+
* this method can emit is capable of writing `embedding` or `text`. The row is not re-inserted, so
|
|
217
|
+
* pgvector never re-parses a vector literal either.
|
|
218
|
+
*
|
|
219
|
+
* `RETURNING` the id column yields the chunk count without a second query.
|
|
220
|
+
*/
|
|
221
|
+
async updateMetadata(documentId, patch) {
|
|
222
|
+
const keys = effectivePatchKeys(patch);
|
|
223
|
+
if (keys.length === 0) {
|
|
224
|
+
return 0;
|
|
225
|
+
}
|
|
226
|
+
const set = {};
|
|
227
|
+
const removed = [];
|
|
228
|
+
for (const key of keys) {
|
|
229
|
+
if (patch[key] === null) {
|
|
230
|
+
removed.push(key);
|
|
231
|
+
}
|
|
232
|
+
else {
|
|
233
|
+
set[key] = patch[key];
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
const c = this.col;
|
|
237
|
+
const raw = await this.db.rawQuery(`UPDATE ${this.table}
|
|
238
|
+
SET ${c.metadata} = (COALESCE(${c.metadata}, '{}'::jsonb) || ?::jsonb) - ?::text[]
|
|
239
|
+
WHERE ${documentIdExpr(c.id)} = ?
|
|
240
|
+
RETURNING ${c.id} AS id`, [JSON.stringify(set), removed, documentId]);
|
|
241
|
+
return normalizeRows(raw).length;
|
|
242
|
+
}
|
|
206
243
|
async listDocuments(filter) {
|
|
207
244
|
const c = this.col;
|
|
208
245
|
const docExpr = documentIdExpr(c.id);
|
|
@@ -219,6 +256,38 @@ export class PgVectorStore {
|
|
|
219
256
|
};
|
|
220
257
|
});
|
|
221
258
|
}
|
|
259
|
+
/**
|
|
260
|
+
* {@link VectorStore.listDocumentIds} — `SELECT DISTINCT` of the document-id expression. Unlike
|
|
261
|
+
* {@link PgVectorStore.listDocuments} it never selects the `jsonb` metadata column, so Postgres does not
|
|
262
|
+
* transfer it and {@link parseMetadata} is never called: on a large corpus that is most of the cost.
|
|
263
|
+
* The `WHERE` comes from the same {@link buildMetadataWhere} `search` uses.
|
|
264
|
+
*/
|
|
265
|
+
async listDocumentIds(filter) {
|
|
266
|
+
const docExpr = documentIdExpr(this.col.id);
|
|
267
|
+
const where = buildMetadataWhere(filter, this.col.metadata);
|
|
268
|
+
const raw = await this.db.rawQuery(`SELECT DISTINCT ${docExpr} AS doc_id
|
|
269
|
+
FROM ${this.table}
|
|
270
|
+
${where.sql}
|
|
271
|
+
ORDER BY doc_id`, where.bindings);
|
|
272
|
+
return normalizeRows(raw).map((row) => String(row.doc_id));
|
|
273
|
+
}
|
|
274
|
+
/**
|
|
275
|
+
* {@link VectorStore.removeWhere} — one `DELETE … RETURNING`, so the chunk count comes back from the
|
|
276
|
+
* same statement that does the deleting: no count-then-delete, no race between the two, one round trip.
|
|
277
|
+
*
|
|
278
|
+
* Filter parity with {@link PgVectorStore.search} is by construction — the `WHERE` fragment is produced
|
|
279
|
+
* by the identical {@link buildMetadataWhere} call, so an empty-array value emits the same `false`
|
|
280
|
+
* clause here as it does there, and a scalar the same `@> ?::jsonb` containment.
|
|
281
|
+
*/
|
|
282
|
+
async removeWhere(filter) {
|
|
283
|
+
assertRemovalFilter(filter);
|
|
284
|
+
if (filterDeniesAll(filter)) {
|
|
285
|
+
return 0;
|
|
286
|
+
}
|
|
287
|
+
const where = buildMetadataWhere(filter, this.col.metadata);
|
|
288
|
+
const raw = await this.db.rawQuery(`DELETE FROM ${this.table} ${where.sql} RETURNING ${this.col.id} AS id`, where.bindings);
|
|
289
|
+
return normalizeRows(raw).length;
|
|
290
|
+
}
|
|
222
291
|
async search(embedding, options) {
|
|
223
292
|
const c = this.col;
|
|
224
293
|
const vector = toVectorLiteral(embedding);
|