@adonis-agora/agent 0.16.0 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/CHANGELOG.md +139 -0
  2. package/README.md +210 -0
  3. package/dist/providers/agent_provider.d.ts +6 -4
  4. package/dist/providers/agent_provider.d.ts.map +1 -1
  5. package/dist/providers/agent_provider.js +23 -16
  6. package/dist/providers/agent_provider.js.map +1 -1
  7. package/dist/src/agent-deps-factory.d.ts +6 -0
  8. package/dist/src/agent-deps-factory.d.ts.map +1 -1
  9. package/dist/src/agent-deps-factory.js +5 -1
  10. package/dist/src/agent-deps-factory.js.map +1 -1
  11. package/dist/src/agent-deps.d.ts +6 -1
  12. package/dist/src/agent-deps.d.ts.map +1 -1
  13. package/dist/src/agent-deps.js.map +1 -1
  14. package/dist/src/agent-loop.d.ts +11 -2
  15. package/dist/src/agent-loop.d.ts.map +1 -1
  16. package/dist/src/agent-loop.js +69 -2
  17. package/dist/src/agent-loop.js.map +1 -1
  18. package/dist/src/data/tenant-scope.d.ts +16 -1
  19. package/dist/src/data/tenant-scope.d.ts.map +1 -1
  20. package/dist/src/data/tenant-scope.js +50 -7
  21. package/dist/src/data/tenant-scope.js.map +1 -1
  22. package/dist/src/define_config.d.ts +25 -9
  23. package/dist/src/define_config.d.ts.map +1 -1
  24. package/dist/src/define_config.js.map +1 -1
  25. package/dist/src/governance-gate.d.ts +24 -8
  26. package/dist/src/governance-gate.d.ts.map +1 -1
  27. package/dist/src/governance-gate.js +14 -3
  28. package/dist/src/governance-gate.js.map +1 -1
  29. package/dist/src/index.d.ts +2 -1
  30. package/dist/src/index.d.ts.map +1 -1
  31. package/dist/src/index.js +5 -1
  32. package/dist/src/index.js.map +1 -1
  33. package/dist/src/rag/chunk.d.ts +35 -0
  34. package/dist/src/rag/chunk.d.ts.map +1 -1
  35. package/dist/src/rag/chunk.js +60 -0
  36. package/dist/src/rag/chunk.js.map +1 -1
  37. package/dist/src/rag/index.d.ts +1 -1
  38. package/dist/src/rag/index.d.ts.map +1 -1
  39. package/dist/src/rag/index.js +1 -1
  40. package/dist/src/rag/index.js.map +1 -1
  41. package/dist/src/rag/memory-vector-store.d.ts +20 -1
  42. package/dist/src/rag/memory-vector-store.d.ts.map +1 -1
  43. package/dist/src/rag/memory-vector-store.js +56 -1
  44. package/dist/src/rag/memory-vector-store.js.map +1 -1
  45. package/dist/src/rag/pg-vector-store.d.ts +32 -1
  46. package/dist/src/rag/pg-vector-store.d.ts.map +1 -1
  47. package/dist/src/rag/pg-vector-store.js +69 -0
  48. package/dist/src/rag/pg-vector-store.js.map +1 -1
  49. package/dist/src/rag/qdrant-store.d.ts +89 -2
  50. package/dist/src/rag/qdrant-store.d.ts.map +1 -1
  51. package/dist/src/rag/qdrant-store.js +155 -2
  52. package/dist/src/rag/qdrant-store.js.map +1 -1
  53. package/dist/src/rag/vector-store.d.ts +144 -0
  54. package/dist/src/rag/vector-store.d.ts.map +1 -1
  55. package/dist/src/rag/vector-store.js +73 -0
  56. package/dist/src/rag/vector-store.js.map +1 -1
  57. package/package.json +14 -9
@@ -1,11 +1,16 @@
1
1
  import type { HttpContext } from '@adonisjs/core/http';
2
2
  import type { Actor } from './types.js';
3
3
  /**
4
- * Optional authorization gate for the cross-actor `/agent/governance/*` read routes. It runs AFTER
5
- * the actor is resolved (so the caller is already authenticated), and decides whether THIS actor may
6
- * read the platform-wide governance read-model (every actor's spend/usage/threads/approvals). Return
7
- * `false` to deny (the route replies `403`). Omit the gate entirely and any resolved actor may read
8
- * governance — the historical behavior, correct only when every authenticated caller is trusted staff.
4
+ * Authorization gate for the cross-actor `/agent/governance/*` read routes. It runs AFTER the actor
5
+ * is resolved (so the caller is already authenticated), and decides whether THIS actor may read the
6
+ * platform-wide governance read-model (every actor's spend/usage/threads/approvals). Return `false`
7
+ * to deny (the route replies `403`).
8
+ *
9
+ * The gate is what makes those routes exist: omit it and the provider does NOT mount
10
+ * `/agent/governance/*` at all (every route answers `404`). Set it to mount them gated, or set
11
+ * `governanceAuthorize: () => true` to deliberately restore the old behaviour of letting ANY
12
+ * authenticated actor read them. `GET /agent/approvals/mine` is unaffected — it stays mounted and
13
+ * scoped to the calling actor's own pending approvals.
9
14
  *
10
15
  * Mirrors `@adonis-agora/agent-dashboard`'s `authorize` hook so the JSON routes and the console SPA
11
16
  * that reads them can be gated with the SAME predicate (typically "actor is ADMIN").
@@ -19,13 +24,24 @@ export interface GovernanceGateVerdict {
19
24
  error?: string;
20
25
  }
21
26
  /**
22
- * Decide whether an already-resolved actor may read the governance routes — the router-free core of
23
- * the provider's governance gate, extracted so it can be unit tested without booting an app.
27
+ * Decide whether an already-resolved actor holds cross-actor governance privilege — the router-free
28
+ * core of the provider's governance gate, extracted so it can be unit tested without booting an app.
24
29
  *
25
- * - No `authorize` configured → `{ ok: true }` (governance open to any resolved actor).
30
+ * - No `authorize` configured → `{ ok: true }` (see the caveat below — the provider never gets here).
26
31
  * - `authorize` returns truthy → `{ ok: true }`.
27
32
  * - `authorize` returns falsy → `{ ok: false, status: 403 }`.
28
33
  * - `authorize` throws → `{ ok: false, status: 403 }` (fail-closed; the error message is surfaced).
34
+ *
35
+ * The `undefined` branch is a pure-function convenience, NOT a claim that governance is readable
36
+ * without a gate. Neither provider caller can reach it: `#resolveGovernanceActor` backs the
37
+ * `/agent/governance/*` routes, which are NOT mounted at all when no gate is configured (they answer
38
+ * `404`), so it always passes one; and `#assertOwner` — which reuses this predicate as the "may act
39
+ * across actors" seam for the per-actor run/thread routes — short-circuits to "not privileged" when
40
+ * no gate is configured, leaving ownership strict, rather than calling in. To mount the governance
41
+ * routes, set `governanceAuthorize` (typically an ADMIN check); to deliberately restore the old
42
+ * behaviour of letting ANY authenticated actor read them, set `governanceAuthorize: () => true`.
43
+ * Either way `GET /agent/approvals/mine` is unaffected — it stays mounted and scoped to the calling
44
+ * actor's own pending approvals.
29
45
  */
30
46
  export declare function evaluateGovernanceGate(actor: Actor, ctx: HttpContext, authorize?: AgentGovernanceAuthorize): Promise<GovernanceGateVerdict>;
31
47
  //# sourceMappingURL=governance-gate.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"governance-gate.d.ts","sourceRoot":"","sources":["../../src/governance-gate.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAExC;;;;;;;;;GASG;AACH,MAAM,MAAM,wBAAwB,GAAG,CACrC,KAAK,EAAE,KAAK,EACZ,GAAG,EAAE,WAAW,KACb,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;AAEhC,oGAAoG;AACpG,MAAM,WAAW,qBAAqB;IACpC,EAAE,EAAE,OAAO,CAAC;IACZ,qGAAqG;IACrG,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;;;GAQG;AACH,wBAAsB,sBAAsB,CAC1C,KAAK,EAAE,KAAK,EACZ,GAAG,EAAE,WAAW,EAChB,SAAS,CAAC,EAAE,wBAAwB,GACnC,OAAO,CAAC,qBAAqB,CAAC,CAYhC"}
1
+ {"version":3,"file":"governance-gate.d.ts","sourceRoot":"","sources":["../../src/governance-gate.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AAExC;;;;;;;;;;;;;;GAcG;AACH,MAAM,MAAM,wBAAwB,GAAG,CACrC,KAAK,EAAE,KAAK,EACZ,GAAG,EAAE,WAAW,KACb,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;AAEhC,oGAAoG;AACpG,MAAM,WAAW,qBAAqB;IACpC,EAAE,EAAE,OAAO,CAAC;IACZ,qGAAqG;IACrG,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAsB,sBAAsB,CAC1C,KAAK,EAAE,KAAK,EACZ,GAAG,EAAE,WAAW,EAChB,SAAS,CAAC,EAAE,wBAAwB,GACnC,OAAO,CAAC,qBAAqB,CAAC,CAYhC"}
@@ -1,11 +1,22 @@
1
1
  /**
2
- * Decide whether an already-resolved actor may read the governance routes — the router-free core of
3
- * the provider's governance gate, extracted so it can be unit tested without booting an app.
2
+ * Decide whether an already-resolved actor holds cross-actor governance privilege — the router-free
3
+ * core of the provider's governance gate, extracted so it can be unit tested without booting an app.
4
4
  *
5
- * - No `authorize` configured → `{ ok: true }` (governance open to any resolved actor).
5
+ * - No `authorize` configured → `{ ok: true }` (see the caveat below — the provider never gets here).
6
6
  * - `authorize` returns truthy → `{ ok: true }`.
7
7
  * - `authorize` returns falsy → `{ ok: false, status: 403 }`.
8
8
  * - `authorize` throws → `{ ok: false, status: 403 }` (fail-closed; the error message is surfaced).
9
+ *
10
+ * The `undefined` branch is a pure-function convenience, NOT a claim that governance is readable
11
+ * without a gate. Neither provider caller can reach it: `#resolveGovernanceActor` backs the
12
+ * `/agent/governance/*` routes, which are NOT mounted at all when no gate is configured (they answer
13
+ * `404`), so it always passes one; and `#assertOwner` — which reuses this predicate as the "may act
14
+ * across actors" seam for the per-actor run/thread routes — short-circuits to "not privileged" when
15
+ * no gate is configured, leaving ownership strict, rather than calling in. To mount the governance
16
+ * routes, set `governanceAuthorize` (typically an ADMIN check); to deliberately restore the old
17
+ * behaviour of letting ANY authenticated actor read them, set `governanceAuthorize: () => true`.
18
+ * Either way `GET /agent/approvals/mine` is unaffected — it stays mounted and scoped to the calling
19
+ * actor's own pending approvals.
9
20
  */
10
21
  export async function evaluateGovernanceGate(actor, ctx, authorize) {
11
22
  if (authorize === undefined)
@@ -1 +1 @@
1
- {"version":3,"file":"governance-gate.js","sourceRoot":"","sources":["../../src/governance-gate.ts"],"names":[],"mappings":"AA0BA;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,KAAY,EACZ,GAAgB,EAChB,SAAoC;IAEpC,IAAI,SAAS,KAAK,SAAS;QAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;IAC9D,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAC5C,OAAO,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC;IAC9F,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO;YACL,EAAE,EAAE,KAAK;YACT,MAAM,EAAE,GAAG;YACX,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW;SAC5D,CAAC;IACJ,CAAC;AACH,CAAC"}
1
+ {"version":3,"file":"governance-gate.js","sourceRoot":"","sources":["../../src/governance-gate.ts"],"names":[],"mappings":"AA+BA;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,KAAY,EACZ,GAAgB,EAChB,SAAoC;IAEpC,IAAI,SAAS,KAAK,SAAS;QAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;IAC9D,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAC5C,OAAO,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC;IAC9F,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO;YACL,EAAE,EAAE,KAAK;YACT,MAAM,EAAE,GAAG;YACX,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW;SAC5D,CAAC;IACJ,CAAC;AACH,CAAC"}
@@ -1,4 +1,4 @@
1
- export declare const VERSION = "0.16.0";
1
+ export declare const VERSION = "0.18.0";
2
2
  export * from './types.js';
3
3
  export * from './spi/tool.js';
4
4
  export * from './base-tool.js';
@@ -62,4 +62,5 @@ export { LucidPricingStore } from './stores/lucid-pricing.js';
62
62
  export { LucidGovernanceQueries } from './stores/lucid-governance-queries.js';
63
63
  export { dataTool, SqlValidator, SqlValidationError, GroupTableAccessPolicy, TenantScopeRewriter, injectLimit, loadSqlParser, } from './data/index.js';
64
64
  export type { DataToolConfig, DataToolResult, QueryRunner, SqlValidationResult, TableAccessPolicy, GroupTableAccessConfig, TenantScopeConfig, SqlParserLike, } from './data/index.js';
65
+ export { configure } from '../configure.js';
65
66
  //# sourceMappingURL=index.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,OAAO,WAAW,CAAC;AAEhC,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC;AAC9B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,yBAAyB,CAAC;AACxC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,sBAAsB,CAAC;AACrC,cAAc,uBAAuB,CAAC;AACtC,cAAc,sBAAsB,CAAC;AACrC,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AACtC,cAAc,yBAAyB,CAAC;AACxC,cAAc,0BAA0B,CAAC;AACzC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,6BAA6B,CAAC;AAC5C,cAAc,6BAA6B,CAAC;AAC5C,cAAc,oBAAoB,CAAC;AACnC,cAAc,mBAAmB,CAAC;AAClC,cAAc,gBAAgB,CAAC;AAC/B,OAAO,EACL,WAAW,EACX,eAAe,EACf,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,uBAAuB,CAAC;AAC/B,YAAY,EACV,gBAAgB,EAChB,kBAAkB,EAClB,YAAY,EACZ,WAAW,EACX,YAAY,GACb,MAAM,uBAAuB,CAAC;AAC/B,YAAY,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AACjE,cAAc,sBAAsB,CAAC;AACrC,cAAc,eAAe,CAAC;AAC9B,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EACL,YAAY,EACZ,kBAAkB,EAClB,kBAAkB,EAClB,iBAAiB,EACjB,qBAAqB,GACtB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,YAAY,EACZ,kBAAkB,EAClB,KAAK,aAAa,EAClB,KAAK,cAAc,GACpB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,wBAAwB,EACxB,oBAAoB,EACpB,qCAAqC,EACrC,uCAAuC,GACxC,MAAM,iBAAiB,CAAC;AACzB,YAAY,EACV,yBAAyB,EACzB,yBAAyB,EACzB,+BAA+B,GAChC,MAAM,iBAAiB,CAAC;AACzB,cAAc,kBAAkB,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAGtC,OAAO,EACL,YAAY,EACZ,MAAM,EACN,MAAM,EACN,aAAa,EACb,iBAAiB,EACjB,UAAU,EACV,gBAAgB,EAChB,UAAU,EACV,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAC3D,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,YAAY,EAAE,wBAAwB,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAC5F,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACnD,YAAY,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AACvD,YAAY,EACV,WAAW,EACX,mBAAmB,EACnB,YAAY,EACZ,WAAW,EACX,gBAAgB,EAChB,oBAAoB,EACpB,qBAAqB,EACrB,0BAA0B,EAC1B,YAAY,EACZ,YAAY,EACZ,WAAW,EACX,cAAc,EACd,cAAc,EACd,kBAAkB,EAClB,wBAAwB,EACxB,wBAAwB,EACxB,qBAAqB,EACrB,gBAAgB,EAChB,gBAAgB,EAChB,qBAAqB,EACrB,uBAAuB,EACvB,gBAAgB,EAChB,wBAAwB,EACxB,wBAAwB,EACxB,YAAY,EACZ,YAAY,EACZ,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,MAAM,EACN,UAAU,EACV,cAAc,EACd,uBAAuB,EACvB,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACV,aAAa,EACb,UAAU,EACV,SAAS,EACT,cAAc,EACd,qBAAqB,GACtB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EACL,aAAa,EACb,kBAAkB,EAClB,sBAAsB,EACtB,uBAAuB,GACxB,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACvE,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAC;AACpG,YAAY,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAC;AACtE,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACzC,YAAY,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAC;AAChE,OAAO,EACL,oBAAoB,EACpB,KAAK,2BAA2B,GACjC,MAAM,8BAA8B,CAAC;AACtC,YAAY,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,EACL,yBAAyB,EACzB,mBAAmB,EACnB,iBAAiB,EACjB,oBAAoB,GACrB,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AACpE,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,YAAY,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AACxD,YAAY,EAAE,WAAW,IAAI,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAC3E,OAAO,EAAE,iBAAiB,EAAE,MAAM,kCAAkC,CAAC;AACrE,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,YAAY,EACV,sBAAsB,EACtB,iBAAiB,EACjB,eAAe,EACf,qBAAqB,EACrB,sBAAsB,GACvB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACL,qBAAqB,EACrB,iBAAiB,EACjB,iBAAiB,EACjB,YAAY,GACb,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAC5D,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,MAAM,sCAAsC,CAAC;AAG9E,OAAO,EACL,QAAQ,EACR,YAAY,EACZ,kBAAkB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,WAAW,EACX,aAAa,GACd,MAAM,iBAAiB,CAAC;AACzB,YAAY,EACV,cAAc,EACd,cAAc,EACd,WAAW,EACX,mBAAmB,EACnB,iBAAiB,EACjB,sBAAsB,EACtB,iBAAiB,EACjB,aAAa,GACd,MAAM,iBAAiB,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,OAAO,WAAW,CAAC;AAEhC,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC;AAC9B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,yBAAyB,CAAC;AACxC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,sBAAsB,CAAC;AACrC,cAAc,uBAAuB,CAAC;AACtC,cAAc,sBAAsB,CAAC;AACrC,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AACtC,cAAc,yBAAyB,CAAC;AACxC,cAAc,0BAA0B,CAAC;AACzC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,6BAA6B,CAAC;AAC5C,cAAc,6BAA6B,CAAC;AAC5C,cAAc,oBAAoB,CAAC;AACnC,cAAc,mBAAmB,CAAC;AAClC,cAAc,gBAAgB,CAAC;AAC/B,OAAO,EACL,WAAW,EACX,eAAe,EACf,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,uBAAuB,CAAC;AAC/B,YAAY,EACV,gBAAgB,EAChB,kBAAkB,EAClB,YAAY,EACZ,WAAW,EACX,YAAY,GACb,MAAM,uBAAuB,CAAC;AAC/B,YAAY,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AACjE,cAAc,sBAAsB,CAAC;AACrC,cAAc,eAAe,CAAC;AAC9B,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EACL,YAAY,EACZ,kBAAkB,EAClB,kBAAkB,EAClB,iBAAiB,EACjB,qBAAqB,GACtB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,YAAY,EACZ,kBAAkB,EAClB,KAAK,aAAa,EAClB,KAAK,cAAc,GACpB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,wBAAwB,EACxB,oBAAoB,EACpB,qCAAqC,EACrC,uCAAuC,GACxC,MAAM,iBAAiB,CAAC;AACzB,YAAY,EACV,yBAAyB,EACzB,yBAAyB,EACzB,+BAA+B,GAChC,MAAM,iBAAiB,CAAC;AACzB,cAAc,kBAAkB,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAGtC,OAAO,EACL,YAAY,EACZ,MAAM,EACN,MAAM,EACN,aAAa,EACb,iBAAiB,EACjB,UAAU,EACV,gBAAgB,EAChB,UAAU,EACV,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAC3D,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAC9D,YAAY,EAAE,wBAAwB,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAC5F,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AACnD,YAAY,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AACvD,YAAY,EACV,WAAW,EACX,mBAAmB,EACnB,YAAY,EACZ,WAAW,EACX,gBAAgB,EAChB,oBAAoB,EACpB,qBAAqB,EACrB,0BAA0B,EAC1B,YAAY,EACZ,YAAY,EACZ,WAAW,EACX,cAAc,EACd,cAAc,EACd,kBAAkB,EAClB,wBAAwB,EACxB,wBAAwB,EACxB,qBAAqB,EACrB,gBAAgB,EAChB,gBAAgB,EAChB,qBAAqB,EACrB,uBAAuB,EACvB,gBAAgB,EAChB,wBAAwB,EACxB,wBAAwB,EACxB,YAAY,EACZ,YAAY,EACZ,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,MAAM,EACN,UAAU,EACV,cAAc,EACd,uBAAuB,EACvB,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACV,aAAa,EACb,UAAU,EACV,SAAS,EACT,cAAc,EACd,qBAAqB,GACtB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EACL,aAAa,EACb,kBAAkB,EAClB,sBAAsB,EACtB,uBAAuB,GACxB,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AACvE,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAC;AACpG,YAAY,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAC;AACtE,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACzC,YAAY,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAC;AAChE,OAAO,EACL,oBAAoB,EACpB,KAAK,2BAA2B,GACjC,MAAM,8BAA8B,CAAC;AACtC,YAAY,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,EACL,yBAAyB,EACzB,mBAAmB,EACnB,iBAAiB,EACjB,oBAAoB,GACrB,MAAM,qBAAqB,CAAC;AAC7B,YAAY,EAAE,wBAAwB,EAAE,MAAM,qBAAqB,CAAC;AACpE,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,YAAY,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AACxD,YAAY,EAAE,WAAW,IAAI,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAC3E,OAAO,EAAE,iBAAiB,EAAE,MAAM,kCAAkC,CAAC;AACrE,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACpD,YAAY,EACV,sBAAsB,EACtB,iBAAiB,EACjB,eAAe,EACf,qBAAqB,EACrB,sBAAsB,GACvB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EACL,qBAAqB,EACrB,iBAAiB,EACjB,iBAAiB,EACjB,YAAY,GACb,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAC5D,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,MAAM,sCAAsC,CAAC;AAG9E,OAAO,EACL,QAAQ,EACR,YAAY,EACZ,kBAAkB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,WAAW,EACX,aAAa,GACd,MAAM,iBAAiB,CAAC;AACzB,YAAY,EACV,cAAc,EACd,cAAc,EACd,WAAW,EACX,mBAAmB,EACnB,iBAAiB,EACjB,sBAAsB,EACtB,iBAAiB,EACjB,aAAa,GACd,MAAM,iBAAiB,CAAC;AAKzB,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC"}
package/dist/src/index.js CHANGED
@@ -1,4 +1,4 @@
1
- export const VERSION = '0.16.0';
1
+ export const VERSION = '0.18.0';
2
2
  export * from './types.js';
3
3
  export * from './spi/tool.js';
4
4
  export * from './base-tool.js';
@@ -48,4 +48,8 @@ export { LucidPricingStore } from './stores/lucid-pricing.js';
48
48
  export { LucidGovernanceQueries } from './stores/lucid-governance-queries.js';
49
49
  // ── Data satellite: governed read-only SQL tool ──────────────────────────────
50
50
  export { dataTool, SqlValidator, SqlValidationError, GroupTableAccessPolicy, TenantScopeRewriter, injectLimit, loadSqlParser, } from './data/index.js';
51
+ // Re-export the configure hook from the package root so `node ace configure` finds it.
52
+ // AdonisJS imports the package MAIN and reads `configure` off the module namespace —
53
+ // the `./configure` subpath alone is never consulted.
54
+ export { configure } from '../configure.js';
51
55
  //# sourceMappingURL=index.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,OAAO,GAAG,QAAQ,CAAC;AAEhC,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC;AAC9B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,yBAAyB,CAAC;AACxC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,sBAAsB,CAAC;AACrC,cAAc,uBAAuB,CAAC;AACtC,cAAc,sBAAsB,CAAC;AACrC,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AACtC,cAAc,yBAAyB,CAAC;AACxC,cAAc,0BAA0B,CAAC;AACzC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,6BAA6B,CAAC;AAC5C,cAAc,6BAA6B,CAAC;AAC5C,cAAc,oBAAoB,CAAC;AACnC,cAAc,mBAAmB,CAAC;AAClC,cAAc,gBAAgB,CAAC;AAC/B,OAAO,EACL,WAAW,EACX,eAAe,EACf,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,uBAAuB,CAAC;AAS/B,cAAc,sBAAsB,CAAC;AACrC,cAAc,eAAe,CAAC;AAC9B,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EACL,YAAY,EACZ,kBAAkB,EAClB,kBAAkB,EAClB,iBAAiB,EACjB,qBAAqB,GACtB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,YAAY,EACZ,kBAAkB,GAGnB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,wBAAwB,EACxB,oBAAoB,EACpB,qCAAqC,EACrC,uCAAuC,GACxC,MAAM,iBAAiB,CAAC;AAMzB,cAAc,kBAAkB,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAEtC,gFAAgF;AAChF,OAAO,EACL,YAAY,EACZ,MAAM,EACN,MAAM,EACN,aAAa,EACb,iBAAiB,EACjB,UAAU,EACV,gBAAgB,EAChB,UAAU,EACV,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAC3D,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAE9D,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAgCnD,OAAO,EACL,MAAM,EACN,UAAU,EACV,cAAc,EACd,uBAAuB,EACvB,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,kBAAkB,CAAC;AAQ1B,OAAO,EACL,aAAa,EACb,kBAAkB,EAClB,sBAAsB,EACtB,uBAAuB,GACxB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAC;AAEpG,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAEzC,OAAO,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAC;AAChE,OAAO,EACL,oBAAoB,GAErB,MAAM,8BAA8B,CAAC;AAEtC,OAAO,EACL,yBAAyB,EACzB,mBAAmB,EACnB,iBAAiB,EACjB,oBAAoB,GACrB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAElD,OAAO,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AAExD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kCAAkC,CAAC;AACrE,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAQpD,OAAO,EACL,qBAAqB,EACrB,iBAAiB,EACjB,iBAAiB,EACjB,YAAY,GACb,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAC5D,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,MAAM,sCAAsC,CAAC;AAE9E,gFAAgF;AAChF,OAAO,EACL,QAAQ,EACR,YAAY,EACZ,kBAAkB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,WAAW,EACX,aAAa,GACd,MAAM,iBAAiB,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,OAAO,GAAG,QAAQ,CAAC;AAEhC,cAAc,YAAY,CAAC;AAC3B,cAAc,eAAe,CAAC;AAC9B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,yBAAyB,CAAC;AACxC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,sBAAsB,CAAC;AACrC,cAAc,uBAAuB,CAAC;AACtC,cAAc,sBAAsB,CAAC;AACrC,cAAc,wBAAwB,CAAC;AACvC,cAAc,uBAAuB,CAAC;AACtC,cAAc,yBAAyB,CAAC;AACxC,cAAc,0BAA0B,CAAC;AACzC,cAAc,6BAA6B,CAAC;AAC5C,cAAc,6BAA6B,CAAC;AAC5C,cAAc,6BAA6B,CAAC;AAC5C,cAAc,oBAAoB,CAAC;AACnC,cAAc,mBAAmB,CAAC;AAClC,cAAc,gBAAgB,CAAC;AAC/B,OAAO,EACL,WAAW,EACX,eAAe,EACf,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,uBAAuB,CAAC;AAS/B,cAAc,sBAAsB,CAAC;AACrC,cAAc,eAAe,CAAC;AAC9B,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EACL,YAAY,EACZ,kBAAkB,EAClB,kBAAkB,EAClB,iBAAiB,EACjB,qBAAqB,GACtB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EACL,YAAY,EACZ,kBAAkB,GAGnB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,wBAAwB,EACxB,oBAAoB,EACpB,qCAAqC,EACrC,uCAAuC,GACxC,MAAM,iBAAiB,CAAC;AAMzB,cAAc,kBAAkB,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,UAAU,CAAC;AAEtC,gFAAgF;AAChF,OAAO,EACL,YAAY,EACZ,MAAM,EACN,MAAM,EACN,aAAa,EACb,iBAAiB,EACjB,UAAU,EACV,gBAAgB,EAChB,UAAU,EACV,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAC3D,OAAO,EAAE,sBAAsB,EAAE,MAAM,sBAAsB,CAAC;AAE9D,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAgCnD,OAAO,EACL,MAAM,EACN,UAAU,EACV,cAAc,EACd,uBAAuB,EACvB,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,kBAAkB,CAAC;AAQ1B,OAAO,EACL,aAAa,EACb,kBAAkB,EAClB,sBAAsB,EACtB,uBAAuB,GACxB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,qBAAqB,EAAE,MAAM,yBAAyB,CAAC;AAEpG,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAEzC,OAAO,EAAE,wBAAwB,EAAE,MAAM,sBAAsB,CAAC;AAChE,OAAO,EACL,oBAAoB,GAErB,MAAM,8BAA8B,CAAC;AAEtC,OAAO,EACL,yBAAyB,EACzB,mBAAmB,EACnB,iBAAiB,EACjB,oBAAoB,GACrB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAElD,OAAO,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAC;AAExD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kCAAkC,CAAC;AACrE,OAAO,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAQpD,OAAO,EACL,qBAAqB,EACrB,iBAAiB,EACjB,iBAAiB,EACjB,YAAY,GACb,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAC5D,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,MAAM,sCAAsC,CAAC;AAE9E,gFAAgF;AAChF,OAAO,EACL,QAAQ,EACR,YAAY,EACZ,kBAAkB,EAClB,sBAAsB,EACtB,mBAAmB,EACnB,WAAW,EACX,aAAa,GACd,MAAM,iBAAiB,CAAC;AAYzB,uFAAuF;AACvF,qFAAqF;AACrF,sDAAsD;AACtD,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC"}
@@ -3,12 +3,47 @@ export interface ChunkOptions {
3
3
  chunkSize?: number;
4
4
  /** Characters of overlap carried from the end of one chunk into the next. Default 100. */
5
5
  overlap?: number;
6
+ /**
7
+ * The record delimiter, when the text has boundaries that *mean* something. Pass it and the chunker
8
+ * cuts ONLY on this string — never inside a record.
9
+ *
10
+ * Prose has no real boundaries, so the default heuristic (paragraph → sentence → word, in the back
11
+ * half of the window) is the right guess. Machine-shaped text is the opposite case: a spreadsheet
12
+ * flattened to one field-labelled record per line, a log, a JSONL export. There the boundaries are
13
+ * known exactly, and a heuristic cut lands mid-record — stranding the half that carries the row
14
+ * identifier in a different chunk from the half that carries the value, so neither chunk can answer
15
+ * a question about that row. `separator: '\n'` says "the lines are the records".
16
+ *
17
+ * Two consequences worth knowing before you pass it, both by design:
18
+ *
19
+ * 1. **`chunkSize` becomes a target, not a cap.** A single record longer than `chunkSize` is emitted
20
+ * whole, as its own over-size chunk. It is NOT split, and it does NOT fall back to the prose
21
+ * heuristic — a mid-record cut is the exact failure `separator` exists to prevent, so silently
22
+ * performing one to respect a size limit would defeat the feature. An over-size chunk is visible
23
+ * (you can see the record, and shorten it or raise `chunkSize`); a mangled record is not. It never
24
+ * drags its neighbours over the limit either: it always chunks alone.
25
+ * 2. **`overlap` is spent on whole records.** It stops being a character count and becomes a
26
+ * character *budget*, filled with entire trailing records of the previous chunk — as many as fit,
27
+ * never a partial one, and never so many that a chunk fails to advance. Half a record is not
28
+ * useful context; it is the same mid-record cut by another name. `overlap: 0` carries nothing,
29
+ * which is usually what self-contained records want.
30
+ *
31
+ * Empty or whitespace-only records are dropped. An empty string is treated as "not passed" (it would
32
+ * otherwise split every character), leaving the prose path. Omit it and NOTHING changes: the prose
33
+ * path below is untouched, byte-for-byte, so no existing corpus is re-chunked (and so none has to be
34
+ * re-embedded).
35
+ */
36
+ separator?: string;
6
37
  }
7
38
  /**
8
39
  * Splits text into overlapping chunks for embedding. Greedy up to `chunkSize`, but prefers to break on
9
40
  * a paragraph → sentence → word boundary in the back half of the window (so a chunk rarely cuts
10
41
  * mid-sentence), then carries `overlap` characters into the next chunk to preserve context across the
11
42
  * seam. Pure and deterministic.
43
+ *
44
+ * Pass {@link ChunkOptions.separator} for text whose record boundaries are known rather than guessed;
45
+ * that path cuts only on the separator, and its two deviations (`chunkSize` becomes a target,
46
+ * `overlap` is spent on whole records) are documented there.
12
47
  */
13
48
  export declare function chunkText(text: string, options?: ChunkOptions): string[];
14
49
  //# sourceMappingURL=chunk.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"chunk.d.ts","sourceRoot":"","sources":["../../../src/rag/chunk.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,YAAY;IAC3B,oDAAoD;IACpD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0FAA0F;IAC1F,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;;;GAKG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,YAAiB,GAAG,MAAM,EAAE,CAuC5E"}
1
+ {"version":3,"file":"chunk.d.ts","sourceRoot":"","sources":["../../../src/rag/chunk.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,YAAY;IAC3B,oDAAoD;IACpD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0FAA0F;IAC1F,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OA6BG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;;;;GASG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE,YAAiB,GAAG,MAAM,EAAE,CA2C5E"}
@@ -3,10 +3,18 @@
3
3
  * a paragraph → sentence → word boundary in the back half of the window (so a chunk rarely cuts
4
4
  * mid-sentence), then carries `overlap` characters into the next chunk to preserve context across the
5
5
  * seam. Pure and deterministic.
6
+ *
7
+ * Pass {@link ChunkOptions.separator} for text whose record boundaries are known rather than guessed;
8
+ * that path cuts only on the separator, and its two deviations (`chunkSize` becomes a target,
9
+ * `overlap` is spent on whole records) are documented there.
6
10
  */
7
11
  export function chunkText(text, options = {}) {
8
12
  const chunkSize = options.chunkSize ?? 800;
9
13
  const overlap = options.overlap ?? 100;
14
+ const separator = options.separator;
15
+ if (separator !== undefined && separator.length > 0) {
16
+ return chunkRecords(text, separator, chunkSize, overlap);
17
+ }
10
18
  const normalized = text.trim();
11
19
  if (normalized.length === 0) {
12
20
  return [];
@@ -39,4 +47,56 @@ export function chunkText(text, options = {}) {
39
47
  }
40
48
  return chunks;
41
49
  }
50
+ /**
51
+ * The {@link ChunkOptions.separator} path: pack WHOLE records into chunks. Records are the atoms here —
52
+ * no chunk ever contains a fragment of one — so `chunkSize` is honoured by taking fewer records rather
53
+ * than by cutting one, and `overlap` is honoured by repeating trailing records rather than by repeating
54
+ * characters. Pure and deterministic; see {@link ChunkOptions.separator} for the two trades.
55
+ */
56
+ function chunkRecords(text, separator, chunkSize, overlap) {
57
+ const records = text
58
+ .split(separator)
59
+ .map((record) => record.trim())
60
+ .filter((record) => record.length > 0);
61
+ if (records.length === 0) {
62
+ return [];
63
+ }
64
+ const chunks = [];
65
+ let start = 0;
66
+ while (start < records.length) {
67
+ // Pack forward while the joined length fits. The FIRST record is taken unconditionally: that is
68
+ // what emits an over-long record whole instead of cutting it (trade 1).
69
+ let end = start;
70
+ let length = 0;
71
+ while (end < records.length) {
72
+ const record = records[end] ?? '';
73
+ const cost = end === start ? record.length : separator.length + record.length;
74
+ if (end > start && length + cost > chunkSize) {
75
+ break;
76
+ }
77
+ length += cost;
78
+ end += 1;
79
+ }
80
+ chunks.push(records.slice(start, end).join(separator));
81
+ if (end >= records.length) {
82
+ break;
83
+ }
84
+ // Overlap as a whole-record budget: walk back over trailing records while they fit in `overlap`.
85
+ // `back > start + 1` keeps at least one record consumed by this chunk, so `start` strictly
86
+ // increases — an over-long record (which always chunks alone) is therefore never carried forward,
87
+ // and no `overlap`, however large, can stall the loop.
88
+ let back = end;
89
+ let carried = 0;
90
+ while (back > start + 1) {
91
+ const cost = (records[back - 1] ?? '').length + separator.length;
92
+ if (carried + cost > overlap) {
93
+ break;
94
+ }
95
+ carried += cost;
96
+ back -= 1;
97
+ }
98
+ start = back;
99
+ }
100
+ return chunks;
101
+ }
42
102
  //# sourceMappingURL=chunk.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"chunk.js","sourceRoot":"","sources":["../../../src/rag/chunk.ts"],"names":[],"mappings":"AAOA;;;;;GAKG;AACH,MAAM,UAAU,SAAS,CAAC,IAAY,EAAE,UAAwB,EAAE;IAChE,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,GAAG,CAAC;IAC3C,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,GAAG,CAAC;IACvC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC/B,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,IAAI,UAAU,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;QACnC,OAAO,CAAC,UAAU,CAAC,CAAC;IACtB,CAAC;IAED,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,OAAO,KAAK,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC;QACjC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,SAAS,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;QAC3D,IAAI,OAAO,GAAG,GAAG,CAAC;QAClB,IAAI,GAAG,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC;YAC5B,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;YAC5C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CACvB,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,EAC1B,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,EACxB,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,CACxB,CAAC;YACF,uEAAuE;YACvE,IAAI,QAAQ,GAAG,SAAS,GAAG,CAAC,EAAE,CAAC;gBAC7B,OAAO,GAAG,KAAK,GAAG,QAAQ,GAAG,CAAC,CAAC;YACjC,CAAC;QACH,CAAC;QACD,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;QACtD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;QACD,IAAI,OAAO,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;YACjC,MAAM;QACR,CAAC;QACD,yFAAyF;QACzF,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,OAAO,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;IACjD,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
1
+ {"version":3,"file":"chunk.js","sourceRoot":"","sources":["../../../src/rag/chunk.ts"],"names":[],"mappings":"AAsCA;;;;;;;;;GASG;AACH,MAAM,UAAU,SAAS,CAAC,IAAY,EAAE,UAAwB,EAAE;IAChE,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,GAAG,CAAC;IAC3C,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,GAAG,CAAC;IACvC,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;IACpC,IAAI,SAAS,KAAK,SAAS,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpD,OAAO,YAAY,CAAC,IAAI,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IAC3D,CAAC;IACD,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC/B,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,IAAI,UAAU,CAAC,MAAM,IAAI,SAAS,EAAE,CAAC;QACnC,OAAO,CAAC,UAAU,CAAC,CAAC;IACtB,CAAC;IAED,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,OAAO,KAAK,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC;QACjC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,SAAS,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;QAC3D,IAAI,OAAO,GAAG,GAAG,CAAC;QAClB,IAAI,GAAG,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC;YAC5B,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;YAC5C,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CACvB,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,EAC1B,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,EACxB,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,CACxB,CAAC;YACF,uEAAuE;YACvE,IAAI,QAAQ,GAAG,SAAS,GAAG,CAAC,EAAE,CAAC;gBAC7B,OAAO,GAAG,KAAK,GAAG,QAAQ,GAAG,CAAC,CAAC;YACjC,CAAC;QACH,CAAC;QACD,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;QACtD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC;QACD,IAAI,OAAO,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;YACjC,MAAM;QACR,CAAC;QACD,yFAAyF;QACzF,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,GAAG,OAAO,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;IACjD,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;GAKG;AACH,SAAS,YAAY,CACnB,IAAY,EACZ,SAAiB,EACjB,SAAiB,EACjB,OAAe;IAEf,MAAM,OAAO,GAAG,IAAI;SACjB,KAAK,CAAC,SAAS,CAAC;SAChB,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;SAC9B,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACzC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,OAAO,KAAK,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;QAC9B,gGAAgG;QAChG,wEAAwE;QACxE,IAAI,GAAG,GAAG,KAAK,CAAC;QAChB,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,OAAO,GAAG,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;YAC5B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;YAClC,MAAM,IAAI,GAAG,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;YAC9E,IAAI,GAAG,GAAG,KAAK,IAAI,MAAM,GAAG,IAAI,GAAG,SAAS,EAAE,CAAC;gBAC7C,MAAM;YACR,CAAC;YACD,MAAM,IAAI,IAAI,CAAC;YACf,GAAG,IAAI,CAAC,CAAC;QACX,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;QACvD,IAAI,GAAG,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YAC1B,MAAM;QACR,CAAC;QACD,iGAAiG;QACjG,2FAA2F;QAC3F,kGAAkG;QAClG,uDAAuD;QACvD,IAAI,IAAI,GAAG,GAAG,CAAC;QACf,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,OAAO,IAAI,GAAG,KAAK,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,IAAI,GAAG,CAAC,OAAO,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC;YACjE,IAAI,OAAO,GAAG,IAAI,GAAG,OAAO,EAAE,CAAC;gBAC7B,MAAM;YACR,CAAC;YACD,OAAO,IAAI,IAAI,CAAC;YAChB,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;QACD,KAAK,GAAG,IAAI,CAAC;IACf,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
@@ -1,5 +1,5 @@
1
1
  export { chunkText, type ChunkOptions } from './chunk.js';
2
- export { documentIdOf, type IndexedDocument, type VectorRecord, type VectorSearchOptions, type VectorStore, } from './vector-store.js';
2
+ export { applyMetadataPatch, assertRemovalFilter, documentIdOf, effectivePatchKeys, filterDeniesAll, UnsafeRemovalError, type IndexedDocument, type MetadataPatch, type VectorRecord, type VectorSearchOptions, type VectorStore, } from './vector-store.js';
3
3
  export { matchesFilter } from './filter.js';
4
4
  export { MemoryVectorStore, cosineSimilarity } from './memory-vector-store.js';
5
5
  export { EmbeddingRetriever } from './embedding-retriever.js';
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/rag/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,KAAK,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1D,OAAO,EACL,YAAY,EACZ,KAAK,eAAe,EACpB,KAAK,YAAY,EACjB,KAAK,mBAAmB,EACxB,KAAK,WAAW,GACjB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAC/E,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EACL,kBAAkB,EAClB,KAAK,yBAAyB,GAC/B,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,gBAAgB,EAAE,KAAK,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AACxF,OAAO,EAAE,eAAe,EAAE,KAAK,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AACrF,OAAO,EACL,aAAa,EACb,iBAAiB,EACjB,eAAe,EACf,KAAK,cAAc,EACnB,KAAK,eAAe,EACpB,KAAK,oBAAoB,GAC1B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,cAAc,EACd,YAAY,EACZ,eAAe,EACf,KAAK,WAAW,EAChB,KAAK,mBAAmB,EACxB,KAAK,cAAc,EACnB,KAAK,aAAa,GACnB,MAAM,aAAa,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/rag/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,KAAK,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1D,OAAO,EACL,kBAAkB,EAClB,mBAAmB,EACnB,YAAY,EACZ,kBAAkB,EAClB,eAAe,EACf,kBAAkB,EAClB,KAAK,eAAe,EACpB,KAAK,aAAa,EAClB,KAAK,YAAY,EACjB,KAAK,mBAAmB,EACxB,KAAK,WAAW,GACjB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAC/E,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EACL,kBAAkB,EAClB,KAAK,yBAAyB,GAC/B,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,gBAAgB,EAAE,KAAK,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AACxF,OAAO,EAAE,eAAe,EAAE,KAAK,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AACrF,OAAO,EACL,aAAa,EACb,iBAAiB,EACjB,eAAe,EACf,KAAK,cAAc,EACnB,KAAK,eAAe,EACpB,KAAK,oBAAoB,GAC1B,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,cAAc,EACd,YAAY,EACZ,eAAe,EACf,KAAK,WAAW,EAChB,KAAK,mBAAmB,EACxB,KAAK,cAAc,EACnB,KAAK,aAAa,GACnB,MAAM,aAAa,CAAC"}
@@ -1,5 +1,5 @@
1
1
  export { chunkText } from './chunk.js';
2
- export { documentIdOf, } from './vector-store.js';
2
+ export { applyMetadataPatch, assertRemovalFilter, documentIdOf, effectivePatchKeys, filterDeniesAll, UnsafeRemovalError, } from './vector-store.js';
3
3
  export { matchesFilter } from './filter.js';
4
4
  export { MemoryVectorStore, cosineSimilarity } from './memory-vector-store.js';
5
5
  export { EmbeddingRetriever } from './embedding-retriever.js';
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/rag/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAqB,MAAM,YAAY,CAAC;AAC1D,OAAO,EACL,YAAY,GAKb,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAC/E,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EACL,kBAAkB,GAEnB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,gBAAgB,EAAgC,MAAM,wBAAwB,CAAC;AACxF,OAAO,EAAE,eAAe,EAA+B,MAAM,uBAAuB,CAAC;AACrF,OAAO,EACL,aAAa,EACb,iBAAiB,EACjB,eAAe,GAIhB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,cAAc,EACd,YAAY,EACZ,eAAe,GAKhB,MAAM,aAAa,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/rag/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAqB,MAAM,YAAY,CAAC;AAC1D,OAAO,EACL,kBAAkB,EAClB,mBAAmB,EACnB,YAAY,EACZ,kBAAkB,EAClB,eAAe,EACf,kBAAkB,GAMnB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAC/E,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EACL,kBAAkB,GAEnB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,gBAAgB,EAAgC,MAAM,wBAAwB,CAAC;AACxF,OAAO,EAAE,eAAe,EAA+B,MAAM,uBAAuB,CAAC;AACrF,OAAO,EACL,aAAa,EACb,iBAAiB,EACjB,eAAe,GAIhB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,cAAc,EACd,YAAY,EACZ,eAAe,GAKhB,MAAM,aAAa,CAAC"}
@@ -1,5 +1,5 @@
1
1
  import type { Passage } from '../spi/retriever.js';
2
- import { type IndexedDocument, type VectorRecord, type VectorSearchOptions, type VectorStore } from './vector-store.js';
2
+ import { type IndexedDocument, type MetadataPatch, type VectorRecord, type VectorSearchOptions, type VectorStore } from './vector-store.js';
3
3
  /**
4
4
  * An in-process {@link VectorStore} — cosine similarity over a Map, no infra. The reference adapter for
5
5
  * tests and small/embedded corpora; a pgvector/Lucid-backed store for production scale is deferred.
@@ -8,7 +8,26 @@ export declare class MemoryVectorStore implements VectorStore {
8
8
  private readonly records;
9
9
  upsert(records: VectorRecord[]): Promise<void>;
10
10
  remove(documentId: string): Promise<void>;
11
+ /**
12
+ * {@link VectorStore.updateMetadata} — rewrite every chunk's metadata for one document, touching
13
+ * neither `text` nor `embedding`. Both are carried across by the spread, so the stored `embedding`
14
+ * stays the **same array instance** it was upserted with: nothing here can perturb a vector even in
15
+ * principle. Merge semantics come from the shared {@link applyMetadataPatch}, so this adapter cannot
16
+ * drift from the other two.
17
+ */
18
+ updateMetadata(documentId: string, patch: MetadataPatch): Promise<number>;
11
19
  listDocuments(filter?: Record<string, unknown>): Promise<IndexedDocument[]>;
20
+ /**
21
+ * {@link VectorStore.listDocumentIds} — the ids `listDocuments` would report, without building the
22
+ * per-document metadata entries. Filtering goes through the same {@link matchesFilter} `search` uses.
23
+ */
24
+ listDocumentIds(filter?: Record<string, unknown>): Promise<string[]>;
25
+ /**
26
+ * {@link VectorStore.removeWhere} — delete every chunk the same filter would let `search` reach.
27
+ * Parity is by construction: the predicate below is the identical {@link matchesFilter} call, on the
28
+ * identical argument, as the one in {@link MemoryVectorStore.search}.
29
+ */
30
+ removeWhere(filter: Record<string, unknown>): Promise<number>;
12
31
  search(embedding: number[], options: VectorSearchOptions): Promise<Passage[]>;
13
32
  }
14
33
  /** Cosine similarity of two vectors; `0` when either has zero magnitude. Pure. */
@@ -1 +1 @@
1
- {"version":3,"file":"memory-vector-store.d.ts","sourceRoot":"","sources":["../../../src/rag/memory-vector-store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AAEnD,OAAO,EACL,KAAK,eAAe,EACpB,KAAK,YAAY,EACjB,KAAK,mBAAmB,EACxB,KAAK,WAAW,EAEjB,MAAM,mBAAmB,CAAC;AAE3B;;;GAGG;AACH,qBAAa,iBAAkB,YAAW,WAAW;IACnD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmC;IAErD,MAAM,CAAC,OAAO,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAM9C,MAAM,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAQzC,aAAa,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAiB3E,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;CAsBpF;AAED,kFAAkF;AAClF,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,CAcjE"}
1
+ {"version":3,"file":"memory-vector-store.d.ts","sourceRoot":"","sources":["../../../src/rag/memory-vector-store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AAEnD,OAAO,EACL,KAAK,eAAe,EACpB,KAAK,aAAa,EAClB,KAAK,YAAY,EACjB,KAAK,mBAAmB,EACxB,KAAK,WAAW,EAMjB,MAAM,mBAAmB,CAAC;AAE3B;;;GAGG;AACH,qBAAa,iBAAkB,YAAW,WAAW;IACnD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmC;IAErD,MAAM,CAAC,OAAO,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAM9C,MAAM,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAQ/C;;;;;;OAMG;IACG,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC;IAezE,aAAa,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAiBjF;;;OAGG;IACG,eAAe,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAW1E;;;;OAIG;IACG,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IAgB7D,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;CAsBpF;AAED,kFAAkF;AAClF,wBAAgB,gBAAgB,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,CAcjE"}
@@ -1,5 +1,5 @@
1
1
  import { matchesFilter } from './filter.js';
2
- import { documentIdOf, } from './vector-store.js';
2
+ import { applyMetadataPatch, assertRemovalFilter, documentIdOf, effectivePatchKeys, filterDeniesAll, } from './vector-store.js';
3
3
  /**
4
4
  * An in-process {@link VectorStore} — cosine similarity over a Map, no infra. The reference adapter for
5
5
  * tests and small/embedded corpora; a pgvector/Lucid-backed store for production scale is deferred.
@@ -18,6 +18,27 @@ export class MemoryVectorStore {
18
18
  }
19
19
  }
20
20
  }
21
+ /**
22
+ * {@link VectorStore.updateMetadata} — rewrite every chunk's metadata for one document, touching
23
+ * neither `text` nor `embedding`. Both are carried across by the spread, so the stored `embedding`
24
+ * stays the **same array instance** it was upserted with: nothing here can perturb a vector even in
25
+ * principle. Merge semantics come from the shared {@link applyMetadataPatch}, so this adapter cannot
26
+ * drift from the other two.
27
+ */
28
+ async updateMetadata(documentId, patch) {
29
+ if (effectivePatchKeys(patch).length === 0) {
30
+ return 0;
31
+ }
32
+ let written = 0;
33
+ for (const [id, record] of this.records) {
34
+ if (documentIdOf(id) !== documentId) {
35
+ continue;
36
+ }
37
+ this.records.set(id, { ...record, metadata: applyMetadataPatch(record.metadata, patch) });
38
+ written += 1;
39
+ }
40
+ return written;
41
+ }
21
42
  async listDocuments(filter) {
22
43
  const documents = new Map();
23
44
  for (const record of this.records.values()) {
@@ -34,6 +55,40 @@ export class MemoryVectorStore {
34
55
  }
35
56
  return [...documents.values()];
36
57
  }
58
+ /**
59
+ * {@link VectorStore.listDocumentIds} — the ids `listDocuments` would report, without building the
60
+ * per-document metadata entries. Filtering goes through the same {@link matchesFilter} `search` uses.
61
+ */
62
+ async listDocumentIds(filter) {
63
+ const ids = new Set();
64
+ for (const record of this.records.values()) {
65
+ if (filter !== undefined && !matchesFilter(record.metadata, filter)) {
66
+ continue;
67
+ }
68
+ ids.add(documentIdOf(record.id));
69
+ }
70
+ return [...ids];
71
+ }
72
+ /**
73
+ * {@link VectorStore.removeWhere} — delete every chunk the same filter would let `search` reach.
74
+ * Parity is by construction: the predicate below is the identical {@link matchesFilter} call, on the
75
+ * identical argument, as the one in {@link MemoryVectorStore.search}.
76
+ */
77
+ async removeWhere(filter) {
78
+ assertRemovalFilter(filter);
79
+ if (filterDeniesAll(filter)) {
80
+ return 0;
81
+ }
82
+ let removed = 0;
83
+ for (const [id, record] of [...this.records]) {
84
+ if (!matchesFilter(record.metadata, filter)) {
85
+ continue;
86
+ }
87
+ this.records.delete(id);
88
+ removed += 1;
89
+ }
90
+ return removed;
91
+ }
37
92
  async search(embedding, options) {
38
93
  const scored = [];
39
94
  for (const record of this.records.values()) {
@@ -1 +1 @@
1
- {"version":3,"file":"memory-vector-store.js","sourceRoot":"","sources":["../../../src/rag/memory-vector-store.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAKL,YAAY,GACb,MAAM,mBAAmB,CAAC;AAE3B;;;GAGG;AACH,MAAM,OAAO,iBAAiB;IACX,OAAO,GAAG,IAAI,GAAG,EAAwB,CAAC;IAE3D,KAAK,CAAC,MAAM,CAAC,OAAuB;QAClC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,UAAkB;QAC7B,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;YACrC,IAAI,YAAY,CAAC,EAAE,CAAC,KAAK,UAAU,EAAE,CAAC;gBACpC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,MAAgC;QAClD,MAAM,SAAS,GAAG,IAAI,GAAG,EAA2B,CAAC;QACrD,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YAC3C,IAAI,MAAM,KAAK,SAAS,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC;gBACpE,SAAS;YACX,CAAC;YACD,MAAM,EAAE,GAAG,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACnC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;gBACvB,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE;oBAChB,EAAE;oBACF,GAAG,CAAC,MAAM,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBACxE,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,OAAO,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;IACjC,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,SAAmB,EAAE,OAA4B;QAC5D,MAAM,MAAM,GAAc,EAAE,CAAC;QAC7B,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YAC3C,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBACpF,SAAS;YACX,CAAC;YACD,MAAM,KAAK,GAAG,gBAAgB,CAAC,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;YAC5D,2FAA2F;YAC3F,4DAA4D;YAC5D,IAAI,OAAO,CAAC,QAAQ,KAAK,SAAS,IAAI,KAAK,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;gBAC/D,SAAS;YACX,CAAC;YACD,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,MAAM,CAAC,EAAE;gBACb,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,KAAK;gBACL,GAAG,CAAC,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACjE,GAAG,CAAC,MAAM,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACxE,CAAC,CAAC;QACL,CAAC;QACD,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IACzE,CAAC;CACF;AAED,kFAAkF;AAClF,MAAM,UAAU,gBAAgB,CAAC,CAAW,EAAE,CAAW;IACvD,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;IAC5C,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QAC/C,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC7B,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC7B,GAAG,IAAI,MAAM,GAAG,MAAM,CAAC;QACvB,KAAK,IAAI,MAAM,GAAG,MAAM,CAAC;QACzB,KAAK,IAAI,MAAM,GAAG,MAAM,CAAC;IAC3B,CAAC;IACD,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACxD,OAAO,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,WAAW,CAAC;AACnD,CAAC"}
1
+ {"version":3,"file":"memory-vector-store.js","sourceRoot":"","sources":["../../../src/rag/memory-vector-store.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAML,kBAAkB,EAClB,mBAAmB,EACnB,YAAY,EACZ,kBAAkB,EAClB,eAAe,GAChB,MAAM,mBAAmB,CAAC;AAE3B;;;GAGG;AACH,MAAM,OAAO,iBAAiB;IACX,OAAO,GAAG,IAAI,GAAG,EAAwB,CAAC;IAE3D,KAAK,CAAC,MAAM,CAAC,OAAuB;QAClC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,UAAkB;QAC7B,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;YACrC,IAAI,YAAY,CAAC,EAAE,CAAC,KAAK,UAAU,EAAE,CAAC;gBACpC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,cAAc,CAAC,UAAkB,EAAE,KAAoB;QAC3D,IAAI,kBAAkB,CAAC,KAAK,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3C,OAAO,CAAC,CAAC;QACX,CAAC;QACD,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACxC,IAAI,YAAY,CAAC,EAAE,CAAC,KAAK,UAAU,EAAE,CAAC;gBACpC,SAAS;YACX,CAAC;YACD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,GAAG,MAAM,EAAE,QAAQ,EAAE,kBAAkB,CAAC,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,EAAE,CAAC,CAAC;YAC1F,OAAO,IAAI,CAAC,CAAC;QACf,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,MAAgC;QAClD,MAAM,SAAS,GAAG,IAAI,GAAG,EAA2B,CAAC;QACrD,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YAC3C,IAAI,MAAM,KAAK,SAAS,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC;gBACpE,SAAS;YACX,CAAC;YACD,MAAM,EAAE,GAAG,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACnC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;gBACvB,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE;oBAChB,EAAE;oBACF,GAAG,CAAC,MAAM,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBACxE,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QACD,OAAO,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC;IACjC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,eAAe,CAAC,MAAgC;QACpD,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;QAC9B,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YAC3C,IAAI,MAAM,KAAK,SAAS,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC;gBACpE,SAAS;YACX,CAAC;YACD,GAAG,CAAC,GAAG,CAAC,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QACnC,CAAC;QACD,OAAO,CAAC,GAAG,GAAG,CAAC,CAAC;IAClB,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,WAAW,CAAC,MAA+B;QAC/C,mBAAmB,CAAC,MAAM,CAAC,CAAC;QAC5B,IAAI,eAAe,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5B,OAAO,CAAC,CAAC;QACX,CAAC;QACD,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC7C,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC;gBAC5C,SAAS;YACX,CAAC;YACD,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACxB,OAAO,IAAI,CAAC,CAAC;QACf,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,SAAmB,EAAE,OAA4B;QAC5D,MAAM,MAAM,GAAc,EAAE,CAAC;QAC7B,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YAC3C,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBACpF,SAAS;YACX,CAAC;YACD,MAAM,KAAK,GAAG,gBAAgB,CAAC,SAAS,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;YAC5D,2FAA2F;YAC3F,4DAA4D;YAC5D,IAAI,OAAO,CAAC,QAAQ,KAAK,SAAS,IAAI,KAAK,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;gBAC/D,SAAS;YACX,CAAC;YACD,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,MAAM,CAAC,EAAE;gBACb,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,KAAK;gBACL,GAAG,CAAC,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACjE,GAAG,CAAC,MAAM,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACxE,CAAC,CAAC;QACL,CAAC;QACD,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IACzE,CAAC;CACF;AAED,kFAAkF;AAClF,MAAM,UAAU,gBAAgB,CAAC,CAAW,EAAE,CAAW;IACvD,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;IAC5C,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QAC/C,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC7B,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC7B,GAAG,IAAI,MAAM,GAAG,MAAM,CAAC;QACvB,KAAK,IAAI,MAAM,GAAG,MAAM,CAAC;QACzB,KAAK,IAAI,MAAM,GAAG,MAAM,CAAC;IAC3B,CAAC;IACD,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACxD,OAAO,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,WAAW,CAAC;AACnD,CAAC"}
@@ -2,7 +2,7 @@ import type { EmbeddingProvider } from '../spi/embedding-provider.js';
2
2
  import type { Passage } from '../spi/retriever.js';
3
3
  import type { LucidDatabaseLike } from '../stores/lucid.js';
4
4
  import { EmbeddingRetriever } from './embedding-retriever.js';
5
- import type { IndexedDocument, VectorRecord, VectorSearchOptions, VectorStore } from './vector-store.js';
5
+ import type { IndexedDocument, MetadataPatch, VectorRecord, VectorSearchOptions, VectorStore } from './vector-store.js';
6
6
  /**
7
7
  * The similarity metric — which pgvector distance operator ranks results (and which index opclass the
8
8
  * migration should build). `cosine` (the default) is scale-invariant and the right choice for most
@@ -71,7 +71,38 @@ export declare class PgVectorStore implements VectorStore {
71
71
  schemaStatements(): string[];
72
72
  upsert(records: VectorRecord[]): Promise<void>;
73
73
  remove(documentId: string): Promise<void>;
74
+ /**
75
+ * {@link VectorStore.updateMetadata} — one `UPDATE` for the whole document, with the
76
+ * {@link import('./vector-store.js').MetadataPatch} merge done **in Postgres** rather than read back
77
+ * into JS: `(COALESCE(metadata, '{}') || <set>) - <removed keys>`. `||` is jsonb concatenation, which
78
+ * at the top level is exactly a shallow merge (right-hand keys win, wholesale); `jsonb - text[]`
79
+ * removes keys, which is how a `null` in the patch deletes. So there is no read-modify-write and no
80
+ * lost-update window between a `SELECT` and an `UPDATE`.
81
+ *
82
+ * The embedding is untouched **structurally**: `SET` names only the metadata column, so no statement
83
+ * this method can emit is capable of writing `embedding` or `text`. The row is not re-inserted, so
84
+ * pgvector never re-parses a vector literal either.
85
+ *
86
+ * `RETURNING` the id column yields the chunk count without a second query.
87
+ */
88
+ updateMetadata(documentId: string, patch: MetadataPatch): Promise<number>;
74
89
  listDocuments(filter?: Record<string, unknown>): Promise<IndexedDocument[]>;
90
+ /**
91
+ * {@link VectorStore.listDocumentIds} — `SELECT DISTINCT` of the document-id expression. Unlike
92
+ * {@link PgVectorStore.listDocuments} it never selects the `jsonb` metadata column, so Postgres does not
93
+ * transfer it and {@link parseMetadata} is never called: on a large corpus that is most of the cost.
94
+ * The `WHERE` comes from the same {@link buildMetadataWhere} `search` uses.
95
+ */
96
+ listDocumentIds(filter?: Record<string, unknown>): Promise<string[]>;
97
+ /**
98
+ * {@link VectorStore.removeWhere} — one `DELETE … RETURNING`, so the chunk count comes back from the
99
+ * same statement that does the deleting: no count-then-delete, no race between the two, one round trip.
100
+ *
101
+ * Filter parity with {@link PgVectorStore.search} is by construction — the `WHERE` fragment is produced
102
+ * by the identical {@link buildMetadataWhere} call, so an empty-array value emits the same `false`
103
+ * clause here as it does there, and a scalar the same `@> ?::jsonb` containment.
104
+ */
105
+ removeWhere(filter: Record<string, unknown>): Promise<number>;
75
106
  search(embedding: number[], options: VectorSearchOptions): Promise<Passage[]>;
76
107
  }
77
108
  /**
@@ -1 +1 @@
1
- {"version":3,"file":"pg-vector-store.d.ts","sourceRoot":"","sources":["../../../src/rag/pg-vector-store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AACtE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,KAAK,EACV,eAAe,EACf,YAAY,EACZ,mBAAmB,EACnB,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAE3B;;;;GAIG;AACH,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,IAAI,GAAG,OAAO,CAAC;AAEvD;;;;;GAKG;AACH,MAAM,WAAW,eAAe;IAC9B,6EAA6E;IAC7E,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,yCAAyC;IACzC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,2DAA2D;IAC3D,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,8DAA8D;IAC9D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,yDAAyD;IACzD,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,oBAAoB;IACnC,sFAAsF;IACtF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,oGAAoG;IACpG,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,+DAA+D;IAC/D,MAAM,CAAC,EAAE,cAAc,CAAC;IACxB,2DAA2D;IAC3D,OAAO,CAAC,EAAE,eAAe,CAAC;CAC3B;AAsDD,yGAAyG;AACzG,wBAAgB,eAAe,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,MAAM,CAE3D;AAyFD;;;;;;;;;;;;;;GAcG;AACH,qBAAa,aAAc,YAAW,WAAW;IAQ7C,OAAO,CAAC,QAAQ,CAAC,EAAE;IAPrB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAa;IACpC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAiB;IAC5C,OAAO,CAAC,QAAQ,CAAC,GAAG,CAA4B;gBAG7B,EAAE,EAAE,iBAAiB,EACtC,OAAO,GAAE,oBAAyB;IAgBpC;;;OAGG;IACG,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAMnC,uGAAuG;IACvG,gBAAgB,IAAI,MAAM,EAAE;IAgBtB,MAAM,CAAC,OAAO,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAsB9C,MAAM,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAMzC,aAAa,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAoB3E,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;CA+CpF;AAED;;;;GAIG;AACH,qBAAa,iBAAkB,SAAQ,kBAAkB;gBAE3C,QAAQ,EAAE,iBAAiB,EAAE,KAAK,EAAE,aAAa;CAG9D"}
1
+ {"version":3,"file":"pg-vector-store.d.ts","sourceRoot":"","sources":["../../../src/rag/pg-vector-store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AACtE,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAC5D,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAE9D,OAAO,KAAK,EACV,eAAe,EACf,aAAa,EACb,YAAY,EACZ,mBAAmB,EACnB,WAAW,EACZ,MAAM,mBAAmB,CAAC;AAE3B;;;;GAIG;AACH,MAAM,MAAM,cAAc,GAAG,QAAQ,GAAG,IAAI,GAAG,OAAO,CAAC;AAEvD;;;;;GAKG;AACH,MAAM,WAAW,eAAe;IAC9B,6EAA6E;IAC7E,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,yCAAyC;IACzC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,2DAA2D;IAC3D,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,8DAA8D;IAC9D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,yDAAyD;IACzD,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,oBAAoB;IACnC,sFAAsF;IACtF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,oGAAoG;IACpG,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,+DAA+D;IAC/D,MAAM,CAAC,EAAE,cAAc,CAAC;IACxB,2DAA2D;IAC3D,OAAO,CAAC,EAAE,eAAe,CAAC;CAC3B;AAsDD,yGAAyG;AACzG,wBAAgB,eAAe,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,MAAM,CAE3D;AAyFD;;;;;;;;;;;;;;GAcG;AACH,qBAAa,aAAc,YAAW,WAAW;IAQ7C,OAAO,CAAC,QAAQ,CAAC,EAAE;IAPrB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAa;IACpC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAiB;IAC5C,OAAO,CAAC,QAAQ,CAAC,GAAG,CAA4B;gBAG7B,EAAE,EAAE,iBAAiB,EACtC,OAAO,GAAE,oBAAyB;IAgBpC;;;OAGG;IACG,YAAY,IAAI,OAAO,CAAC,IAAI,CAAC;IAMnC,uGAAuG;IACvG,gBAAgB,IAAI,MAAM,EAAE;IAgBtB,MAAM,CAAC,OAAO,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAsB9C,MAAM,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAM/C;;;;;;;;;;;;;OAaG;IACG,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC;IAyBzE,aAAa,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAoBjF;;;;;OAKG;IACG,eAAe,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAa1E;;;;;;;OAOG;IACG,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IAa7D,MAAM,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,mBAAmB,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;CA+CpF;AAED;;;;GAIG;AACH,qBAAa,iBAAkB,SAAQ,kBAAkB;gBAE3C,QAAQ,EAAE,iBAAiB,EAAE,KAAK,EAAE,aAAa;CAG9D"}
@@ -1,4 +1,5 @@
1
1
  import { EmbeddingRetriever } from './embedding-retriever.js';
2
+ import { assertRemovalFilter, effectivePatchKeys, filterDeniesAll } from './vector-store.js';
2
3
  const METRICS = {
3
4
  cosine: {
4
5
  operator: '<=>',
@@ -203,6 +204,42 @@ export class PgVectorStore {
203
204
  documentId,
204
205
  ]);
205
206
  }
207
+ /**
208
+ * {@link VectorStore.updateMetadata} — one `UPDATE` for the whole document, with the
209
+ * {@link import('./vector-store.js').MetadataPatch} merge done **in Postgres** rather than read back
210
+ * into JS: `(COALESCE(metadata, '{}') || <set>) - <removed keys>`. `||` is jsonb concatenation, which
211
+ * at the top level is exactly a shallow merge (right-hand keys win, wholesale); `jsonb - text[]`
212
+ * removes keys, which is how a `null` in the patch deletes. So there is no read-modify-write and no
213
+ * lost-update window between a `SELECT` and an `UPDATE`.
214
+ *
215
+ * The embedding is untouched **structurally**: `SET` names only the metadata column, so no statement
216
+ * this method can emit is capable of writing `embedding` or `text`. The row is not re-inserted, so
217
+ * pgvector never re-parses a vector literal either.
218
+ *
219
+ * `RETURNING` the id column yields the chunk count without a second query.
220
+ */
221
+ async updateMetadata(documentId, patch) {
222
+ const keys = effectivePatchKeys(patch);
223
+ if (keys.length === 0) {
224
+ return 0;
225
+ }
226
+ const set = {};
227
+ const removed = [];
228
+ for (const key of keys) {
229
+ if (patch[key] === null) {
230
+ removed.push(key);
231
+ }
232
+ else {
233
+ set[key] = patch[key];
234
+ }
235
+ }
236
+ const c = this.col;
237
+ const raw = await this.db.rawQuery(`UPDATE ${this.table}
238
+ SET ${c.metadata} = (COALESCE(${c.metadata}, '{}'::jsonb) || ?::jsonb) - ?::text[]
239
+ WHERE ${documentIdExpr(c.id)} = ?
240
+ RETURNING ${c.id} AS id`, [JSON.stringify(set), removed, documentId]);
241
+ return normalizeRows(raw).length;
242
+ }
206
243
  async listDocuments(filter) {
207
244
  const c = this.col;
208
245
  const docExpr = documentIdExpr(c.id);
@@ -219,6 +256,38 @@ export class PgVectorStore {
219
256
  };
220
257
  });
221
258
  }
259
+ /**
260
+ * {@link VectorStore.listDocumentIds} — `SELECT DISTINCT` of the document-id expression. Unlike
261
+ * {@link PgVectorStore.listDocuments} it never selects the `jsonb` metadata column, so Postgres does not
262
+ * transfer it and {@link parseMetadata} is never called: on a large corpus that is most of the cost.
263
+ * The `WHERE` comes from the same {@link buildMetadataWhere} `search` uses.
264
+ */
265
+ async listDocumentIds(filter) {
266
+ const docExpr = documentIdExpr(this.col.id);
267
+ const where = buildMetadataWhere(filter, this.col.metadata);
268
+ const raw = await this.db.rawQuery(`SELECT DISTINCT ${docExpr} AS doc_id
269
+ FROM ${this.table}
270
+ ${where.sql}
271
+ ORDER BY doc_id`, where.bindings);
272
+ return normalizeRows(raw).map((row) => String(row.doc_id));
273
+ }
274
+ /**
275
+ * {@link VectorStore.removeWhere} — one `DELETE … RETURNING`, so the chunk count comes back from the
276
+ * same statement that does the deleting: no count-then-delete, no race between the two, one round trip.
277
+ *
278
+ * Filter parity with {@link PgVectorStore.search} is by construction — the `WHERE` fragment is produced
279
+ * by the identical {@link buildMetadataWhere} call, so an empty-array value emits the same `false`
280
+ * clause here as it does there, and a scalar the same `@> ?::jsonb` containment.
281
+ */
282
+ async removeWhere(filter) {
283
+ assertRemovalFilter(filter);
284
+ if (filterDeniesAll(filter)) {
285
+ return 0;
286
+ }
287
+ const where = buildMetadataWhere(filter, this.col.metadata);
288
+ const raw = await this.db.rawQuery(`DELETE FROM ${this.table} ${where.sql} RETURNING ${this.col.id} AS id`, where.bindings);
289
+ return normalizeRows(raw).length;
290
+ }
222
291
  async search(embedding, options) {
223
292
  const c = this.col;
224
293
  const vector = toVectorLiteral(embedding);