@functional-systems/memdrive 0.1.1-dev.2 → 0.1.1-dev.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (189) hide show
  1. package/CHANGELOG.md +6 -0
  2. package/README.md +20 -279
  3. package/dist/commonjs/chunking.d.ts +1 -2
  4. package/dist/commonjs/chunking.d.ts.map +1 -1
  5. package/dist/commonjs/chunking.js +2 -5
  6. package/dist/commonjs/chunking.js.map +1 -1
  7. package/dist/commonjs/config.d.ts +17 -0
  8. package/dist/commonjs/config.d.ts.map +1 -0
  9. package/dist/commonjs/config.js +116 -0
  10. package/dist/commonjs/config.js.map +1 -0
  11. package/dist/commonjs/errors.d.ts +10 -6
  12. package/dist/commonjs/errors.d.ts.map +1 -1
  13. package/dist/commonjs/errors.js +25 -5
  14. package/dist/commonjs/errors.js.map +1 -1
  15. package/dist/commonjs/index.d.ts +15 -5
  16. package/dist/commonjs/index.d.ts.map +1 -1
  17. package/dist/commonjs/index.js +332 -280
  18. package/dist/commonjs/index.js.map +1 -1
  19. package/dist/commonjs/records.d.ts +44 -0
  20. package/dist/commonjs/records.d.ts.map +1 -0
  21. package/dist/commonjs/records.js +88 -0
  22. package/dist/commonjs/records.js.map +1 -0
  23. package/dist/commonjs/search.d.ts +8 -0
  24. package/dist/commonjs/search.d.ts.map +1 -0
  25. package/dist/commonjs/search.js +56 -0
  26. package/dist/commonjs/search.js.map +1 -0
  27. package/dist/commonjs/types.d.ts +84 -252
  28. package/dist/commonjs/types.d.ts.map +1 -1
  29. package/dist/esm/chunking.d.ts +1 -2
  30. package/dist/esm/chunking.d.ts.map +1 -1
  31. package/dist/esm/chunking.js +2 -4
  32. package/dist/esm/chunking.js.map +1 -1
  33. package/dist/esm/config.d.ts +17 -0
  34. package/dist/esm/config.d.ts.map +1 -0
  35. package/dist/esm/config.js +107 -0
  36. package/dist/esm/config.js.map +1 -0
  37. package/dist/esm/errors.d.ts +10 -6
  38. package/dist/esm/errors.d.ts.map +1 -1
  39. package/dist/esm/errors.js +22 -4
  40. package/dist/esm/errors.js.map +1 -1
  41. package/dist/esm/index.d.ts +15 -5
  42. package/dist/esm/index.d.ts.map +1 -1
  43. package/dist/esm/index.js +332 -278
  44. package/dist/esm/index.js.map +1 -1
  45. package/dist/esm/records.d.ts +44 -0
  46. package/dist/esm/records.d.ts.map +1 -0
  47. package/dist/esm/records.js +77 -0
  48. package/dist/esm/records.js.map +1 -0
  49. package/dist/esm/search.d.ts +8 -0
  50. package/dist/esm/search.d.ts.map +1 -0
  51. package/dist/esm/search.js +52 -0
  52. package/dist/esm/search.js.map +1 -0
  53. package/dist/esm/types.d.ts +84 -252
  54. package/dist/esm/types.d.ts.map +1 -1
  55. package/package.json +7 -42
  56. package/src/chunking.ts +2 -3
  57. package/src/config.ts +121 -0
  58. package/src/errors.ts +20 -6
  59. package/src/index.ts +352 -260
  60. package/src/records.ts +83 -0
  61. package/src/search.ts +57 -0
  62. package/src/types.ts +74 -253
  63. package/dist/commonjs/filter.d.ts +0 -30
  64. package/dist/commonjs/filter.d.ts.map +0 -1
  65. package/dist/commonjs/filter.js +0 -224
  66. package/dist/commonjs/filter.js.map +0 -1
  67. package/dist/commonjs/recall-input.d.ts +0 -8
  68. package/dist/commonjs/recall-input.d.ts.map +0 -1
  69. package/dist/commonjs/recall-input.js +0 -45
  70. package/dist/commonjs/recall-input.js.map +0 -1
  71. package/dist/commonjs/retriever/index.d.ts +0 -7
  72. package/dist/commonjs/retriever/index.d.ts.map +0 -1
  73. package/dist/commonjs/retriever/index.js +0 -167
  74. package/dist/commonjs/retriever/index.js.map +0 -1
  75. package/dist/commonjs/retriever/scope.d.ts +0 -5
  76. package/dist/commonjs/retriever/scope.d.ts.map +0 -1
  77. package/dist/commonjs/retriever/scope.js +0 -69
  78. package/dist/commonjs/retriever/scope.js.map +0 -1
  79. package/dist/commonjs/retriever/types.d.ts +0 -89
  80. package/dist/commonjs/retriever/types.d.ts.map +0 -1
  81. package/dist/commonjs/retriever/types.js +0 -3
  82. package/dist/commonjs/retriever/types.js.map +0 -1
  83. package/dist/commonjs/storage.d.ts +0 -274
  84. package/dist/commonjs/storage.d.ts.map +0 -1
  85. package/dist/commonjs/storage.js +0 -861
  86. package/dist/commonjs/storage.js.map +0 -1
  87. package/dist/commonjs/tools/definitions.d.ts +0 -5
  88. package/dist/commonjs/tools/definitions.d.ts.map +0 -1
  89. package/dist/commonjs/tools/definitions.js +0 -147
  90. package/dist/commonjs/tools/definitions.js.map +0 -1
  91. package/dist/commonjs/tools/evidence-ids.d.ts +0 -7
  92. package/dist/commonjs/tools/evidence-ids.d.ts.map +0 -1
  93. package/dist/commonjs/tools/evidence-ids.js +0 -71
  94. package/dist/commonjs/tools/evidence-ids.js.map +0 -1
  95. package/dist/commonjs/tools/factory.d.ts +0 -7
  96. package/dist/commonjs/tools/factory.d.ts.map +0 -1
  97. package/dist/commonjs/tools/factory.js +0 -169
  98. package/dist/commonjs/tools/factory.js.map +0 -1
  99. package/dist/commonjs/tools/index.d.ts +0 -6
  100. package/dist/commonjs/tools/index.d.ts.map +0 -1
  101. package/dist/commonjs/tools/index.js +0 -25
  102. package/dist/commonjs/tools/index.js.map +0 -1
  103. package/dist/commonjs/tools/internal-types.d.ts +0 -18
  104. package/dist/commonjs/tools/internal-types.d.ts.map +0 -1
  105. package/dist/commonjs/tools/internal-types.js +0 -3
  106. package/dist/commonjs/tools/internal-types.js.map +0 -1
  107. package/dist/commonjs/tools/short-ids.d.ts +0 -8
  108. package/dist/commonjs/tools/short-ids.d.ts.map +0 -1
  109. package/dist/commonjs/tools/short-ids.js +0 -81
  110. package/dist/commonjs/tools/short-ids.js.map +0 -1
  111. package/dist/commonjs/tools/types.d.ts +0 -53
  112. package/dist/commonjs/tools/types.d.ts.map +0 -1
  113. package/dist/commonjs/tools/types.js +0 -3
  114. package/dist/commonjs/tools/types.js.map +0 -1
  115. package/dist/commonjs/values.d.ts +0 -17
  116. package/dist/commonjs/values.d.ts.map +0 -1
  117. package/dist/commonjs/values.js +0 -90
  118. package/dist/commonjs/values.js.map +0 -1
  119. package/dist/esm/filter.d.ts +0 -30
  120. package/dist/esm/filter.d.ts.map +0 -1
  121. package/dist/esm/filter.js +0 -209
  122. package/dist/esm/filter.js.map +0 -1
  123. package/dist/esm/recall-input.d.ts +0 -8
  124. package/dist/esm/recall-input.d.ts.map +0 -1
  125. package/dist/esm/recall-input.js +0 -40
  126. package/dist/esm/recall-input.js.map +0 -1
  127. package/dist/esm/retriever/index.d.ts +0 -7
  128. package/dist/esm/retriever/index.d.ts.map +0 -1
  129. package/dist/esm/retriever/index.js +0 -163
  130. package/dist/esm/retriever/index.js.map +0 -1
  131. package/dist/esm/retriever/scope.d.ts +0 -5
  132. package/dist/esm/retriever/scope.d.ts.map +0 -1
  133. package/dist/esm/retriever/scope.js +0 -66
  134. package/dist/esm/retriever/scope.js.map +0 -1
  135. package/dist/esm/retriever/types.d.ts +0 -89
  136. package/dist/esm/retriever/types.d.ts.map +0 -1
  137. package/dist/esm/retriever/types.js +0 -2
  138. package/dist/esm/retriever/types.js.map +0 -1
  139. package/dist/esm/storage.d.ts +0 -274
  140. package/dist/esm/storage.d.ts.map +0 -1
  141. package/dist/esm/storage.js +0 -845
  142. package/dist/esm/storage.js.map +0 -1
  143. package/dist/esm/tools/definitions.d.ts +0 -5
  144. package/dist/esm/tools/definitions.d.ts.map +0 -1
  145. package/dist/esm/tools/definitions.js +0 -144
  146. package/dist/esm/tools/definitions.js.map +0 -1
  147. package/dist/esm/tools/evidence-ids.d.ts +0 -7
  148. package/dist/esm/tools/evidence-ids.d.ts.map +0 -1
  149. package/dist/esm/tools/evidence-ids.js +0 -68
  150. package/dist/esm/tools/evidence-ids.js.map +0 -1
  151. package/dist/esm/tools/factory.d.ts +0 -7
  152. package/dist/esm/tools/factory.d.ts.map +0 -1
  153. package/dist/esm/tools/factory.js +0 -166
  154. package/dist/esm/tools/factory.js.map +0 -1
  155. package/dist/esm/tools/index.d.ts +0 -6
  156. package/dist/esm/tools/index.d.ts.map +0 -1
  157. package/dist/esm/tools/index.js +0 -22
  158. package/dist/esm/tools/index.js.map +0 -1
  159. package/dist/esm/tools/internal-types.d.ts +0 -18
  160. package/dist/esm/tools/internal-types.d.ts.map +0 -1
  161. package/dist/esm/tools/internal-types.js +0 -2
  162. package/dist/esm/tools/internal-types.js.map +0 -1
  163. package/dist/esm/tools/short-ids.d.ts +0 -8
  164. package/dist/esm/tools/short-ids.d.ts.map +0 -1
  165. package/dist/esm/tools/short-ids.js +0 -78
  166. package/dist/esm/tools/short-ids.js.map +0 -1
  167. package/dist/esm/tools/types.d.ts +0 -53
  168. package/dist/esm/tools/types.d.ts.map +0 -1
  169. package/dist/esm/tools/types.js +0 -2
  170. package/dist/esm/tools/types.js.map +0 -1
  171. package/dist/esm/values.d.ts +0 -17
  172. package/dist/esm/values.d.ts.map +0 -1
  173. package/dist/esm/values.js +0 -87
  174. package/dist/esm/values.js.map +0 -1
  175. package/docs/tools.md +0 -264
  176. package/src/filter.ts +0 -189
  177. package/src/recall-input.ts +0 -46
  178. package/src/retriever/index.ts +0 -142
  179. package/src/retriever/scope.ts +0 -56
  180. package/src/retriever/types.ts +0 -68
  181. package/src/storage.ts +0 -765
  182. package/src/tools/definitions.ts +0 -175
  183. package/src/tools/evidence-ids.ts +0 -57
  184. package/src/tools/factory.ts +0 -153
  185. package/src/tools/index.ts +0 -25
  186. package/src/tools/internal-types.ts +0 -12
  187. package/src/tools/short-ids.ts +0 -58
  188. package/src/tools/types.ts +0 -48
  189. package/src/values.ts +0 -103
package/docs/tools.md DELETED
@@ -1,264 +0,0 @@
1
- # Common memory tools
2
-
3
- `@functional-systems/memdrive/tools` supplies provider-independent tool definitions
4
- and execution functions. Calling agents plan queries, choose evidence and generate
5
- answers. A separate hosted MCP service owns authentication, authorization,
6
- protocol handling, deployment and operational limits. No MCP or model client is
7
- bundled with MemDrive.
8
-
9
- ## Bind tools
10
-
11
- ```ts
12
- import { createTools } from '@functional-systems/memdrive/tools';
13
-
14
- // The application has already opened and authorized this drive.
15
- const tools = await createTools({ drive });
16
- const result = await tools.execute('memdrive_recall', {
17
- requests: [{ query: 'What was agreed about the meeting?', limit: 10 }],
18
- }, { signal });
19
- ```
20
-
21
- Creation reads `drive.describe()` once to build schemas from the actual embedding
22
- configuration and metadata declarations, including field descriptions. Pass an
23
- optional `branch` to bind data operations to that branch; otherwise the supplied
24
- handle's branch is used. Model arguments cannot change the drive, connection or
25
- branch. `describe` remains collection-scoped, as it is in the SDK.
26
-
27
- The default tools are read-only. `createTools({ drive, writable: true })` also
28
- exposes `remember` and `forget`. This switch and tool annotations are not access
29
- control: the service must authorize each call and select the correct handle.
30
- Collection creation/destruction, configuration and branch administration are not
31
- part of this memory-tool surface.
32
-
33
- `definitions` returns detached JSON data. Recreate the registry after changing
34
- the metadata schema; existing input schemas describe the creation-time fields.
35
- A `describe` call reads fresh metadata but does not silently change the registered
36
- tool definitions. Hosted services own any tool-list refresh notification.
37
-
38
- ## Tool catalog
39
-
40
- | Tool | Purpose | Input and result |
41
- | --- | --- | --- |
42
- | `memdrive_recall` | Retrieve evidence for questions or missing facts | `{requests: [...]}` with 1–5 SDK recall requests; `value.results` follows input order, including per-request failures. |
43
- | `memdrive_read` | Expand excerpts to complete originals | `{ids: [...]}` using canonical `memId`; returns `value.mems`, omitting missing IDs. |
44
- | `memdrive_describe` | Discover fields, meanings and search configuration | `{}`; returns the existing SDK drive descriptor. |
45
- | `memdrive_count_values` | Inspect keyword value distributions | `{fields, filter?, excludeIds?, limit?}`; returns buckets, per-field `hasMore` and `asOf`. |
46
- | `memdrive_list` | Browse inventory without a relevance query | `{limit?, cursor?}`; returns original mems and optional `nextCursor`. Continue even after an empty page when a cursor exists. |
47
- | `memdrive_remember` | Add memories or fully replace supplied IDs | `{mems: [...]}`; returns IDs in input order. Available only when writable. |
48
- | `memdrive_forget` | Delete whole memories and their chunks | Exactly `{ids: [...]}` or `{filter: ...}`; success returns `{acknowledged: true}`, not a deletion count. Available only when writable. |
49
-
50
- Descriptions state when to use each operation, parameter meanings and important
51
- limitations. Field schemas carry the actual metadata types and descriptions.
52
- Closed objects reject extra top-level arguments, filters use explicit Boolean
53
- expressions, and SDK validation remains authoritative for dates, byte limits,
54
- vector values and other semantic constraints. No silent coercion or null-to-omit
55
- conversion is performed. Input is detached before asynchronous execution.
56
-
57
- By default, recall uses the SDK's existing vector or hybrid pipeline and native reranking
58
- settings. It does not add a query-rewriting model, select evidence or execute a
59
- second search. The tool's five-request bound is separate from the SDK's uncapped
60
- batch length; SDK concurrency remains shared across branch views.
61
-
62
- External-vector drives have vector-input schemas. They return references and
63
- metadata without vectors or source text. A model must not invent embeddings;
64
- the caller provides values from the configured vector space.
65
-
66
- Counts accept up to five keyword fields and 100 values per field, matching native
67
- facet limits. They count original memories once rather than counting chunks.
68
- An array may contribute to several buckets. Counts can be cached for 60 seconds;
69
- `asOf` is the fetch time, and `hasMore: null` means completeness is unknown.
70
-
71
- ## Canonical source identifiers
72
-
73
- Public `createTools` inputs and results always use canonical stored memory IDs.
74
- Returned text matches retain their UTF-16 offsets, content, timestamp and metadata.
75
- The public registry exposes only `definitions` and `execute`; it has no alias
76
- options or resolver methods. A real stored ID such as `m1` remains an ordinary
77
- canonical ID. Writable tools use the same canonical-ID contract.
78
-
79
- The optional agentic retriever uses short memory/evidence references internally
80
- for its injected model session. Those references are resolved before returning
81
- selected evidence to the caller. They are visible to the model adapter and
82
- host audit events, not public memory-tool results. The adapter's
83
- `ModelTurn.evidenceIds` field belongs to that internal model protocol; it does not
84
- add an ID mode to public tools. Unknown or undelivered selections fail. There is
85
- no duplicate-body suppression or automatic exclusion.
86
-
87
- ## Evidence and writes
88
-
89
- A match with `kind: 'mem'` is already complete; `kind: 'chunk'` contains an excerpt
90
- with UTF-16 offsets. Pass its canonical `memId` to `memdrive_read` for surrounding context. There are no session-specific evidence tables, automatic deduplication
91
- or finish/report tools. Callers retain earlier results and decide
92
- whether more retrieval is useful. `excludeIds` excludes whole memories and all
93
- their chunks, so it is not equivalent to hiding only previously seen excerpts.
94
-
95
- Remember without an ID creates a new memory, including when repeated. Supplying
96
- an existing ID fully replaces content and metadata; it is not a patch. Errors may
97
- contain `completedIds`, `uncertainIds` and `notAttemptedIds`. Preserve these outcomes
98
- and reconcile them before retrying; neither tools nor SDK automatically retry a
99
- multi-request write. Deletion success acknowledges the request and does not prove
100
- that any specified ID existed. Cancellation propagates through `AbortSignal` but
101
- does not undo accepted writes.
102
-
103
- ## One execution contract, different transports
104
-
105
- Every execution returns JSON data in one of these envelopes:
106
-
107
- ```ts
108
- { ok: true, value: /* SDK result, or forget acknowledgement */ }
109
- { ok: false, error: { code, message, status?, details? } }
110
- ```
111
-
112
- The tool's `outputSchema` describes both. Errors expose actionable SDK codes and
113
- safe write outcomes, without causes, stack traces or raw transport bodies.
114
- Unexpected exceptions use `TOOL_EXECUTION_FAILED` with a generic message. A batch
115
- recall that ran has `ok: true` even when some `value.results` items contain errors;
116
- inspect each item. Unknown or disabled tools return `UNKNOWN_TOOL` before dispatch.
117
-
118
- Direct function calling can register the same definitions. For OpenAI Responses,
119
- the registration shape is:
120
-
121
- ```ts
122
- const functions = tools.definitions.map(tool => ({
123
- type: 'function', name: tool.name, description: tool.description,
124
- parameters: tool.inputSchema, strict: false,
125
- }));
126
- // On a function call, parse its arguments and pass the same object to execute.
127
- const result = await tools.execute(call.name, JSON.parse(call.arguments), { signal });
128
- const output = JSON.stringify(result);
129
- ```
130
-
131
- These portable schemas preserve optional fields and SDK omission semantics.
132
- They are not advertised as OpenAI strict schemas: that mode requires every
133
- property to be required and uses nullable fields for optionality. Explicitly use
134
- `strict: false`; a future provider-specific strict conversion must also define
135
- and test its argument normalization. Claude registration uses the same `name`
136
- and `description` with `input_schema: tool.inputSchema`.
137
-
138
- A hosted MCP service lists the same definitions and packages the execution data:
139
-
140
- ```ts
141
- const result = await tools.execute(name, argumentsObject, { signal });
142
- const mcpResult = {
143
- isError: !result.ok,
144
- structuredContent: result,
145
- content: [{ type: 'text', text: JSON.stringify(result) }],
146
- };
147
- ```
148
-
149
- This is a mapping example, not an MCP server implementation. The MCP service
150
- handles protocol errors, unknown-tool routing, sessions, cancellation messages
151
- and authentication. In particular, it maps unknown names to the protocol's
152
- unknown-tool error instead of treating them as successful tool dispatches.
153
- It must not change defaults, truncate results silently or implement separate
154
- search logic. Direct function-call output and MCP `structuredContent`/JSON text
155
- carry the same canonical execution result. Provider context limits and differing
156
- agent prompts can still change the model's choices and final answer.
157
-
158
- ## Standards and validation
159
-
160
- Definitions use JSON Schema 2020-12, structured output schemas and tool annotations
161
- following the [MCP tools specification](https://modelcontextprotocol.io/specification/2025-11-25/server/tools).
162
- The [MCP annotation reference](https://modelcontextprotocol.io/specification/2025-11-25/schema#toolannotations)
163
- defines effect hints, not authorization. Reads are marked read-only; remember and
164
- forget can destroy prior data, and remember is not generally idempotent because
165
- IDs may be generated. The tools operate on a bound memory database rather than
166
- arbitrary external destinations.
167
-
168
- The [OpenAI function-calling guide](https://developers.openai.com/api/docs/guides/function-calling)
169
- documents strict-mode requirements and explicit non-strict registration.
170
- The [Claude tool-definition guide](https://platform.claude.com/docs/en/agents-and-tools/tool-use/define-tools)
171
- emphasizes clear purposes, parameter meanings and limitations. These sources
172
- inform the shared contract; they do not imply live certification by a provider.
173
-
174
- Maintained tests compile the schemas with Ajv 2020, validate real SDK fixture
175
- results, exercise branch binding, writable selection, batch correspondence,
176
- partial failures and cancellation, and compare direct/MCP result packaging.
177
- Package checks exercise ESM, CommonJS, public TypeScript types and browser bundles.
178
- Simulated envelope equality is not a live MCP transport or model-quality test.
179
-
180
- ## Optional retriever
181
-
182
- `createRetriever` from `@functional-systems/memdrive/retriever` provides direct
183
- `.recall(request, {signal})`. It defaults to `single_pass`, which invokes one base
184
- recall without creating a model session. To configure recall within the standard
185
- tool catalog, use the host-owned `retrieval` option:
186
-
187
- ```ts
188
- const tools = await createTools({ drive, branch: 'main',
189
- retrieval: { strategy: 'agentic', agent: { model, countTokens } } });
190
- const result = await tools.execute('memdrive_recall', {
191
- requests: [{ query: 'What changed?', limit: 10 }, { query: 'What was agreed?' }],
192
- }, { signal });
193
- ```
194
-
195
- The tool inventory and all input schemas stay unchanged: recall accepts 1–5
196
- requests, while read/list/describe/count and optional writable tools remain
197
- available. Only configured recall execution/output changes. Each result slot
198
- includes `completion` and orchestration `usage`; invalid requests retain an
199
- `error` slot with explicit zero work, and independent siblings still execute.
200
- Requests run concurrently in input order, with isolated model sessions, evidence
201
- registries and per-request budgets. A failed/canceled retrieval has no fallback
202
- matches. Native accounting may be unknown (`null`). Returned canonical IDs work
203
- with the same registry's read/count/filter operations. Enabling retrieval does
204
- not remove writable tools or change their input/output IDs.
205
-
206
- Omitting `retrieval` preserves basic execution and output schemas exactly. Base
207
- `drive.recall` is unchanged. Internal agent tools always use primitive recall;
208
- configuration does not recursively invoke another agent. Strategy, adapter,
209
- guidance and budgets are snapshotted host configuration, never model arguments.
210
-
211
- For `strategy: 'agentic'`, provide `agent.model`, `agent.countTokens` and optional
212
- `agent.instructions` / `agent.limits`. Managed text memories are required; an
213
- external-vector drive is rejected because the SDK cannot generate query vectors.
214
- Each recall creates an isolated read-only tool session with short memory and
215
- evidence IDs. The model can search, read, list, describe and count values, then
216
- select delivered evidence IDs. The result contains canonical `matches`,
217
- `completion` and `usage`. The caller's `limit` bounds the final selection;
218
- individual searches retain their own request limits. There is no automatic
219
- fallback to unselected candidates, duplicate suppression or answer generation.
220
-
221
- The adapter implements `createSession({request, instructions, tools})`, returning
222
- `prepare(replies)`. Both are strictly local: no network calls. `prepare` returns
223
- `{inputTokens, respond({signal, maxOutputTokens})}`. Estimate **the entire provider
224
- request**, including instructions, tool schemas, history and opaque state;
225
- reservation is only as conservative as this estimate. `respond` performs one
226
- provider call and returns `{status, usage, calls?}` or
227
- `{status, usage, evidenceIds?}`. Each call has `{id, name, arguments}`; replies have
228
- `{id, name, result}`. IDs must be unique within the retrieval. Preserve provider
229
- conversation/reasoning state in the adapter, and pass the supplied signal to its
230
- transport. Return `status: 'incomplete'` with known usage when parsing a completed
231
- provider response fails. Network failures may throw and are counted as unknown
232
- usage. No provider package is included in MemDrive.
233
-
234
- Defaults are 8 model rounds, 24 tool calls, 120 data calls, 48,000 context tokens,
235
- 160,000 total model tokens, 4,096 output tokens per call, 16,000 tokens per delivered
236
- tool result and 900,000 milliseconds. Each recall batch item counts as one data
237
- call; read/list/count/describe count as one each. These are **not native database
238
- HTTP limits**. `countTokens` measures the complete serialized tool result; an
239
- oversized result is replaced by an explicit size-limit error, never truncated.
240
- The SDK reserves estimated input plus maximum output before dispatch and checks
241
- reported usage afterward; it cannot undo provider overuse. Adapters must honor
242
- cancellation and output bounds. Synchronous adapter work cannot be preempted.
243
- Async model, tool and record-hook waits are bounded by the retrieval signal.
244
-
245
- `completion.status` is `completed`, `budget_exhausted`, `aborted` or `failed`.
246
- Noncompleted outcomes contain no matches and an explicit reason, preserving
247
- usage already observed. Invalid caller requests reject before model work.
248
- `usage` reports model calls, known input/output tokens, unknown-usage calls,
249
- tool/data calls and elapsed milliseconds. `databaseRequests` is null after any
250
- primitive without exact native request accounting; `knownRecallDatabaseRequests`
251
- retains the subtotal from completed base recall results. Initialization performs
252
- one `describe` before retrieval timing and is outside per-recall usage. A native
253
- transport hook can impose a separate hard HTTP budget.
254
-
255
- The original caller filter and exclusions constrain every internal search/count
256
- and every original exposed through read/list; the supplied drive fixes the
257
- branch. Scope uses the SDK's filter semantics. Browsing can return an empty page
258
- with a continuation cursor after filtering. Read/list may physically fetch
259
- excluded originals before filtering. This is retrieval selection, **not an
260
- authorization boundary**: authorize the underlying drive separately. Underlying
261
- drive/native diagnostics may contain data outside this selection. Optional
262
- retriever `record` events are detached and scoped; tool events include host-only
263
- canonical source mappings for audit, including oversized results that were not
264
- delivered. Only actually delivered evidence is eligible for final selection.
package/src/filter.ts DELETED
@@ -1,189 +0,0 @@
1
- import type { Filter, FilterScalar, Metadata, MetadataSchema, MetadataType } from './types.js';
2
-
3
- import { requireInput } from './errors.js';
4
-
5
- export const isObject = (value: unknown): value is Record<string, unknown> => value !== null && typeof value === 'object'
6
- && !Array.isArray(value) && [Object.prototype, null].includes(Object.getPrototypeOf(value));
7
- const own = (value: object, key: PropertyKey) => Object.hasOwn(value, key);
8
- const kinds = new Set(['string', 'number', 'boolean', 'datetime', 'string[]']);
9
- export const matchAll = () => ({ queryString: { query: '*:*' } });
10
-
11
- // LambdaDB datetime indexes store epoch milliseconds. Reject finer precision
12
- // instead of silently changing equality/range semantics; trailing zeroes are exact.
13
- export function dateValue(value: unknown) {
14
- if (typeof value !== 'string') return undefined;
15
- const m = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.(\d{1,9}))?(Z|[+-]\d{2}:\d{2})$/.exec(value);
16
- if (!m) return undefined;
17
- const year = m[1]!, month = m[2]!, day = m[3]!, hour = m[4]!, minute = m[5]!, second = m[6]!, fraction = m[7] ?? '', zone = m[8]!;
18
- if (/[1-9]/.test(fraction.slice(3))) return undefined;
19
- const y = Number(year), mo = Number(month), d = Number(day);
20
- const days = [31, y % 4 === 0 && (y % 100 !== 0 || y % 400 === 0) ? 29 : 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31];
21
- if (mo < 1 || mo > 12 || d < 1 || d > days[mo - 1]! || Number(hour) > 23
22
- || Number(minute) > 59 || Number(second) > 59
23
- || (zone !== 'Z' && (Number(zone.slice(1, 3)) > 23 || Number(zone.slice(4)) > 59))) return undefined;
24
- const millis = Date.parse(`${year}-${month}-${day}T${hour}:${minute}:${second}${zone}`);
25
- return Number.isFinite(millis) ? BigInt(millis) + BigInt(fraction.padEnd(3, '0').slice(0, 3)) : undefined;
26
- }
27
-
28
- export function validateDescription(description: unknown): string | undefined {
29
- requireInput(description === undefined || (typeof description === 'string' && description.length <= 255),
30
- 'Description must be text of at most 255 characters; use an empty string to clear it.');
31
- return description;
32
- }
33
-
34
- export function validateMetadataSchema(schema: unknown): MetadataSchema {
35
- requireInput(isObject(schema), 'metadataSchema must be an object.');
36
- for (const [name, declaration] of Object.entries(schema)) {
37
- requireInput(/^[A-Za-z_][A-Za-z0-9_]*$/.test(name)
38
- && !['id', '__proto__', 'prototype', 'constructor'].includes(name), 'Metadata field names must be unambiguous identifiers and cannot be reserved.');
39
- requireInput(isObject(declaration) && Object.keys(declaration).every(key => ['type', 'description'].includes(key))
40
- && own(declaration, 'type') && typeof declaration.type === 'string' && kinds.has(declaration.type), 'Declare each metadata field with a supported type and optional description.');
41
- validateDescription(declaration.description);
42
- }
43
- return structuredClone(schema) as MetadataSchema;
44
- }
45
-
46
- export function fieldType(field: unknown, schema: MetadataSchema): MetadataType | undefined {
47
- if (field === 'id') return 'string';
48
- if (field === 'timestamp') return 'datetime';
49
- if (typeof field === 'string' && field.startsWith('metadata.')) {
50
- const name = field.slice(9);
51
- if (own(schema, name)) return schema[name]!.type;
52
- }
53
- return undefined;
54
- }
55
-
56
- export function validScalar(value: unknown, type: MetadataType | undefined) {
57
- if (type === 'string' || type === 'string[]') return typeof value === 'string' && value.length <= 4096;
58
- if (type === 'number') return typeof value === 'number' && Number.isFinite(value);
59
- if (type === 'boolean') return typeof value === 'boolean';
60
- return type === 'datetime' && dateValue(value) !== undefined;
61
- }
62
-
63
- export function validateFilter(filter: unknown, schema: MetadataSchema): Filter | undefined {
64
- if (filter === undefined) return undefined;
65
- const seen = new Set<object>();
66
- const visit = (value: unknown): void => {
67
- requireInput(isObject(value) && !seen.has(value), 'Filter must be an acyclic explicit Boolean expression.');
68
- seen.add(value);
69
- const keys = Object.keys(value);
70
- if (keys.length === 1 && ['and', 'or'].includes(keys[0]!)) {
71
- const children = value[keys[0]!];
72
- requireInput(Array.isArray(children) && children.length > 0, 'and/or require nonempty arrays.');
73
- children.forEach(visit);
74
- } else {
75
- requireInput(keys.length === 3 && ['field', 'op', 'value'].every(key => own(value, key)), 'Filter leaves require exactly field, op and value.');
76
- const type = fieldType(value.field, schema);
77
- requireInput(type !== undefined, 'Filter field must be id, timestamp, or declared metadata.');
78
- if (value.op === 'exists') requireInput(typeof value.value === 'boolean', 'exists requires a boolean.');
79
- else if (value.op === 'in') requireInput(Array.isArray(value.value) && value.value.length > 0
80
- && value.value.every(item => validScalar(item, type)), 'in requires a nonempty list of correctly typed scalar values.');
81
- else {
82
- requireInput(typeof value.op === 'string' && ['eq', 'neq', 'gt', 'gte', 'lt', 'lte'].includes(value.op), 'Unsupported filter operator.');
83
- requireInput(validScalar(value.value, type), 'Filter value does not match the declared field type.');
84
- requireInput(['eq', 'neq'].includes(value.op) || ['number', 'datetime'].includes(type), 'Ordering is supported only for numbers and datetimes.');
85
- }
86
- }
87
- seen.delete(value);
88
- };
89
- visit(filter);
90
- return structuredClone(filter) as Filter;
91
- }
92
-
93
- export interface NativeFilter { queryString?: {query: string; defaultField?: string}; bool?: NativeFilter[]; occur?: string | undefined }
94
-
95
- const queryString = (field: string, query: string) => ({ queryString: { query, defaultField: field } });
96
- // Quote even literal *:*: LambdaDB treats that exact string as match-all with skipSyntax.
97
- const quote = (value: unknown) => `"${String(value).replace(/[\\"]/g, '\\$&')}"`;
98
- const bool = (clauses: NativeFilter[], occur: string): NativeFilter => ({ bool: clauses.map(clause => ({ ...clause, occur })) });
99
- function exists(field: string, type: MetadataType | undefined): NativeFilter {
100
- if (type === 'boolean') return bool([queryString(field, 'true'), queryString(field, 'false')], 'should');
101
- // A bare wildcard is rejected by the native parser's leading-wildcard policy.
102
- return queryString(field, '[* TO *]');
103
- }
104
- function decimal(value: FilterScalar) {
105
- const text = String(value);
106
- if (!text.includes('e')) return text;
107
- // Native numeric queries use DecimalFormat, which does not parse JS's
108
- // lowercase exponent syntax. Expand the shortest round-trippable decimal.
109
- const [mantissa, exponent] = text.split('e') as [string, string];
110
- const sign = mantissa.startsWith('-') ? '-' : '';
111
- const unsigned = sign ? mantissa.slice(1) : mantissa;
112
- const point = (unsigned.indexOf('.') === -1 ? unsigned.length : unsigned.indexOf('.')) + Number(exponent);
113
- const digits = unsigned.replace('.', '');
114
- return sign + (point <= 0 ? `0.${'0'.repeat(-point)}${digits}`
115
- : point >= digits.length ? `${digits}${'0'.repeat(point - digits.length)}`
116
- : `${digits.slice(0, point)}.${digits.slice(point)}`);
117
- }
118
- function equal(field: string, value: FilterScalar, type: MetadataType | undefined): NativeFilter {
119
- const literal = type === 'number' ? decimal(value) : type === 'boolean' ? String(value) : quote(value);
120
- return queryString(field, type === 'number' || type === 'datetime' ? `[${literal} TO ${literal}]` : literal);
121
- }
122
- function nativeFilter(filter: Filter | undefined, schema: MetadataSchema): NativeFilter {
123
- if (filter === undefined) return matchAll();
124
- if (filter.and) return bool(filter.and.map(item => nativeFilter(item, schema)), 'filter');
125
- if (filter.or) return bool(filter.or.map(item => nativeFilter(item, schema)), 'should');
126
- const { op, value } = filter, type = fieldType(filter.field, schema);
127
- const field = filter.field === 'id' ? 'memId' : filter.field;
128
- if (op === 'exists') return value ? exists(field, type) : { bool: [
129
- { ...matchAll(), occur: 'filter' }, { ...exists(field, type), occur: 'must_not' },
130
- ] };
131
- if (op === 'eq') return equal(field, value, type);
132
- if (op === 'neq') return { bool: [
133
- { ...exists(field, type), occur: 'filter' }, { ...equal(field, value, type), occur: 'must_not' },
134
- ] };
135
- if (op === 'in') return bool(value.map(item => equal(field, item, type)), 'should');
136
- const literal = type === 'datetime' ? quote(value) : decimal(value);
137
- const inclusive = ['gte', 'lte'].includes(op);
138
- return queryString(field, `${inclusive ? '[' : '{'}${op.startsWith('g') ? `${literal} TO *` : `* TO ${literal}`}${inclusive ? ']' : '}'}`);
139
- }
140
-
141
- export function compileFilter(filter: unknown, schema: MetadataSchema): NativeFilter {
142
- return nativeFilter(validateFilter(filter, schema), schema);
143
- }
144
-
145
- export function buildFilter(filters: unknown[] = [], excludeIds: string[] = [], schema: MetadataSchema = {}): NativeFilter {
146
- requireInput(Array.isArray(filters) && Array.isArray(excludeIds)
147
- && excludeIds.every(id => typeof id === 'string' && id.length > 0 && id.length <= 512), 'Invalid filters or excluded IDs.');
148
- const clauses = filters.filter(item => item !== undefined).map(item => ({ ...compileFilter(item, schema), occur: 'filter' }));
149
- if (clauses.length === 0) clauses.push({ ...matchAll(), occur: 'filter' });
150
- clauses.push(...excludeIds.map(id => ({ ...equal('memId', id, 'string'), occur: 'must_not' })));
151
- if (clauses.length === 1) { const { occur: _occur, ...clause } = clauses[0]!; return clause; }
152
- return { bool: clauses };
153
- }
154
-
155
- export function buildSearchFilter(filters: unknown[] = [], excludeIds: string[] = [], seenChunkIds: string[] = [], schema: MetadataSchema = {}): NativeFilter {
156
- requireInput(Array.isArray(seenChunkIds) && seenChunkIds.every(id => typeof id === 'string' && id.length > 0),
157
- 'Invalid seen chunk IDs.');
158
- return { bool: [
159
- { ...buildFilter(filters, excludeIds, schema), occur: 'filter' },
160
- ...seenChunkIds.map(id => ({ ...equal('id', id, 'string'), occur: 'must_not' })),
161
- ] };
162
- }
163
-
164
- export function matchesFilter(mem: {id: string; timestamp?: string | undefined; metadata?: Metadata | undefined}, filter: Filter | undefined, schema: MetadataSchema): boolean {
165
- validateFilter(filter, schema);
166
- const visit = (node: Filter | undefined): boolean => {
167
- if (node === undefined) return true;
168
- if (node.and) return node.and.every(visit);
169
- if (node.or) return node.or.some(visit);
170
- const { field, op, value } = node;
171
- const stored = field.startsWith('metadata.') ? mem.metadata?.[field.slice(9)] : mem[field as 'id' | 'timestamp'];
172
- const values = stored === undefined ? [] : Array.isArray(stored) ? stored : [stored];
173
- if (op === 'exists') return (values.length > 0) === value;
174
- const type = fieldType(field, schema);
175
- const convert = (item: unknown) => type === 'datetime' ? dateValue(item) : item;
176
- const same = (expected: FilterScalar) => values.some(item => convert(item) === convert(expected));
177
- if (op === 'eq') return same(value);
178
- if (op === 'neq') return values.length > 0 && !same(value);
179
- if (op === 'in') return value.some(same);
180
- return values.some(item => {
181
- const a = convert(item) as number | bigint, b = convert(value) as number | bigint;
182
- if (op === 'gt') return a > b;
183
- if (op === 'gte') return a >= b;
184
- if (op === 'lt') return a < b;
185
- return a <= b;
186
- });
187
- };
188
- return visit(filter);
189
- }
@@ -1,46 +0,0 @@
1
- import type { RecallRequest, TextRecallRequest, MetadataSchema, EmbeddingConfig } from './types.js';
2
- import { validateVector } from './storage.js';
3
- import { validateFilter } from './filter.js';
4
- import { requireInput } from './errors.js';
5
-
6
- export type ValidRecall = RecallRequest & { limit: number };
7
- const keys = (value: unknown, allowed: string[]): value is Record<string, unknown> => value !== null && typeof value === 'object' && !Array.isArray(value)
8
- && Object.keys(value).every(key => allowed.includes(key));
9
- const nonempty = (value: unknown): value is string => typeof value === 'string' && value.trim().length > 0;
10
-
11
- export function recallInput(request: unknown, schema: MetadataSchema, embedding: EmbeddingConfig): ValidRecall {
12
- const external = embedding.mode === 'external';
13
- requireInput(keys(request, [...(external ? ['queryVector'] : ['query', 'recentDialogue', 'context']), 'filter', 'excludeIds', 'limit', 'rerank']),
14
- 'Use the documented recall arguments.');
15
- if (external) {
16
- validateVector(request.queryVector, embedding);
17
- } else requireInput(nonempty(request.query) && request.query.trim() !== '*:*', 'query must be nonempty text other than the exact match-all expression *:*.');
18
- requireInput(request.context === undefined || typeof request.context === 'string', 'context must be text.');
19
- requireInput(request.recentDialogue === undefined || (Array.isArray(request.recentDialogue)
20
- && request.recentDialogue.every(message => keys(message, ['role', 'content'])
21
- && typeof message.role === 'string' && ['user', 'assistant'].includes(message.role) && typeof message.content === 'string')),
22
- 'recentDialogue must contain role-labelled user/assistant text.');
23
- requireInput(request.rerank === undefined || typeof request.rerank === 'boolean', 'rerank must be a boolean.');
24
- requireInput(request.excludeIds === undefined || (Array.isArray(request.excludeIds)
25
- && request.excludeIds.every(nonempty) && new Set(request.excludeIds).size === request.excludeIds.length),
26
- 'excludeIds must contain distinct canonical IDs.');
27
- const limit = request.limit ?? 10;
28
- requireInput(typeof limit === 'number' && Number.isSafeInteger(limit) && limit >= 1 && limit <= 1000, 'limit must be between 1 and 1000.');
29
- validateFilter(request.filter, schema);
30
- return { ...structuredClone(request), limit } as ValidRecall;
31
- }
32
-
33
- export function contextualQuery(request: TextRecallRequest): string {
34
- if (!request.recentDialogue?.length && !request.context) return request.query;
35
- return JSON.stringify({ question: request.query,
36
- ...(request.recentDialogue?.length ? { recent_dialogue: request.recentDialogue } : {}),
37
- ...(request.context ? { context: request.context } : {}) });
38
- }
39
-
40
- export function textRecallQuery(request: TextRecallRequest): string {
41
- const context = contextualQuery(request);
42
- // Native text-query input bound, not a silent SDK context truncation policy.
43
- requireInput(new TextEncoder().encode(request.query).length <= 8192 && new TextEncoder().encode(context).length <= 8192,
44
- 'The contextual search query exceeds the native 8 KiB limit.');
45
- return context;
46
- }