@functional-systems/memdrive 0.1.1-dev.1 → 0.1.1-dev.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (117) hide show
  1. package/CHANGELOG.md +6 -0
  2. package/README.md +20 -250
  3. package/dist/commonjs/chunking.d.ts +1 -2
  4. package/dist/commonjs/chunking.d.ts.map +1 -1
  5. package/dist/commonjs/chunking.js +2 -5
  6. package/dist/commonjs/chunking.js.map +1 -1
  7. package/dist/commonjs/config.d.ts +17 -0
  8. package/dist/commonjs/config.d.ts.map +1 -0
  9. package/dist/commonjs/config.js +116 -0
  10. package/dist/commonjs/config.js.map +1 -0
  11. package/dist/commonjs/errors.d.ts +10 -6
  12. package/dist/commonjs/errors.d.ts.map +1 -1
  13. package/dist/commonjs/errors.js +25 -5
  14. package/dist/commonjs/errors.js.map +1 -1
  15. package/dist/commonjs/index.d.ts +15 -5
  16. package/dist/commonjs/index.d.ts.map +1 -1
  17. package/dist/commonjs/index.js +331 -307
  18. package/dist/commonjs/index.js.map +1 -1
  19. package/dist/commonjs/records.d.ts +44 -0
  20. package/dist/commonjs/records.d.ts.map +1 -0
  21. package/dist/commonjs/records.js +88 -0
  22. package/dist/commonjs/records.js.map +1 -0
  23. package/dist/commonjs/search.d.ts +8 -0
  24. package/dist/commonjs/search.d.ts.map +1 -0
  25. package/dist/commonjs/search.js +56 -0
  26. package/dist/commonjs/search.js.map +1 -0
  27. package/dist/commonjs/types.d.ts +84 -252
  28. package/dist/commonjs/types.d.ts.map +1 -1
  29. package/dist/esm/chunking.d.ts +1 -2
  30. package/dist/esm/chunking.d.ts.map +1 -1
  31. package/dist/esm/chunking.js +2 -4
  32. package/dist/esm/chunking.js.map +1 -1
  33. package/dist/esm/config.d.ts +17 -0
  34. package/dist/esm/config.d.ts.map +1 -0
  35. package/dist/esm/config.js +107 -0
  36. package/dist/esm/config.js.map +1 -0
  37. package/dist/esm/errors.d.ts +10 -6
  38. package/dist/esm/errors.d.ts.map +1 -1
  39. package/dist/esm/errors.js +22 -4
  40. package/dist/esm/errors.js.map +1 -1
  41. package/dist/esm/index.d.ts +15 -5
  42. package/dist/esm/index.d.ts.map +1 -1
  43. package/dist/esm/index.js +331 -305
  44. package/dist/esm/index.js.map +1 -1
  45. package/dist/esm/records.d.ts +44 -0
  46. package/dist/esm/records.d.ts.map +1 -0
  47. package/dist/esm/records.js +77 -0
  48. package/dist/esm/records.js.map +1 -0
  49. package/dist/esm/search.d.ts +8 -0
  50. package/dist/esm/search.d.ts.map +1 -0
  51. package/dist/esm/search.js +52 -0
  52. package/dist/esm/search.js.map +1 -0
  53. package/dist/esm/types.d.ts +84 -252
  54. package/dist/esm/types.d.ts.map +1 -1
  55. package/package.json +10 -32
  56. package/src/chunking.ts +2 -3
  57. package/src/config.ts +121 -0
  58. package/src/errors.ts +20 -6
  59. package/src/index.ts +351 -289
  60. package/src/records.ts +83 -0
  61. package/src/search.ts +57 -0
  62. package/src/types.ts +74 -253
  63. package/dist/commonjs/filter.d.ts +0 -30
  64. package/dist/commonjs/filter.d.ts.map +0 -1
  65. package/dist/commonjs/filter.js +0 -224
  66. package/dist/commonjs/filter.js.map +0 -1
  67. package/dist/commonjs/storage.d.ts +0 -274
  68. package/dist/commonjs/storage.d.ts.map +0 -1
  69. package/dist/commonjs/storage.js +0 -861
  70. package/dist/commonjs/storage.js.map +0 -1
  71. package/dist/commonjs/tools/definitions.d.ts +0 -5
  72. package/dist/commonjs/tools/definitions.d.ts.map +0 -1
  73. package/dist/commonjs/tools/definitions.js +0 -129
  74. package/dist/commonjs/tools/definitions.js.map +0 -1
  75. package/dist/commonjs/tools/index.d.ts +0 -6
  76. package/dist/commonjs/tools/index.d.ts.map +0 -1
  77. package/dist/commonjs/tools/index.js +0 -104
  78. package/dist/commonjs/tools/index.js.map +0 -1
  79. package/dist/commonjs/tools/types.d.ts +0 -50
  80. package/dist/commonjs/tools/types.d.ts.map +0 -1
  81. package/dist/commonjs/tools/types.js +0 -3
  82. package/dist/commonjs/tools/types.js.map +0 -1
  83. package/dist/commonjs/values.d.ts +0 -17
  84. package/dist/commonjs/values.d.ts.map +0 -1
  85. package/dist/commonjs/values.js +0 -90
  86. package/dist/commonjs/values.js.map +0 -1
  87. package/dist/esm/filter.d.ts +0 -30
  88. package/dist/esm/filter.d.ts.map +0 -1
  89. package/dist/esm/filter.js +0 -209
  90. package/dist/esm/filter.js.map +0 -1
  91. package/dist/esm/storage.d.ts +0 -274
  92. package/dist/esm/storage.d.ts.map +0 -1
  93. package/dist/esm/storage.js +0 -845
  94. package/dist/esm/storage.js.map +0 -1
  95. package/dist/esm/tools/definitions.d.ts +0 -5
  96. package/dist/esm/tools/definitions.d.ts.map +0 -1
  97. package/dist/esm/tools/definitions.js +0 -126
  98. package/dist/esm/tools/definitions.js.map +0 -1
  99. package/dist/esm/tools/index.d.ts +0 -6
  100. package/dist/esm/tools/index.d.ts.map +0 -1
  101. package/dist/esm/tools/index.js +0 -101
  102. package/dist/esm/tools/index.js.map +0 -1
  103. package/dist/esm/tools/types.d.ts +0 -50
  104. package/dist/esm/tools/types.d.ts.map +0 -1
  105. package/dist/esm/tools/types.js +0 -2
  106. package/dist/esm/tools/types.js.map +0 -1
  107. package/dist/esm/values.d.ts +0 -17
  108. package/dist/esm/values.d.ts.map +0 -1
  109. package/dist/esm/values.js +0 -87
  110. package/dist/esm/values.js.map +0 -1
  111. package/docs/tools.md +0 -163
  112. package/src/filter.ts +0 -189
  113. package/src/storage.ts +0 -765
  114. package/src/tools/definitions.ts +0 -156
  115. package/src/tools/index.ts +0 -97
  116. package/src/tools/types.ts +0 -45
  117. package/src/values.ts +0 -103
package/src/index.ts CHANGED
@@ -1,323 +1,385 @@
1
- // Application-facing memory storage and retrieval. No answer assembly lives here.
2
- import type { Backend, DataBackend, Evidence, VectorEvidence } from './storage.js';
3
- import type {
4
- CreateOptions, OpenOptions, RequestOptions, RecallRequest, TextRecallRequest, RecallResult,
5
- BatchRecallResult, MetadataSchema, EmbeddingConfig, TextSearch, Usage, Observation, RecordEvent,
6
- MemInput, ForgetRequest, ConfigureRequest, BranchRequest, CatalogRequest, CatalogResult,
7
- ListRequest, MemDrive, DriveData, CountValuesRequest,
8
- } from './types.js';
9
- import { createBackend, openBackend, listBackends, validateChunk, publicMatch, validateVector, vectorMatch } from './storage.js';
10
- import { buildSearchFilter, validateFilter, matchesFilter } from './filter.js';
11
- import { MemDriveError, requireInput, operationOptions, checkAbort, abortError } from './errors.js';
12
- import { createValueCounter } from './values.js';
13
-
1
+ import type { ConnectionOptions, CreateFileRequest, CreateFileStoreOptions, DriveFile, FileDrive, FileSearchHit,
2
+ FileStoreDescription, MainFileDrive, OpenFileStoreOptions, ReadFileRequest, RequestOptions, SearchFilesRequest,
3
+ WorkingFileDrive } from './types.js';
4
+ import { checkAbort, databaseError, MemDriveError, nativeOptions, nativeRequest, operationOptions, requireInput } from './errors.js';
5
+ import { keysOnly, parseStore, storeConfiguration } from './config.js';
6
+ import { bytes, decodeFile, fileRecords, originalId, resultDocs, validFileId, validPath } from './records.js';
7
+ import { searchQuery, validateReranking } from './search.js';
14
8
  export type * from './types.js';
15
9
  export { MemDriveError } from './errors.js';
16
10
 
17
- type Options = (CreateOptions | OpenOptions) & { branch?: string | undefined; embedding?: CreateOptions['embedding']; textSearch?: TextSearch | undefined };
18
- type Queue = <T>(operation: () => Promise<T>, signal?: AbortSignal) => Promise<T>;
19
- type QueryBody = Parameters<DataBackend['query']>[0];
20
- type NativeResponse = Awaited<ReturnType<DataBackend['query']>>;
21
- type NormalizedResponse = Omit<NativeResponse, 'docs'> & { docs: Array<Omit<NativeResponse['docs'][number], 'doc'> & { doc: Evidence | VectorEvidence }> };
22
- type ValidRecall = RecallRequest & { limit: number };
23
- type PreparedRecall = { input: unknown; start: number; error?: never } | { error: MemDriveError; input?: never; start?: never };
24
-
25
- const keys = (value: unknown, allowed: string[]): value is Record<string, unknown> => value !== null && typeof value === 'object' && !Array.isArray(value)
26
- && Object.keys(value).every(key => allowed.includes(key));
27
- const nonempty = (value: unknown): value is string => typeof value === 'string' && value.trim().length > 0;
11
+ type Handle = ReturnType<ConnectionOptions['client']['collection']>;
12
+ type Client = ConnectionOptions['client'];
28
13
 
29
- function validateOptions<T extends Options>(options: T, creating: boolean): T {
30
- requireInput(keys(options, ['client', 'name', 'maxConcurrentRequests', 'observe', 'record',
31
- ...(creating ? ['metadataSchema', 'textSearch', 'embedding', 'labels', 'description'] : ['branch'])]), 'Use the documented MemDrive options.');
32
- requireInput(creating || options.branch === undefined || (typeof options.branch === 'string'
33
- && /^[A-Za-z0-9_-]{3,52}$/.test(options.branch)), 'branch must be a valid branch name.');
34
- if (creating && options.embedding?.mode === 'external')
35
- requireInput(options.textSearch === undefined, 'External embeddings do not support textSearch.');
36
- const concurrency = options.maxConcurrentRequests ?? 5;
37
- requireInput(Number.isSafeInteger(concurrency) && concurrency > 0, 'maxConcurrentRequests must be a positive integer.');
38
- for (const name of ['observe', 'record'] as const)
39
- requireInput(options[name] === undefined || typeof options[name] === 'function', `${name} must be a callback.`);
40
- return { ...options };
14
+ const main = 'main';
15
+ const prefix = 'fork-';
16
+ const clients = new WeakMap<Client, Map<string, Store>>();
17
+ const requestMaxBytes = 6_029_312;
18
+ const maximumInputBytes = 300_000;
19
+ const maximumRequestDocuments = 1000;
20
+ const writeBatchLimit = 100;
21
+ const validName = (name: unknown, maxLength = 52): name is string =>
22
+ typeof name === 'string' && name.length >= 3 && name.length <= maxLength && /^[A-Za-z0-9_-]+$/.test(name);
23
+ const validForkName = (name: unknown): name is string => validName(name, 52 - prefix.length) && name !== main;
24
+ function validateConnection(options: unknown, keys: string[]): asserts options is ConnectionOptions {
25
+ requireInput(keysOnly(options, keys) && options.client !== null && typeof options.client === 'object'
26
+ && 'collection' in options.client && typeof options.client.collection === 'function',
27
+ 'Provide a LambdaDB client and documented file store options.');
28
+ requireInput(validName(options.name), 'Store name must be 3–52 letters, digits, underscores or hyphens.');
41
29
  }
42
30
 
43
- // Separate queues bound logical recalls and database work across all branch views.
44
- function requestQueue(limit: number): Queue {
45
- let active = 0;
46
- const waiting: Array<{ start(): void }> = [];
47
- return <T>(operation: () => Promise<T>, signal?: AbortSignal) => new Promise<T>((resolve, reject) => {
48
- try { checkAbort(signal); } catch (error) { reject(error); return; }
49
- const entry = { start() {
50
- signal?.removeEventListener('abort', cancel);
51
- active++;
52
- Promise.resolve().then(() => { checkAbort(signal); return operation(); }).then(resolve, reject).finally(() => {
53
- active--;
54
- waiting.shift()?.start();
55
- });
56
- } };
57
- const cancel = () => {
58
- const index = waiting.indexOf(entry);
59
- if (index < 0) return;
60
- waiting.splice(index, 1);
61
- signal?.removeEventListener('abort', cancel);
62
- reject(abortError(undefined, signal));
63
- };
64
- if (active < limit) entry.start();
65
- else {
66
- waiting.push(entry);
67
- signal?.addEventListener('abort', cancel, { once: true });
68
- if (signal?.aborted) cancel();
69
- }
70
- });
31
+ async function remote<T>(operation: () => Promise<T>, message: string, signal?: AbortSignal, codes: Record<number, string> = {}) {
32
+ try { return await nativeRequest(operation, signal); }
33
+ catch (cause) {
34
+ const error = databaseError(cause, message);
35
+ if (error.status !== undefined && codes[error.status] !== undefined) error.code = codes[error.status]!;
36
+ throw error;
37
+ }
71
38
  }
72
39
 
73
- function recallInput(request: unknown, schema: MetadataSchema, embedding: EmbeddingConfig): ValidRecall {
74
- const external = embedding.mode === 'external';
75
- requireInput(keys(request, [...(external ? ['queryVector'] : ['query', 'recentDialogue', 'context']), 'filter', 'excludeIds', 'limit', 'rerank']),
76
- 'Use the documented recall arguments.');
77
- if (external) {
78
- validateVector(request.queryVector, embedding);
79
- } else requireInput(nonempty(request.query) && request.query.trim() !== '*:*', 'query must be nonempty text other than the exact match-all expression *:*.');
80
- requireInput(request.context === undefined || typeof request.context === 'string', 'context must be text.');
81
- requireInput(request.recentDialogue === undefined || (Array.isArray(request.recentDialogue)
82
- && request.recentDialogue.every(message => keys(message, ['role', 'content'])
83
- && typeof message.role === 'string' && ['user', 'assistant'].includes(message.role) && typeof message.content === 'string')),
84
- 'recentDialogue must contain role-labelled user/assistant text.');
85
- requireInput(request.rerank === undefined || typeof request.rerank === 'boolean', 'rerank must be a boolean.');
86
- requireInput(request.excludeIds === undefined || (Array.isArray(request.excludeIds)
87
- && request.excludeIds.every(nonempty) && new Set(request.excludeIds).size === request.excludeIds.length),
88
- 'excludeIds must contain distinct canonical IDs.');
89
- const limit = request.limit ?? 10;
90
- requireInput(typeof limit === 'number' && Number.isSafeInteger(limit) && limit >= 1 && limit <= 1000, 'limit must be between 1 and 1000.');
91
- validateFilter(request.filter, schema);
92
- return { ...structuredClone(request), limit } as ValidRecall;
93
- }
40
+ const branchRead = (branch: string) => ({
41
+ ref: { kind: 'branch' as const, name: branch }, consistentRead: true, includeVectors: false,
42
+ });
43
+ const kindClause = { queryString: { query: '"file"', defaultField: 'kind' }, occur: 'filter' };
44
+ const pathClause = (path: string) => ({ queryString: { query: JSON.stringify(path), defaultField: 'path' } });
94
45
 
95
- export function contextualQuery(request: TextRecallRequest): string {
96
- if (!request.recentDialogue?.length && !request.context) return request.query;
97
- return JSON.stringify({ question: request.query,
98
- ...(request.recentDialogue?.length ? { recent_dialogue: request.recentDialogue } : {}),
99
- ...(request.context ? { context: request.context } : {}) });
46
+ function pathQuery(paths: string[]) {
47
+ return { bool: [
48
+ kindClause,
49
+ paths.length === 1 ? { ...pathClause(paths[0]!), occur: 'filter' }
50
+ : { bool: paths.map(path => ({ ...pathClause(path), occur: 'should' })), occur: 'filter' },
51
+ ] };
100
52
  }
101
53
 
102
- function recallQuery(request: ValidRecall, schema: MetadataSchema, textSearch: TextSearch | undefined, embedding: EmbeddingConfig): QueryBody {
103
- if (embedding.mode === 'external') return { size: request.limit, query: { knn: {
104
- field: 'embedding', queryVector: request.queryVector, k: request.limit,
105
- filter: buildSearchFilter([request.filter], request.excludeIds ?? [], [], schema),
106
- } } };
107
- const context = contextualQuery(request as TextRecallRequest);
108
- // Native text-query input bound, not a silent SDK context truncation policy.
109
- requireInput(new TextEncoder().encode(request.query).length <= 8192 && new TextEncoder().encode(context).length <= 8192,
110
- 'The contextual search query exceeds the native 8 KiB limit.');
111
- const filter = buildSearchFilter([request.filter], request.excludeIds ?? [], [], schema);
112
- const candidates = Math.max(200, request.limit);
113
- const semantic = { knn: { field: 'embedding', queryText: context, k: candidates, filter } };
114
- const query = textSearch ? { bayesian: [semantic,
115
- { bool: [{ ...filter, occur: 'filter' },
116
- { queryString: { query: request.query, defaultField: 'content', skipSyntax: true }, occur: 'must' }] },
117
- ] } : semantic;
118
- return { size: request.limit, query, ...(request.rerank === false ? (textSearch ? { candidateSize: candidates } : {}) : {
119
- rerank: { provider: 'typesafe', model: 'jev-1.13.0', queryText: context,
120
- fields: ['content'], candidateSize: candidates, onFailure: 'error' },
121
- }) };
54
+ type FileRecords = Awaited<ReturnType<typeof fileRecords>>;
55
+ type WriteRecord = FileRecords['docs'][number];
56
+ function writeBatches(records: FileRecords[], branch: string): WriteRecord[][] {
57
+ const envelope = bytes({ docs: [], branch });
58
+ const batches: WriteRecord[][] = [];
59
+ let batch: WriteRecord[] = [];
60
+ let inputBytes = 0, wireBytes = envelope;
61
+ const flush = () => {
62
+ if (batch.length) batches.push(batch);
63
+ batch = []; inputBytes = 0; wireBytes = envelope;
64
+ };
65
+ const fits = (docs: WriteRecord[]) => batch.length + docs.length <= maximumRequestDocuments
66
+ && inputBytes + docs.reduce((sum, doc) => sum + doc.inputBytes, 0) <= maximumInputBytes
67
+ && wireBytes + docs.reduce((sum, doc) => sum + doc.wireBytes, 0) <= requestMaxBytes;
68
+ const append = (docs: WriteRecord[]) => {
69
+ batch.push(...docs);
70
+ for (const doc of docs) {
71
+ inputBytes += doc.inputBytes;
72
+ wireBytes += doc.wireBytes;
73
+ }
74
+ };
75
+ for (const { docs } of records) {
76
+ if (!fits(docs)) flush();
77
+ if (fits(docs)) append(docs);
78
+ else {
79
+ // Keep each ordinary file together. For a large file, stage its
80
+ // chunks first and include the original only in the final request.
81
+ for (const doc of docs) {
82
+ const next = [doc];
83
+ if (!fits(next)) flush();
84
+ requireInput(fits(next), 'A file record exceeds the limit for a native write.');
85
+ append(next);
86
+ }
87
+ }
88
+ }
89
+ flush();
90
+ return batches;
122
91
  }
123
92
 
124
- async function validatedHits(response: NativeResponse, request: ValidRecall, schema: MetadataSchema, name: string, max: number, embedding: EmbeddingConfig): Promise<Array<Evidence | VectorEvidence>> {
125
- if (response?.isDocsInline === false || !Array.isArray(response?.docs) || response.docs.length > max)
126
- throw new MemDriveError('INVALID_RESULT', 'Search returned an invalid document envelope.');
127
- const seen = new Set<string>();
128
- return Promise.all(response.docs.map(async hit => {
129
- if (hit.collection !== undefined && hit.collection !== name)
130
- throw new MemDriveError('INVALID_RESULT', 'Search returned a different collection.');
131
- let mem;
132
- try { mem = await (embedding.mode === 'external' ? vectorMatch(hit.doc, schema) : validateChunk(hit.doc, schema)); }
133
- catch (cause) { throw new MemDriveError('INVALID_RESULT', 'Search returned invalid Mem evidence.', { cause }); }
134
- if (seen.has(mem.id) || request.excludeIds?.includes(mem.memId) || !matchesFilter({ ...mem, id: mem.memId }, request.filter, schema))
135
- throw new MemDriveError('INVALID_RESULT', 'Search returned duplicate, excluded or filtered-out evidence.');
136
- seen.add(mem.id);
137
- return mem;
138
- }));
139
- }
93
+ type Store = ReturnType<typeof store>;
94
+ function store(handle: Handle, description: FileStoreDescription) {
95
+ const { name, metadataSchema } = description;
96
+ const forks = new Map<string, WorkingFileDrive>();
140
97
 
141
- function verifyRerank(response: NormalizedResponse, body: QueryBody) {
142
- const result = response.rerank;
143
- const same = result?.provider === body.rerank!.provider && result?.model === body.rerank!.model;
144
- const applied = result?.status === 'applied' && Number.isSafeInteger(result.candidateCount)
145
- && result.scoredCount === result.candidateCount && result.candidateCount !== undefined && result.candidateCount >= response.docs.length
146
- && result.candidateCount <= body.rerank!.candidateSize!
147
- && response.docs.every(hit => typeof hit.score === 'number' && Number.isFinite(hit.score) && hit.score >= 0 && hit.score <= 1);
148
- const empty = result?.status === 'skipped' && result.reason === 'noCandidates'
149
- && result.candidateCount === 0 && result.scoredCount === 0 && response.docs.length === 0;
150
- if (!same || (!applied && !empty)) throw new MemDriveError('RERANK_FAILED', 'Native reranking was not confirmed.');
151
- }
98
+ async function describe(options?: RequestOptions) {
99
+ const { signal } = operationOptions(options);
100
+ checkAbort(signal);
101
+ return structuredClone(description);
102
+ }
152
103
 
153
- function makeDataHandle(backend: DataBackend, options: Options, queue: Queue, recallQueue: Queue): DriveData {
154
- const counter = createValueCounter({ describe: options => backend.describe(options), query: (body, options) => backend.query(body, options),
155
- metadataSchema: () => backend.metadataSchema });
156
- let generation = backend.generation;
157
- const invalidate = () => { generation = backend.generation; counter.invalidate(); };
158
- const inspect = async <T>(operation: () => Promise<T>): Promise<T> => {
159
- backend.assertActive();
160
- if (generation !== backend.generation) invalidate();
161
- try { const result = await operation(); backend.assertActive(); return result; }
162
- finally { if (generation !== backend.generation) invalidate(); }
163
- };
164
- const mutation = async <T>(operation: () => Promise<T>): Promise<T> => {
165
- try { return await operation(); }
166
- finally { invalidate(); }
167
- };
168
- const executeRecall = async (input: unknown, start: number, requestIndex?: number, signal?: AbortSignal): Promise<RecallResult> => {
169
- const usage: Usage = { elapsedMs: 0, databaseRequests: 0 };
170
- const hook = <T extends Observation | RecordEvent>(callback?: ((event: T) => void | Promise<void>)) => async (event: T) => {
171
- checkAbort(signal);
172
- await callback?.(requestIndex === undefined ? event : { ...event, requestIndex });
104
+ async function fetchFiles(branch: string, fileIds: string[], signal?: AbortSignal) {
105
+ const response = await remote(() => handle.docs.fetch({ ids: fileIds.map(originalId), ...branchRead(branch) }, nativeOptions(signal)),
106
+ 'Could not read file originals.', signal);
107
+ checkAbort(signal);
108
+ const files = new Map<string, DriveFile>();
109
+ const expected = new Set(fileIds);
110
+ for (const doc of resultDocs(response, name, fileIds.length)) {
111
+ const file = decodeFile(doc, metadataSchema);
112
+ if (!expected.has(file.fileId) || files.has(file.fileId))
113
+ throw new MemDriveError('INVALID_RESULT', 'Read returned unexpected or duplicate file originals.');
114
+ files.set(file.fileId, file);
115
+ }
116
+ return files;
117
+ }
118
+
119
+ function reader(branch: string) {
120
+ function read(request: ReadFileRequest, options?: RequestOptions): Promise<DriveFile | null>;
121
+ function read(request: { paths: string[] }, options?: RequestOptions): Promise<DriveFile[]>;
122
+ async function read(request: ReadFileRequest | { paths: string[] }, options?: RequestOptions) {
123
+ const { signal } = operationOptions(options);
173
124
  checkAbort(signal);
174
- };
175
- const record = hook(options.record);
176
- const observe = hook(options.observe);
177
- const executeDb = <T>(operation: (options?: RequestOptions) => Promise<T>): Promise<T> => queue(async () => {
178
- backend.assertActive(); checkAbort(signal);
179
- usage.databaseRequests++;
180
- try { const result = await operation({ signal }); checkAbort(signal); return result; }
181
- catch (cause) {
182
- const failure = abortError(cause, signal);
183
- if (failure instanceof MemDriveError) throw failure;
184
- const error = new MemDriveError('DATABASE_ERROR', 'MemDrive database request failed.', { cause: failure });
185
- const remote = failure as { statusCode?: number; status?: number };
186
- error.status = remote.statusCode ?? remote.status;
187
- throw error;
125
+ requireInput(keysOnly(request, ['fileId', 'path', 'paths']) && Object.keys(request).length === 1,
126
+ 'Read requires exactly one of fileId, path or paths.');
127
+ if ('fileId' in request) {
128
+ const fileId = request.fileId;
129
+ requireInput(validFileId(fileId), 'Provide a fileId returned by MemDrive.');
130
+ return (await fetchFiles(branch, [fileId], signal)).get(fileId) ?? null;
188
131
  }
189
- }, signal);
190
- try {
132
+ const multi = 'paths' in request;
133
+ const value = multi ? request.paths : [request.path];
134
+ requireInput(Array.isArray(value) && value.length >= 1 && value.length <= writeBatchLimit && [...value].every(validPath)
135
+ && new Set(value).size === value.length, 'Provide 1–100 distinct valid absolute file paths.');
136
+ const paths: string[] = [...value];
137
+ const response = await remote(() => handle.query({ size: paths.length * 2, ...branchRead(branch), query: pathQuery(paths) },
138
+ nativeOptions(signal)), 'Could not read files by path.', signal);
191
139
  checkAbort(signal);
192
- backend.assertActive();
193
- if (generation !== backend.generation) invalidate();
194
- const schema = () => backend.metadataSchema;
195
- const request = recallInput(input, schema(), backend.embedding);
196
- requireInput(request.rerank !== true || backend.embedding.mode === 'managed',
197
- 'rerank: true requires managed embeddings.');
198
- const query = async (body: QueryBody): Promise<NormalizedResponse> => {
199
- const response = await executeDb(options => backend.query(body, options));
200
- const mems = await validatedHits(response, request, schema(), options.name, body.size ?? 10, backend.embedding);
201
- return { ...response, docs: response.docs.map((hit, i) => ({ ...hit, doc: mems[i]! })) };
202
- };
203
- const body = recallQuery(request, schema(), backend.textSearch, backend.embedding);
204
- await record({ phase: 'query', request: structuredClone(body) });
205
- const response = await query(body);
206
- if (body.rerank) verifyRerank(response, body);
207
- const mems = response.docs.map(hit => hit.doc);
208
- await record({ phase: 'query_result', response: structuredClone(response) });
209
- await observe({ type: 'retrieved', ids: mems.map(mem => mem.id), matches: mems.map(publicMatch) });
210
- backend.assertActive(); checkAbort(signal);
211
- usage.elapsedMs = performance.now() - start;
212
- return { matches: mems.map(publicMatch), usage };
213
- } catch (cause) {
214
- const failure = abortError(cause, signal);
215
- usage.elapsedMs = performance.now() - start;
216
- const error = failure instanceof MemDriveError ? failure
217
- : new MemDriveError('RECALL_FAILED', 'MemDrive retrieval failed.', { cause: failure });
218
- error.usage = structuredClone(usage);
219
- throw error;
220
- }
221
- };
222
- const snapshot = (input: unknown): PreparedRecall => {
223
- const start = performance.now();
224
- try { return { input: structuredClone(input), start }; }
225
- catch (cause) {
226
- const error = new MemDriveError('INVALID_INPUT', 'Recall input must be cloneable data.', { cause });
227
- error.usage = { elapsedMs: 0, databaseRequests: 0 };
228
- return { error };
229
- }
230
- };
231
- const submit = async (prepared: PreparedRecall, requestIndex?: number, signal?: AbortSignal): Promise<RecallResult> => {
232
- if (prepared.error) throw prepared.error;
233
- try { return await recallQueue(() => executeRecall(prepared.input, prepared.start, requestIndex, signal), signal); }
234
- catch (cause) {
235
- // Queue cancellation and executeRecall both reject with MemDriveError.
236
- const error = cause as MemDriveError;
237
- error.usage ??= { elapsedMs: performance.now() - prepared.start, databaseRequests: 0 };
238
- throw error;
140
+ const expected = new Set(paths), files = new Map<string, DriveFile>();
141
+ for (const doc of resultDocs(response, name, paths.length)) {
142
+ const file = decodeFile(doc, metadataSchema);
143
+ if (!expected.has(file.path) || files.has(file.path))
144
+ throw new MemDriveError('INVALID_RESULT', 'Read returned unexpected or duplicate paths.');
145
+ files.set(file.path, file);
146
+ }
147
+ return multi ? paths.flatMap(path => files.has(path) ? [files.get(path)!] : []) : files.get(paths[0]!) ?? null;
239
148
  }
240
- };
241
- async function recall(input: RecallRequest, options?: RequestOptions): Promise<RecallResult>;
242
- async function recall(input: { requests: RecallRequest[] }, options?: RequestOptions): Promise<BatchRecallResult>;
243
- async function recall(input: unknown, options?: RequestOptions): Promise<RecallResult | BatchRecallResult> {
149
+ return read;
150
+ }
151
+
152
+ function searcher(branch: string) {
153
+ return async (request: SearchFilesRequest, options?: RequestOptions) => {
244
154
  const { signal } = operationOptions(options);
245
- if (input !== null && typeof input === 'object' && Object.hasOwn(input, 'requests')) {
246
- requireInput(keys(input, ['requests']) && Array.isArray(input.requests) && input.requests.length > 0,
247
- 'Batch recall requires only a nonempty requests array.');
248
- // Snapshot every item before any operation can yield or mutate caller data.
249
- const prepared = Array.from(input.requests, snapshot);
250
- const results = await Promise.all(prepared.map(async (item, requestIndex) => {
251
- try { return await submit(item, requestIndex, signal); }
252
- catch (cause) {
253
- const error = cause as MemDriveError;
254
- return { error: { code: error.code, message: error.message,
255
- ...(Number.isInteger(error.status) ? { status: error.status! } : {}) },
256
- usage: structuredClone(error.usage!) };
257
- }
258
- }));
259
- return { results };
155
+ checkAbort(signal);
156
+ const body = searchQuery(request, metadataSchema);
157
+ const response = await remote(() => handle.query({ ...body, ...branchRead(branch),
158
+ fields: { include: ['id', 'kind', 'fileId', 'path', 'modifiedAt', 'content', 'start', 'end'] } },
159
+ nativeOptions(signal)), 'File search failed.', signal);
160
+ checkAbort(signal);
161
+ const docs = resultDocs(response, name, body.size!);
162
+ const reranked = validateReranking(response, body);
163
+ const fileIds = new Set<string>(), chunkIds = new Set<string>();
164
+ const chunks = docs.map(doc => {
165
+ if (doc.kind !== 'chunk' || !validFileId(doc.fileId) || typeof doc.id !== 'string'
166
+ || !doc.id.startsWith(`chunk:${doc.fileId}:`) || chunkIds.has(doc.id) || !validPath(doc.path)
167
+ || typeof doc.content !== 'string' || !doc.content.trim() || /\p{Surrogate}/u.test(doc.content))
168
+ throw new MemDriveError('INVALID_RESULT', 'Search returned an invalid or duplicate chunk.');
169
+ fileIds.add(doc.fileId);
170
+ chunkIds.add(doc.id);
171
+ return { fileId: doc.fileId, path: doc.path, modifiedAt: doc.modifiedAt, content: doc.content,
172
+ start: doc.start, end: doc.end };
173
+ });
174
+ if (!chunks.length) return { hits: [], reranked };
175
+ const files = await fetchFiles(branch, [...fileIds], signal);
176
+ const hits: FileSearchHit[] = [];
177
+ for (const chunk of chunks) {
178
+ const file = files.get(chunk.fileId);
179
+ // An orphan or stale chunk must not expose body text from another file or revision.
180
+ if (!file || file.path !== chunk.path || file.modifiedAt !== chunk.modifiedAt) continue;
181
+ const { start, end } = chunk;
182
+ const verified = typeof start === 'number' && typeof end === 'number'
183
+ && Number.isSafeInteger(start) && Number.isSafeInteger(end) && start >= 0 && end > start && end <= file.content.length
184
+ && file.content.slice(start, end) === chunk.content;
185
+ hits.push({ ...file, content: chunk.content, ...(verified ? { range: { start, end } } : {}) });
260
186
  }
261
- return submit(snapshot(input), undefined, signal);
187
+ return { hits, reranked };
188
+ };
189
+ }
262
190
 
191
+ function working(fork: string): WorkingFileDrive {
192
+ const existing = forks.get(fork);
193
+ if (existing) return existing;
194
+ const branch = prefix + fork;
195
+ let writes = Promise.resolve();
196
+ const unconfirmed = new Map<string, string>();
197
+
198
+ function write(request: CreateFileRequest, options?: RequestOptions): Promise<DriveFile>;
199
+ function write(request: CreateFileRequest[], options?: RequestOptions): Promise<DriveFile[]>;
200
+ async function write(request: CreateFileRequest | CreateFileRequest[], options?: RequestOptions) {
201
+ const { signal } = operationOptions(options);
202
+ checkAbort(signal);
203
+ const multi = Array.isArray(request);
204
+ const operations = multi ? [...request] : [request];
205
+ requireInput(operations.length >= 1 && operations.length <= writeBatchLimit,
206
+ 'Submit 1–100 file create operations.');
207
+ const prepared = Promise.all(operations.map(operation => fileRecords(operation, metadataSchema)));
208
+ // Validation and hashing begin immediately. Attach the queue now so overlapping writes keep call order.
209
+ void prepared.catch(() => {});
210
+ const write = writes.then(async () => {
211
+ checkAbort(signal);
212
+ const records = await prepared;
213
+ const paths = records.map(record => record.file.path);
214
+ const expectedPaths = new Set(paths);
215
+ requireInput(expectedPaths.size === paths.length, 'Create paths must be distinct within one write.');
216
+ const batches = writeBatches(records, branch);
217
+ const response = await remote(() => handle.query({ size: 2 * paths.length, ...branchRead(branch),
218
+ query: pathQuery(paths), fields: { include: ['id', 'kind', 'fileId', 'path'] } }, nativeOptions(signal)),
219
+ 'Could not check create destinations.', signal);
220
+ checkAbort(signal);
221
+ const occupied = new Set<string>();
222
+ for (const doc of resultDocs(response, name, 2 * paths.length)) {
223
+ if (doc.kind !== 'file' || !validFileId(doc.fileId) || doc.id !== originalId(doc.fileId)
224
+ || typeof doc.path !== 'string' || !expectedPaths.has(doc.path) || occupied.has(doc.path))
225
+ throw new MemDriveError('INVALID_RESULT', 'Destination lookup returned an unexpected or duplicate file.');
226
+ occupied.add(doc.path);
227
+ unconfirmed.delete(doc.path);
228
+ }
229
+ if (occupied.size) throw new MemDriveError('PATH_EXISTS', 'A file already exists at a requested path.',
230
+ { details: { paths: [...occupied] } });
231
+ for (const path of paths) {
232
+ const fileId = unconfirmed.get(path);
233
+ if (fileId) throw new MemDriveError('WRITE_UNCONFIRMED', 'An earlier create at this path is still unconfirmed.',
234
+ { details: { fileId, path } });
235
+ }
236
+
237
+ const completed: DriveFile[] = [];
238
+ const staged = new Map<string, string>();
239
+ const filesById = new Map(records.map(({ file }) => [file.fileId, file]));
240
+ const details = () => ({
241
+ completedFileIds: completed.map(file => file.fileId),
242
+ incompleteTargets: [...staged].map(([fileId, path]) => ({ fileId, path })),
243
+ });
244
+ for (const batch of batches) {
245
+ const docs = batch.map(record => record.doc);
246
+ const ids = new Set(docs.map(doc => doc.id));
247
+ const targets = [...new Map(docs.map(({ fileId, path }) => [fileId, { fileId, path }])).values()];
248
+ const uncertain = (cause: unknown) => {
249
+ for (const { kind, fileId, path } of docs) if (kind === 'file') unconfirmed.set(path, fileId);
250
+ return new MemDriveError('WRITE_UNCONFIRMED', 'The create may have been accepted. Inspect the files on this fork.', {
251
+ cause, details: { ...(targets.length === 1 ? targets[0] : {}), targets, ...details() },
252
+ });
253
+ };
254
+ try { checkAbort(signal); }
255
+ catch (cause) {
256
+ throw new MemDriveError('ABORTED', 'Write aborted before the next request.', { cause, details: details() });
257
+ }
258
+ try {
259
+ await nativeRequest(() => handle.docs.upsert({ docs, branch }, nativeOptions(signal)), signal);
260
+ } catch (cause) {
261
+ const error = databaseError(cause, 'The file create was rejected.');
262
+ if ([400, 401, 403, 404, 413, 422, 429].includes(error.status ?? 0)) {
263
+ if (completed.length || staged.size) {
264
+ const partial = new MemDriveError(error.code, error.message, { cause: error, details: details() });
265
+ partial.status = error.status;
266
+ throw partial;
267
+ }
268
+ throw error;
269
+ }
270
+ throw uncertain(cause);
271
+ }
272
+ try {
273
+ const confirmation = await nativeRequest(() => handle.docs.fetch({ ids: [...ids], ...branchRead(branch),
274
+ fields: { include: ['id'] } }, nativeOptions(signal)), signal);
275
+ checkAbort(signal);
276
+ const found = resultDocs(confirmation, name, ids.size);
277
+ if (found.some(doc => typeof doc.id !== 'string' || !ids.delete(doc.id)) || ids.size)
278
+ throw new MemDriveError('INVALID_RESULT', 'The original and its complete chunk set were not confirmed.');
279
+ } catch (cause) {
280
+ throw uncertain(cause);
281
+ }
282
+ for (const { fileId, kind, path } of docs) {
283
+ if (kind === 'file') {
284
+ staged.delete(fileId);
285
+ completed.push(filesById.get(fileId)!);
286
+ } else staged.set(fileId, path);
287
+ }
288
+ }
289
+ return multi ? completed : completed[0]!;
290
+ });
291
+ writes = write.then(() => undefined, () => undefined);
292
+ return write;
293
+ }
294
+ const drive: WorkingFileDrive = Object.freeze({
295
+ name, at: Object.freeze({ kind: 'fork' as const, name: fork }), read: reader(branch), search: searcher(branch), describe, write,
296
+ });
297
+ forks.set(fork, drive);
298
+ return drive;
263
299
  }
264
- return Object.freeze({
265
- remember: (request: { mems: MemInput[] }, options?: RequestOptions) => mutation(() => backend.remember(request, options)),
266
- forget: (request: ForgetRequest, options?: RequestOptions) => mutation(() => backend.forget(request, options)),
267
- read: (request: { ids: string[] }, options?: RequestOptions) => backend.read(request, undefined, options),
268
- list: (request?: ListRequest, options?: RequestOptions) => backend.list(request, options),
269
- embedding: Object.freeze(structuredClone(backend.embedding)),
270
- recall,
271
- async countValues(request: CountValuesRequest, options?: RequestOptions) {
300
+
301
+ const mainView = Object.freeze({
302
+ name, at: Object.freeze({ kind: 'main' as const }), read: reader(main), search: searcher(main), describe,
303
+ async fork(request: { name: string }, options?: RequestOptions) {
272
304
  const { signal } = operationOptions(options);
273
305
  checkAbort(signal);
274
- return inspect(() => counter.countValues(request, operation => queue(async () => {
275
- backend.assertActive(); checkAbort(signal);
276
- const result = await operation({ signal });
277
- backend.assertActive(); checkAbort(signal);
278
- return result;
279
- }, signal), signal));
306
+ requireInput(keysOnly(request, ['name']) && validForkName(request.name),
307
+ 'Fork name must be 3–47 letters, digits, underscores or hyphens and cannot be main.');
308
+ const fork = request.name;
309
+ const branch = prefix + fork;
310
+ const result = await remote(() => handle.branches.create({ branchName: branch, source: { kind: 'branch', name: main } },
311
+ nativeOptions(signal)), 'Could not create the working fork.', signal, { 409: 'BRANCH_CONFLICT' });
312
+ checkAbort(signal);
313
+ if (result?.branch?.name !== branch) throw new MemDriveError('INVALID_RESULT', 'Fork creation returned an invalid branch.');
314
+ return working(fork);
280
315
  },
281
- });
282
- }
316
+ }) satisfies MainFileDrive;
283
317
 
284
- function makeDrive(backend: Backend, options: Options): MemDrive {
285
- const concurrency = options.maxConcurrentRequests ?? 5;
286
- const queue = requestQueue(concurrency), recallQueue = requestQueue(concurrency);
287
- const at = (ref: { branch: string }) => {
288
- requireInput(keys(ref, ['branch']) && Object.hasOwn(ref, 'branch'), 'Select a branch with { branch }.');
289
- return makeDataHandle(backend.at(ref.branch), options, queue, recallQueue);
318
+ return {
319
+ main: mainView,
320
+ async openFork(fork: string, signal?: AbortSignal) {
321
+ const cached = forks.get(fork);
322
+ if (cached) return cached;
323
+ const response = await remote(() => handle.branches.list(nativeOptions(signal)), 'Could not find the working fork.', signal);
324
+ checkAbort(signal);
325
+ if (!Array.isArray(response?.branches) || !response.branches.every(branch => typeof branch?.name === 'string'))
326
+ throw new MemDriveError('INVALID_RESULT', 'Fork lookup returned invalid branches.');
327
+ if (!response.branches.some(branch => branch.name === prefix + fork))
328
+ throw new MemDriveError('BRANCH_NOT_FOUND', 'The working fork does not exist.');
329
+ return working(fork);
330
+ },
290
331
  };
291
- return Object.freeze({
292
- ...at({ branch: options.branch ?? 'main' }),
293
- at,
294
- branches: Object.freeze({
295
- create: (request: BranchRequest, options?: RequestOptions) => backend.branches.create(request, options),
296
- list: (options?: RequestOptions) => backend.branches.list(options),
297
- delete: (name: string, options?: RequestOptions) => backend.branches.delete(name, options),
298
- }),
299
- configure: (request: ConfigureRequest, options?: RequestOptions) => backend.configure(request, options),
300
- destroy: (options?: RequestOptions) => backend.destroy(options),
301
- describe: (options?: RequestOptions) => backend.describeDescriptor(options),
302
- });
303
332
  }
304
333
 
305
- export async function create(options: CreateOptions, requestOptions?: RequestOptions): Promise<MemDrive> {
306
- requestOptions = operationOptions(requestOptions); checkAbort(requestOptions.signal);
307
- options = validateOptions(options, true);
308
- const backend = await createBackend({ client: options.client, name: options.name, metadataSchema: options.metadataSchema, textSearch: options.textSearch, embedding: options.embedding, labels: options.labels,
309
- description: options.description }, requestOptions);
310
- return makeDrive(backend, options);
334
+ function storesFor(client: Client) {
335
+ const existing = clients.get(client);
336
+ if (existing) return existing;
337
+ const stores = new Map<string, Store>();
338
+ clients.set(client, stores);
339
+ return stores;
311
340
  }
312
341
 
313
- export async function open(options: OpenOptions, requestOptions?: RequestOptions): Promise<MemDrive> {
314
- requestOptions = operationOptions(requestOptions); checkAbort(requestOptions.signal);
315
- options = validateOptions(options, false);
316
- const backend = await openBackend({ client: options.client, name: options.name }, requestOptions);
317
- if (options.branch !== undefined && options.branch !== 'main'
318
- && !(await backend.branches.list(requestOptions)).some(branch => branch.name === options.branch))
319
- throw new MemDriveError('BRANCH_NOT_FOUND', 'The selected MemDrive branch does not exist.');
320
- return makeDrive(backend, options);
342
+ export async function create(options: CreateFileStoreOptions, requestOptions?: RequestOptions): Promise<MainFileDrive> {
343
+ const { signal } = operationOptions(requestOptions);
344
+ checkAbort(signal);
345
+ validateConnection(options, ['client', 'name', 'labels', 'textSearch', 'metadataSchema']);
346
+ const { client, name } = options;
347
+ requireInput(typeof client.createCollection === 'function', 'Client must support collection creation.');
348
+ const { indexConfigs, tags, description } = storeConfiguration(options);
349
+ await remote(() => client.createCollection({ collectionName: name, indexConfigs, tags },
350
+ nativeOptions(signal)), 'Could not create the file store.', signal, { 409: 'STORE_EXISTS' });
351
+ checkAbort(signal);
352
+ const stores = storesFor(client);
353
+ const created = stores.get(name) ?? store(client.collection(name), description);
354
+ stores.set(name, created);
355
+ return created.main;
356
+ }
357
+
358
+ export function open(options: ConnectionOptions & { at?: undefined }, requestOptions?: RequestOptions): Promise<MainFileDrive>;
359
+ export function open(options: ConnectionOptions & { at: { fork: string } }, requestOptions?: RequestOptions): Promise<WorkingFileDrive>;
360
+ export function open(options: OpenFileStoreOptions, requestOptions?: RequestOptions): Promise<FileDrive>;
361
+ export async function open(options: OpenFileStoreOptions, requestOptions?: RequestOptions): Promise<FileDrive> {
362
+ const { signal } = operationOptions(requestOptions);
363
+ checkAbort(signal);
364
+ validateConnection(options, ['client', 'name', 'at']);
365
+ const { client, name, at } = options;
366
+ let fork: string | undefined;
367
+ if (at !== undefined) {
368
+ requireInput(keysOnly(at, ['fork']) && validForkName(at.fork),
369
+ 'Select a working fork by its name.');
370
+ fork = at.fork;
371
+ }
372
+ const stores = storesFor(client);
373
+ let selected = stores.get(name);
374
+ if (!selected) {
375
+ const handle = client.collection(name);
376
+ const response = await remote(() => handle.get(nativeOptions(signal)), 'Could not inspect the file store.', signal, { 404: 'STORE_NOT_FOUND' });
377
+ checkAbort(signal);
378
+ const description = parseStore(response?.collection, name);
379
+ selected = stores.get(name) ?? store(handle, description);
380
+ stores.set(name, selected);
381
+ }
382
+ return fork === undefined ? selected.main : selected.openFork(fork, signal);
321
383
  }
322
384
 
323
- export const list = (request: CatalogRequest, options?: RequestOptions): Promise<CatalogResult> => listBackends(request, options);
385
+ export const MemDrive = Object.freeze({ create, open });