@functional-systems/memdrive 0.1.1-dev.2 → 0.1.1-dev.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (189) hide show
  1. package/CHANGELOG.md +6 -0
  2. package/README.md +20 -279
  3. package/dist/commonjs/chunking.d.ts +1 -2
  4. package/dist/commonjs/chunking.d.ts.map +1 -1
  5. package/dist/commonjs/chunking.js +2 -5
  6. package/dist/commonjs/chunking.js.map +1 -1
  7. package/dist/commonjs/config.d.ts +17 -0
  8. package/dist/commonjs/config.d.ts.map +1 -0
  9. package/dist/commonjs/config.js +116 -0
  10. package/dist/commonjs/config.js.map +1 -0
  11. package/dist/commonjs/errors.d.ts +10 -6
  12. package/dist/commonjs/errors.d.ts.map +1 -1
  13. package/dist/commonjs/errors.js +25 -5
  14. package/dist/commonjs/errors.js.map +1 -1
  15. package/dist/commonjs/index.d.ts +15 -5
  16. package/dist/commonjs/index.d.ts.map +1 -1
  17. package/dist/commonjs/index.js +332 -280
  18. package/dist/commonjs/index.js.map +1 -1
  19. package/dist/commonjs/records.d.ts +44 -0
  20. package/dist/commonjs/records.d.ts.map +1 -0
  21. package/dist/commonjs/records.js +88 -0
  22. package/dist/commonjs/records.js.map +1 -0
  23. package/dist/commonjs/search.d.ts +8 -0
  24. package/dist/commonjs/search.d.ts.map +1 -0
  25. package/dist/commonjs/search.js +56 -0
  26. package/dist/commonjs/search.js.map +1 -0
  27. package/dist/commonjs/types.d.ts +84 -252
  28. package/dist/commonjs/types.d.ts.map +1 -1
  29. package/dist/esm/chunking.d.ts +1 -2
  30. package/dist/esm/chunking.d.ts.map +1 -1
  31. package/dist/esm/chunking.js +2 -4
  32. package/dist/esm/chunking.js.map +1 -1
  33. package/dist/esm/config.d.ts +17 -0
  34. package/dist/esm/config.d.ts.map +1 -0
  35. package/dist/esm/config.js +107 -0
  36. package/dist/esm/config.js.map +1 -0
  37. package/dist/esm/errors.d.ts +10 -6
  38. package/dist/esm/errors.d.ts.map +1 -1
  39. package/dist/esm/errors.js +22 -4
  40. package/dist/esm/errors.js.map +1 -1
  41. package/dist/esm/index.d.ts +15 -5
  42. package/dist/esm/index.d.ts.map +1 -1
  43. package/dist/esm/index.js +332 -278
  44. package/dist/esm/index.js.map +1 -1
  45. package/dist/esm/records.d.ts +44 -0
  46. package/dist/esm/records.d.ts.map +1 -0
  47. package/dist/esm/records.js +77 -0
  48. package/dist/esm/records.js.map +1 -0
  49. package/dist/esm/search.d.ts +8 -0
  50. package/dist/esm/search.d.ts.map +1 -0
  51. package/dist/esm/search.js +52 -0
  52. package/dist/esm/search.js.map +1 -0
  53. package/dist/esm/types.d.ts +84 -252
  54. package/dist/esm/types.d.ts.map +1 -1
  55. package/package.json +7 -42
  56. package/src/chunking.ts +2 -3
  57. package/src/config.ts +121 -0
  58. package/src/errors.ts +20 -6
  59. package/src/index.ts +352 -260
  60. package/src/records.ts +83 -0
  61. package/src/search.ts +57 -0
  62. package/src/types.ts +74 -253
  63. package/dist/commonjs/filter.d.ts +0 -30
  64. package/dist/commonjs/filter.d.ts.map +0 -1
  65. package/dist/commonjs/filter.js +0 -224
  66. package/dist/commonjs/filter.js.map +0 -1
  67. package/dist/commonjs/recall-input.d.ts +0 -8
  68. package/dist/commonjs/recall-input.d.ts.map +0 -1
  69. package/dist/commonjs/recall-input.js +0 -45
  70. package/dist/commonjs/recall-input.js.map +0 -1
  71. package/dist/commonjs/retriever/index.d.ts +0 -7
  72. package/dist/commonjs/retriever/index.d.ts.map +0 -1
  73. package/dist/commonjs/retriever/index.js +0 -167
  74. package/dist/commonjs/retriever/index.js.map +0 -1
  75. package/dist/commonjs/retriever/scope.d.ts +0 -5
  76. package/dist/commonjs/retriever/scope.d.ts.map +0 -1
  77. package/dist/commonjs/retriever/scope.js +0 -69
  78. package/dist/commonjs/retriever/scope.js.map +0 -1
  79. package/dist/commonjs/retriever/types.d.ts +0 -89
  80. package/dist/commonjs/retriever/types.d.ts.map +0 -1
  81. package/dist/commonjs/retriever/types.js +0 -3
  82. package/dist/commonjs/retriever/types.js.map +0 -1
  83. package/dist/commonjs/storage.d.ts +0 -274
  84. package/dist/commonjs/storage.d.ts.map +0 -1
  85. package/dist/commonjs/storage.js +0 -861
  86. package/dist/commonjs/storage.js.map +0 -1
  87. package/dist/commonjs/tools/definitions.d.ts +0 -5
  88. package/dist/commonjs/tools/definitions.d.ts.map +0 -1
  89. package/dist/commonjs/tools/definitions.js +0 -147
  90. package/dist/commonjs/tools/definitions.js.map +0 -1
  91. package/dist/commonjs/tools/evidence-ids.d.ts +0 -7
  92. package/dist/commonjs/tools/evidence-ids.d.ts.map +0 -1
  93. package/dist/commonjs/tools/evidence-ids.js +0 -71
  94. package/dist/commonjs/tools/evidence-ids.js.map +0 -1
  95. package/dist/commonjs/tools/factory.d.ts +0 -7
  96. package/dist/commonjs/tools/factory.d.ts.map +0 -1
  97. package/dist/commonjs/tools/factory.js +0 -169
  98. package/dist/commonjs/tools/factory.js.map +0 -1
  99. package/dist/commonjs/tools/index.d.ts +0 -6
  100. package/dist/commonjs/tools/index.d.ts.map +0 -1
  101. package/dist/commonjs/tools/index.js +0 -25
  102. package/dist/commonjs/tools/index.js.map +0 -1
  103. package/dist/commonjs/tools/internal-types.d.ts +0 -18
  104. package/dist/commonjs/tools/internal-types.d.ts.map +0 -1
  105. package/dist/commonjs/tools/internal-types.js +0 -3
  106. package/dist/commonjs/tools/internal-types.js.map +0 -1
  107. package/dist/commonjs/tools/short-ids.d.ts +0 -8
  108. package/dist/commonjs/tools/short-ids.d.ts.map +0 -1
  109. package/dist/commonjs/tools/short-ids.js +0 -81
  110. package/dist/commonjs/tools/short-ids.js.map +0 -1
  111. package/dist/commonjs/tools/types.d.ts +0 -53
  112. package/dist/commonjs/tools/types.d.ts.map +0 -1
  113. package/dist/commonjs/tools/types.js +0 -3
  114. package/dist/commonjs/tools/types.js.map +0 -1
  115. package/dist/commonjs/values.d.ts +0 -17
  116. package/dist/commonjs/values.d.ts.map +0 -1
  117. package/dist/commonjs/values.js +0 -90
  118. package/dist/commonjs/values.js.map +0 -1
  119. package/dist/esm/filter.d.ts +0 -30
  120. package/dist/esm/filter.d.ts.map +0 -1
  121. package/dist/esm/filter.js +0 -209
  122. package/dist/esm/filter.js.map +0 -1
  123. package/dist/esm/recall-input.d.ts +0 -8
  124. package/dist/esm/recall-input.d.ts.map +0 -1
  125. package/dist/esm/recall-input.js +0 -40
  126. package/dist/esm/recall-input.js.map +0 -1
  127. package/dist/esm/retriever/index.d.ts +0 -7
  128. package/dist/esm/retriever/index.d.ts.map +0 -1
  129. package/dist/esm/retriever/index.js +0 -163
  130. package/dist/esm/retriever/index.js.map +0 -1
  131. package/dist/esm/retriever/scope.d.ts +0 -5
  132. package/dist/esm/retriever/scope.d.ts.map +0 -1
  133. package/dist/esm/retriever/scope.js +0 -66
  134. package/dist/esm/retriever/scope.js.map +0 -1
  135. package/dist/esm/retriever/types.d.ts +0 -89
  136. package/dist/esm/retriever/types.d.ts.map +0 -1
  137. package/dist/esm/retriever/types.js +0 -2
  138. package/dist/esm/retriever/types.js.map +0 -1
  139. package/dist/esm/storage.d.ts +0 -274
  140. package/dist/esm/storage.d.ts.map +0 -1
  141. package/dist/esm/storage.js +0 -845
  142. package/dist/esm/storage.js.map +0 -1
  143. package/dist/esm/tools/definitions.d.ts +0 -5
  144. package/dist/esm/tools/definitions.d.ts.map +0 -1
  145. package/dist/esm/tools/definitions.js +0 -144
  146. package/dist/esm/tools/definitions.js.map +0 -1
  147. package/dist/esm/tools/evidence-ids.d.ts +0 -7
  148. package/dist/esm/tools/evidence-ids.d.ts.map +0 -1
  149. package/dist/esm/tools/evidence-ids.js +0 -68
  150. package/dist/esm/tools/evidence-ids.js.map +0 -1
  151. package/dist/esm/tools/factory.d.ts +0 -7
  152. package/dist/esm/tools/factory.d.ts.map +0 -1
  153. package/dist/esm/tools/factory.js +0 -166
  154. package/dist/esm/tools/factory.js.map +0 -1
  155. package/dist/esm/tools/index.d.ts +0 -6
  156. package/dist/esm/tools/index.d.ts.map +0 -1
  157. package/dist/esm/tools/index.js +0 -22
  158. package/dist/esm/tools/index.js.map +0 -1
  159. package/dist/esm/tools/internal-types.d.ts +0 -18
  160. package/dist/esm/tools/internal-types.d.ts.map +0 -1
  161. package/dist/esm/tools/internal-types.js +0 -2
  162. package/dist/esm/tools/internal-types.js.map +0 -1
  163. package/dist/esm/tools/short-ids.d.ts +0 -8
  164. package/dist/esm/tools/short-ids.d.ts.map +0 -1
  165. package/dist/esm/tools/short-ids.js +0 -78
  166. package/dist/esm/tools/short-ids.js.map +0 -1
  167. package/dist/esm/tools/types.d.ts +0 -53
  168. package/dist/esm/tools/types.d.ts.map +0 -1
  169. package/dist/esm/tools/types.js +0 -2
  170. package/dist/esm/tools/types.js.map +0 -1
  171. package/dist/esm/values.d.ts +0 -17
  172. package/dist/esm/values.d.ts.map +0 -1
  173. package/dist/esm/values.js +0 -87
  174. package/dist/esm/values.js.map +0 -1
  175. package/docs/tools.md +0 -264
  176. package/src/filter.ts +0 -189
  177. package/src/recall-input.ts +0 -46
  178. package/src/retriever/index.ts +0 -142
  179. package/src/retriever/scope.ts +0 -56
  180. package/src/retriever/types.ts +0 -68
  181. package/src/storage.ts +0 -765
  182. package/src/tools/definitions.ts +0 -175
  183. package/src/tools/evidence-ids.ts +0 -57
  184. package/src/tools/factory.ts +0 -153
  185. package/src/tools/index.ts +0 -25
  186. package/src/tools/internal-types.ts +0 -12
  187. package/src/tools/short-ids.ts +0 -58
  188. package/src/tools/types.ts +0 -48
  189. package/src/values.ts +0 -103
@@ -1,315 +1,367 @@
1
1
  "use strict";
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.list = exports.contextualQuery = exports.MemDriveError = void 0;
3
+ exports.MemDrive = exports.MemDriveError = void 0;
4
4
  exports.create = create;
5
5
  exports.open = open;
6
- const storage_js_1 = require("./storage.js");
7
- const filter_js_1 = require("./filter.js");
8
6
  const errors_js_1 = require("./errors.js");
9
- const recall_input_js_1 = require("./recall-input.js");
10
- const values_js_1 = require("./values.js");
7
+ const config_js_1 = require("./config.js");
8
+ const records_js_1 = require("./records.js");
9
+ const search_js_1 = require("./search.js");
11
10
  var errors_js_2 = require("./errors.js");
12
11
  Object.defineProperty(exports, "MemDriveError", { enumerable: true, get: function () { return errors_js_2.MemDriveError; } });
13
- const keys = (value, allowed) => value !== null && typeof value === 'object' && !Array.isArray(value)
14
- && Object.keys(value).every(key => allowed.includes(key));
15
- function validateOptions(options, creating) {
16
- (0, errors_js_1.requireInput)(keys(options, ['client', 'name', 'maxConcurrentRequests', 'observe', 'record',
17
- ...(creating ? ['metadataSchema', 'textSearch', 'embedding', 'labels', 'description'] : ['branch'])]), 'Use the documented MemDrive options.');
18
- (0, errors_js_1.requireInput)(creating || options.branch === undefined || (typeof options.branch === 'string'
19
- && /^[A-Za-z0-9_-]{3,52}$/.test(options.branch)), 'branch must be a valid branch name.');
20
- if (creating && options.embedding?.mode === 'external')
21
- (0, errors_js_1.requireInput)(options.textSearch === undefined, 'External embeddings do not support textSearch.');
22
- const concurrency = options.maxConcurrentRequests ?? 5;
23
- (0, errors_js_1.requireInput)(Number.isSafeInteger(concurrency) && concurrency > 0, 'maxConcurrentRequests must be a positive integer.');
24
- for (const name of ['observe', 'record'])
25
- (0, errors_js_1.requireInput)(options[name] === undefined || typeof options[name] === 'function', `${name} must be a callback.`);
26
- return { ...options };
12
+ const main = 'main';
13
+ const prefix = 'fork-';
14
+ const clients = new WeakMap();
15
+ const requestMaxBytes = 6_029_312;
16
+ const maximumInputBytes = 300_000;
17
+ const maximumRequestDocuments = 1000;
18
+ const writeBatchLimit = 100;
19
+ const validName = (name, maxLength = 52) => typeof name === 'string' && name.length >= 3 && name.length <= maxLength && /^[A-Za-z0-9_-]+$/.test(name);
20
+ const validForkName = (name) => validName(name, 52 - prefix.length) && name !== main;
21
+ function validateConnection(options, keys) {
22
+ (0, errors_js_1.requireInput)((0, config_js_1.keysOnly)(options, keys) && options.client !== null && typeof options.client === 'object'
23
+ && 'collection' in options.client && typeof options.client.collection === 'function', 'Provide a LambdaDB client and documented file store options.');
24
+ (0, errors_js_1.requireInput)(validName(options.name), 'Store name must be 3–52 letters, digits, underscores or hyphens.');
27
25
  }
28
- // Separate queues bound logical recalls and database work across all branch views.
29
- function requestQueue(limit) {
30
- let active = 0;
31
- const waiting = [];
32
- return (operation, signal) => new Promise((resolve, reject) => {
33
- try {
34
- (0, errors_js_1.checkAbort)(signal);
35
- }
36
- catch (error) {
37
- reject(error);
38
- return;
39
- }
40
- const entry = { start() {
41
- signal?.removeEventListener('abort', cancel);
42
- active++;
43
- Promise.resolve().then(() => { (0, errors_js_1.checkAbort)(signal); return operation(); }).then(resolve, reject).finally(() => {
44
- active--;
45
- waiting.shift()?.start();
46
- });
47
- } };
48
- const cancel = () => {
49
- const index = waiting.indexOf(entry);
50
- if (index < 0)
51
- return;
52
- waiting.splice(index, 1);
53
- signal?.removeEventListener('abort', cancel);
54
- reject((0, errors_js_1.abortError)(undefined, signal));
55
- };
56
- if (active < limit)
57
- entry.start();
58
- else {
59
- waiting.push(entry);
60
- signal?.addEventListener('abort', cancel, { once: true });
61
- if (signal?.aborted)
62
- cancel();
63
- }
64
- });
26
+ async function remote(operation, message, signal, codes = {}) {
27
+ try {
28
+ return await (0, errors_js_1.nativeRequest)(operation, signal);
29
+ }
30
+ catch (cause) {
31
+ const error = (0, errors_js_1.databaseError)(cause, message);
32
+ if (error.status !== undefined && codes[error.status] !== undefined)
33
+ error.code = codes[error.status];
34
+ throw error;
35
+ }
65
36
  }
66
- var recall_input_js_2 = require("./recall-input.js");
67
- Object.defineProperty(exports, "contextualQuery", { enumerable: true, get: function () { return recall_input_js_2.contextualQuery; } });
68
- function recallQuery(request, schema, textSearch, embedding) {
69
- if (embedding.mode === 'external')
70
- return { size: request.limit, query: { knn: {
71
- field: 'embedding', queryVector: request.queryVector, k: request.limit,
72
- filter: (0, filter_js_1.buildSearchFilter)([request.filter], request.excludeIds ?? [], [], schema),
73
- } } };
74
- const context = (0, recall_input_js_1.textRecallQuery)(request);
75
- const filter = (0, filter_js_1.buildSearchFilter)([request.filter], request.excludeIds ?? [], [], schema);
76
- const candidates = Math.max(200, request.limit);
77
- const semantic = { knn: { field: 'embedding', queryText: context, k: candidates, filter } };
78
- const query = textSearch ? { bayesian: [semantic,
79
- { bool: [{ ...filter, occur: 'filter' },
80
- { queryString: { query: request.query, defaultField: 'content', skipSyntax: true }, occur: 'must' }] },
81
- ] } : semantic;
82
- return { size: request.limit, query, ...(request.rerank === false ? (textSearch ? { candidateSize: candidates } : {}) : {
83
- rerank: { provider: 'typesafe', model: 'jev-1.13.0', queryText: context,
84
- fields: ['content'], candidateSize: candidates, onFailure: 'error' },
85
- }) };
37
+ const branchRead = (branch) => ({
38
+ ref: { kind: 'branch', name: branch }, consistentRead: true, includeVectors: false,
39
+ });
40
+ const kindClause = { queryString: { query: '"file"', defaultField: 'kind' }, occur: 'filter' };
41
+ const pathClause = (path) => ({ queryString: { query: JSON.stringify(path), defaultField: 'path' } });
42
+ function pathQuery(paths) {
43
+ return { bool: [
44
+ kindClause,
45
+ paths.length === 1 ? { ...pathClause(paths[0]), occur: 'filter' }
46
+ : { bool: paths.map(path => ({ ...pathClause(path), occur: 'should' })), occur: 'filter' },
47
+ ] };
86
48
  }
87
- async function validatedHits(response, request, schema, name, max, embedding) {
88
- if (response?.isDocsInline === false || !Array.isArray(response?.docs) || response.docs.length > max)
89
- throw new errors_js_1.MemDriveError('INVALID_RESULT', 'Search returned an invalid document envelope.');
90
- const seen = new Set();
91
- return Promise.all(response.docs.map(async (hit) => {
92
- if (hit.collection !== undefined && hit.collection !== name)
93
- throw new errors_js_1.MemDriveError('INVALID_RESULT', 'Search returned a different collection.');
94
- let mem;
95
- try {
96
- mem = await (embedding.mode === 'external' ? (0, storage_js_1.vectorMatch)(hit.doc, schema) : (0, storage_js_1.validateChunk)(hit.doc, schema));
49
+ function writeBatches(records, branch) {
50
+ const envelope = (0, records_js_1.bytes)({ docs: [], branch });
51
+ const batches = [];
52
+ let batch = [];
53
+ let inputBytes = 0, wireBytes = envelope;
54
+ const flush = () => {
55
+ if (batch.length)
56
+ batches.push(batch);
57
+ batch = [];
58
+ inputBytes = 0;
59
+ wireBytes = envelope;
60
+ };
61
+ const fits = (docs) => batch.length + docs.length <= maximumRequestDocuments
62
+ && inputBytes + docs.reduce((sum, doc) => sum + doc.inputBytes, 0) <= maximumInputBytes
63
+ && wireBytes + docs.reduce((sum, doc) => sum + doc.wireBytes, 0) <= requestMaxBytes;
64
+ const append = (docs) => {
65
+ batch.push(...docs);
66
+ for (const doc of docs) {
67
+ inputBytes += doc.inputBytes;
68
+ wireBytes += doc.wireBytes;
97
69
  }
98
- catch (cause) {
99
- throw new errors_js_1.MemDriveError('INVALID_RESULT', 'Search returned invalid Mem evidence.', { cause });
70
+ };
71
+ for (const { docs } of records) {
72
+ if (!fits(docs))
73
+ flush();
74
+ if (fits(docs))
75
+ append(docs);
76
+ else {
77
+ // Keep each ordinary file together. For a large file, stage its
78
+ // chunks first and include the original only in the final request.
79
+ for (const doc of docs) {
80
+ const next = [doc];
81
+ if (!fits(next))
82
+ flush();
83
+ (0, errors_js_1.requireInput)(fits(next), 'A file record exceeds the limit for a native write.');
84
+ append(next);
85
+ }
100
86
  }
101
- if (seen.has(mem.id) || request.excludeIds?.includes(mem.memId) || !(0, filter_js_1.matchesFilter)({ ...mem, id: mem.memId }, request.filter, schema))
102
- throw new errors_js_1.MemDriveError('INVALID_RESULT', 'Search returned duplicate, excluded or filtered-out evidence.');
103
- seen.add(mem.id);
104
- return mem;
105
- }));
106
- }
107
- function verifyRerank(response, body) {
108
- const result = response.rerank;
109
- const same = result?.provider === body.rerank.provider && result?.model === body.rerank.model;
110
- const applied = result?.status === 'applied' && Number.isSafeInteger(result.candidateCount)
111
- && result.scoredCount === result.candidateCount && result.candidateCount !== undefined && result.candidateCount >= response.docs.length
112
- && result.candidateCount <= body.rerank.candidateSize
113
- && response.docs.every(hit => typeof hit.score === 'number' && Number.isFinite(hit.score) && hit.score >= 0 && hit.score <= 1);
114
- const empty = result?.status === 'skipped' && result.reason === 'noCandidates'
115
- && result.candidateCount === 0 && result.scoredCount === 0 && response.docs.length === 0;
116
- if (!same || (!applied && !empty))
117
- throw new errors_js_1.MemDriveError('RERANK_FAILED', 'Native reranking was not confirmed.');
87
+ }
88
+ flush();
89
+ return batches;
118
90
  }
119
- function makeDataHandle(backend, options, queue, recallQueue) {
120
- const counter = (0, values_js_1.createValueCounter)({ describe: options => backend.describe(options), query: (body, options) => backend.query(body, options),
121
- metadataSchema: () => backend.metadataSchema });
122
- let generation = backend.generation;
123
- const invalidate = () => { generation = backend.generation; counter.invalidate(); };
124
- const inspect = async (operation) => {
125
- backend.assertActive();
126
- if (generation !== backend.generation)
127
- invalidate();
128
- try {
129
- const result = await operation();
130
- backend.assertActive();
131
- return result;
132
- }
133
- finally {
134
- if (generation !== backend.generation)
135
- invalidate();
136
- }
137
- };
138
- const mutation = async (operation) => {
139
- try {
140
- return await operation();
91
+ function store(handle, description) {
92
+ const { name, metadataSchema } = description;
93
+ const forks = new Map();
94
+ async function describe(options) {
95
+ const { signal } = (0, errors_js_1.operationOptions)(options);
96
+ (0, errors_js_1.checkAbort)(signal);
97
+ return structuredClone(description);
98
+ }
99
+ async function fetchFiles(branch, fileIds, signal) {
100
+ const response = await remote(() => handle.docs.fetch({ ids: fileIds.map(records_js_1.originalId), ...branchRead(branch) }, (0, errors_js_1.nativeOptions)(signal)), 'Could not read file originals.', signal);
101
+ (0, errors_js_1.checkAbort)(signal);
102
+ const files = new Map();
103
+ const expected = new Set(fileIds);
104
+ for (const doc of (0, records_js_1.resultDocs)(response, name, fileIds.length)) {
105
+ const file = (0, records_js_1.decodeFile)(doc, metadataSchema);
106
+ if (!expected.has(file.fileId) || files.has(file.fileId))
107
+ throw new errors_js_1.MemDriveError('INVALID_RESULT', 'Read returned unexpected or duplicate file originals.');
108
+ files.set(file.fileId, file);
141
109
  }
142
- finally {
143
- invalidate();
110
+ return files;
111
+ }
112
+ function reader(branch) {
113
+ async function read(request, options) {
114
+ const { signal } = (0, errors_js_1.operationOptions)(options);
115
+ (0, errors_js_1.checkAbort)(signal);
116
+ (0, errors_js_1.requireInput)((0, config_js_1.keysOnly)(request, ['fileId', 'path', 'paths']) && Object.keys(request).length === 1, 'Read requires exactly one of fileId, path or paths.');
117
+ if ('fileId' in request) {
118
+ const fileId = request.fileId;
119
+ (0, errors_js_1.requireInput)((0, records_js_1.validFileId)(fileId), 'Provide a fileId returned by MemDrive.');
120
+ return (await fetchFiles(branch, [fileId], signal)).get(fileId) ?? null;
121
+ }
122
+ const multi = 'paths' in request;
123
+ const value = multi ? request.paths : [request.path];
124
+ (0, errors_js_1.requireInput)(Array.isArray(value) && value.length >= 1 && value.length <= writeBatchLimit && [...value].every(records_js_1.validPath)
125
+ && new Set(value).size === value.length, 'Provide 1–100 distinct valid absolute file paths.');
126
+ const paths = [...value];
127
+ const response = await remote(() => handle.query({ size: paths.length * 2, ...branchRead(branch), query: pathQuery(paths) }, (0, errors_js_1.nativeOptions)(signal)), 'Could not read files by path.', signal);
128
+ (0, errors_js_1.checkAbort)(signal);
129
+ const expected = new Set(paths), files = new Map();
130
+ for (const doc of (0, records_js_1.resultDocs)(response, name, paths.length)) {
131
+ const file = (0, records_js_1.decodeFile)(doc, metadataSchema);
132
+ if (!expected.has(file.path) || files.has(file.path))
133
+ throw new errors_js_1.MemDriveError('INVALID_RESULT', 'Read returned unexpected or duplicate paths.');
134
+ files.set(file.path, file);
135
+ }
136
+ return multi ? paths.flatMap(path => files.has(path) ? [files.get(path)] : []) : files.get(paths[0]) ?? null;
144
137
  }
145
- };
146
- const executeRecall = async (input, start, requestIndex, signal) => {
147
- const usage = { elapsedMs: 0, databaseRequests: 0 };
148
- const hook = (callback) => async (event) => {
138
+ return read;
139
+ }
140
+ function searcher(branch) {
141
+ return async (request, options) => {
142
+ const { signal } = (0, errors_js_1.operationOptions)(options);
149
143
  (0, errors_js_1.checkAbort)(signal);
150
- await callback?.(requestIndex === undefined ? event : { ...event, requestIndex });
144
+ const body = (0, search_js_1.searchQuery)(request, metadataSchema);
145
+ const response = await remote(() => handle.query({ ...body, ...branchRead(branch),
146
+ fields: { include: ['id', 'kind', 'fileId', 'path', 'modifiedAt', 'content', 'start', 'end'] } }, (0, errors_js_1.nativeOptions)(signal)), 'File search failed.', signal);
151
147
  (0, errors_js_1.checkAbort)(signal);
148
+ const docs = (0, records_js_1.resultDocs)(response, name, body.size);
149
+ const reranked = (0, search_js_1.validateReranking)(response, body);
150
+ const fileIds = new Set(), chunkIds = new Set();
151
+ const chunks = docs.map(doc => {
152
+ if (doc.kind !== 'chunk' || !(0, records_js_1.validFileId)(doc.fileId) || typeof doc.id !== 'string'
153
+ || !doc.id.startsWith(`chunk:${doc.fileId}:`) || chunkIds.has(doc.id) || !(0, records_js_1.validPath)(doc.path)
154
+ || typeof doc.content !== 'string' || !doc.content.trim() || /\p{Surrogate}/u.test(doc.content))
155
+ throw new errors_js_1.MemDriveError('INVALID_RESULT', 'Search returned an invalid or duplicate chunk.');
156
+ fileIds.add(doc.fileId);
157
+ chunkIds.add(doc.id);
158
+ return { fileId: doc.fileId, path: doc.path, modifiedAt: doc.modifiedAt, content: doc.content,
159
+ start: doc.start, end: doc.end };
160
+ });
161
+ if (!chunks.length)
162
+ return { hits: [], reranked };
163
+ const files = await fetchFiles(branch, [...fileIds], signal);
164
+ const hits = [];
165
+ for (const chunk of chunks) {
166
+ const file = files.get(chunk.fileId);
167
+ // An orphan or stale chunk must not expose body text from another file or revision.
168
+ if (!file || file.path !== chunk.path || file.modifiedAt !== chunk.modifiedAt)
169
+ continue;
170
+ const { start, end } = chunk;
171
+ const verified = typeof start === 'number' && typeof end === 'number'
172
+ && Number.isSafeInteger(start) && Number.isSafeInteger(end) && start >= 0 && end > start && end <= file.content.length
173
+ && file.content.slice(start, end) === chunk.content;
174
+ hits.push({ ...file, content: chunk.content, ...(verified ? { range: { start, end } } : {}) });
175
+ }
176
+ return { hits, reranked };
152
177
  };
153
- const record = hook(options.record);
154
- const observe = hook(options.observe);
155
- const executeDb = (operation) => queue(async () => {
156
- backend.assertActive();
178
+ }
179
+ function working(fork) {
180
+ const existing = forks.get(fork);
181
+ if (existing)
182
+ return existing;
183
+ const branch = prefix + fork;
184
+ let writes = Promise.resolve();
185
+ const unconfirmed = new Map();
186
+ async function write(request, options) {
187
+ const { signal } = (0, errors_js_1.operationOptions)(options);
157
188
  (0, errors_js_1.checkAbort)(signal);
158
- usage.databaseRequests++;
159
- try {
160
- const result = await operation({ signal });
189
+ const multi = Array.isArray(request);
190
+ const operations = multi ? [...request] : [request];
191
+ (0, errors_js_1.requireInput)(operations.length >= 1 && operations.length <= writeBatchLimit, 'Submit 1–100 file create operations.');
192
+ const prepared = Promise.all(operations.map(operation => (0, records_js_1.fileRecords)(operation, metadataSchema)));
193
+ // Validation and hashing begin immediately. Attach the queue now so overlapping writes keep call order.
194
+ void prepared.catch(() => { });
195
+ const write = writes.then(async () => {
161
196
  (0, errors_js_1.checkAbort)(signal);
162
- return result;
163
- }
164
- catch (cause) {
165
- const failure = (0, errors_js_1.abortError)(cause, signal);
166
- if (failure instanceof errors_js_1.MemDriveError)
167
- throw failure;
168
- const error = new errors_js_1.MemDriveError('DATABASE_ERROR', 'MemDrive database request failed.', { cause: failure });
169
- const remote = failure;
170
- error.status = remote.statusCode ?? remote.status;
171
- throw error;
172
- }
173
- }, signal);
174
- try {
175
- (0, errors_js_1.checkAbort)(signal);
176
- backend.assertActive();
177
- if (generation !== backend.generation)
178
- invalidate();
179
- const schema = () => backend.metadataSchema;
180
- const request = (0, recall_input_js_1.recallInput)(input, schema(), backend.embedding);
181
- (0, errors_js_1.requireInput)(request.rerank !== true || backend.embedding.mode === 'managed', 'rerank: true requires managed embeddings.');
182
- const query = async (body) => {
183
- const response = await executeDb(options => backend.query(body, options));
184
- const mems = await validatedHits(response, request, schema(), options.name, body.size ?? 10, backend.embedding);
185
- return { ...response, docs: response.docs.map((hit, i) => ({ ...hit, doc: mems[i] })) };
186
- };
187
- const body = recallQuery(request, schema(), backend.textSearch, backend.embedding);
188
- await record({ phase: 'query', request: structuredClone(body) });
189
- const response = await query(body);
190
- if (body.rerank)
191
- verifyRerank(response, body);
192
- const mems = response.docs.map(hit => hit.doc);
193
- await record({ phase: 'query_result', response: structuredClone(response) });
194
- await observe({ type: 'retrieved', ids: mems.map(mem => mem.id), matches: mems.map(storage_js_1.publicMatch) });
195
- backend.assertActive();
196
- (0, errors_js_1.checkAbort)(signal);
197
- usage.elapsedMs = performance.now() - start;
198
- return { matches: mems.map(storage_js_1.publicMatch), usage };
199
- }
200
- catch (cause) {
201
- const failure = (0, errors_js_1.abortError)(cause, signal);
202
- usage.elapsedMs = performance.now() - start;
203
- const error = failure instanceof errors_js_1.MemDriveError ? failure
204
- : new errors_js_1.MemDriveError('RECALL_FAILED', 'MemDrive retrieval failed.', { cause: failure });
205
- error.usage = structuredClone(usage);
206
- throw error;
207
- }
208
- };
209
- const snapshot = (input) => {
210
- const start = performance.now();
211
- try {
212
- return { input: structuredClone(input), start };
213
- }
214
- catch (cause) {
215
- const error = new errors_js_1.MemDriveError('INVALID_INPUT', 'Recall input must be cloneable data.', { cause });
216
- error.usage = { elapsedMs: 0, databaseRequests: 0 };
217
- return { error };
218
- }
219
- };
220
- const submit = async (prepared, requestIndex, signal) => {
221
- if (prepared.error)
222
- throw prepared.error;
223
- try {
224
- return await recallQueue(() => executeRecall(prepared.input, prepared.start, requestIndex, signal), signal);
225
- }
226
- catch (cause) {
227
- // Queue cancellation and executeRecall both reject with MemDriveError.
228
- const error = cause;
229
- error.usage ??= { elapsedMs: performance.now() - prepared.start, databaseRequests: 0 };
230
- throw error;
231
- }
232
- };
233
- async function recall(input, options) {
234
- const { signal } = (0, errors_js_1.operationOptions)(options);
235
- if (input !== null && typeof input === 'object' && Object.hasOwn(input, 'requests')) {
236
- (0, errors_js_1.requireInput)(keys(input, ['requests']) && Array.isArray(input.requests) && input.requests.length > 0, 'Batch recall requires only a nonempty requests array.');
237
- // Snapshot every item before any operation can yield or mutate caller data.
238
- const prepared = Array.from(input.requests, snapshot);
239
- const results = await Promise.all(prepared.map(async (item, requestIndex) => {
240
- try {
241
- return await submit(item, requestIndex, signal);
197
+ const records = await prepared;
198
+ const paths = records.map(record => record.file.path);
199
+ const expectedPaths = new Set(paths);
200
+ (0, errors_js_1.requireInput)(expectedPaths.size === paths.length, 'Create paths must be distinct within one write.');
201
+ const batches = writeBatches(records, branch);
202
+ const response = await remote(() => handle.query({ size: 2 * paths.length, ...branchRead(branch),
203
+ query: pathQuery(paths), fields: { include: ['id', 'kind', 'fileId', 'path'] } }, (0, errors_js_1.nativeOptions)(signal)), 'Could not check create destinations.', signal);
204
+ (0, errors_js_1.checkAbort)(signal);
205
+ const occupied = new Set();
206
+ for (const doc of (0, records_js_1.resultDocs)(response, name, 2 * paths.length)) {
207
+ if (doc.kind !== 'file' || !(0, records_js_1.validFileId)(doc.fileId) || doc.id !== (0, records_js_1.originalId)(doc.fileId)
208
+ || typeof doc.path !== 'string' || !expectedPaths.has(doc.path) || occupied.has(doc.path))
209
+ throw new errors_js_1.MemDriveError('INVALID_RESULT', 'Destination lookup returned an unexpected or duplicate file.');
210
+ occupied.add(doc.path);
211
+ unconfirmed.delete(doc.path);
212
+ }
213
+ if (occupied.size)
214
+ throw new errors_js_1.MemDriveError('PATH_EXISTS', 'A file already exists at a requested path.', { details: { paths: [...occupied] } });
215
+ for (const path of paths) {
216
+ const fileId = unconfirmed.get(path);
217
+ if (fileId)
218
+ throw new errors_js_1.MemDriveError('WRITE_UNCONFIRMED', 'An earlier create at this path is still unconfirmed.', { details: { fileId, path } });
242
219
  }
243
- catch (cause) {
244
- const error = cause;
245
- return { error: { code: error.code, message: error.message,
246
- ...(Number.isInteger(error.status) ? { status: error.status } : {}) },
247
- usage: structuredClone(error.usage) };
220
+ const completed = [];
221
+ const staged = new Map();
222
+ const filesById = new Map(records.map(({ file }) => [file.fileId, file]));
223
+ const details = () => ({
224
+ completedFileIds: completed.map(file => file.fileId),
225
+ incompleteTargets: [...staged].map(([fileId, path]) => ({ fileId, path })),
226
+ });
227
+ for (const batch of batches) {
228
+ const docs = batch.map(record => record.doc);
229
+ const ids = new Set(docs.map(doc => doc.id));
230
+ const targets = [...new Map(docs.map(({ fileId, path }) => [fileId, { fileId, path }])).values()];
231
+ const uncertain = (cause) => {
232
+ for (const { kind, fileId, path } of docs)
233
+ if (kind === 'file')
234
+ unconfirmed.set(path, fileId);
235
+ return new errors_js_1.MemDriveError('WRITE_UNCONFIRMED', 'The create may have been accepted. Inspect the files on this fork.', {
236
+ cause, details: { ...(targets.length === 1 ? targets[0] : {}), targets, ...details() },
237
+ });
238
+ };
239
+ try {
240
+ (0, errors_js_1.checkAbort)(signal);
241
+ }
242
+ catch (cause) {
243
+ throw new errors_js_1.MemDriveError('ABORTED', 'Write aborted before the next request.', { cause, details: details() });
244
+ }
245
+ try {
246
+ await (0, errors_js_1.nativeRequest)(() => handle.docs.upsert({ docs, branch }, (0, errors_js_1.nativeOptions)(signal)), signal);
247
+ }
248
+ catch (cause) {
249
+ const error = (0, errors_js_1.databaseError)(cause, 'The file create was rejected.');
250
+ if ([400, 401, 403, 404, 413, 422, 429].includes(error.status ?? 0)) {
251
+ if (completed.length || staged.size) {
252
+ const partial = new errors_js_1.MemDriveError(error.code, error.message, { cause: error, details: details() });
253
+ partial.status = error.status;
254
+ throw partial;
255
+ }
256
+ throw error;
257
+ }
258
+ throw uncertain(cause);
259
+ }
260
+ try {
261
+ const confirmation = await (0, errors_js_1.nativeRequest)(() => handle.docs.fetch({ ids: [...ids], ...branchRead(branch),
262
+ fields: { include: ['id'] } }, (0, errors_js_1.nativeOptions)(signal)), signal);
263
+ (0, errors_js_1.checkAbort)(signal);
264
+ const found = (0, records_js_1.resultDocs)(confirmation, name, ids.size);
265
+ if (found.some(doc => typeof doc.id !== 'string' || !ids.delete(doc.id)) || ids.size)
266
+ throw new errors_js_1.MemDriveError('INVALID_RESULT', 'The original and its complete chunk set were not confirmed.');
267
+ }
268
+ catch (cause) {
269
+ throw uncertain(cause);
270
+ }
271
+ for (const { fileId, kind, path } of docs) {
272
+ if (kind === 'file') {
273
+ staged.delete(fileId);
274
+ completed.push(filesById.get(fileId));
275
+ }
276
+ else
277
+ staged.set(fileId, path);
278
+ }
248
279
  }
249
- }));
250
- return { results };
280
+ return multi ? completed : completed[0];
281
+ });
282
+ writes = write.then(() => undefined, () => undefined);
283
+ return write;
251
284
  }
252
- return submit(snapshot(input), undefined, signal);
285
+ const drive = Object.freeze({
286
+ name, at: Object.freeze({ kind: 'fork', name: fork }), read: reader(branch), search: searcher(branch), describe, write,
287
+ });
288
+ forks.set(fork, drive);
289
+ return drive;
253
290
  }
254
- return Object.freeze({
255
- remember: (request, options) => mutation(() => backend.remember(request, options)),
256
- forget: (request, options) => mutation(() => backend.forget(request, options)),
257
- read: (request, options) => backend.read(request, undefined, options),
258
- list: (request, options) => backend.list(request, options),
259
- embedding: Object.freeze(structuredClone(backend.embedding)),
260
- recall,
261
- async countValues(request, options) {
291
+ const mainView = Object.freeze({
292
+ name, at: Object.freeze({ kind: 'main' }), read: reader(main), search: searcher(main), describe,
293
+ async fork(request, options) {
262
294
  const { signal } = (0, errors_js_1.operationOptions)(options);
263
295
  (0, errors_js_1.checkAbort)(signal);
264
- return inspect(() => counter.countValues(request, operation => queue(async () => {
265
- backend.assertActive();
266
- (0, errors_js_1.checkAbort)(signal);
267
- const result = await operation({ signal });
268
- backend.assertActive();
269
- (0, errors_js_1.checkAbort)(signal);
270
- return result;
271
- }, signal), signal));
296
+ (0, errors_js_1.requireInput)((0, config_js_1.keysOnly)(request, ['name']) && validForkName(request.name), 'Fork name must be 3–47 letters, digits, underscores or hyphens and cannot be main.');
297
+ const fork = request.name;
298
+ const branch = prefix + fork;
299
+ const result = await remote(() => handle.branches.create({ branchName: branch, source: { kind: 'branch', name: main } }, (0, errors_js_1.nativeOptions)(signal)), 'Could not create the working fork.', signal, { 409: 'BRANCH_CONFLICT' });
300
+ (0, errors_js_1.checkAbort)(signal);
301
+ if (result?.branch?.name !== branch)
302
+ throw new errors_js_1.MemDriveError('INVALID_RESULT', 'Fork creation returned an invalid branch.');
303
+ return working(fork);
272
304
  },
273
305
  });
274
- }
275
- function makeDrive(backend, options) {
276
- const concurrency = options.maxConcurrentRequests ?? 5;
277
- const queue = requestQueue(concurrency), recallQueue = requestQueue(concurrency);
278
- const at = (ref) => {
279
- (0, errors_js_1.requireInput)(keys(ref, ['branch']) && Object.hasOwn(ref, 'branch'), 'Select a branch with { branch }.');
280
- return makeDataHandle(backend.at(ref.branch), options, queue, recallQueue);
306
+ return {
307
+ main: mainView,
308
+ async openFork(fork, signal) {
309
+ const cached = forks.get(fork);
310
+ if (cached)
311
+ return cached;
312
+ const response = await remote(() => handle.branches.list((0, errors_js_1.nativeOptions)(signal)), 'Could not find the working fork.', signal);
313
+ (0, errors_js_1.checkAbort)(signal);
314
+ if (!Array.isArray(response?.branches) || !response.branches.every(branch => typeof branch?.name === 'string'))
315
+ throw new errors_js_1.MemDriveError('INVALID_RESULT', 'Fork lookup returned invalid branches.');
316
+ if (!response.branches.some(branch => branch.name === prefix + fork))
317
+ throw new errors_js_1.MemDriveError('BRANCH_NOT_FOUND', 'The working fork does not exist.');
318
+ return working(fork);
319
+ },
281
320
  };
282
- return Object.freeze({
283
- ...at({ branch: options.branch ?? 'main' }),
284
- at,
285
- branches: Object.freeze({
286
- create: (request, options) => backend.branches.create(request, options),
287
- list: (options) => backend.branches.list(options),
288
- delete: (name, options) => backend.branches.delete(name, options),
289
- }),
290
- configure: (request, options) => backend.configure(request, options),
291
- destroy: (options) => backend.destroy(options),
292
- describe: (options) => backend.describeDescriptor(options),
293
- });
321
+ }
322
+ function storesFor(client) {
323
+ const existing = clients.get(client);
324
+ if (existing)
325
+ return existing;
326
+ const stores = new Map();
327
+ clients.set(client, stores);
328
+ return stores;
294
329
  }
295
330
  async function create(options, requestOptions) {
296
- requestOptions = (0, errors_js_1.operationOptions)(requestOptions);
297
- (0, errors_js_1.checkAbort)(requestOptions.signal);
298
- options = validateOptions(options, true);
299
- const backend = await (0, storage_js_1.createBackend)({ client: options.client, name: options.name, metadataSchema: options.metadataSchema, textSearch: options.textSearch, embedding: options.embedding, labels: options.labels,
300
- description: options.description }, requestOptions);
301
- return makeDrive(backend, options);
331
+ const { signal } = (0, errors_js_1.operationOptions)(requestOptions);
332
+ (0, errors_js_1.checkAbort)(signal);
333
+ validateConnection(options, ['client', 'name', 'labels', 'textSearch', 'metadataSchema']);
334
+ const { client, name } = options;
335
+ (0, errors_js_1.requireInput)(typeof client.createCollection === 'function', 'Client must support collection creation.');
336
+ const { indexConfigs, tags, description } = (0, config_js_1.storeConfiguration)(options);
337
+ await remote(() => client.createCollection({ collectionName: name, indexConfigs, tags }, (0, errors_js_1.nativeOptions)(signal)), 'Could not create the file store.', signal, { 409: 'STORE_EXISTS' });
338
+ (0, errors_js_1.checkAbort)(signal);
339
+ const stores = storesFor(client);
340
+ const created = stores.get(name) ?? store(client.collection(name), description);
341
+ stores.set(name, created);
342
+ return created.main;
302
343
  }
303
344
  async function open(options, requestOptions) {
304
- requestOptions = (0, errors_js_1.operationOptions)(requestOptions);
305
- (0, errors_js_1.checkAbort)(requestOptions.signal);
306
- options = validateOptions(options, false);
307
- const backend = await (0, storage_js_1.openBackend)({ client: options.client, name: options.name }, requestOptions);
308
- if (options.branch !== undefined && options.branch !== 'main'
309
- && !(await backend.branches.list(requestOptions)).some(branch => branch.name === options.branch))
310
- throw new errors_js_1.MemDriveError('BRANCH_NOT_FOUND', 'The selected MemDrive branch does not exist.');
311
- return makeDrive(backend, options);
345
+ const { signal } = (0, errors_js_1.operationOptions)(requestOptions);
346
+ (0, errors_js_1.checkAbort)(signal);
347
+ validateConnection(options, ['client', 'name', 'at']);
348
+ const { client, name, at } = options;
349
+ let fork;
350
+ if (at !== undefined) {
351
+ (0, errors_js_1.requireInput)((0, config_js_1.keysOnly)(at, ['fork']) && validForkName(at.fork), 'Select a working fork by its name.');
352
+ fork = at.fork;
353
+ }
354
+ const stores = storesFor(client);
355
+ let selected = stores.get(name);
356
+ if (!selected) {
357
+ const handle = client.collection(name);
358
+ const response = await remote(() => handle.get((0, errors_js_1.nativeOptions)(signal)), 'Could not inspect the file store.', signal, { 404: 'STORE_NOT_FOUND' });
359
+ (0, errors_js_1.checkAbort)(signal);
360
+ const description = (0, config_js_1.parseStore)(response?.collection, name);
361
+ selected = stores.get(name) ?? store(handle, description);
362
+ stores.set(name, selected);
363
+ }
364
+ return fork === undefined ? selected.main : selected.openFork(fork, signal);
312
365
  }
313
- const list = (request, options) => (0, storage_js_1.listBackends)(request, options);
314
- exports.list = list;
366
+ exports.MemDrive = Object.freeze({ create, open });
315
367
  //# sourceMappingURL=index.js.map