@functional-systems/memdrive 0.1.1-dev.2 → 0.1.1-dev.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/README.md +20 -279
- package/dist/commonjs/chunking.d.ts +1 -2
- package/dist/commonjs/chunking.d.ts.map +1 -1
- package/dist/commonjs/chunking.js +2 -5
- package/dist/commonjs/chunking.js.map +1 -1
- package/dist/commonjs/config.d.ts +17 -0
- package/dist/commonjs/config.d.ts.map +1 -0
- package/dist/commonjs/config.js +116 -0
- package/dist/commonjs/config.js.map +1 -0
- package/dist/commonjs/errors.d.ts +10 -6
- package/dist/commonjs/errors.d.ts.map +1 -1
- package/dist/commonjs/errors.js +25 -5
- package/dist/commonjs/errors.js.map +1 -1
- package/dist/commonjs/index.d.ts +15 -5
- package/dist/commonjs/index.d.ts.map +1 -1
- package/dist/commonjs/index.js +332 -280
- package/dist/commonjs/index.js.map +1 -1
- package/dist/commonjs/records.d.ts +44 -0
- package/dist/commonjs/records.d.ts.map +1 -0
- package/dist/commonjs/records.js +88 -0
- package/dist/commonjs/records.js.map +1 -0
- package/dist/commonjs/search.d.ts +8 -0
- package/dist/commonjs/search.d.ts.map +1 -0
- package/dist/commonjs/search.js +56 -0
- package/dist/commonjs/search.js.map +1 -0
- package/dist/commonjs/types.d.ts +84 -252
- package/dist/commonjs/types.d.ts.map +1 -1
- package/dist/esm/chunking.d.ts +1 -2
- package/dist/esm/chunking.d.ts.map +1 -1
- package/dist/esm/chunking.js +2 -4
- package/dist/esm/chunking.js.map +1 -1
- package/dist/esm/config.d.ts +17 -0
- package/dist/esm/config.d.ts.map +1 -0
- package/dist/esm/config.js +107 -0
- package/dist/esm/config.js.map +1 -0
- package/dist/esm/errors.d.ts +10 -6
- package/dist/esm/errors.d.ts.map +1 -1
- package/dist/esm/errors.js +22 -4
- package/dist/esm/errors.js.map +1 -1
- package/dist/esm/index.d.ts +15 -5
- package/dist/esm/index.d.ts.map +1 -1
- package/dist/esm/index.js +332 -278
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/records.d.ts +44 -0
- package/dist/esm/records.d.ts.map +1 -0
- package/dist/esm/records.js +77 -0
- package/dist/esm/records.js.map +1 -0
- package/dist/esm/search.d.ts +8 -0
- package/dist/esm/search.d.ts.map +1 -0
- package/dist/esm/search.js +52 -0
- package/dist/esm/search.js.map +1 -0
- package/dist/esm/types.d.ts +84 -252
- package/dist/esm/types.d.ts.map +1 -1
- package/package.json +7 -42
- package/src/chunking.ts +2 -3
- package/src/config.ts +121 -0
- package/src/errors.ts +20 -6
- package/src/index.ts +352 -260
- package/src/records.ts +83 -0
- package/src/search.ts +57 -0
- package/src/types.ts +74 -253
- package/dist/commonjs/filter.d.ts +0 -30
- package/dist/commonjs/filter.d.ts.map +0 -1
- package/dist/commonjs/filter.js +0 -224
- package/dist/commonjs/filter.js.map +0 -1
- package/dist/commonjs/recall-input.d.ts +0 -8
- package/dist/commonjs/recall-input.d.ts.map +0 -1
- package/dist/commonjs/recall-input.js +0 -45
- package/dist/commonjs/recall-input.js.map +0 -1
- package/dist/commonjs/retriever/index.d.ts +0 -7
- package/dist/commonjs/retriever/index.d.ts.map +0 -1
- package/dist/commonjs/retriever/index.js +0 -167
- package/dist/commonjs/retriever/index.js.map +0 -1
- package/dist/commonjs/retriever/scope.d.ts +0 -5
- package/dist/commonjs/retriever/scope.d.ts.map +0 -1
- package/dist/commonjs/retriever/scope.js +0 -69
- package/dist/commonjs/retriever/scope.js.map +0 -1
- package/dist/commonjs/retriever/types.d.ts +0 -89
- package/dist/commonjs/retriever/types.d.ts.map +0 -1
- package/dist/commonjs/retriever/types.js +0 -3
- package/dist/commonjs/retriever/types.js.map +0 -1
- package/dist/commonjs/storage.d.ts +0 -274
- package/dist/commonjs/storage.d.ts.map +0 -1
- package/dist/commonjs/storage.js +0 -861
- package/dist/commonjs/storage.js.map +0 -1
- package/dist/commonjs/tools/definitions.d.ts +0 -5
- package/dist/commonjs/tools/definitions.d.ts.map +0 -1
- package/dist/commonjs/tools/definitions.js +0 -147
- package/dist/commonjs/tools/definitions.js.map +0 -1
- package/dist/commonjs/tools/evidence-ids.d.ts +0 -7
- package/dist/commonjs/tools/evidence-ids.d.ts.map +0 -1
- package/dist/commonjs/tools/evidence-ids.js +0 -71
- package/dist/commonjs/tools/evidence-ids.js.map +0 -1
- package/dist/commonjs/tools/factory.d.ts +0 -7
- package/dist/commonjs/tools/factory.d.ts.map +0 -1
- package/dist/commonjs/tools/factory.js +0 -169
- package/dist/commonjs/tools/factory.js.map +0 -1
- package/dist/commonjs/tools/index.d.ts +0 -6
- package/dist/commonjs/tools/index.d.ts.map +0 -1
- package/dist/commonjs/tools/index.js +0 -25
- package/dist/commonjs/tools/index.js.map +0 -1
- package/dist/commonjs/tools/internal-types.d.ts +0 -18
- package/dist/commonjs/tools/internal-types.d.ts.map +0 -1
- package/dist/commonjs/tools/internal-types.js +0 -3
- package/dist/commonjs/tools/internal-types.js.map +0 -1
- package/dist/commonjs/tools/short-ids.d.ts +0 -8
- package/dist/commonjs/tools/short-ids.d.ts.map +0 -1
- package/dist/commonjs/tools/short-ids.js +0 -81
- package/dist/commonjs/tools/short-ids.js.map +0 -1
- package/dist/commonjs/tools/types.d.ts +0 -53
- package/dist/commonjs/tools/types.d.ts.map +0 -1
- package/dist/commonjs/tools/types.js +0 -3
- package/dist/commonjs/tools/types.js.map +0 -1
- package/dist/commonjs/values.d.ts +0 -17
- package/dist/commonjs/values.d.ts.map +0 -1
- package/dist/commonjs/values.js +0 -90
- package/dist/commonjs/values.js.map +0 -1
- package/dist/esm/filter.d.ts +0 -30
- package/dist/esm/filter.d.ts.map +0 -1
- package/dist/esm/filter.js +0 -209
- package/dist/esm/filter.js.map +0 -1
- package/dist/esm/recall-input.d.ts +0 -8
- package/dist/esm/recall-input.d.ts.map +0 -1
- package/dist/esm/recall-input.js +0 -40
- package/dist/esm/recall-input.js.map +0 -1
- package/dist/esm/retriever/index.d.ts +0 -7
- package/dist/esm/retriever/index.d.ts.map +0 -1
- package/dist/esm/retriever/index.js +0 -163
- package/dist/esm/retriever/index.js.map +0 -1
- package/dist/esm/retriever/scope.d.ts +0 -5
- package/dist/esm/retriever/scope.d.ts.map +0 -1
- package/dist/esm/retriever/scope.js +0 -66
- package/dist/esm/retriever/scope.js.map +0 -1
- package/dist/esm/retriever/types.d.ts +0 -89
- package/dist/esm/retriever/types.d.ts.map +0 -1
- package/dist/esm/retriever/types.js +0 -2
- package/dist/esm/retriever/types.js.map +0 -1
- package/dist/esm/storage.d.ts +0 -274
- package/dist/esm/storage.d.ts.map +0 -1
- package/dist/esm/storage.js +0 -845
- package/dist/esm/storage.js.map +0 -1
- package/dist/esm/tools/definitions.d.ts +0 -5
- package/dist/esm/tools/definitions.d.ts.map +0 -1
- package/dist/esm/tools/definitions.js +0 -144
- package/dist/esm/tools/definitions.js.map +0 -1
- package/dist/esm/tools/evidence-ids.d.ts +0 -7
- package/dist/esm/tools/evidence-ids.d.ts.map +0 -1
- package/dist/esm/tools/evidence-ids.js +0 -68
- package/dist/esm/tools/evidence-ids.js.map +0 -1
- package/dist/esm/tools/factory.d.ts +0 -7
- package/dist/esm/tools/factory.d.ts.map +0 -1
- package/dist/esm/tools/factory.js +0 -166
- package/dist/esm/tools/factory.js.map +0 -1
- package/dist/esm/tools/index.d.ts +0 -6
- package/dist/esm/tools/index.d.ts.map +0 -1
- package/dist/esm/tools/index.js +0 -22
- package/dist/esm/tools/index.js.map +0 -1
- package/dist/esm/tools/internal-types.d.ts +0 -18
- package/dist/esm/tools/internal-types.d.ts.map +0 -1
- package/dist/esm/tools/internal-types.js +0 -2
- package/dist/esm/tools/internal-types.js.map +0 -1
- package/dist/esm/tools/short-ids.d.ts +0 -8
- package/dist/esm/tools/short-ids.d.ts.map +0 -1
- package/dist/esm/tools/short-ids.js +0 -78
- package/dist/esm/tools/short-ids.js.map +0 -1
- package/dist/esm/tools/types.d.ts +0 -53
- package/dist/esm/tools/types.d.ts.map +0 -1
- package/dist/esm/tools/types.js +0 -2
- package/dist/esm/tools/types.js.map +0 -1
- package/dist/esm/values.d.ts +0 -17
- package/dist/esm/values.d.ts.map +0 -1
- package/dist/esm/values.js +0 -87
- package/dist/esm/values.js.map +0 -1
- package/docs/tools.md +0 -264
- package/src/filter.ts +0 -189
- package/src/recall-input.ts +0 -46
- package/src/retriever/index.ts +0 -142
- package/src/retriever/scope.ts +0 -56
- package/src/retriever/types.ts +0 -68
- package/src/storage.ts +0 -765
- package/src/tools/definitions.ts +0 -175
- package/src/tools/evidence-ids.ts +0 -57
- package/src/tools/factory.ts +0 -153
- package/src/tools/index.ts +0 -25
- package/src/tools/internal-types.ts +0 -12
- package/src/tools/short-ids.ts +0 -58
- package/src/tools/types.ts +0 -48
- package/src/values.ts +0 -103
package/dist/esm/index.js
CHANGED
|
@@ -1,307 +1,361 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import { createValueCounter } from './values.js';
|
|
1
|
+
import { checkAbort, databaseError, MemDriveError, nativeOptions, nativeRequest, operationOptions, requireInput } from './errors.js';
|
|
2
|
+
import { keysOnly, parseStore, storeConfiguration } from './config.js';
|
|
3
|
+
import { bytes, decodeFile, fileRecords, originalId, resultDocs, validFileId, validPath } from './records.js';
|
|
4
|
+
import { searchQuery, validateReranking } from './search.js';
|
|
6
5
|
export { MemDriveError } from './errors.js';
|
|
7
|
-
const
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
requireInput(
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
return { ...options };
|
|
6
|
+
const main = 'main';
|
|
7
|
+
const prefix = 'fork-';
|
|
8
|
+
const clients = new WeakMap();
|
|
9
|
+
const requestMaxBytes = 6_029_312;
|
|
10
|
+
const maximumInputBytes = 300_000;
|
|
11
|
+
const maximumRequestDocuments = 1000;
|
|
12
|
+
const writeBatchLimit = 100;
|
|
13
|
+
const validName = (name, maxLength = 52) => typeof name === 'string' && name.length >= 3 && name.length <= maxLength && /^[A-Za-z0-9_-]+$/.test(name);
|
|
14
|
+
const validForkName = (name) => validName(name, 52 - prefix.length) && name !== main;
|
|
15
|
+
function validateConnection(options, keys) {
|
|
16
|
+
requireInput(keysOnly(options, keys) && options.client !== null && typeof options.client === 'object'
|
|
17
|
+
&& 'collection' in options.client && typeof options.client.collection === 'function', 'Provide a LambdaDB client and documented file store options.');
|
|
18
|
+
requireInput(validName(options.name), 'Store name must be 3–52 letters, digits, underscores or hyphens.');
|
|
21
19
|
}
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
return;
|
|
33
|
-
}
|
|
34
|
-
const entry = { start() {
|
|
35
|
-
signal?.removeEventListener('abort', cancel);
|
|
36
|
-
active++;
|
|
37
|
-
Promise.resolve().then(() => { checkAbort(signal); return operation(); }).then(resolve, reject).finally(() => {
|
|
38
|
-
active--;
|
|
39
|
-
waiting.shift()?.start();
|
|
40
|
-
});
|
|
41
|
-
} };
|
|
42
|
-
const cancel = () => {
|
|
43
|
-
const index = waiting.indexOf(entry);
|
|
44
|
-
if (index < 0)
|
|
45
|
-
return;
|
|
46
|
-
waiting.splice(index, 1);
|
|
47
|
-
signal?.removeEventListener('abort', cancel);
|
|
48
|
-
reject(abortError(undefined, signal));
|
|
49
|
-
};
|
|
50
|
-
if (active < limit)
|
|
51
|
-
entry.start();
|
|
52
|
-
else {
|
|
53
|
-
waiting.push(entry);
|
|
54
|
-
signal?.addEventListener('abort', cancel, { once: true });
|
|
55
|
-
if (signal?.aborted)
|
|
56
|
-
cancel();
|
|
57
|
-
}
|
|
58
|
-
});
|
|
20
|
+
async function remote(operation, message, signal, codes = {}) {
|
|
21
|
+
try {
|
|
22
|
+
return await nativeRequest(operation, signal);
|
|
23
|
+
}
|
|
24
|
+
catch (cause) {
|
|
25
|
+
const error = databaseError(cause, message);
|
|
26
|
+
if (error.status !== undefined && codes[error.status] !== undefined)
|
|
27
|
+
error.code = codes[error.status];
|
|
28
|
+
throw error;
|
|
29
|
+
}
|
|
59
30
|
}
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
const query = textSearch ? { bayesian: [semantic,
|
|
72
|
-
{ bool: [{ ...filter, occur: 'filter' },
|
|
73
|
-
{ queryString: { query: request.query, defaultField: 'content', skipSyntax: true }, occur: 'must' }] },
|
|
74
|
-
] } : semantic;
|
|
75
|
-
return { size: request.limit, query, ...(request.rerank === false ? (textSearch ? { candidateSize: candidates } : {}) : {
|
|
76
|
-
rerank: { provider: 'typesafe', model: 'jev-1.13.0', queryText: context,
|
|
77
|
-
fields: ['content'], candidateSize: candidates, onFailure: 'error' },
|
|
78
|
-
}) };
|
|
31
|
+
const branchRead = (branch) => ({
|
|
32
|
+
ref: { kind: 'branch', name: branch }, consistentRead: true, includeVectors: false,
|
|
33
|
+
});
|
|
34
|
+
const kindClause = { queryString: { query: '"file"', defaultField: 'kind' }, occur: 'filter' };
|
|
35
|
+
const pathClause = (path) => ({ queryString: { query: JSON.stringify(path), defaultField: 'path' } });
|
|
36
|
+
function pathQuery(paths) {
|
|
37
|
+
return { bool: [
|
|
38
|
+
kindClause,
|
|
39
|
+
paths.length === 1 ? { ...pathClause(paths[0]), occur: 'filter' }
|
|
40
|
+
: { bool: paths.map(path => ({ ...pathClause(path), occur: 'should' })), occur: 'filter' },
|
|
41
|
+
] };
|
|
79
42
|
}
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
43
|
+
function writeBatches(records, branch) {
|
|
44
|
+
const envelope = bytes({ docs: [], branch });
|
|
45
|
+
const batches = [];
|
|
46
|
+
let batch = [];
|
|
47
|
+
let inputBytes = 0, wireBytes = envelope;
|
|
48
|
+
const flush = () => {
|
|
49
|
+
if (batch.length)
|
|
50
|
+
batches.push(batch);
|
|
51
|
+
batch = [];
|
|
52
|
+
inputBytes = 0;
|
|
53
|
+
wireBytes = envelope;
|
|
54
|
+
};
|
|
55
|
+
const fits = (docs) => batch.length + docs.length <= maximumRequestDocuments
|
|
56
|
+
&& inputBytes + docs.reduce((sum, doc) => sum + doc.inputBytes, 0) <= maximumInputBytes
|
|
57
|
+
&& wireBytes + docs.reduce((sum, doc) => sum + doc.wireBytes, 0) <= requestMaxBytes;
|
|
58
|
+
const append = (docs) => {
|
|
59
|
+
batch.push(...docs);
|
|
60
|
+
for (const doc of docs) {
|
|
61
|
+
inputBytes += doc.inputBytes;
|
|
62
|
+
wireBytes += doc.wireBytes;
|
|
90
63
|
}
|
|
91
|
-
|
|
92
|
-
|
|
64
|
+
};
|
|
65
|
+
for (const { docs } of records) {
|
|
66
|
+
if (!fits(docs))
|
|
67
|
+
flush();
|
|
68
|
+
if (fits(docs))
|
|
69
|
+
append(docs);
|
|
70
|
+
else {
|
|
71
|
+
// Keep each ordinary file together. For a large file, stage its
|
|
72
|
+
// chunks first and include the original only in the final request.
|
|
73
|
+
for (const doc of docs) {
|
|
74
|
+
const next = [doc];
|
|
75
|
+
if (!fits(next))
|
|
76
|
+
flush();
|
|
77
|
+
requireInput(fits(next), 'A file record exceeds the limit for a native write.');
|
|
78
|
+
append(next);
|
|
79
|
+
}
|
|
93
80
|
}
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
return mem;
|
|
98
|
-
}));
|
|
99
|
-
}
|
|
100
|
-
function verifyRerank(response, body) {
|
|
101
|
-
const result = response.rerank;
|
|
102
|
-
const same = result?.provider === body.rerank.provider && result?.model === body.rerank.model;
|
|
103
|
-
const applied = result?.status === 'applied' && Number.isSafeInteger(result.candidateCount)
|
|
104
|
-
&& result.scoredCount === result.candidateCount && result.candidateCount !== undefined && result.candidateCount >= response.docs.length
|
|
105
|
-
&& result.candidateCount <= body.rerank.candidateSize
|
|
106
|
-
&& response.docs.every(hit => typeof hit.score === 'number' && Number.isFinite(hit.score) && hit.score >= 0 && hit.score <= 1);
|
|
107
|
-
const empty = result?.status === 'skipped' && result.reason === 'noCandidates'
|
|
108
|
-
&& result.candidateCount === 0 && result.scoredCount === 0 && response.docs.length === 0;
|
|
109
|
-
if (!same || (!applied && !empty))
|
|
110
|
-
throw new MemDriveError('RERANK_FAILED', 'Native reranking was not confirmed.');
|
|
81
|
+
}
|
|
82
|
+
flush();
|
|
83
|
+
return batches;
|
|
111
84
|
}
|
|
112
|
-
function
|
|
113
|
-
const
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
if (
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
};
|
|
131
|
-
const mutation = async (operation) => {
|
|
132
|
-
try {
|
|
133
|
-
return await operation();
|
|
85
|
+
function store(handle, description) {
|
|
86
|
+
const { name, metadataSchema } = description;
|
|
87
|
+
const forks = new Map();
|
|
88
|
+
async function describe(options) {
|
|
89
|
+
const { signal } = operationOptions(options);
|
|
90
|
+
checkAbort(signal);
|
|
91
|
+
return structuredClone(description);
|
|
92
|
+
}
|
|
93
|
+
async function fetchFiles(branch, fileIds, signal) {
|
|
94
|
+
const response = await remote(() => handle.docs.fetch({ ids: fileIds.map(originalId), ...branchRead(branch) }, nativeOptions(signal)), 'Could not read file originals.', signal);
|
|
95
|
+
checkAbort(signal);
|
|
96
|
+
const files = new Map();
|
|
97
|
+
const expected = new Set(fileIds);
|
|
98
|
+
for (const doc of resultDocs(response, name, fileIds.length)) {
|
|
99
|
+
const file = decodeFile(doc, metadataSchema);
|
|
100
|
+
if (!expected.has(file.fileId) || files.has(file.fileId))
|
|
101
|
+
throw new MemDriveError('INVALID_RESULT', 'Read returned unexpected or duplicate file originals.');
|
|
102
|
+
files.set(file.fileId, file);
|
|
134
103
|
}
|
|
135
|
-
|
|
136
|
-
|
|
104
|
+
return files;
|
|
105
|
+
}
|
|
106
|
+
function reader(branch) {
|
|
107
|
+
async function read(request, options) {
|
|
108
|
+
const { signal } = operationOptions(options);
|
|
109
|
+
checkAbort(signal);
|
|
110
|
+
requireInput(keysOnly(request, ['fileId', 'path', 'paths']) && Object.keys(request).length === 1, 'Read requires exactly one of fileId, path or paths.');
|
|
111
|
+
if ('fileId' in request) {
|
|
112
|
+
const fileId = request.fileId;
|
|
113
|
+
requireInput(validFileId(fileId), 'Provide a fileId returned by MemDrive.');
|
|
114
|
+
return (await fetchFiles(branch, [fileId], signal)).get(fileId) ?? null;
|
|
115
|
+
}
|
|
116
|
+
const multi = 'paths' in request;
|
|
117
|
+
const value = multi ? request.paths : [request.path];
|
|
118
|
+
requireInput(Array.isArray(value) && value.length >= 1 && value.length <= writeBatchLimit && [...value].every(validPath)
|
|
119
|
+
&& new Set(value).size === value.length, 'Provide 1–100 distinct valid absolute file paths.');
|
|
120
|
+
const paths = [...value];
|
|
121
|
+
const response = await remote(() => handle.query({ size: paths.length * 2, ...branchRead(branch), query: pathQuery(paths) }, nativeOptions(signal)), 'Could not read files by path.', signal);
|
|
122
|
+
checkAbort(signal);
|
|
123
|
+
const expected = new Set(paths), files = new Map();
|
|
124
|
+
for (const doc of resultDocs(response, name, paths.length)) {
|
|
125
|
+
const file = decodeFile(doc, metadataSchema);
|
|
126
|
+
if (!expected.has(file.path) || files.has(file.path))
|
|
127
|
+
throw new MemDriveError('INVALID_RESULT', 'Read returned unexpected or duplicate paths.');
|
|
128
|
+
files.set(file.path, file);
|
|
129
|
+
}
|
|
130
|
+
return multi ? paths.flatMap(path => files.has(path) ? [files.get(path)] : []) : files.get(paths[0]) ?? null;
|
|
137
131
|
}
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
132
|
+
return read;
|
|
133
|
+
}
|
|
134
|
+
function searcher(branch) {
|
|
135
|
+
return async (request, options) => {
|
|
136
|
+
const { signal } = operationOptions(options);
|
|
142
137
|
checkAbort(signal);
|
|
143
|
-
|
|
138
|
+
const body = searchQuery(request, metadataSchema);
|
|
139
|
+
const response = await remote(() => handle.query({ ...body, ...branchRead(branch),
|
|
140
|
+
fields: { include: ['id', 'kind', 'fileId', 'path', 'modifiedAt', 'content', 'start', 'end'] } }, nativeOptions(signal)), 'File search failed.', signal);
|
|
144
141
|
checkAbort(signal);
|
|
142
|
+
const docs = resultDocs(response, name, body.size);
|
|
143
|
+
const reranked = validateReranking(response, body);
|
|
144
|
+
const fileIds = new Set(), chunkIds = new Set();
|
|
145
|
+
const chunks = docs.map(doc => {
|
|
146
|
+
if (doc.kind !== 'chunk' || !validFileId(doc.fileId) || typeof doc.id !== 'string'
|
|
147
|
+
|| !doc.id.startsWith(`chunk:${doc.fileId}:`) || chunkIds.has(doc.id) || !validPath(doc.path)
|
|
148
|
+
|| typeof doc.content !== 'string' || !doc.content.trim() || /\p{Surrogate}/u.test(doc.content))
|
|
149
|
+
throw new MemDriveError('INVALID_RESULT', 'Search returned an invalid or duplicate chunk.');
|
|
150
|
+
fileIds.add(doc.fileId);
|
|
151
|
+
chunkIds.add(doc.id);
|
|
152
|
+
return { fileId: doc.fileId, path: doc.path, modifiedAt: doc.modifiedAt, content: doc.content,
|
|
153
|
+
start: doc.start, end: doc.end };
|
|
154
|
+
});
|
|
155
|
+
if (!chunks.length)
|
|
156
|
+
return { hits: [], reranked };
|
|
157
|
+
const files = await fetchFiles(branch, [...fileIds], signal);
|
|
158
|
+
const hits = [];
|
|
159
|
+
for (const chunk of chunks) {
|
|
160
|
+
const file = files.get(chunk.fileId);
|
|
161
|
+
// An orphan or stale chunk must not expose body text from another file or revision.
|
|
162
|
+
if (!file || file.path !== chunk.path || file.modifiedAt !== chunk.modifiedAt)
|
|
163
|
+
continue;
|
|
164
|
+
const { start, end } = chunk;
|
|
165
|
+
const verified = typeof start === 'number' && typeof end === 'number'
|
|
166
|
+
&& Number.isSafeInteger(start) && Number.isSafeInteger(end) && start >= 0 && end > start && end <= file.content.length
|
|
167
|
+
&& file.content.slice(start, end) === chunk.content;
|
|
168
|
+
hits.push({ ...file, content: chunk.content, ...(verified ? { range: { start, end } } : {}) });
|
|
169
|
+
}
|
|
170
|
+
return { hits, reranked };
|
|
145
171
|
};
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
const
|
|
149
|
-
|
|
172
|
+
}
|
|
173
|
+
function working(fork) {
|
|
174
|
+
const existing = forks.get(fork);
|
|
175
|
+
if (existing)
|
|
176
|
+
return existing;
|
|
177
|
+
const branch = prefix + fork;
|
|
178
|
+
let writes = Promise.resolve();
|
|
179
|
+
const unconfirmed = new Map();
|
|
180
|
+
async function write(request, options) {
|
|
181
|
+
const { signal } = operationOptions(options);
|
|
150
182
|
checkAbort(signal);
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
183
|
+
const multi = Array.isArray(request);
|
|
184
|
+
const operations = multi ? [...request] : [request];
|
|
185
|
+
requireInput(operations.length >= 1 && operations.length <= writeBatchLimit, 'Submit 1–100 file create operations.');
|
|
186
|
+
const prepared = Promise.all(operations.map(operation => fileRecords(operation, metadataSchema)));
|
|
187
|
+
// Validation and hashing begin immediately. Attach the queue now so overlapping writes keep call order.
|
|
188
|
+
void prepared.catch(() => { });
|
|
189
|
+
const write = writes.then(async () => {
|
|
154
190
|
checkAbort(signal);
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
const mems = await validatedHits(response, request, schema(), options.name, body.size ?? 10, backend.embedding);
|
|
178
|
-
return { ...response, docs: response.docs.map((hit, i) => ({ ...hit, doc: mems[i] })) };
|
|
179
|
-
};
|
|
180
|
-
const body = recallQuery(request, schema(), backend.textSearch, backend.embedding);
|
|
181
|
-
await record({ phase: 'query', request: structuredClone(body) });
|
|
182
|
-
const response = await query(body);
|
|
183
|
-
if (body.rerank)
|
|
184
|
-
verifyRerank(response, body);
|
|
185
|
-
const mems = response.docs.map(hit => hit.doc);
|
|
186
|
-
await record({ phase: 'query_result', response: structuredClone(response) });
|
|
187
|
-
await observe({ type: 'retrieved', ids: mems.map(mem => mem.id), matches: mems.map(publicMatch) });
|
|
188
|
-
backend.assertActive();
|
|
189
|
-
checkAbort(signal);
|
|
190
|
-
usage.elapsedMs = performance.now() - start;
|
|
191
|
-
return { matches: mems.map(publicMatch), usage };
|
|
192
|
-
}
|
|
193
|
-
catch (cause) {
|
|
194
|
-
const failure = abortError(cause, signal);
|
|
195
|
-
usage.elapsedMs = performance.now() - start;
|
|
196
|
-
const error = failure instanceof MemDriveError ? failure
|
|
197
|
-
: new MemDriveError('RECALL_FAILED', 'MemDrive retrieval failed.', { cause: failure });
|
|
198
|
-
error.usage = structuredClone(usage);
|
|
199
|
-
throw error;
|
|
200
|
-
}
|
|
201
|
-
};
|
|
202
|
-
const snapshot = (input) => {
|
|
203
|
-
const start = performance.now();
|
|
204
|
-
try {
|
|
205
|
-
return { input: structuredClone(input), start };
|
|
206
|
-
}
|
|
207
|
-
catch (cause) {
|
|
208
|
-
const error = new MemDriveError('INVALID_INPUT', 'Recall input must be cloneable data.', { cause });
|
|
209
|
-
error.usage = { elapsedMs: 0, databaseRequests: 0 };
|
|
210
|
-
return { error };
|
|
211
|
-
}
|
|
212
|
-
};
|
|
213
|
-
const submit = async (prepared, requestIndex, signal) => {
|
|
214
|
-
if (prepared.error)
|
|
215
|
-
throw prepared.error;
|
|
216
|
-
try {
|
|
217
|
-
return await recallQueue(() => executeRecall(prepared.input, prepared.start, requestIndex, signal), signal);
|
|
218
|
-
}
|
|
219
|
-
catch (cause) {
|
|
220
|
-
// Queue cancellation and executeRecall both reject with MemDriveError.
|
|
221
|
-
const error = cause;
|
|
222
|
-
error.usage ??= { elapsedMs: performance.now() - prepared.start, databaseRequests: 0 };
|
|
223
|
-
throw error;
|
|
224
|
-
}
|
|
225
|
-
};
|
|
226
|
-
async function recall(input, options) {
|
|
227
|
-
const { signal } = operationOptions(options);
|
|
228
|
-
if (input !== null && typeof input === 'object' && Object.hasOwn(input, 'requests')) {
|
|
229
|
-
requireInput(keys(input, ['requests']) && Array.isArray(input.requests) && input.requests.length > 0, 'Batch recall requires only a nonempty requests array.');
|
|
230
|
-
// Snapshot every item before any operation can yield or mutate caller data.
|
|
231
|
-
const prepared = Array.from(input.requests, snapshot);
|
|
232
|
-
const results = await Promise.all(prepared.map(async (item, requestIndex) => {
|
|
233
|
-
try {
|
|
234
|
-
return await submit(item, requestIndex, signal);
|
|
191
|
+
const records = await prepared;
|
|
192
|
+
const paths = records.map(record => record.file.path);
|
|
193
|
+
const expectedPaths = new Set(paths);
|
|
194
|
+
requireInput(expectedPaths.size === paths.length, 'Create paths must be distinct within one write.');
|
|
195
|
+
const batches = writeBatches(records, branch);
|
|
196
|
+
const response = await remote(() => handle.query({ size: 2 * paths.length, ...branchRead(branch),
|
|
197
|
+
query: pathQuery(paths), fields: { include: ['id', 'kind', 'fileId', 'path'] } }, nativeOptions(signal)), 'Could not check create destinations.', signal);
|
|
198
|
+
checkAbort(signal);
|
|
199
|
+
const occupied = new Set();
|
|
200
|
+
for (const doc of resultDocs(response, name, 2 * paths.length)) {
|
|
201
|
+
if (doc.kind !== 'file' || !validFileId(doc.fileId) || doc.id !== originalId(doc.fileId)
|
|
202
|
+
|| typeof doc.path !== 'string' || !expectedPaths.has(doc.path) || occupied.has(doc.path))
|
|
203
|
+
throw new MemDriveError('INVALID_RESULT', 'Destination lookup returned an unexpected or duplicate file.');
|
|
204
|
+
occupied.add(doc.path);
|
|
205
|
+
unconfirmed.delete(doc.path);
|
|
206
|
+
}
|
|
207
|
+
if (occupied.size)
|
|
208
|
+
throw new MemDriveError('PATH_EXISTS', 'A file already exists at a requested path.', { details: { paths: [...occupied] } });
|
|
209
|
+
for (const path of paths) {
|
|
210
|
+
const fileId = unconfirmed.get(path);
|
|
211
|
+
if (fileId)
|
|
212
|
+
throw new MemDriveError('WRITE_UNCONFIRMED', 'An earlier create at this path is still unconfirmed.', { details: { fileId, path } });
|
|
235
213
|
}
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
214
|
+
const completed = [];
|
|
215
|
+
const staged = new Map();
|
|
216
|
+
const filesById = new Map(records.map(({ file }) => [file.fileId, file]));
|
|
217
|
+
const details = () => ({
|
|
218
|
+
completedFileIds: completed.map(file => file.fileId),
|
|
219
|
+
incompleteTargets: [...staged].map(([fileId, path]) => ({ fileId, path })),
|
|
220
|
+
});
|
|
221
|
+
for (const batch of batches) {
|
|
222
|
+
const docs = batch.map(record => record.doc);
|
|
223
|
+
const ids = new Set(docs.map(doc => doc.id));
|
|
224
|
+
const targets = [...new Map(docs.map(({ fileId, path }) => [fileId, { fileId, path }])).values()];
|
|
225
|
+
const uncertain = (cause) => {
|
|
226
|
+
for (const { kind, fileId, path } of docs)
|
|
227
|
+
if (kind === 'file')
|
|
228
|
+
unconfirmed.set(path, fileId);
|
|
229
|
+
return new MemDriveError('WRITE_UNCONFIRMED', 'The create may have been accepted. Inspect the files on this fork.', {
|
|
230
|
+
cause, details: { ...(targets.length === 1 ? targets[0] : {}), targets, ...details() },
|
|
231
|
+
});
|
|
232
|
+
};
|
|
233
|
+
try {
|
|
234
|
+
checkAbort(signal);
|
|
235
|
+
}
|
|
236
|
+
catch (cause) {
|
|
237
|
+
throw new MemDriveError('ABORTED', 'Write aborted before the next request.', { cause, details: details() });
|
|
238
|
+
}
|
|
239
|
+
try {
|
|
240
|
+
await nativeRequest(() => handle.docs.upsert({ docs, branch }, nativeOptions(signal)), signal);
|
|
241
|
+
}
|
|
242
|
+
catch (cause) {
|
|
243
|
+
const error = databaseError(cause, 'The file create was rejected.');
|
|
244
|
+
if ([400, 401, 403, 404, 413, 422, 429].includes(error.status ?? 0)) {
|
|
245
|
+
if (completed.length || staged.size) {
|
|
246
|
+
const partial = new MemDriveError(error.code, error.message, { cause: error, details: details() });
|
|
247
|
+
partial.status = error.status;
|
|
248
|
+
throw partial;
|
|
249
|
+
}
|
|
250
|
+
throw error;
|
|
251
|
+
}
|
|
252
|
+
throw uncertain(cause);
|
|
253
|
+
}
|
|
254
|
+
try {
|
|
255
|
+
const confirmation = await nativeRequest(() => handle.docs.fetch({ ids: [...ids], ...branchRead(branch),
|
|
256
|
+
fields: { include: ['id'] } }, nativeOptions(signal)), signal);
|
|
257
|
+
checkAbort(signal);
|
|
258
|
+
const found = resultDocs(confirmation, name, ids.size);
|
|
259
|
+
if (found.some(doc => typeof doc.id !== 'string' || !ids.delete(doc.id)) || ids.size)
|
|
260
|
+
throw new MemDriveError('INVALID_RESULT', 'The original and its complete chunk set were not confirmed.');
|
|
261
|
+
}
|
|
262
|
+
catch (cause) {
|
|
263
|
+
throw uncertain(cause);
|
|
264
|
+
}
|
|
265
|
+
for (const { fileId, kind, path } of docs) {
|
|
266
|
+
if (kind === 'file') {
|
|
267
|
+
staged.delete(fileId);
|
|
268
|
+
completed.push(filesById.get(fileId));
|
|
269
|
+
}
|
|
270
|
+
else
|
|
271
|
+
staged.set(fileId, path);
|
|
272
|
+
}
|
|
241
273
|
}
|
|
242
|
-
|
|
243
|
-
|
|
274
|
+
return multi ? completed : completed[0];
|
|
275
|
+
});
|
|
276
|
+
writes = write.then(() => undefined, () => undefined);
|
|
277
|
+
return write;
|
|
244
278
|
}
|
|
245
|
-
|
|
279
|
+
const drive = Object.freeze({
|
|
280
|
+
name, at: Object.freeze({ kind: 'fork', name: fork }), read: reader(branch), search: searcher(branch), describe, write,
|
|
281
|
+
});
|
|
282
|
+
forks.set(fork, drive);
|
|
283
|
+
return drive;
|
|
246
284
|
}
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
read: (request, options) => backend.read(request, undefined, options),
|
|
251
|
-
list: (request, options) => backend.list(request, options),
|
|
252
|
-
embedding: Object.freeze(structuredClone(backend.embedding)),
|
|
253
|
-
recall,
|
|
254
|
-
async countValues(request, options) {
|
|
285
|
+
const mainView = Object.freeze({
|
|
286
|
+
name, at: Object.freeze({ kind: 'main' }), read: reader(main), search: searcher(main), describe,
|
|
287
|
+
async fork(request, options) {
|
|
255
288
|
const { signal } = operationOptions(options);
|
|
256
289
|
checkAbort(signal);
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
290
|
+
requireInput(keysOnly(request, ['name']) && validForkName(request.name), 'Fork name must be 3–47 letters, digits, underscores or hyphens and cannot be main.');
|
|
291
|
+
const fork = request.name;
|
|
292
|
+
const branch = prefix + fork;
|
|
293
|
+
const result = await remote(() => handle.branches.create({ branchName: branch, source: { kind: 'branch', name: main } }, nativeOptions(signal)), 'Could not create the working fork.', signal, { 409: 'BRANCH_CONFLICT' });
|
|
294
|
+
checkAbort(signal);
|
|
295
|
+
if (result?.branch?.name !== branch)
|
|
296
|
+
throw new MemDriveError('INVALID_RESULT', 'Fork creation returned an invalid branch.');
|
|
297
|
+
return working(fork);
|
|
265
298
|
},
|
|
266
299
|
});
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
300
|
+
return {
|
|
301
|
+
main: mainView,
|
|
302
|
+
async openFork(fork, signal) {
|
|
303
|
+
const cached = forks.get(fork);
|
|
304
|
+
if (cached)
|
|
305
|
+
return cached;
|
|
306
|
+
const response = await remote(() => handle.branches.list(nativeOptions(signal)), 'Could not find the working fork.', signal);
|
|
307
|
+
checkAbort(signal);
|
|
308
|
+
if (!Array.isArray(response?.branches) || !response.branches.every(branch => typeof branch?.name === 'string'))
|
|
309
|
+
throw new MemDriveError('INVALID_RESULT', 'Fork lookup returned invalid branches.');
|
|
310
|
+
if (!response.branches.some(branch => branch.name === prefix + fork))
|
|
311
|
+
throw new MemDriveError('BRANCH_NOT_FOUND', 'The working fork does not exist.');
|
|
312
|
+
return working(fork);
|
|
313
|
+
},
|
|
274
314
|
};
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
configure: (request, options) => backend.configure(request, options),
|
|
284
|
-
destroy: (options) => backend.destroy(options),
|
|
285
|
-
describe: (options) => backend.describeDescriptor(options),
|
|
286
|
-
});
|
|
315
|
+
}
|
|
316
|
+
function storesFor(client) {
|
|
317
|
+
const existing = clients.get(client);
|
|
318
|
+
if (existing)
|
|
319
|
+
return existing;
|
|
320
|
+
const stores = new Map();
|
|
321
|
+
clients.set(client, stores);
|
|
322
|
+
return stores;
|
|
287
323
|
}
|
|
288
324
|
export async function create(options, requestOptions) {
|
|
289
|
-
|
|
290
|
-
checkAbort(
|
|
291
|
-
options
|
|
292
|
-
const
|
|
293
|
-
|
|
294
|
-
|
|
325
|
+
const { signal } = operationOptions(requestOptions);
|
|
326
|
+
checkAbort(signal);
|
|
327
|
+
validateConnection(options, ['client', 'name', 'labels', 'textSearch', 'metadataSchema']);
|
|
328
|
+
const { client, name } = options;
|
|
329
|
+
requireInput(typeof client.createCollection === 'function', 'Client must support collection creation.');
|
|
330
|
+
const { indexConfigs, tags, description } = storeConfiguration(options);
|
|
331
|
+
await remote(() => client.createCollection({ collectionName: name, indexConfigs, tags }, nativeOptions(signal)), 'Could not create the file store.', signal, { 409: 'STORE_EXISTS' });
|
|
332
|
+
checkAbort(signal);
|
|
333
|
+
const stores = storesFor(client);
|
|
334
|
+
const created = stores.get(name) ?? store(client.collection(name), description);
|
|
335
|
+
stores.set(name, created);
|
|
336
|
+
return created.main;
|
|
295
337
|
}
|
|
296
338
|
export async function open(options, requestOptions) {
|
|
297
|
-
|
|
298
|
-
checkAbort(
|
|
299
|
-
options
|
|
300
|
-
const
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
339
|
+
const { signal } = operationOptions(requestOptions);
|
|
340
|
+
checkAbort(signal);
|
|
341
|
+
validateConnection(options, ['client', 'name', 'at']);
|
|
342
|
+
const { client, name, at } = options;
|
|
343
|
+
let fork;
|
|
344
|
+
if (at !== undefined) {
|
|
345
|
+
requireInput(keysOnly(at, ['fork']) && validForkName(at.fork), 'Select a working fork by its name.');
|
|
346
|
+
fork = at.fork;
|
|
347
|
+
}
|
|
348
|
+
const stores = storesFor(client);
|
|
349
|
+
let selected = stores.get(name);
|
|
350
|
+
if (!selected) {
|
|
351
|
+
const handle = client.collection(name);
|
|
352
|
+
const response = await remote(() => handle.get(nativeOptions(signal)), 'Could not inspect the file store.', signal, { 404: 'STORE_NOT_FOUND' });
|
|
353
|
+
checkAbort(signal);
|
|
354
|
+
const description = parseStore(response?.collection, name);
|
|
355
|
+
selected = stores.get(name) ?? store(handle, description);
|
|
356
|
+
stores.set(name, selected);
|
|
357
|
+
}
|
|
358
|
+
return fork === undefined ? selected.main : selected.openFork(fork, signal);
|
|
305
359
|
}
|
|
306
|
-
export const
|
|
360
|
+
export const MemDrive = Object.freeze({ create, open });
|
|
307
361
|
//# sourceMappingURL=index.js.map
|