@functional-systems/memdrive 0.1.1-dev.1 → 0.1.1-dev.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/README.md +20 -250
- package/dist/commonjs/chunking.d.ts +1 -2
- package/dist/commonjs/chunking.d.ts.map +1 -1
- package/dist/commonjs/chunking.js +2 -5
- package/dist/commonjs/chunking.js.map +1 -1
- package/dist/commonjs/config.d.ts +17 -0
- package/dist/commonjs/config.d.ts.map +1 -0
- package/dist/commonjs/config.js +116 -0
- package/dist/commonjs/config.js.map +1 -0
- package/dist/commonjs/errors.d.ts +10 -6
- package/dist/commonjs/errors.d.ts.map +1 -1
- package/dist/commonjs/errors.js +25 -5
- package/dist/commonjs/errors.js.map +1 -1
- package/dist/commonjs/index.d.ts +15 -5
- package/dist/commonjs/index.d.ts.map +1 -1
- package/dist/commonjs/index.js +331 -307
- package/dist/commonjs/index.js.map +1 -1
- package/dist/commonjs/records.d.ts +44 -0
- package/dist/commonjs/records.d.ts.map +1 -0
- package/dist/commonjs/records.js +88 -0
- package/dist/commonjs/records.js.map +1 -0
- package/dist/commonjs/search.d.ts +8 -0
- package/dist/commonjs/search.d.ts.map +1 -0
- package/dist/commonjs/search.js +56 -0
- package/dist/commonjs/search.js.map +1 -0
- package/dist/commonjs/types.d.ts +84 -252
- package/dist/commonjs/types.d.ts.map +1 -1
- package/dist/esm/chunking.d.ts +1 -2
- package/dist/esm/chunking.d.ts.map +1 -1
- package/dist/esm/chunking.js +2 -4
- package/dist/esm/chunking.js.map +1 -1
- package/dist/esm/config.d.ts +17 -0
- package/dist/esm/config.d.ts.map +1 -0
- package/dist/esm/config.js +107 -0
- package/dist/esm/config.js.map +1 -0
- package/dist/esm/errors.d.ts +10 -6
- package/dist/esm/errors.d.ts.map +1 -1
- package/dist/esm/errors.js +22 -4
- package/dist/esm/errors.js.map +1 -1
- package/dist/esm/index.d.ts +15 -5
- package/dist/esm/index.d.ts.map +1 -1
- package/dist/esm/index.js +331 -305
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/records.d.ts +44 -0
- package/dist/esm/records.d.ts.map +1 -0
- package/dist/esm/records.js +77 -0
- package/dist/esm/records.js.map +1 -0
- package/dist/esm/search.d.ts +8 -0
- package/dist/esm/search.d.ts.map +1 -0
- package/dist/esm/search.js +52 -0
- package/dist/esm/search.js.map +1 -0
- package/dist/esm/types.d.ts +84 -252
- package/dist/esm/types.d.ts.map +1 -1
- package/package.json +10 -32
- package/src/chunking.ts +2 -3
- package/src/config.ts +121 -0
- package/src/errors.ts +20 -6
- package/src/index.ts +351 -289
- package/src/records.ts +83 -0
- package/src/search.ts +57 -0
- package/src/types.ts +74 -253
- package/dist/commonjs/filter.d.ts +0 -30
- package/dist/commonjs/filter.d.ts.map +0 -1
- package/dist/commonjs/filter.js +0 -224
- package/dist/commonjs/filter.js.map +0 -1
- package/dist/commonjs/storage.d.ts +0 -274
- package/dist/commonjs/storage.d.ts.map +0 -1
- package/dist/commonjs/storage.js +0 -861
- package/dist/commonjs/storage.js.map +0 -1
- package/dist/commonjs/tools/definitions.d.ts +0 -5
- package/dist/commonjs/tools/definitions.d.ts.map +0 -1
- package/dist/commonjs/tools/definitions.js +0 -129
- package/dist/commonjs/tools/definitions.js.map +0 -1
- package/dist/commonjs/tools/index.d.ts +0 -6
- package/dist/commonjs/tools/index.d.ts.map +0 -1
- package/dist/commonjs/tools/index.js +0 -104
- package/dist/commonjs/tools/index.js.map +0 -1
- package/dist/commonjs/tools/types.d.ts +0 -50
- package/dist/commonjs/tools/types.d.ts.map +0 -1
- package/dist/commonjs/tools/types.js +0 -3
- package/dist/commonjs/tools/types.js.map +0 -1
- package/dist/commonjs/values.d.ts +0 -17
- package/dist/commonjs/values.d.ts.map +0 -1
- package/dist/commonjs/values.js +0 -90
- package/dist/commonjs/values.js.map +0 -1
- package/dist/esm/filter.d.ts +0 -30
- package/dist/esm/filter.d.ts.map +0 -1
- package/dist/esm/filter.js +0 -209
- package/dist/esm/filter.js.map +0 -1
- package/dist/esm/storage.d.ts +0 -274
- package/dist/esm/storage.d.ts.map +0 -1
- package/dist/esm/storage.js +0 -845
- package/dist/esm/storage.js.map +0 -1
- package/dist/esm/tools/definitions.d.ts +0 -5
- package/dist/esm/tools/definitions.d.ts.map +0 -1
- package/dist/esm/tools/definitions.js +0 -126
- package/dist/esm/tools/definitions.js.map +0 -1
- package/dist/esm/tools/index.d.ts +0 -6
- package/dist/esm/tools/index.d.ts.map +0 -1
- package/dist/esm/tools/index.js +0 -101
- package/dist/esm/tools/index.js.map +0 -1
- package/dist/esm/tools/types.d.ts +0 -50
- package/dist/esm/tools/types.d.ts.map +0 -1
- package/dist/esm/tools/types.js +0 -2
- package/dist/esm/tools/types.js.map +0 -1
- package/dist/esm/values.d.ts +0 -17
- package/dist/esm/values.d.ts.map +0 -1
- package/dist/esm/values.js +0 -87
- package/dist/esm/values.js.map +0 -1
- package/docs/tools.md +0 -163
- package/src/filter.ts +0 -189
- package/src/storage.ts +0 -765
- package/src/tools/definitions.ts +0 -156
- package/src/tools/index.ts +0 -97
- package/src/tools/types.ts +0 -45
- package/src/values.ts +0 -103
package/dist/esm/storage.js
DELETED
|
@@ -1,845 +0,0 @@
|
|
|
1
|
-
import { Analyzer } from '@functional-systems/lambdadb/models/indexconfigsunion.js';
|
|
2
|
-
import { chunkSpans, CHUNKING_POLICY } from './chunking.js';
|
|
3
|
-
import { MemDriveError, requireInput, operationOptions, checkAbort, abortError } from './errors.js';
|
|
4
|
-
import { compileFilter, isObject, validScalar, validateMetadataSchema, validateDescription } from './filter.js';
|
|
5
|
-
// Keep v0 until release; recreate development stores after incompatible layout changes.
|
|
6
|
-
const format = 'mems-v0';
|
|
7
|
-
const encoder = new TextEncoder();
|
|
8
|
-
const requestBytes = 6029312;
|
|
9
|
-
const documentBytes = 5 * 1024 * 1024;
|
|
10
|
-
const defaultBranch = 'main';
|
|
11
|
-
const validBranchName = (name) => typeof name === 'string' && /^[A-Za-z0-9_-]{3,52}$/.test(name);
|
|
12
|
-
function checkBranch(name) { requireInput(validBranchName(name), 'Branch name must be 3–52 letters, digits, underscores or hyphens.'); }
|
|
13
|
-
const systemIndexes = {
|
|
14
|
-
memId: { type: 'keyword' },
|
|
15
|
-
kind: { type: 'keyword' },
|
|
16
|
-
embedding: { type: 'vector', managedEmbedding: true, embedding: {
|
|
17
|
-
provider: 'openai', model: 'text-embedding-3-small', sourceField: 'content', dimensions: 1536, similarity: 'cosine',
|
|
18
|
-
} },
|
|
19
|
-
timestamp: { type: 'datetime' },
|
|
20
|
-
};
|
|
21
|
-
const types = { string: 'keyword', 'string[]': 'keyword', number: 'double', boolean: 'boolean', datetime: 'datetime' };
|
|
22
|
-
const bytes = (value) => encoder.encode(JSON.stringify(value)).length;
|
|
23
|
-
const same = (a, b) => JSON.stringify(a) === JSON.stringify(b);
|
|
24
|
-
const keysOnly = (value, keys) => isObject(value) && Object.keys(value).every(key => keys.includes(key));
|
|
25
|
-
const keywordNames = (schema) => Object.entries(schema).filter(([, value]) => ['string', 'string[]'].includes(value.type)).map(([key]) => key).sort();
|
|
26
|
-
const supportedAnalyzers = new Set(Object.values(Analyzer));
|
|
27
|
-
const validAnalyzers = (names) => Array.isArray(names) && names.length > 0
|
|
28
|
-
&& new Set(names).size === names.length && [...names].every(name => supportedAnalyzers.has(name));
|
|
29
|
-
function validateTextSearch(value) {
|
|
30
|
-
if (value === undefined)
|
|
31
|
-
return undefined;
|
|
32
|
-
requireInput(keysOnly(value, ['analyzers']) && validAnalyzers(value.analyzers), 'textSearch requires a nonempty array of distinct supported lowercase analyzers.');
|
|
33
|
-
return { analyzers: [...value.analyzers] };
|
|
34
|
-
}
|
|
35
|
-
function storedTextSearch(config) {
|
|
36
|
-
if (config === undefined)
|
|
37
|
-
return undefined;
|
|
38
|
-
const names = isObject(config) ? config.analyzers ?? ['standard'] : undefined;
|
|
39
|
-
if (!isObject(config) || config.type !== 'text' || !validAnalyzers(names))
|
|
40
|
-
throw new MemDriveError('INCOMPATIBLE_STORE', 'The content text index must use distinct supported analyzers.');
|
|
41
|
-
return { analyzers: [...names] };
|
|
42
|
-
}
|
|
43
|
-
const managedEmbedding = Object.freeze({ mode: 'managed', provider: 'openai', model: 'text-embedding-3-small',
|
|
44
|
-
dimensions: 1536, similarity: 'cosine' });
|
|
45
|
-
function validateEmbedding(value) {
|
|
46
|
-
if (value === undefined || (isObject(value) && value.mode === 'managed')) {
|
|
47
|
-
requireInput(value === undefined || keysOnly(value, ['mode']), 'Managed embedding accepts only mode; its model settings are fixed.');
|
|
48
|
-
return managedEmbedding;
|
|
49
|
-
}
|
|
50
|
-
requireInput(keysOnly(value, ['mode', 'dimensions', 'similarity', 'identity']) && value.mode === 'external'
|
|
51
|
-
&& typeof value.dimensions === 'number' && Number.isInteger(value.dimensions) && value.dimensions >= 64 && value.dimensions <= 4096
|
|
52
|
-
&& value.similarity === 'cosine' && typeof value.identity === 'string' && value.identity.trim().length > 0
|
|
53
|
-
&& value.identity.length <= 127 && !/[:#,]/.test(value.identity), 'External embedding requires dimensions 64–4096, cosine similarity and a nonblank identity of at most 127 characters without :, # or comma.');
|
|
54
|
-
return Object.freeze({ mode: 'external', dimensions: value.dimensions, similarity: 'cosine', identity: value.identity });
|
|
55
|
-
}
|
|
56
|
-
function storedEmbedding(collection) {
|
|
57
|
-
const candidate = collection.indexConfigs?.embedding;
|
|
58
|
-
const vector = isObject(candidate) ? candidate : undefined;
|
|
59
|
-
const rawEmbedding = vector?.embedding;
|
|
60
|
-
const embedding = isObject(rawEmbedding) ? rawEmbedding : undefined;
|
|
61
|
-
if (vector?.type === 'vector' && rawEmbedding === undefined && vector.managedEmbedding !== true) {
|
|
62
|
-
try {
|
|
63
|
-
return validateEmbedding({ mode: 'external', dimensions: vector.dimensions,
|
|
64
|
-
similarity: vector.similarity, identity: collection.tags?.['memory.embedding'] });
|
|
65
|
-
}
|
|
66
|
-
catch (cause) {
|
|
67
|
-
throw new MemDriveError('INCOMPATIBLE_STORE', 'Invalid external embedding configuration.', { cause });
|
|
68
|
-
}
|
|
69
|
-
}
|
|
70
|
-
if (vector?.type !== 'vector' || vector.managedEmbedding === false || embedding?.provider !== managedEmbedding.provider
|
|
71
|
-
|| embedding.model !== managedEmbedding.model || embedding.sourceField !== 'content'
|
|
72
|
-
|| embedding.dimensions !== managedEmbedding.dimensions || embedding.similarity !== managedEmbedding.similarity
|
|
73
|
-
|| vector.dimensions !== undefined || vector.similarity !== undefined || collection.tags?.['memory.embedding'] !== undefined)
|
|
74
|
-
throw new MemDriveError('INCOMPATIBLE_STORE', 'Invalid managed embedding configuration.');
|
|
75
|
-
return managedEmbedding;
|
|
76
|
-
}
|
|
77
|
-
function indexConfigs(schema, textSearch, embedding) {
|
|
78
|
-
const result = structuredClone(systemIndexes);
|
|
79
|
-
if (embedding.mode === 'external')
|
|
80
|
-
result.embedding = { type: 'vector', dimensions: embedding.dimensions, similarity: embedding.similarity };
|
|
81
|
-
if (textSearch)
|
|
82
|
-
result.content = { type: 'text', analyzers: [...textSearch.analyzers] };
|
|
83
|
-
if (Object.keys(schema).length)
|
|
84
|
-
result.metadata = { type: 'object', objectIndexConfigs: Object.fromEntries(Object.entries(schema).map(([key, { type, description }]) => [key, { type: types[type],
|
|
85
|
-
...(description === undefined ? {} : { description }) }])) };
|
|
86
|
-
return result;
|
|
87
|
-
}
|
|
88
|
-
// Native tags allow five short values, not an arbitrary JSON schema. Native index
|
|
89
|
-
// types recover scalar types; this canonical bitmap preserves array cardinality
|
|
90
|
-
// over sorted keyword metadata names. Changing that order regenerates the bitmap.
|
|
91
|
-
function arrayTag(schema) {
|
|
92
|
-
const names = keywordNames(schema), bits = new Uint8Array(Math.ceil(names.length / 8));
|
|
93
|
-
names.forEach((key, i) => { if (schema[key].type === 'string[]')
|
|
94
|
-
bits[i >> 3] = bits[i >> 3] | 1 << (i % 8); });
|
|
95
|
-
const encoded = bits.length ? btoa(String.fromCharCode(...bits)).replaceAll('+', '-').replaceAll('/', '_').replace(/=+$/, '') : '-';
|
|
96
|
-
requireInput(encoded.length <= 127, 'Metadata keyword declarations exceed the native format-tag capacity.');
|
|
97
|
-
return encoded;
|
|
98
|
-
}
|
|
99
|
-
function validateLabels(labels = {}) {
|
|
100
|
-
// Match native Java whitespace exactly, including control separators.
|
|
101
|
-
// eslint-disable-next-line no-control-regex
|
|
102
|
-
const blank = /^[\u0009-\u000d\u001c-\u0020\u1680\u2000-\u2006\u2008-\u200a\u2028\u2029\u205f\u3000]*$/u;
|
|
103
|
-
requireInput(isObject(labels) && Object.keys(labels).length <= 5 && Object.entries(labels).every(([key, value]) => /^[A-Za-z0-9_.-]{1,63}$/.test(key) && !key.startsWith('memory.') && typeof value === 'string'
|
|
104
|
-
&& value.length >= 1 && value.length <= 127 && !/[:#,]/.test(value) && !blank.test(value)), 'Labels must use native-safe tag names and values; memory.* names are reserved.');
|
|
105
|
-
return structuredClone(labels);
|
|
106
|
-
}
|
|
107
|
-
function collectionLabels(collection) {
|
|
108
|
-
try {
|
|
109
|
-
requireInput(isObject(collection.tags) && Object.keys(collection.tags).length <= 5, 'Invalid collection tags.');
|
|
110
|
-
return validateLabels(Object.fromEntries(Object.entries(collection.tags).filter(([key]) => !key.startsWith('memory.'))));
|
|
111
|
-
}
|
|
112
|
-
catch (cause) {
|
|
113
|
-
throw new MemDriveError('INCOMPATIBLE_STORE', 'Invalid MemDrive labels.', { cause });
|
|
114
|
-
}
|
|
115
|
-
}
|
|
116
|
-
function descriptor(collection) {
|
|
117
|
-
const metadataSchema = schemaFromCollection(collection, collection.collectionName);
|
|
118
|
-
const textSearch = storedTextSearch(collection.indexConfigs.content);
|
|
119
|
-
return { name: collection.collectionName, ...(collection.description === undefined ? {} : { description: collection.description }),
|
|
120
|
-
labels: collectionLabels(collection), metadataSchema,
|
|
121
|
-
...(textSearch ? { textSearch } : {}), embedding: structuredClone(storedEmbedding(collection)) };
|
|
122
|
-
}
|
|
123
|
-
function tags(schema, embedding) {
|
|
124
|
-
return { 'memory.format': format, 'memory.arrays': arrayTag(schema),
|
|
125
|
-
...(embedding.mode === 'external' ? { 'memory.embedding': embedding.identity } : {}) };
|
|
126
|
-
}
|
|
127
|
-
function schemaFromCollection(collection, name) {
|
|
128
|
-
const fail = condition => { if (!condition)
|
|
129
|
-
throw new MemDriveError('INCOMPATIBLE_STORE', 'Collection is not a compatible MemDrive.'); };
|
|
130
|
-
fail(isObject(collection) && collection.collectionName === name && collection.defaultBranchName === defaultBranch
|
|
131
|
-
&& collection.tags?.['memory.format'] === format
|
|
132
|
-
&& collection.partitionConfig === undefined && (collection.numPartitions === undefined || collection.numPartitions === 1));
|
|
133
|
-
try {
|
|
134
|
-
validateDescription(collection.description);
|
|
135
|
-
}
|
|
136
|
-
catch {
|
|
137
|
-
fail(false);
|
|
138
|
-
}
|
|
139
|
-
const configs = collection.indexConfigs;
|
|
140
|
-
fail(isObject(configs) && Object.keys(configs).every(key => ['id', 'content', ...Object.keys(systemIndexes), 'metadata'].includes(key)));
|
|
141
|
-
storedTextSearch(configs.content);
|
|
142
|
-
fail(configs.memId?.type === 'keyword' && configs.kind?.type === 'keyword'
|
|
143
|
-
&& configs.timestamp?.type === 'datetime' && (configs.id === undefined || configs.id.type === 'keyword'));
|
|
144
|
-
const embedding = storedEmbedding(collection);
|
|
145
|
-
fail(embedding.mode !== 'external' || configs.content === undefined);
|
|
146
|
-
fail(configs.metadata === undefined || (configs.metadata.type === 'object' && isObject(configs.metadata.objectIndexConfigs)));
|
|
147
|
-
fail(Object.values(configs.metadata?.type === 'object' ? configs.metadata.objectIndexConfigs : {}).every((config) => isObject(config)
|
|
148
|
-
&& typeof config.type === 'string' && ['keyword', 'double', 'boolean', 'datetime'].includes(config.type)));
|
|
149
|
-
const metadataConfigs = configs.metadata?.type === 'object' ? configs.metadata.objectIndexConfigs : {};
|
|
150
|
-
const publicTypes = { keyword: 'string', double: 'number', boolean: 'boolean', datetime: 'datetime' };
|
|
151
|
-
const declarations = Object.fromEntries(Object.entries(metadataConfigs).map(([key, config]) => {
|
|
152
|
-
// The allowed native types were checked above; validate the complete public declaration below.
|
|
153
|
-
fail(isObject(config) && typeof config.type === 'string' && Object.hasOwn(publicTypes, config.type));
|
|
154
|
-
return [key, { type: publicTypes[config.type],
|
|
155
|
-
...(config.description === undefined ? {} : { description: config.description }) }];
|
|
156
|
-
}));
|
|
157
|
-
let schema;
|
|
158
|
-
try {
|
|
159
|
-
schema = validateMetadataSchema(declarations);
|
|
160
|
-
}
|
|
161
|
-
catch {
|
|
162
|
-
fail(false);
|
|
163
|
-
}
|
|
164
|
-
const names = keywordNames(schema), encoded = collection.tags['memory.arrays'];
|
|
165
|
-
fail(typeof encoded === 'string' && encoded.length <= 127 && (encoded === '-' || /^[A-Za-z0-9_-]+$/.test(encoded)));
|
|
166
|
-
let raw = '';
|
|
167
|
-
try {
|
|
168
|
-
raw = encoded === '-' ? '' : atob(encoded.replaceAll('-', '+').replaceAll('_', '/'));
|
|
169
|
-
}
|
|
170
|
-
catch {
|
|
171
|
-
fail(false);
|
|
172
|
-
}
|
|
173
|
-
fail(raw.length === Math.ceil(names.length / 8));
|
|
174
|
-
names.forEach((key, i) => { if (raw.charCodeAt(i >> 3) & (1 << (i % 8)))
|
|
175
|
-
schema[key].type = 'string[]'; });
|
|
176
|
-
fail(arrayTag(schema) === encoded); // Reject padding bits and noncanonical encodings.
|
|
177
|
-
return schema;
|
|
178
|
-
}
|
|
179
|
-
function validateAttributes(mem, schema, requireId) {
|
|
180
|
-
if (requireId || Object.hasOwn(mem, 'id'))
|
|
181
|
-
requireInput(typeof mem.id === 'string'
|
|
182
|
-
&& mem.id.trim().length > 0 && mem.id.length <= 512, 'Mem id must be nonempty text of at most 512 characters.');
|
|
183
|
-
if (Object.hasOwn(mem, 'timestamp'))
|
|
184
|
-
requireInput(validScalar(mem.timestamp, 'datetime'), 'timestamp must be RFC 3339 text.');
|
|
185
|
-
if (Object.hasOwn(mem, 'metadata')) {
|
|
186
|
-
requireInput(isObject(mem.metadata), 'metadata must be an object.');
|
|
187
|
-
for (const [key, value] of Object.entries(mem.metadata)) {
|
|
188
|
-
requireInput(Object.hasOwn(schema, key), 'Mem includes undeclared metadata.');
|
|
189
|
-
const { type } = schema[key];
|
|
190
|
-
requireInput(type === 'string[]' ? Array.isArray(value) && value.every(item => validScalar(item, 'string'))
|
|
191
|
-
: validScalar(value, type), 'Metadata value does not match its declared type or indexing limits.');
|
|
192
|
-
}
|
|
193
|
-
}
|
|
194
|
-
}
|
|
195
|
-
export function validateMem(mem, schema, { requireId = false } = {}) {
|
|
196
|
-
requireInput(keysOnly(mem, ['id', 'content', 'timestamp', 'metadata']), 'Mem accepts only id, content, timestamp and metadata.');
|
|
197
|
-
requireInput(typeof mem.content === 'string' && mem.content.trim().length > 0, 'Mem content must be nonempty text.');
|
|
198
|
-
validateAttributes(mem, schema, requireId);
|
|
199
|
-
requireInput(bytes(mem) <= documentBytes, 'Mem exceeds the native document byte limit.');
|
|
200
|
-
return structuredClone(mem);
|
|
201
|
-
}
|
|
202
|
-
export function validateVector(vector, embedding) {
|
|
203
|
-
requireInput(Array.isArray(vector) && vector.length === embedding.dimensions
|
|
204
|
-
&& [...vector].every(value => typeof value === 'number' && Number.isFinite(value))
|
|
205
|
-
&& vector.some(value => value !== 0), 'Embedding must be a dense, finite, nonzero vector with the configured dimensions.');
|
|
206
|
-
return [...vector];
|
|
207
|
-
}
|
|
208
|
-
export function validateVectorMem(mem, schema, embedding, { requireId = false } = {}) {
|
|
209
|
-
requireInput(keysOnly(mem, ['id', 'embedding', 'timestamp', 'metadata']), 'External Mem accepts only id, embedding, timestamp and metadata.');
|
|
210
|
-
validateAttributes(mem, schema, requireId);
|
|
211
|
-
const result = { ...structuredClone(mem), embedding: validateVector(mem.embedding, embedding) };
|
|
212
|
-
requireInput(bytes(result) <= documentBytes, 'Mem exceeds the native document byte limit.');
|
|
213
|
-
return result;
|
|
214
|
-
}
|
|
215
|
-
const digest = async (id) => [...new Uint8Array(await globalThis.crypto.subtle.digest('SHA-256', encoder.encode(id)))]
|
|
216
|
-
.map(value => value.toString(16).padStart(2, '0')).join('');
|
|
217
|
-
export const sourceDocumentId = async (id) => `mem:${await digest(id)}`;
|
|
218
|
-
const chunkDocumentId = (sourceId, ordinal) => `chunk:${sourceId.slice(4)}:${ordinal}`;
|
|
219
|
-
function attributes(mem) {
|
|
220
|
-
return { ...(mem.timestamp === undefined ? {} : { timestamp: mem.timestamp }),
|
|
221
|
-
...(mem.metadata === undefined ? {} : { metadata: structuredClone(mem.metadata) }) };
|
|
222
|
-
}
|
|
223
|
-
export function publicMatch(evidence) {
|
|
224
|
-
const { id: _id, ...match } = evidence;
|
|
225
|
-
return structuredClone(match);
|
|
226
|
-
}
|
|
227
|
-
export async function sourceMatch(mem) {
|
|
228
|
-
return { id: await sourceDocumentId(mem.id), kind: 'mem', memId: mem.id, content: mem.content,
|
|
229
|
-
start: 0, end: mem.content.length, ...attributes(mem) };
|
|
230
|
-
}
|
|
231
|
-
export async function validateChunk(doc, schema) {
|
|
232
|
-
if (isObject(doc) && doc.kind === 'mem') {
|
|
233
|
-
requireInput(Object.hasOwn(doc, 'content'), 'A Mem without searchable content is not a search result.');
|
|
234
|
-
return sourceMatch(await validateSource(doc, schema));
|
|
235
|
-
}
|
|
236
|
-
requireInput(keysOnly(doc, ['id', 'kind', 'memId', 'content', 'start', 'end', 'timestamp', 'metadata'])
|
|
237
|
-
&& doc.kind === 'chunk', 'Expected a stored chunk.');
|
|
238
|
-
const mem = validateMem({ id: doc.memId, content: doc.content, ...attributes(doc) }, schema, { requireId: true });
|
|
239
|
-
requireInput(typeof doc.start === 'number' && Number.isSafeInteger(doc.start) && doc.start >= 0 && typeof doc.end === 'number' && Number.isSafeInteger(doc.end)
|
|
240
|
-
&& doc.end > doc.start && doc.end - doc.start === mem.content.length
|
|
241
|
-
&& [...mem.content].length <= 800, 'Invalid chunk source span.');
|
|
242
|
-
const prefix = `chunk:${await digest(mem.id)}:`;
|
|
243
|
-
requireInput(typeof doc.id === 'string' && doc.id.startsWith(prefix)
|
|
244
|
-
&& /^(0|[1-9][0-9]*)$/.test(doc.id.slice(prefix.length))
|
|
245
|
-
&& Number.isSafeInteger(Number(doc.id.slice(prefix.length))), 'Invalid chunk identity.');
|
|
246
|
-
return { id: doc.id, kind: 'chunk', memId: mem.id, content: mem.content, start: doc.start, end: doc.end, ...attributes(mem) };
|
|
247
|
-
}
|
|
248
|
-
export async function validateSource(doc, schema) {
|
|
249
|
-
const whole = isObject(doc) && Object.hasOwn(doc, 'content');
|
|
250
|
-
requireInput(keysOnly(doc, ['id', 'kind', 'memId', whole ? 'content' : 'memContent', 'chunkCount', 'chunkSlots', 'chunkingPolicy', 'timestamp', 'metadata'])
|
|
251
|
-
&& doc.kind === 'mem' && doc.chunkingPolicy === CHUNKING_POLICY, 'Expected a compatible stored original Mem.');
|
|
252
|
-
const mem = validateMem({ id: doc.memId, content: whole ? doc.content : doc.memContent, ...attributes(doc) }, schema, { requireId: true });
|
|
253
|
-
requireInput(typeof doc.id === 'string' && doc.id === await sourceDocumentId(mem.id) && typeof doc.chunkCount === 'number' && Number.isSafeInteger(doc.chunkCount) && doc.chunkCount >= 1
|
|
254
|
-
&& typeof doc.chunkSlots === 'number' && Number.isSafeInteger(doc.chunkSlots) && doc.chunkSlots >= (whole ? 0 : doc.chunkCount)
|
|
255
|
-
&& (!whole || (doc.chunkCount === 1 && [...mem.content].length <= 800)), 'Invalid original Mem manifest.');
|
|
256
|
-
const smallestSlot = { id: chunkDocumentId(doc.id, 0), kind: 'chunk', memId: mem.id, ...attributes(mem) };
|
|
257
|
-
requireInput(bytes(doc) <= documentBytes && bytes(doc) + doc.chunkSlots * (bytes(smallestSlot) + 1) <= requestBytes, 'Original Mem manifest exceeds the complete replacement limit.');
|
|
258
|
-
return mem;
|
|
259
|
-
}
|
|
260
|
-
export async function vectorSource(doc, schema) {
|
|
261
|
-
requireInput(keysOnly(doc, ['id', 'kind', 'memId', 'timestamp', 'metadata']) && doc.kind === 'mem', 'Expected an external original Mem without vector values.');
|
|
262
|
-
const candidate = { id: doc.memId, ...attributes(doc) };
|
|
263
|
-
validateAttributes(candidate, schema, true);
|
|
264
|
-
const mem = candidate;
|
|
265
|
-
requireInput(doc.id === await sourceDocumentId(mem.id) && bytes(doc) <= documentBytes, 'Invalid external original Mem.');
|
|
266
|
-
return mem;
|
|
267
|
-
}
|
|
268
|
-
export async function vectorMatch(doc, schema) {
|
|
269
|
-
const mem = await vectorSource(doc, schema);
|
|
270
|
-
return { id: doc.id, kind: 'mem', memId: mem.id, ...attributes(mem) };
|
|
271
|
-
}
|
|
272
|
-
function originalIds(request) {
|
|
273
|
-
requireInput(keysOnly(request, ['ids']) && Array.isArray(request.ids) && request.ids.length > 0
|
|
274
|
-
&& request.ids.every(id => typeof id === 'string' && id.trim().length > 0 && id.length <= 512)
|
|
275
|
-
&& new Set(request.ids).size === request.ids.length, 'read requires distinct nonempty Mem IDs.');
|
|
276
|
-
return [...request.ids];
|
|
277
|
-
}
|
|
278
|
-
function databaseError(error, message, details) {
|
|
279
|
-
if (error instanceof MemDriveError) {
|
|
280
|
-
if (details !== undefined)
|
|
281
|
-
error.details = { ...error.details, ...details };
|
|
282
|
-
return error;
|
|
283
|
-
}
|
|
284
|
-
const result = new MemDriveError('DATABASE_ERROR', message, { cause: error, details });
|
|
285
|
-
const remote = error;
|
|
286
|
-
const status = remote?.statusCode ?? remote?.status ?? remote?.httpStatus;
|
|
287
|
-
if (typeof status === 'number' && Number.isInteger(status))
|
|
288
|
-
result.status = status;
|
|
289
|
-
return result;
|
|
290
|
-
}
|
|
291
|
-
function validateConnection({ client, name }) {
|
|
292
|
-
requireInput(client && typeof client.collection === 'function', 'Provide an initialized LambdaDB client.');
|
|
293
|
-
requireInput(typeof name === 'string' && /^[A-Za-z0-9_-]{3,52}$/.test(name), 'MemDrive name must be 3–52 letters, digits, underscores or hyphens.');
|
|
294
|
-
}
|
|
295
|
-
async function nativeRequest(operation, signal) {
|
|
296
|
-
checkAbort(signal);
|
|
297
|
-
try {
|
|
298
|
-
return await operation();
|
|
299
|
-
}
|
|
300
|
-
catch (cause) {
|
|
301
|
-
throw abortError(cause, signal);
|
|
302
|
-
}
|
|
303
|
-
}
|
|
304
|
-
const nativeOptions = (signal) => signal === undefined ? {} : { signal };
|
|
305
|
-
const guarded = (assertActive, signal) => () => { checkAbort(signal); assertActive(); };
|
|
306
|
-
async function load(handle, name, options) {
|
|
307
|
-
const { signal } = operationOptions(options);
|
|
308
|
-
let result;
|
|
309
|
-
try {
|
|
310
|
-
result = await nativeRequest(() => handle.get(nativeOptions(signal)), signal);
|
|
311
|
-
}
|
|
312
|
-
catch (error) {
|
|
313
|
-
throw databaseError(error, 'Could not inspect the MemDrive.');
|
|
314
|
-
}
|
|
315
|
-
checkAbort(signal);
|
|
316
|
-
const schema = schemaFromCollection(result?.collection, name);
|
|
317
|
-
collectionLabels(result.collection);
|
|
318
|
-
return { result: structuredClone(result), schema, embedding: storedEmbedding(result.collection),
|
|
319
|
-
textSearch: storedTextSearch(result.collection.indexConfigs.content) };
|
|
320
|
-
}
|
|
321
|
-
function backend(handle, name, initial) {
|
|
322
|
-
let schema = initial.schema, textSearch = initial.textSearch, generation = 0, deleted = false;
|
|
323
|
-
const embedding = initial.embedding;
|
|
324
|
-
const decodeSource = (doc) => embedding.mode === 'external' ? vectorSource(doc, schema) : validateSource(doc, schema);
|
|
325
|
-
let schemaUpdates = Promise.resolve();
|
|
326
|
-
const branchEpochs = new Map();
|
|
327
|
-
const epoch = (branch) => branchEpochs.get(branch) ?? 0;
|
|
328
|
-
const retire = (branch) => { branchEpochs.set(branch, epoch(branch) + 1); generation++; };
|
|
329
|
-
const active = () => { if (deleted)
|
|
330
|
-
throw new MemDriveError('STORE_DELETED', 'This MemDrive has been deleted.'); };
|
|
331
|
-
const refresh = async (options) => {
|
|
332
|
-
const { signal } = operationOptions(options);
|
|
333
|
-
const check = guarded(active, signal);
|
|
334
|
-
check();
|
|
335
|
-
const state = await load(handle, name, nativeOptions(signal));
|
|
336
|
-
check();
|
|
337
|
-
if (!same(embedding, state.embedding))
|
|
338
|
-
throw new MemDriveError('INCOMPATIBLE_STORE', 'Embedding configuration changed; reopen the MemDrive.');
|
|
339
|
-
if (!same(schema, state.schema) || !same(textSearch, state.textSearch))
|
|
340
|
-
generation++;
|
|
341
|
-
schema = state.schema;
|
|
342
|
-
textSearch = state.textSearch;
|
|
343
|
-
return state.result;
|
|
344
|
-
};
|
|
345
|
-
const fetchOriginals = async (ids, branch, assertActive, execute = operation => operation(), signal) => {
|
|
346
|
-
const found = new Map();
|
|
347
|
-
for (let start = 0; start < ids.length; start += 1000) {
|
|
348
|
-
assertActive();
|
|
349
|
-
const batch = ids.slice(start, start + 1000), expected = new Set(batch);
|
|
350
|
-
const documentIds = await Promise.all(batch.map(sourceDocumentId));
|
|
351
|
-
assertActive();
|
|
352
|
-
let response;
|
|
353
|
-
try {
|
|
354
|
-
response = await execute(() => {
|
|
355
|
-
assertActive();
|
|
356
|
-
return nativeRequest(() => handle.docs.fetch({ ids: documentIds,
|
|
357
|
-
ref: { kind: 'branch', name: branch }, consistentRead: true, includeVectors: false }, nativeOptions(signal)), signal);
|
|
358
|
-
});
|
|
359
|
-
}
|
|
360
|
-
catch (error) {
|
|
361
|
-
throw databaseError(error, 'Could not read original Mems.');
|
|
362
|
-
}
|
|
363
|
-
assertActive();
|
|
364
|
-
if (response?.isDocsInline === false || !Array.isArray(response?.docs) || response.docs.length > batch.length)
|
|
365
|
-
throw new MemDriveError('INVALID_RESULT', 'Original read returned an invalid document envelope.');
|
|
366
|
-
for (const hit of response.docs) {
|
|
367
|
-
try {
|
|
368
|
-
requireInput(hit.collection === undefined || hit.collection === name, 'Original read returned another collection.');
|
|
369
|
-
const mem = await decodeSource(hit.doc);
|
|
370
|
-
requireInput(expected.has(mem.id) && !found.has(mem.id), 'Original read returned unexpected or duplicate Mems.');
|
|
371
|
-
found.set(mem.id, structuredClone(hit.doc));
|
|
372
|
-
}
|
|
373
|
-
catch (cause) {
|
|
374
|
-
throw new MemDriveError('INVALID_RESULT', 'Original read returned an invalid Mem.', { cause });
|
|
375
|
-
}
|
|
376
|
-
}
|
|
377
|
-
}
|
|
378
|
-
assertActive();
|
|
379
|
-
return found;
|
|
380
|
-
};
|
|
381
|
-
const api = {
|
|
382
|
-
get metadataSchema() { return structuredClone(schema); },
|
|
383
|
-
get textSearch() { return structuredClone(textSearch); },
|
|
384
|
-
get embedding() { return Object.freeze(structuredClone(embedding)); },
|
|
385
|
-
get generation() { return generation; },
|
|
386
|
-
assertActive: active,
|
|
387
|
-
describe: refresh,
|
|
388
|
-
async describeDescriptor(options) {
|
|
389
|
-
const { signal } = operationOptions(options);
|
|
390
|
-
const result = await refresh({ signal });
|
|
391
|
-
guarded(active, signal)();
|
|
392
|
-
return descriptor(result.collection);
|
|
393
|
-
},
|
|
394
|
-
at(branch) {
|
|
395
|
-
active();
|
|
396
|
-
checkBranch(branch);
|
|
397
|
-
const version = epoch(branch);
|
|
398
|
-
const assertActive = () => {
|
|
399
|
-
active();
|
|
400
|
-
if (version !== epoch(branch))
|
|
401
|
-
throw new MemDriveError('BRANCH_INVALIDATED', 'This branch handle was invalidated by deletion; select the branch again.');
|
|
402
|
-
};
|
|
403
|
-
return {
|
|
404
|
-
get metadataSchema() { return api.metadataSchema; },
|
|
405
|
-
get textSearch() { return api.textSearch; },
|
|
406
|
-
get embedding() { return api.embedding; },
|
|
407
|
-
get generation() { return generation; },
|
|
408
|
-
assertActive,
|
|
409
|
-
describe: async (options) => { assertActive(); const result = await refresh(options); guarded(assertActive, options?.signal)(); return result; },
|
|
410
|
-
describeDescriptor: async (options) => { assertActive(); const result = await api.describeDescriptor(options); guarded(assertActive, options?.signal)(); return result; },
|
|
411
|
-
async query(body, options) { assertActive(); const result = await api.query(body, options, branch, assertActive); guarded(assertActive, options?.signal)(); return result; },
|
|
412
|
-
remember: (request, options) => { assertActive(); return api.remember(request, options, branch, assertActive); },
|
|
413
|
-
list: (request, options) => { assertActive(); return api.list(request, options, branch, assertActive); },
|
|
414
|
-
read: (request, execute, options) => { assertActive(); return api.read(request, execute, options, branch, assertActive); },
|
|
415
|
-
forget: (request, options) => { assertActive(); return api.forget(request, options, branch, assertActive); },
|
|
416
|
-
};
|
|
417
|
-
},
|
|
418
|
-
branches: {
|
|
419
|
-
async create(request, options) {
|
|
420
|
-
const { signal } = operationOptions(options);
|
|
421
|
-
const check = guarded(active, signal);
|
|
422
|
-
check();
|
|
423
|
-
requireInput(keysOnly(request, ['name', 'source']), 'Use name and optional source for branch creation.');
|
|
424
|
-
checkBranch(request.name);
|
|
425
|
-
const source = request.source === undefined ? { branch: defaultBranch } : request.source;
|
|
426
|
-
requireInput(keysOnly(source, ['branch']), 'A branch source must contain only branch.');
|
|
427
|
-
checkBranch(source.branch);
|
|
428
|
-
generation++;
|
|
429
|
-
try {
|
|
430
|
-
const response = await nativeRequest(() => handle.branches.create({ branchName: request.name,
|
|
431
|
-
source: { kind: 'branch', name: source.branch } }, nativeOptions(signal)), signal);
|
|
432
|
-
check();
|
|
433
|
-
if (!isObject(response?.branch) || response.branch.name !== request.name)
|
|
434
|
-
throw new MemDriveError('INVALID_RESULT', 'Branch creation returned an invalid branch.');
|
|
435
|
-
return structuredClone(response.branch);
|
|
436
|
-
}
|
|
437
|
-
catch (error) {
|
|
438
|
-
const wrapped = databaseError(error, 'MemDrive branch creation failed.');
|
|
439
|
-
if (wrapped.status === 404)
|
|
440
|
-
wrapped.code = 'BRANCH_NOT_FOUND';
|
|
441
|
-
if (wrapped.status === 409)
|
|
442
|
-
wrapped.code = 'BRANCH_EXISTS';
|
|
443
|
-
throw wrapped;
|
|
444
|
-
}
|
|
445
|
-
finally {
|
|
446
|
-
generation++;
|
|
447
|
-
}
|
|
448
|
-
},
|
|
449
|
-
async list(options) {
|
|
450
|
-
const { signal } = operationOptions(options);
|
|
451
|
-
const check = guarded(active, signal);
|
|
452
|
-
check();
|
|
453
|
-
let response;
|
|
454
|
-
try {
|
|
455
|
-
response = await nativeRequest(() => handle.branches.list(nativeOptions(signal)), signal);
|
|
456
|
-
}
|
|
457
|
-
catch (error) {
|
|
458
|
-
throw databaseError(error, 'Could not list MemDrive branches.');
|
|
459
|
-
}
|
|
460
|
-
check();
|
|
461
|
-
if (!Array.isArray(response?.branches) || !response.branches.every(item => isObject(item) && validBranchName(item.name))
|
|
462
|
-
|| new Set(response.branches.map(item => item.name)).size !== response.branches.length)
|
|
463
|
-
throw new MemDriveError('INVALID_RESULT', 'Branch listing returned invalid branches.');
|
|
464
|
-
return structuredClone(response.branches);
|
|
465
|
-
},
|
|
466
|
-
async delete(branch, options) {
|
|
467
|
-
const { signal } = operationOptions(options);
|
|
468
|
-
const check = guarded(active, signal);
|
|
469
|
-
check();
|
|
470
|
-
checkBranch(branch);
|
|
471
|
-
requireInput(branch !== defaultBranch, 'The main branch cannot be deleted independently of its MemDrive.');
|
|
472
|
-
generation++;
|
|
473
|
-
try {
|
|
474
|
-
await nativeRequest(() => handle.branches.delete(branch, nativeOptions(signal)), signal);
|
|
475
|
-
}
|
|
476
|
-
catch (error) {
|
|
477
|
-
const wrapped = databaseError(error, 'MemDrive branch deletion failed.');
|
|
478
|
-
// A missing or uncertain deletion must not let old handles
|
|
479
|
-
// silently bind to a later branch with the same name.
|
|
480
|
-
if (wrapped.status === 404 || !(wrapped.status !== undefined && wrapped.status >= 400 && wrapped.status < 500))
|
|
481
|
-
retire(branch);
|
|
482
|
-
if (wrapped.status === 404)
|
|
483
|
-
wrapped.code = 'BRANCH_NOT_FOUND';
|
|
484
|
-
if (wrapped.status === 409)
|
|
485
|
-
wrapped.code = 'BRANCH_CONFLICT';
|
|
486
|
-
throw wrapped;
|
|
487
|
-
}
|
|
488
|
-
finally {
|
|
489
|
-
generation++;
|
|
490
|
-
}
|
|
491
|
-
retire(branch);
|
|
492
|
-
check();
|
|
493
|
-
},
|
|
494
|
-
},
|
|
495
|
-
async query(body, options, branch = defaultBranch, assertActive = active) {
|
|
496
|
-
const { signal } = operationOptions(options);
|
|
497
|
-
const check = guarded(assertActive, signal);
|
|
498
|
-
check();
|
|
499
|
-
requireInput(keysOnly(body, ['query', 'size', 'candidateSize', 'rerank', 'facets']), 'Query cannot override the managed read settings.');
|
|
500
|
-
try {
|
|
501
|
-
const result = await nativeRequest(() => handle.query({ ...structuredClone(body),
|
|
502
|
-
ref: { kind: 'branch', name: branch }, consistentRead: true, includeVectors: false }, nativeOptions(signal)), signal);
|
|
503
|
-
check();
|
|
504
|
-
return result;
|
|
505
|
-
}
|
|
506
|
-
catch (error) {
|
|
507
|
-
throw databaseError(error, 'MemDrive query failed.');
|
|
508
|
-
}
|
|
509
|
-
},
|
|
510
|
-
async read(request, execute, options, branch = defaultBranch, assertActive = active) {
|
|
511
|
-
const { signal } = operationOptions(options);
|
|
512
|
-
assertActive = guarded(assertActive, signal);
|
|
513
|
-
assertActive();
|
|
514
|
-
const ids = originalIds(request);
|
|
515
|
-
const originals = await fetchOriginals(ids, branch, assertActive, execute, signal);
|
|
516
|
-
const mems = await Promise.all(ids.filter(id => originals.has(id)).map(id => decodeSource(originals.get(id))));
|
|
517
|
-
assertActive();
|
|
518
|
-
return { mems };
|
|
519
|
-
},
|
|
520
|
-
async list(request = {}, options, branch = defaultBranch, assertActive = active) {
|
|
521
|
-
const { signal } = operationOptions(options);
|
|
522
|
-
assertActive = guarded(assertActive, signal);
|
|
523
|
-
assertActive();
|
|
524
|
-
requireInput(keysOnly(request, ['cursor', 'limit']), 'list accepts cursor and limit only.');
|
|
525
|
-
const limit = request.limit ?? 100;
|
|
526
|
-
requireInput(Number.isInteger(limit) && limit >= 1 && limit <= 1000, 'list limit must be between 1 and 1000.');
|
|
527
|
-
requireInput(request.cursor === undefined || (typeof request.cursor === 'string' && request.cursor.length > 0), 'list cursor must be a nonempty opaque token.');
|
|
528
|
-
let response;
|
|
529
|
-
try {
|
|
530
|
-
response = await nativeRequest(() => handle.docs.list({ size: limit, ...(request.cursor === undefined ? {} : { pageToken: request.cursor }),
|
|
531
|
-
includeVectors: false, ref: { kind: 'branch', name: branch } }, nativeOptions(signal)), signal);
|
|
532
|
-
}
|
|
533
|
-
catch (error) {
|
|
534
|
-
throw databaseError(error, 'Could not list Mems.');
|
|
535
|
-
}
|
|
536
|
-
assertActive();
|
|
537
|
-
if (response?.isDocsInline === false || !Array.isArray(response?.docs) || response.docs.length > limit
|
|
538
|
-
|| (response.nextPageToken !== undefined && (typeof response.nextPageToken !== 'string' || !response.nextPageToken
|
|
539
|
-
|| response.nextPageToken === request.cursor)))
|
|
540
|
-
throw new MemDriveError('INVALID_RESULT', 'Mem listing returned an invalid page.');
|
|
541
|
-
const mems = [], seen = new Set();
|
|
542
|
-
for (const hit of response.docs) {
|
|
543
|
-
try {
|
|
544
|
-
requireInput((hit.collection === undefined || hit.collection === name) && isObject(hit.doc)
|
|
545
|
-
&& typeof hit.doc.kind === 'string' && ['mem', 'chunk'].includes(hit.doc.kind) && typeof hit.doc.id === 'string' && !seen.has(hit.doc.id), 'Listing returned an unexpected or duplicate document.');
|
|
546
|
-
seen.add(hit.doc.id);
|
|
547
|
-
if (hit.doc.kind === 'mem')
|
|
548
|
-
mems.push(await decodeSource(hit.doc));
|
|
549
|
-
}
|
|
550
|
-
catch (cause) {
|
|
551
|
-
throw new MemDriveError('INVALID_RESULT', 'Mem listing returned an invalid document.', { cause });
|
|
552
|
-
}
|
|
553
|
-
}
|
|
554
|
-
assertActive();
|
|
555
|
-
return { mems, ...(response.nextPageToken === undefined ? {} : { nextCursor: response.nextPageToken }) };
|
|
556
|
-
},
|
|
557
|
-
async remember(request, options, branch = defaultBranch, assertActive = active) {
|
|
558
|
-
const { signal } = operationOptions(options);
|
|
559
|
-
assertActive = guarded(assertActive, signal);
|
|
560
|
-
assertActive();
|
|
561
|
-
requireInput(keysOnly(request, ['mems']) && Array.isArray(request.mems) && request.mems.length > 0, 'remember requires a nonempty mems array.');
|
|
562
|
-
const external = embedding.mode === 'external';
|
|
563
|
-
const validate = (mem, options) => external ? validateVectorMem(mem, schema, embedding, options) : validateMem(mem, schema, options);
|
|
564
|
-
const mems = request.mems.map(mem => {
|
|
565
|
-
const validated = validate(mem);
|
|
566
|
-
validated.id ??= globalThis.crypto.randomUUID();
|
|
567
|
-
return (external ? validateVectorMem(validated, schema, embedding, { requireId: true })
|
|
568
|
-
: validateMem(validated, schema, { requireId: true }));
|
|
569
|
-
});
|
|
570
|
-
const ids = mems.map(mem => mem.id);
|
|
571
|
-
requireInput(new Set(ids).size === ids.length, 'Mem IDs must be unique within remember.');
|
|
572
|
-
let originals;
|
|
573
|
-
try {
|
|
574
|
-
originals = external ? new Map() : await fetchOriginals(ids, branch, assertActive, undefined, signal);
|
|
575
|
-
}
|
|
576
|
-
catch (error) {
|
|
577
|
-
if (error instanceof MemDriveError && error.code === 'ABORTED')
|
|
578
|
-
throw databaseError(error, 'MemDrive write was aborted before dispatch.', {
|
|
579
|
-
ids, completedIds: [], uncertainIds: [], notAttemptedIds: [...ids],
|
|
580
|
-
});
|
|
581
|
-
throw error;
|
|
582
|
-
}
|
|
583
|
-
const replacements = await Promise.all(mems.map(async (mem) => {
|
|
584
|
-
const id = await sourceDocumentId(mem.id), shared = { memId: mem.id, ...attributes(mem) };
|
|
585
|
-
let docs;
|
|
586
|
-
if ('embedding' in mem)
|
|
587
|
-
docs = [{ id, kind: 'mem', ...shared, embedding: mem.embedding }];
|
|
588
|
-
else {
|
|
589
|
-
const spans = chunkSpans(mem.content).filter(span => span.text.trim());
|
|
590
|
-
const whole = spans.length === 1 && spans[0].start === 0 && spans[0].end === mem.content.length;
|
|
591
|
-
const chunkSlots = Math.max(originals.get(mem.id)?.chunkSlots ?? 0, whole ? 0 : spans.length);
|
|
592
|
-
docs = [{ id, kind: 'mem', ...shared, ...(whole ? { content: mem.content } : { memContent: mem.content }),
|
|
593
|
-
chunkCount: spans.length, chunkSlots, chunkingPolicy: CHUNKING_POLICY }];
|
|
594
|
-
for (let ordinal = 0; ordinal < chunkSlots; ordinal++) {
|
|
595
|
-
const span = whole ? undefined : spans[ordinal];
|
|
596
|
-
docs.push({ id: chunkDocumentId(id, ordinal), kind: 'chunk', ...shared,
|
|
597
|
-
...(span ? { content: span.text, start: span.start, end: span.end } : {}) });
|
|
598
|
-
}
|
|
599
|
-
}
|
|
600
|
-
requireInput(docs.every(doc => bytes(doc) <= documentBytes) && bytes({ docs, branch }) <= requestBytes, 'A complete Mem replacement exceeds the native request or document byte limit.');
|
|
601
|
-
return { ids: [mem.id], docs };
|
|
602
|
-
}));
|
|
603
|
-
// Greedily pack complete Mem replacements. No request may contain
|
|
604
|
-
// only part of a replacement, and all requests validate before writes.
|
|
605
|
-
const batches = [];
|
|
606
|
-
let batch = { ids: [], docs: [] }, size = bytes({ docs: [], branch });
|
|
607
|
-
for (const replacement of replacements) {
|
|
608
|
-
const addition = replacement.docs.reduce((sum, doc) => sum + bytes(doc) + 1, 0);
|
|
609
|
-
const nextSize = size + addition - (batch.docs.length ? 0 : 1);
|
|
610
|
-
if (batch.docs.length && nextSize > requestBytes) {
|
|
611
|
-
batches.push(batch);
|
|
612
|
-
batch = { ids: [], docs: [] };
|
|
613
|
-
size = bytes({ docs: [], branch });
|
|
614
|
-
}
|
|
615
|
-
size += addition - (batch.docs.length ? 0 : 1);
|
|
616
|
-
batch.ids.push(...replacement.ids);
|
|
617
|
-
batch.docs.push(...replacement.docs);
|
|
618
|
-
}
|
|
619
|
-
if (batch.docs.length)
|
|
620
|
-
batches.push(batch);
|
|
621
|
-
const completedIds = [];
|
|
622
|
-
const checkWritable = () => {
|
|
623
|
-
try {
|
|
624
|
-
assertActive();
|
|
625
|
-
}
|
|
626
|
-
catch (error) {
|
|
627
|
-
throw databaseError(error, 'MemDrive branch changed during write.', {
|
|
628
|
-
ids, completedIds: [...completedIds], uncertainIds: [], notAttemptedIds: ids.slice(completedIds.length),
|
|
629
|
-
});
|
|
630
|
-
}
|
|
631
|
-
};
|
|
632
|
-
for (const batch of batches) {
|
|
633
|
-
checkWritable();
|
|
634
|
-
generation++;
|
|
635
|
-
try {
|
|
636
|
-
await nativeRequest(() => handle.docs.upsert({ docs: batch.docs, branch }, nativeOptions(signal)), signal);
|
|
637
|
-
}
|
|
638
|
-
catch (error) {
|
|
639
|
-
throw databaseError(error, 'MemDrive write failed; some mems may already have been accepted.', {
|
|
640
|
-
ids, completedIds: [...completedIds], uncertainIds: [...batch.ids],
|
|
641
|
-
notAttemptedIds: ids.slice(completedIds.length + batch.ids.length),
|
|
642
|
-
});
|
|
643
|
-
}
|
|
644
|
-
finally {
|
|
645
|
-
generation++;
|
|
646
|
-
}
|
|
647
|
-
completedIds.push(...batch.ids);
|
|
648
|
-
}
|
|
649
|
-
checkWritable();
|
|
650
|
-
return { ids };
|
|
651
|
-
},
|
|
652
|
-
async forget(request, options, branch = defaultBranch, assertActive = active) {
|
|
653
|
-
const { signal } = operationOptions(options);
|
|
654
|
-
const check = guarded(assertActive, signal);
|
|
655
|
-
check();
|
|
656
|
-
requireInput(keysOnly(request, ['ids', 'filter']) && Object.keys(request).length === 1, 'forget requires exactly one of ids or filter.');
|
|
657
|
-
let body;
|
|
658
|
-
if (Object.hasOwn(request, 'ids')) {
|
|
659
|
-
requireInput(Array.isArray(request.ids) && request.ids.length > 0
|
|
660
|
-
&& request.ids.every(id => typeof id === 'string' && id.trim().length > 0 && id.length <= 512)
|
|
661
|
-
&& new Set(request.ids).size === request.ids.length, 'forget IDs must be nonempty and unique.');
|
|
662
|
-
body = { filter: compileFilter({ field: 'id', op: 'in', value: [...request.ids] }, schema), branch };
|
|
663
|
-
}
|
|
664
|
-
else {
|
|
665
|
-
requireInput(request.filter !== undefined, 'forget filter must be an explicit expression.');
|
|
666
|
-
body = { filter: compileFilter(request.filter, schema), branch };
|
|
667
|
-
}
|
|
668
|
-
generation++;
|
|
669
|
-
try {
|
|
670
|
-
await nativeRequest(() => handle.docs.delete(body, nativeOptions(signal)), signal);
|
|
671
|
-
}
|
|
672
|
-
catch (error) {
|
|
673
|
-
throw databaseError(error, 'Mem deletion failed.');
|
|
674
|
-
}
|
|
675
|
-
finally {
|
|
676
|
-
generation++;
|
|
677
|
-
}
|
|
678
|
-
check();
|
|
679
|
-
},
|
|
680
|
-
async configure(request, options) {
|
|
681
|
-
const { signal } = operationOptions(options);
|
|
682
|
-
const check = guarded(active, signal);
|
|
683
|
-
check();
|
|
684
|
-
requireInput(keysOnly(request, ['metadataSchema', 'description'])
|
|
685
|
-
&& (Object.hasOwn(request, 'metadataSchema') || request.description !== undefined), 'configure requires metadataSchema, description, or both.');
|
|
686
|
-
const desired = Object.hasOwn(request, 'metadataSchema') ? validateMetadataSchema(request.metadataSchema) : undefined;
|
|
687
|
-
const description = validateDescription(request.description);
|
|
688
|
-
const desiredTags = desired === undefined ? undefined : tags(desired, embedding); // Check format capacity before remote operations.
|
|
689
|
-
// Serialize this handle's read/validate/write/verify sequence. Native
|
|
690
|
-
// PATCH has no CAS for tags: applications must use one schema writer
|
|
691
|
-
// across handles/processes; this queue is not a distributed lock.
|
|
692
|
-
const update = schemaUpdates.then(async () => {
|
|
693
|
-
check();
|
|
694
|
-
const current = await refresh({ signal });
|
|
695
|
-
const before = schemaFromCollection(current.collection, name);
|
|
696
|
-
requireInput(desired === undefined || Object.entries(before).every(([key, value]) => desired[key]?.type === value.type), 'Schema updates may add fields but cannot omit or change existing fields.');
|
|
697
|
-
const expected = desired === undefined ? before : Object.fromEntries(Object.entries(desired).map(([key, value]) => [key,
|
|
698
|
-
{ ...before[key], ...value, ...(value.description === undefined && before[key]?.description !== undefined
|
|
699
|
-
? { description: before[key].description } : {}) }]));
|
|
700
|
-
const nextIndexes = structuredClone(current.collection.indexConfigs);
|
|
701
|
-
const previousMetadata = nextIndexes.metadata?.type === 'object' ? nextIndexes.metadata : undefined;
|
|
702
|
-
if (desired !== undefined && Object.keys(desired).length)
|
|
703
|
-
nextIndexes.metadata = {
|
|
704
|
-
...(previousMetadata ?? { type: 'object' }),
|
|
705
|
-
objectIndexConfigs: { ...previousMetadata?.objectIndexConfigs,
|
|
706
|
-
...Object.fromEntries(Object.entries(expected).map(([key, value]) => [key, {
|
|
707
|
-
...previousMetadata?.objectIndexConfigs?.[key], type: types[value.type],
|
|
708
|
-
...(value.description === undefined ? {} : { description: value.description }),
|
|
709
|
-
}])) },
|
|
710
|
-
};
|
|
711
|
-
const nextTags = { ...current.collection.tags, ...desiredTags };
|
|
712
|
-
requireInput(Object.keys(nextTags).length <= 5, 'Collection tags exceed the native limit.');
|
|
713
|
-
generation++;
|
|
714
|
-
const updateRequest = desired === undefined
|
|
715
|
-
? { description: description }
|
|
716
|
-
: { indexConfigs: nextIndexes, tags: nextTags, ...(description === undefined ? {} : { description }) };
|
|
717
|
-
try {
|
|
718
|
-
await nativeRequest(() => handle.update(updateRequest, nativeOptions(signal)), signal);
|
|
719
|
-
}
|
|
720
|
-
catch (error) {
|
|
721
|
-
throw databaseError(error, 'MemDrive schema update failed; reopen to verify its outcome.');
|
|
722
|
-
}
|
|
723
|
-
// Catch an observed conflicting writer; a later writer still needs
|
|
724
|
-
// application coordination because native PATCH has no condition.
|
|
725
|
-
check();
|
|
726
|
-
const updated = (await refresh({ signal })).collection;
|
|
727
|
-
const actual = schemaFromCollection(updated, name);
|
|
728
|
-
if (Object.keys(actual).length !== Object.keys(expected).length
|
|
729
|
-
|| !Object.entries(expected).every(([key, value]) => actual[key]?.type === value.type
|
|
730
|
-
&& (actual[key]?.description ?? '') === (value.description ?? ''))
|
|
731
|
-
|| (updated.description ?? '') !== (description ?? current.collection.description ?? ''))
|
|
732
|
-
throw new MemDriveError('SCHEMA_CONFLICT', 'Stored schema changed concurrently; reopen before proceeding.');
|
|
733
|
-
});
|
|
734
|
-
const settled = update.finally(() => { generation++; });
|
|
735
|
-
schemaUpdates = settled.catch(() => { });
|
|
736
|
-
return settled;
|
|
737
|
-
},
|
|
738
|
-
async destroy(options) {
|
|
739
|
-
const { signal } = operationOptions(options);
|
|
740
|
-
const check = guarded(active, signal);
|
|
741
|
-
check();
|
|
742
|
-
generation++;
|
|
743
|
-
try {
|
|
744
|
-
await nativeRequest(() => handle.delete(nativeOptions(signal)), signal);
|
|
745
|
-
deleted = true;
|
|
746
|
-
checkAbort(signal);
|
|
747
|
-
}
|
|
748
|
-
catch (error) {
|
|
749
|
-
throw databaseError(error, 'MemDrive deletion failed.');
|
|
750
|
-
}
|
|
751
|
-
},
|
|
752
|
-
};
|
|
753
|
-
return api;
|
|
754
|
-
}
|
|
755
|
-
export async function createBackend(options, requestOptions) {
|
|
756
|
-
const { signal } = operationOptions(requestOptions);
|
|
757
|
-
checkAbort(signal);
|
|
758
|
-
validateConnection(options);
|
|
759
|
-
requireInput(typeof options.client.createCollection === 'function', 'Client must support collection creation.');
|
|
760
|
-
const schema = validateMetadataSchema(options.metadataSchema);
|
|
761
|
-
const textSearch = validateTextSearch(options.textSearch);
|
|
762
|
-
const embedding = validateEmbedding(options.embedding);
|
|
763
|
-
requireInput(embedding.mode !== 'external' || textSearch === undefined, 'External embedding does not support a content text index.');
|
|
764
|
-
const labels = validateLabels(options.labels);
|
|
765
|
-
const description = validateDescription(options.description);
|
|
766
|
-
const allTags = { ...labels, ...tags(schema, embedding) };
|
|
767
|
-
requireInput(Object.keys(allTags).length <= 5, 'Labels and reserved MemDrive tags exceed the native five-tag limit.');
|
|
768
|
-
const request = { collectionName: options.name, indexConfigs: indexConfigs(schema, textSearch, embedding), tags: allTags,
|
|
769
|
-
...(description === undefined ? {} : { description }) };
|
|
770
|
-
try {
|
|
771
|
-
await nativeRequest(() => options.client.createCollection(request, nativeOptions(signal)), signal);
|
|
772
|
-
}
|
|
773
|
-
catch (error) {
|
|
774
|
-
const wrapped = databaseError(error, 'Could not create the MemDrive.');
|
|
775
|
-
if (wrapped.status === 409)
|
|
776
|
-
wrapped.code = 'STORE_EXISTS';
|
|
777
|
-
throw wrapped;
|
|
778
|
-
}
|
|
779
|
-
checkAbort(signal);
|
|
780
|
-
return openBackend(options, nativeOptions(signal));
|
|
781
|
-
}
|
|
782
|
-
export async function openBackend(options, requestOptions) {
|
|
783
|
-
const { signal } = operationOptions(requestOptions);
|
|
784
|
-
checkAbort(signal);
|
|
785
|
-
validateConnection(options);
|
|
786
|
-
const handle = options.client.collection(options.name);
|
|
787
|
-
let state;
|
|
788
|
-
try {
|
|
789
|
-
state = await load(handle, options.name, nativeOptions(signal));
|
|
790
|
-
}
|
|
791
|
-
catch (error) {
|
|
792
|
-
if (error instanceof MemDriveError && error.status === 404)
|
|
793
|
-
error.code = 'STORE_NOT_FOUND';
|
|
794
|
-
throw error;
|
|
795
|
-
}
|
|
796
|
-
checkAbort(signal);
|
|
797
|
-
return backend(handle, options.name, state);
|
|
798
|
-
}
|
|
799
|
-
export async function listBackends(request, options) {
|
|
800
|
-
const { signal } = operationOptions(options);
|
|
801
|
-
checkAbort(signal);
|
|
802
|
-
requireInput(keysOnly(request, ['client', 'limit', 'cursor', 'labels']) && typeof request.client?.listCollections === 'function'
|
|
803
|
-
&& typeof request.client?.collection === 'function', 'Provide a LambdaDB client and documented catalog arguments.');
|
|
804
|
-
const { client, cursor } = request;
|
|
805
|
-
const limit = request.limit ?? 10, labels = validateLabels(request.labels);
|
|
806
|
-
requireInput(Number.isInteger(limit) && limit >= 1 && limit <= 100, 'Catalog limit must be between 1 and 100.');
|
|
807
|
-
requireInput(cursor === undefined || (typeof cursor === 'string' && cursor.length > 0), 'Catalog cursor must be a nonempty opaque token.');
|
|
808
|
-
let page;
|
|
809
|
-
try {
|
|
810
|
-
page = await nativeRequest(() => client.listCollections({ size: limit,
|
|
811
|
-
...(cursor === undefined ? {} : { pageToken: cursor }) }, nativeOptions(signal)), signal);
|
|
812
|
-
}
|
|
813
|
-
catch (error) {
|
|
814
|
-
throw databaseError(error, 'Could not list MemDrives.');
|
|
815
|
-
}
|
|
816
|
-
checkAbort(signal);
|
|
817
|
-
if (!Array.isArray(page?.collections) || page.collections.length > limit
|
|
818
|
-
|| !page.collections.every(item => isObject(item) && typeof item.collectionName === 'string'
|
|
819
|
-
&& /^[A-Za-z0-9_-]{3,52}$/.test(item.collectionName))
|
|
820
|
-
|| new Set(page.collections.map(item => item.collectionName)).size !== page.collections.length
|
|
821
|
-
|| (page.nextPageToken !== undefined && (typeof page.nextPageToken !== 'string'
|
|
822
|
-
|| !page.nextPageToken || page.nextPageToken === cursor)))
|
|
823
|
-
throw new MemDriveError('INVALID_RESULT', 'MemDrive catalog returned an invalid page.');
|
|
824
|
-
const drives = [];
|
|
825
|
-
const matches = (collection) => Object.entries(labels).every(([key, value]) => collection.tags?.[key] === value);
|
|
826
|
-
for (let collection of page.collections) {
|
|
827
|
-
checkAbort(signal);
|
|
828
|
-
if (collection.tags?.['memory.format'] !== format || !matches(collection))
|
|
829
|
-
continue;
|
|
830
|
-
try {
|
|
831
|
-
if (collection.indexConfigs === undefined)
|
|
832
|
-
collection = (await load(client.collection(collection.collectionName), collection.collectionName, nativeOptions(signal))).result.collection;
|
|
833
|
-
const drive = descriptor(collection);
|
|
834
|
-
if (matches(collection))
|
|
835
|
-
drives.push(drive);
|
|
836
|
-
}
|
|
837
|
-
catch (error) {
|
|
838
|
-
if (!(error instanceof MemDriveError) || error.code !== 'INCOMPATIBLE_STORE')
|
|
839
|
-
throw error;
|
|
840
|
-
}
|
|
841
|
-
}
|
|
842
|
-
checkAbort(signal);
|
|
843
|
-
return { drives, ...(page.nextPageToken === undefined ? {} : { nextCursor: page.nextPageToken }) };
|
|
844
|
-
}
|
|
845
|
-
//# sourceMappingURL=storage.js.map
|