@functional-systems/memdrive 0.1.1-dev.1 → 0.1.1-dev.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/README.md +20 -250
- package/dist/commonjs/chunking.d.ts +1 -2
- package/dist/commonjs/chunking.d.ts.map +1 -1
- package/dist/commonjs/chunking.js +2 -5
- package/dist/commonjs/chunking.js.map +1 -1
- package/dist/commonjs/config.d.ts +17 -0
- package/dist/commonjs/config.d.ts.map +1 -0
- package/dist/commonjs/config.js +116 -0
- package/dist/commonjs/config.js.map +1 -0
- package/dist/commonjs/errors.d.ts +10 -6
- package/dist/commonjs/errors.d.ts.map +1 -1
- package/dist/commonjs/errors.js +25 -5
- package/dist/commonjs/errors.js.map +1 -1
- package/dist/commonjs/index.d.ts +15 -5
- package/dist/commonjs/index.d.ts.map +1 -1
- package/dist/commonjs/index.js +331 -307
- package/dist/commonjs/index.js.map +1 -1
- package/dist/commonjs/records.d.ts +44 -0
- package/dist/commonjs/records.d.ts.map +1 -0
- package/dist/commonjs/records.js +88 -0
- package/dist/commonjs/records.js.map +1 -0
- package/dist/commonjs/search.d.ts +8 -0
- package/dist/commonjs/search.d.ts.map +1 -0
- package/dist/commonjs/search.js +56 -0
- package/dist/commonjs/search.js.map +1 -0
- package/dist/commonjs/types.d.ts +84 -252
- package/dist/commonjs/types.d.ts.map +1 -1
- package/dist/esm/chunking.d.ts +1 -2
- package/dist/esm/chunking.d.ts.map +1 -1
- package/dist/esm/chunking.js +2 -4
- package/dist/esm/chunking.js.map +1 -1
- package/dist/esm/config.d.ts +17 -0
- package/dist/esm/config.d.ts.map +1 -0
- package/dist/esm/config.js +107 -0
- package/dist/esm/config.js.map +1 -0
- package/dist/esm/errors.d.ts +10 -6
- package/dist/esm/errors.d.ts.map +1 -1
- package/dist/esm/errors.js +22 -4
- package/dist/esm/errors.js.map +1 -1
- package/dist/esm/index.d.ts +15 -5
- package/dist/esm/index.d.ts.map +1 -1
- package/dist/esm/index.js +331 -305
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/records.d.ts +44 -0
- package/dist/esm/records.d.ts.map +1 -0
- package/dist/esm/records.js +77 -0
- package/dist/esm/records.js.map +1 -0
- package/dist/esm/search.d.ts +8 -0
- package/dist/esm/search.d.ts.map +1 -0
- package/dist/esm/search.js +52 -0
- package/dist/esm/search.js.map +1 -0
- package/dist/esm/types.d.ts +84 -252
- package/dist/esm/types.d.ts.map +1 -1
- package/package.json +10 -32
- package/src/chunking.ts +2 -3
- package/src/config.ts +121 -0
- package/src/errors.ts +20 -6
- package/src/index.ts +351 -289
- package/src/records.ts +83 -0
- package/src/search.ts +57 -0
- package/src/types.ts +74 -253
- package/dist/commonjs/filter.d.ts +0 -30
- package/dist/commonjs/filter.d.ts.map +0 -1
- package/dist/commonjs/filter.js +0 -224
- package/dist/commonjs/filter.js.map +0 -1
- package/dist/commonjs/storage.d.ts +0 -274
- package/dist/commonjs/storage.d.ts.map +0 -1
- package/dist/commonjs/storage.js +0 -861
- package/dist/commonjs/storage.js.map +0 -1
- package/dist/commonjs/tools/definitions.d.ts +0 -5
- package/dist/commonjs/tools/definitions.d.ts.map +0 -1
- package/dist/commonjs/tools/definitions.js +0 -129
- package/dist/commonjs/tools/definitions.js.map +0 -1
- package/dist/commonjs/tools/index.d.ts +0 -6
- package/dist/commonjs/tools/index.d.ts.map +0 -1
- package/dist/commonjs/tools/index.js +0 -104
- package/dist/commonjs/tools/index.js.map +0 -1
- package/dist/commonjs/tools/types.d.ts +0 -50
- package/dist/commonjs/tools/types.d.ts.map +0 -1
- package/dist/commonjs/tools/types.js +0 -3
- package/dist/commonjs/tools/types.js.map +0 -1
- package/dist/commonjs/values.d.ts +0 -17
- package/dist/commonjs/values.d.ts.map +0 -1
- package/dist/commonjs/values.js +0 -90
- package/dist/commonjs/values.js.map +0 -1
- package/dist/esm/filter.d.ts +0 -30
- package/dist/esm/filter.d.ts.map +0 -1
- package/dist/esm/filter.js +0 -209
- package/dist/esm/filter.js.map +0 -1
- package/dist/esm/storage.d.ts +0 -274
- package/dist/esm/storage.d.ts.map +0 -1
- package/dist/esm/storage.js +0 -845
- package/dist/esm/storage.js.map +0 -1
- package/dist/esm/tools/definitions.d.ts +0 -5
- package/dist/esm/tools/definitions.d.ts.map +0 -1
- package/dist/esm/tools/definitions.js +0 -126
- package/dist/esm/tools/definitions.js.map +0 -1
- package/dist/esm/tools/index.d.ts +0 -6
- package/dist/esm/tools/index.d.ts.map +0 -1
- package/dist/esm/tools/index.js +0 -101
- package/dist/esm/tools/index.js.map +0 -1
- package/dist/esm/tools/types.d.ts +0 -50
- package/dist/esm/tools/types.d.ts.map +0 -1
- package/dist/esm/tools/types.js +0 -2
- package/dist/esm/tools/types.js.map +0 -1
- package/dist/esm/values.d.ts +0 -17
- package/dist/esm/values.d.ts.map +0 -1
- package/dist/esm/values.js +0 -87
- package/dist/esm/values.js.map +0 -1
- package/docs/tools.md +0 -163
- package/src/filter.ts +0 -189
- package/src/storage.ts +0 -765
- package/src/tools/definitions.ts +0 -156
- package/src/tools/index.ts +0 -97
- package/src/tools/types.ts +0 -45
- package/src/values.ts +0 -103
package/src/records.ts
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { MemDriveError, requireInput } from './errors.js';
|
|
2
|
+
import { isObject, keysOnly, metadataValues, validDate } from './config.js';
|
|
3
|
+
import { chunkSpans } from './chunking.js';
|
|
4
|
+
import type { CreateFileRequest, DriveFile, MetadataSchema } from './types.js';
|
|
5
|
+
|
|
6
|
+
const encoder = new TextEncoder();
|
|
7
|
+
export const bytes = (value: unknown) => encoder.encode(JSON.stringify(value)).length;
|
|
8
|
+
const fileIdPattern = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
|
|
9
|
+
export const validFileId = (value: unknown): value is string => typeof value === 'string' && fileIdPattern.test(value);
|
|
10
|
+
export const validPath = (value: unknown): value is string => typeof value === 'string' && value.startsWith('/')
|
|
11
|
+
&& !/[\\\p{Cc}\p{Surrogate}]/u.test(value) && encoder.encode(value).length <= 4096
|
|
12
|
+
&& value.slice(1).split('/').every(part => part !== '' && part !== '.' && part !== '..');
|
|
13
|
+
export const originalId = (fileId: string) => `file:${fileId}`;
|
|
14
|
+
|
|
15
|
+
export function requirePath(path: unknown): asserts path is string {
|
|
16
|
+
requireInput(validPath(path), 'File path must be absolute, at most 4096 UTF-8 bytes, and contain valid nonempty components.');
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export function resultDocs(response: unknown, collection: string, maximum: number): Record<string, unknown>[] {
|
|
20
|
+
if (!isObject(response) || response.isDocsInline === false || !Array.isArray(response.docs) || response.docs.length > maximum)
|
|
21
|
+
throw new MemDriveError('INVALID_RESULT', 'Read returned an invalid or ambiguous document envelope.');
|
|
22
|
+
return response.docs.map(hit => {
|
|
23
|
+
if (!isObject(hit) || hit.collection !== collection || !isObject(hit.doc))
|
|
24
|
+
throw new MemDriveError('INVALID_RESULT', 'Read returned an invalid document.');
|
|
25
|
+
return hit.doc;
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export function decodeFile(doc: Record<string, unknown>, schema: MetadataSchema): DriveFile {
|
|
30
|
+
if (doc.kind !== 'file' || !validFileId(doc.fileId) || doc.id !== originalId(doc.fileId)
|
|
31
|
+
|| !validPath(doc.path) || typeof doc.body !== 'string' || /\p{Surrogate}/u.test(doc.body) || !validDate(doc.modifiedAt))
|
|
32
|
+
throw new MemDriveError('INVALID_RESULT', 'Read returned an invalid file original.');
|
|
33
|
+
try {
|
|
34
|
+
const metadata = metadataValues(doc.metadata, schema);
|
|
35
|
+
return { fileId: doc.fileId, path: doc.path, modifiedAt: doc.modifiedAt, content: doc.body,
|
|
36
|
+
...(metadata === undefined ? {} : { metadata }) };
|
|
37
|
+
} catch (cause) {
|
|
38
|
+
throw new MemDriveError('INVALID_RESULT', 'Read returned invalid file metadata.', { cause });
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
async function contentHash(content: string): Promise<string> {
|
|
43
|
+
const hash = new Uint8Array(await globalThis.crypto.subtle.digest('SHA-256', encoder.encode(content)));
|
|
44
|
+
return [...hash].map(byte => byte.toString(16).padStart(2, '0')).join('');
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export async function fileRecords(request: CreateFileRequest, schema: MetadataSchema) {
|
|
48
|
+
requireInput(keysOnly(request, ['op', 'path', 'content', 'metadata']) && request.op === 'create',
|
|
49
|
+
'Write requires a create operation with path and original text.');
|
|
50
|
+
const { path, content } = request;
|
|
51
|
+
requirePath(path);
|
|
52
|
+
requireInput(typeof content === 'string' && !/\p{Surrogate}/u.test(content), 'File content must be UTF-8 text.');
|
|
53
|
+
const metadata = metadataValues(request.metadata, schema);
|
|
54
|
+
const fileId = globalThis.crypto.randomUUID();
|
|
55
|
+
const modifiedAt = new Date().toISOString();
|
|
56
|
+
const parts = path.slice(1).split('/');
|
|
57
|
+
const fileName = parts.pop()!;
|
|
58
|
+
const ancestorPaths = ['/'];
|
|
59
|
+
let ancestor = '/';
|
|
60
|
+
for (const part of parts) {
|
|
61
|
+
ancestor += part + '/';
|
|
62
|
+
ancestorPaths.push(ancestor);
|
|
63
|
+
}
|
|
64
|
+
const shared = { fileId, path, fileName, ancestorPaths, modifiedAt, ...(metadata === undefined ? {} : { metadata }) };
|
|
65
|
+
const original = { id: originalId(fileId), kind: 'file', ...shared, body: content, changed: true };
|
|
66
|
+
const originalBytes = bytes(original);
|
|
67
|
+
requireInput(originalBytes <= 5 * 1024 * 1024, 'The file original exceeds the native 5 MiB document limit.');
|
|
68
|
+
const spans = chunkSpans(content).filter(span => span.text.trim());
|
|
69
|
+
const occurrences = new Map<string, number>();
|
|
70
|
+
const hashes = await Promise.all(spans.map(span => contentHash(span.text)));
|
|
71
|
+
const chunks = spans.map(({ text, start, end }, index) => {
|
|
72
|
+
const hash = hashes[index]!;
|
|
73
|
+
const occurrence = occurrences.get(hash) ?? 0;
|
|
74
|
+
occurrences.set(hash, occurrence + 1);
|
|
75
|
+
return { id: `chunk:${fileId}:${hash}:${occurrence}`, kind: 'chunk', ...shared, content: text, start, end };
|
|
76
|
+
});
|
|
77
|
+
const file: DriveFile = { fileId, path, modifiedAt, content, ...(metadata === undefined ? {} : { metadata }) };
|
|
78
|
+
// Large files stage their chunks before making the original visible.
|
|
79
|
+
const chunkRecords = chunks.map(doc => ({ doc, inputBytes: encoder.encode(doc.content).length, wireBytes: bytes(doc) + 1 }));
|
|
80
|
+
requireInput(chunkRecords.every(record => record.wireBytes - 1 <= 5 * 1024 * 1024), 'A chunk exceeds the native 5 MiB document limit.');
|
|
81
|
+
const docs = [...chunkRecords, { doc: original, inputBytes: 0, wireBytes: originalBytes + 1 }];
|
|
82
|
+
return { file, docs };
|
|
83
|
+
}
|
package/src/search.ts
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { MemDriveError, requireInput } from './errors.js';
|
|
2
|
+
import { keysOnly } from './config.js';
|
|
3
|
+
import type { ConnectionOptions, MetadataSchema, SearchFilesRequest } from './types.js';
|
|
4
|
+
|
|
5
|
+
type NativeQuery = ReturnType<ConnectionOptions['client']['collection']>['query'];
|
|
6
|
+
type QueryCollectionInput = Parameters<NativeQuery>[0];
|
|
7
|
+
type QueryCollectionResponse = Awaited<ReturnType<NativeQuery>>;
|
|
8
|
+
|
|
9
|
+
const encoder = new TextEncoder();
|
|
10
|
+
const reranker = { provider: 'typesafe', model: 'jev-1.13.0' } as const;
|
|
11
|
+
const usableQuery = (value: unknown): value is string => typeof value === 'string' && value.trim().length > 0
|
|
12
|
+
&& !/\p{Surrogate}/u.test(value) && encoder.encode(value).length <= 8192;
|
|
13
|
+
|
|
14
|
+
export function searchQuery(request: SearchFilesRequest, schema: MetadataSchema): QueryCollectionInput {
|
|
15
|
+
requireInput(keysOnly(request, ['query', 'embeddingQuery', 'limit', 'rerank']) && usableQuery(request.query),
|
|
16
|
+
'Search requires a nonblank query of at most 8192 UTF-8 bytes.');
|
|
17
|
+
const { query, embeddingQuery = query, limit = 10, rerank = true } = request;
|
|
18
|
+
requireInput(usableQuery(embeddingQuery), 'Embedding query must be nonblank text of at most 8192 UTF-8 bytes.');
|
|
19
|
+
requireInput(Number.isSafeInteger(limit) && limit >= 1 && limit <= 1000, 'Search limit must be 1–1000.');
|
|
20
|
+
requireInput(typeof rerank === 'boolean' || keysOnly(rerank, ['fields', 'onFailure']), 'Invalid reranking options.');
|
|
21
|
+
const fields = typeof rerank === 'boolean' ? ['content'] : rerank.fields === undefined ? ['content'] : rerank.fields;
|
|
22
|
+
const onFailure = typeof rerank === 'boolean' ? 'error' : rerank.onFailure === undefined ? 'error' : rerank.onFailure;
|
|
23
|
+
requireInput(Array.isArray(fields) && fields.length >= 1 && fields.length <= 8 && new Set(fields).size === fields.length
|
|
24
|
+
&& [...fields].every(field => field === 'content' || (typeof field === 'string' && field.startsWith('metadata.')
|
|
25
|
+
&& Object.hasOwn(schema, field.slice(9)) && schema[field.slice(9)]?.type === 'string')),
|
|
26
|
+
'Reranking fields must be distinct content or declared string metadata paths.');
|
|
27
|
+
requireInput(onFailure === 'error' || onFailure === 'returnOriginal', 'Invalid reranking failure policy.');
|
|
28
|
+
const candidates = Math.max(200, limit);
|
|
29
|
+
const filter = { queryString: { query: '"chunk"', defaultField: 'kind' } };
|
|
30
|
+
return {
|
|
31
|
+
size: limit,
|
|
32
|
+
query: { bayesian: [
|
|
33
|
+
{ knn: { field: 'embedding', queryText: embeddingQuery, k: candidates, filter } },
|
|
34
|
+
{ bool: [
|
|
35
|
+
{ ...filter, occur: 'filter' },
|
|
36
|
+
{ queryString: { query, defaultField: 'content', skipSyntax: true }, occur: 'must' },
|
|
37
|
+
] },
|
|
38
|
+
] },
|
|
39
|
+
...(rerank === false ? { candidateSize: candidates }
|
|
40
|
+
: { rerank: { ...reranker, queryText: embeddingQuery, fields: [...fields], candidateSize: candidates, onFailure } }),
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export function validateReranking(response: QueryCollectionResponse, body: QueryCollectionInput): boolean {
|
|
45
|
+
if (!body.rerank) return false;
|
|
46
|
+
const result = response.rerank;
|
|
47
|
+
if (result?.provider !== reranker.provider || result.model !== reranker.model)
|
|
48
|
+
throw new MemDriveError('RERANK_FAILED', 'Native reranking was not confirmed.');
|
|
49
|
+
const validCount = Number.isSafeInteger(result.candidateCount) && result.candidateCount >= response.docs.length
|
|
50
|
+
&& result.candidateCount <= body.rerank.candidateSize!;
|
|
51
|
+
if (result.status === 'applied' && validCount && result.candidateCount === result.scoredCount
|
|
52
|
+
&& response.docs.every(hit => typeof hit?.score === 'number' && Number.isFinite(hit.score) && hit.score >= 0 && hit.score <= 1)) return true;
|
|
53
|
+
if (result.status === 'skipped' && result.reason === 'noCandidates' && result.candidateCount === 0
|
|
54
|
+
&& result.scoredCount === 0 && response.docs.length === 0) return false;
|
|
55
|
+
if (body.rerank.onFailure === 'returnOriginal' && result.status === 'fallback' && validCount && result.scoredCount === 0) return false;
|
|
56
|
+
throw new MemDriveError('RERANK_FAILED', 'Native reranking was not confirmed.');
|
|
57
|
+
}
|
package/src/types.ts
CHANGED
|
@@ -1,273 +1,94 @@
|
|
|
1
1
|
import type { LambdaDBClient } from '@functional-systems/lambdadb';
|
|
2
|
-
|
|
3
|
-
export
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
}>;
|
|
2
|
+
|
|
3
|
+
export interface ConnectionOptions {
|
|
4
|
+
client: LambdaDBClient;
|
|
5
|
+
name: string;
|
|
6
|
+
}
|
|
7
|
+
|
|
9
8
|
export interface RequestOptions {
|
|
10
9
|
signal?: AbortSignal | undefined;
|
|
11
10
|
}
|
|
12
|
-
|
|
13
|
-
|
|
11
|
+
|
|
12
|
+
export interface OpenFileStoreOptions extends ConnectionOptions {
|
|
13
|
+
at?: { fork: string } | undefined;
|
|
14
14
|
}
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
similarity: 'cosine';
|
|
21
|
-
}
|
|
22
|
-
export interface ExternalEmbedding {
|
|
23
|
-
mode: 'external';
|
|
24
|
-
dimensions: number;
|
|
25
|
-
similarity: 'cosine';
|
|
26
|
-
identity: string;
|
|
27
|
-
}
|
|
28
|
-
export type EmbeddingOptions = {
|
|
29
|
-
mode: 'managed';
|
|
30
|
-
} | ExternalEmbedding;
|
|
31
|
-
export type EmbeddingConfig = ManagedEmbedding | ExternalEmbedding;
|
|
32
|
-
export interface MemAttributes {
|
|
33
|
-
timestamp?: string | undefined;
|
|
34
|
-
metadata?: Metadata | undefined;
|
|
35
|
-
}
|
|
36
|
-
export interface TextMemInput extends MemAttributes {
|
|
37
|
-
id?: string | undefined;
|
|
38
|
-
content: string;
|
|
39
|
-
embedding?: never;
|
|
40
|
-
}
|
|
41
|
-
export interface VectorMemInput extends MemAttributes {
|
|
42
|
-
id?: string | undefined;
|
|
43
|
-
embedding: number[];
|
|
44
|
-
content?: never;
|
|
45
|
-
}
|
|
46
|
-
export type MemInput = TextMemInput | VectorMemInput;
|
|
47
|
-
export interface TextMem extends MemAttributes {
|
|
48
|
-
id: string;
|
|
15
|
+
|
|
16
|
+
export interface DriveFile {
|
|
17
|
+
fileId: string;
|
|
18
|
+
modifiedAt: string;
|
|
19
|
+
path: string;
|
|
49
20
|
content: string;
|
|
21
|
+
metadata?: Metadata | undefined;
|
|
50
22
|
}
|
|
51
|
-
|
|
52
|
-
export
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
export type Mem = TextMem | VectorMem;
|
|
57
|
-
export interface TextMatch extends MemAttributes {
|
|
58
|
-
kind: 'mem' | 'chunk';
|
|
59
|
-
memId: string;
|
|
23
|
+
|
|
24
|
+
export type ReadFileRequest = { fileId: string; path?: never } | { path: string; fileId?: never };
|
|
25
|
+
export interface CreateFileRequest {
|
|
26
|
+
op: 'create';
|
|
27
|
+
path: string;
|
|
60
28
|
content: string;
|
|
61
|
-
|
|
62
|
-
end: number;
|
|
63
|
-
}
|
|
64
|
-
export interface VectorMatch extends MemAttributes {
|
|
65
|
-
kind: 'mem';
|
|
66
|
-
memId: string;
|
|
67
|
-
content?: never;
|
|
68
|
-
start?: never;
|
|
69
|
-
end?: never;
|
|
70
|
-
}
|
|
71
|
-
export type Match = TextMatch | VectorMatch;
|
|
72
|
-
export type FilterScalar = string | number | boolean;
|
|
73
|
-
export type Filter = {
|
|
74
|
-
and: Filter[];
|
|
75
|
-
or?: never;
|
|
76
|
-
field?: never;
|
|
77
|
-
op?: never;
|
|
78
|
-
value?: never;
|
|
79
|
-
} | {
|
|
80
|
-
or: Filter[];
|
|
81
|
-
and?: never;
|
|
82
|
-
field?: never;
|
|
83
|
-
op?: never;
|
|
84
|
-
value?: never;
|
|
85
|
-
} | {
|
|
86
|
-
field: string;
|
|
87
|
-
op: 'eq' | 'neq' | 'gt' | 'gte' | 'lt' | 'lte';
|
|
88
|
-
value: FilterScalar;
|
|
89
|
-
and?: never;
|
|
90
|
-
or?: never;
|
|
91
|
-
} | {
|
|
92
|
-
field: string;
|
|
93
|
-
op: 'in';
|
|
94
|
-
value: FilterScalar[];
|
|
95
|
-
and?: never;
|
|
96
|
-
or?: never;
|
|
97
|
-
} | {
|
|
98
|
-
field: string;
|
|
99
|
-
op: 'exists';
|
|
100
|
-
value: boolean;
|
|
101
|
-
and?: never;
|
|
102
|
-
or?: never;
|
|
103
|
-
};
|
|
104
|
-
export interface RecallOptions {
|
|
105
|
-
filter?: Filter | undefined;
|
|
106
|
-
excludeIds?: string[] | undefined;
|
|
107
|
-
limit?: number | undefined;
|
|
108
|
-
rerank?: boolean | undefined;
|
|
109
|
-
}
|
|
110
|
-
export interface TextRecallRequest extends RecallOptions {
|
|
111
|
-
query: string;
|
|
112
|
-
queryVector?: never;
|
|
113
|
-
context?: string | undefined;
|
|
114
|
-
recentDialogue?: Array<{
|
|
115
|
-
role: 'user' | 'assistant';
|
|
116
|
-
content: string;
|
|
117
|
-
}> | undefined;
|
|
118
|
-
}
|
|
119
|
-
export interface VectorRecallRequest extends Omit<RecallOptions, 'rerank'> {
|
|
120
|
-
queryVector: number[];
|
|
121
|
-
query?: never;
|
|
122
|
-
rerank?: false | undefined;
|
|
123
|
-
context?: never;
|
|
124
|
-
recentDialogue?: never;
|
|
125
|
-
}
|
|
126
|
-
export type RecallRequest = TextRecallRequest | VectorRecallRequest;
|
|
127
|
-
export interface Usage {
|
|
128
|
-
elapsedMs: number;
|
|
129
|
-
databaseRequests: number;
|
|
130
|
-
}
|
|
131
|
-
export interface RecallResult {
|
|
132
|
-
matches: Match[];
|
|
133
|
-
usage: Usage;
|
|
134
|
-
}
|
|
135
|
-
export interface RecallFailure {
|
|
136
|
-
error: {
|
|
137
|
-
code: string;
|
|
138
|
-
message: string;
|
|
139
|
-
status?: number;
|
|
140
|
-
};
|
|
141
|
-
usage: Usage;
|
|
142
|
-
}
|
|
143
|
-
export interface BatchRecallResult {
|
|
144
|
-
results: Array<RecallResult | RecallFailure>;
|
|
145
|
-
}
|
|
146
|
-
export interface Observation {
|
|
147
|
-
type: 'retrieved';
|
|
148
|
-
/** Diagnostic physical evidence IDs, not canonical memId values. */
|
|
149
|
-
ids: string[];
|
|
150
|
-
matches: Match[];
|
|
151
|
-
requestIndex?: number | undefined;
|
|
152
|
-
}
|
|
153
|
-
export type RecordEvent = Record<string, unknown> & {
|
|
154
|
-
phase: string;
|
|
155
|
-
requestIndex?: number | undefined;
|
|
156
|
-
};
|
|
157
|
-
export interface RuntimeOptions {
|
|
158
|
-
maxConcurrentRequests?: number | undefined;
|
|
159
|
-
observe?: ((event: Observation) => void | Promise<void>) | undefined;
|
|
160
|
-
record?: ((event: RecordEvent) => void | Promise<void>) | undefined;
|
|
161
|
-
}
|
|
162
|
-
export interface ConnectionOptions {
|
|
163
|
-
client: LambdaDBClient;
|
|
164
|
-
name: string;
|
|
29
|
+
metadata?: Metadata | undefined;
|
|
165
30
|
}
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
31
|
+
|
|
32
|
+
interface FileView {
|
|
33
|
+
readonly name: string;
|
|
34
|
+
describe(options?: RequestOptions): Promise<FileStoreDescription>;
|
|
35
|
+
search(request: SearchFilesRequest, options?: RequestOptions): Promise<SearchFilesResult>;
|
|
36
|
+
read(request: ReadFileRequest, options?: RequestOptions): Promise<DriveFile | null>;
|
|
37
|
+
read(request: { paths: string[] }, options?: RequestOptions): Promise<DriveFile[]>;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export interface MainFileDrive extends FileView {
|
|
41
|
+
readonly at: { readonly kind: 'main' };
|
|
42
|
+
fork(request: { name: string }, options?: RequestOptions): Promise<WorkingFileDrive>;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export interface WorkingFileDrive extends FileView {
|
|
46
|
+
readonly at: { readonly kind: 'fork'; readonly name: string };
|
|
47
|
+
write(request: CreateFileRequest, options?: RequestOptions): Promise<DriveFile>;
|
|
48
|
+
write(request: CreateFileRequest[], options?: RequestOptions): Promise<DriveFile[]>;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export type FileDrive = MainFileDrive | WorkingFileDrive;
|
|
52
|
+
|
|
53
|
+
export type MetadataType = 'string' | 'number' | 'boolean' | 'datetime';
|
|
54
|
+
export type Metadata = Record<string, string | number | boolean>;
|
|
55
|
+
export type MetadataSchema = Record<string, { type: MetadataType; description?: string | undefined }>;
|
|
56
|
+
|
|
57
|
+
export interface CreateFileStoreOptions extends ConnectionOptions {
|
|
170
58
|
labels?: Record<string, string> | undefined;
|
|
171
|
-
|
|
172
|
-
}
|
|
173
|
-
export interface OpenOptions extends ConnectionOptions, RuntimeOptions {
|
|
174
|
-
branch?: string | undefined;
|
|
59
|
+
metadataSchema?: MetadataSchema | undefined;
|
|
60
|
+
textSearch?: { analyzers: string[] } | undefined;
|
|
175
61
|
}
|
|
176
|
-
|
|
62
|
+
|
|
63
|
+
export interface FileStoreDescription {
|
|
177
64
|
name: string;
|
|
178
|
-
description?: string | undefined;
|
|
179
65
|
labels: Record<string, string>;
|
|
180
66
|
metadataSchema: MetadataSchema;
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
}
|
|
190
|
-
export interface CatalogResult {
|
|
191
|
-
drives: DriveDescriptor[];
|
|
192
|
-
nextCursor?: string | undefined;
|
|
67
|
+
textSearch: { analyzers: string[] };
|
|
68
|
+
embedding: {
|
|
69
|
+
mode: 'managed';
|
|
70
|
+
provider: 'openai';
|
|
71
|
+
model: 'text-embedding-3-small';
|
|
72
|
+
dimensions: 1536;
|
|
73
|
+
similarity: 'cosine';
|
|
74
|
+
};
|
|
193
75
|
}
|
|
194
|
-
|
|
76
|
+
|
|
77
|
+
export interface SearchFilesRequest {
|
|
78
|
+
query: string;
|
|
79
|
+
embeddingQuery?: string | undefined;
|
|
195
80
|
limit?: number | undefined;
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
mems: Mem[];
|
|
200
|
-
nextCursor?: string | undefined;
|
|
201
|
-
}
|
|
202
|
-
export type ForgetRequest = {
|
|
203
|
-
ids: string[];
|
|
204
|
-
filter?: never;
|
|
205
|
-
} | {
|
|
206
|
-
filter: Filter;
|
|
207
|
-
ids?: never;
|
|
208
|
-
};
|
|
209
|
-
export type ConfigureRequest = {
|
|
210
|
-
metadataSchema: MetadataSchema;
|
|
211
|
-
description?: string | undefined;
|
|
212
|
-
} | {
|
|
213
|
-
description: string;
|
|
214
|
-
metadataSchema?: MetadataSchema | undefined;
|
|
215
|
-
};
|
|
216
|
-
export interface BranchRequest {
|
|
217
|
-
name: string;
|
|
218
|
-
source?: {
|
|
219
|
-
branch: string;
|
|
81
|
+
rerank?: boolean | {
|
|
82
|
+
fields?: string[] | undefined;
|
|
83
|
+
onFailure?: 'error' | 'returnOriginal' | undefined;
|
|
220
84
|
} | undefined;
|
|
221
85
|
}
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
fields: string[];
|
|
227
|
-
filter?: Filter | undefined;
|
|
228
|
-
excludeIds?: string[] | undefined;
|
|
229
|
-
/** Values per field: defaults to 10, maximum 100. */
|
|
230
|
-
limit?: number | undefined;
|
|
231
|
-
}
|
|
232
|
-
export interface CountValuesResult {
|
|
233
|
-
values: Record<string, {
|
|
234
|
-
buckets: Array<{ value: string; count: number }>;
|
|
235
|
-
/** Null when the native bucket limit prevents determining completeness. */
|
|
236
|
-
hasMore: boolean | null;
|
|
237
|
-
}>;
|
|
238
|
-
/** Time the cached result was fetched, in ISO 8601 UTC. */
|
|
239
|
-
asOf: string;
|
|
240
|
-
}
|
|
241
|
-
export interface DriveData {
|
|
242
|
-
readonly embedding: Readonly<EmbeddingConfig>;
|
|
243
|
-
remember(request: {
|
|
244
|
-
mems: MemInput[];
|
|
245
|
-
}, options?: RequestOptions): Promise<{
|
|
246
|
-
ids: string[];
|
|
247
|
-
}>;
|
|
248
|
-
read(request: {
|
|
249
|
-
ids: string[];
|
|
250
|
-
}, options?: RequestOptions): Promise<{
|
|
251
|
-
mems: Mem[];
|
|
252
|
-
}>;
|
|
253
|
-
list(request?: ListRequest, options?: RequestOptions): Promise<ListResult>;
|
|
254
|
-
countValues(request: CountValuesRequest, options?: RequestOptions): Promise<CountValuesResult>;
|
|
255
|
-
forget(request: ForgetRequest, options?: RequestOptions): Promise<void>;
|
|
256
|
-
recall(request: RecallRequest, options?: RequestOptions): Promise<RecallResult>;
|
|
257
|
-
recall(request: {
|
|
258
|
-
requests: RecallRequest[];
|
|
259
|
-
}, options?: RequestOptions): Promise<BatchRecallResult>;
|
|
86
|
+
|
|
87
|
+
export interface FileSearchHit extends DriveFile {
|
|
88
|
+
/** Verified UTF-16 offsets into the file's original text. */
|
|
89
|
+
range?: { start: number; end: number } | undefined;
|
|
260
90
|
}
|
|
261
|
-
export interface
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
}): DriveData;
|
|
265
|
-
readonly branches: {
|
|
266
|
-
create(request: BranchRequest, options?: RequestOptions): Promise<Branch>;
|
|
267
|
-
list(options?: RequestOptions): Promise<Branch[]>;
|
|
268
|
-
delete(name: string, options?: RequestOptions): Promise<void>;
|
|
269
|
-
};
|
|
270
|
-
configure(request: ConfigureRequest, options?: RequestOptions): Promise<void>;
|
|
271
|
-
describe(options?: RequestOptions): Promise<DriveDescriptor>;
|
|
272
|
-
destroy(options?: RequestOptions): Promise<void>;
|
|
91
|
+
export interface SearchFilesResult {
|
|
92
|
+
hits: FileSearchHit[];
|
|
93
|
+
reranked: boolean;
|
|
273
94
|
}
|
|
@@ -1,30 +0,0 @@
|
|
|
1
|
-
import type { Filter, Metadata, MetadataSchema, MetadataType } from './types.js';
|
|
2
|
-
export declare const isObject: (value: unknown) => value is Record<string, unknown>;
|
|
3
|
-
export declare const matchAll: () => {
|
|
4
|
-
queryString: {
|
|
5
|
-
query: string;
|
|
6
|
-
};
|
|
7
|
-
};
|
|
8
|
-
export declare function dateValue(value: unknown): bigint | undefined;
|
|
9
|
-
export declare function validateDescription(description: unknown): string | undefined;
|
|
10
|
-
export declare function validateMetadataSchema(schema: unknown): MetadataSchema;
|
|
11
|
-
export declare function fieldType(field: unknown, schema: MetadataSchema): MetadataType | undefined;
|
|
12
|
-
export declare function validScalar(value: unknown, type: MetadataType | undefined): boolean;
|
|
13
|
-
export declare function validateFilter(filter: unknown, schema: MetadataSchema): Filter | undefined;
|
|
14
|
-
export interface NativeFilter {
|
|
15
|
-
queryString?: {
|
|
16
|
-
query: string;
|
|
17
|
-
defaultField?: string;
|
|
18
|
-
};
|
|
19
|
-
bool?: NativeFilter[];
|
|
20
|
-
occur?: string | undefined;
|
|
21
|
-
}
|
|
22
|
-
export declare function compileFilter(filter: unknown, schema: MetadataSchema): NativeFilter;
|
|
23
|
-
export declare function buildFilter(filters?: unknown[], excludeIds?: string[], schema?: MetadataSchema): NativeFilter;
|
|
24
|
-
export declare function buildSearchFilter(filters?: unknown[], excludeIds?: string[], seenChunkIds?: string[], schema?: MetadataSchema): NativeFilter;
|
|
25
|
-
export declare function matchesFilter(mem: {
|
|
26
|
-
id: string;
|
|
27
|
-
timestamp?: string | undefined;
|
|
28
|
-
metadata?: Metadata | undefined;
|
|
29
|
-
}, filter: Filter | undefined, schema: MetadataSchema): boolean;
|
|
30
|
-
//# sourceMappingURL=filter.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"filter.d.ts","sourceRoot":"","sources":["../../src/filter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAgB,QAAQ,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAI/F,eAAO,MAAM,QAAQ,GAAI,OAAO,OAAO,KAAG,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,CACsB,CAAC;AAGhG,eAAO,MAAM,QAAQ;;;;CAA4C,CAAC;AAIlE,wBAAgB,SAAS,CAAC,KAAK,EAAE,OAAO,sBAavC;AAED,wBAAgB,mBAAmB,CAAC,WAAW,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAI5E;AAED,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,OAAO,GAAG,cAAc,CAUtE;AAED,wBAAgB,SAAS,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,cAAc,GAAG,YAAY,GAAG,SAAS,CAQ1F;AAED,wBAAgB,WAAW,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY,GAAG,SAAS,WAKzE;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,cAAc,GAAG,MAAM,GAAG,SAAS,CA4B1F;AAED,MAAM,WAAW,YAAY;IAAG,WAAW,CAAC,EAAE;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAA;KAAC,CAAC;IAAC,IAAI,CAAC,EAAE,YAAY,EAAE,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;CAAE;AAgDzI,wBAAgB,aAAa,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,cAAc,GAAG,YAAY,CAEnF;AAED,wBAAgB,WAAW,CAAC,OAAO,GAAE,OAAO,EAAO,EAAE,UAAU,GAAE,MAAM,EAAO,EAAE,MAAM,GAAE,cAAmB,GAAG,YAAY,CAQzH;AAED,wBAAgB,iBAAiB,CAAC,OAAO,GAAE,OAAO,EAAO,EAAE,UAAU,GAAE,MAAM,EAAO,EAAE,YAAY,GAAE,MAAM,EAAO,EAAE,MAAM,GAAE,cAAmB,GAAG,YAAY,CAO5J;AAED,wBAAgB,aAAa,CAAC,GAAG,EAAE;IAAC,EAAE,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAAC,QAAQ,CAAC,EAAE,QAAQ,GAAG,SAAS,CAAA;CAAC,EAAE,MAAM,EAAE,MAAM,GAAG,SAAS,EAAE,MAAM,EAAE,cAAc,GAAG,OAAO,CAyB7K"}
|