@functional-systems/memdrive 0.1.1-dev.2 → 0.1.1-dev.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (189) hide show
  1. package/CHANGELOG.md +6 -0
  2. package/README.md +20 -279
  3. package/dist/commonjs/chunking.d.ts +1 -2
  4. package/dist/commonjs/chunking.d.ts.map +1 -1
  5. package/dist/commonjs/chunking.js +2 -5
  6. package/dist/commonjs/chunking.js.map +1 -1
  7. package/dist/commonjs/config.d.ts +17 -0
  8. package/dist/commonjs/config.d.ts.map +1 -0
  9. package/dist/commonjs/config.js +116 -0
  10. package/dist/commonjs/config.js.map +1 -0
  11. package/dist/commonjs/errors.d.ts +10 -6
  12. package/dist/commonjs/errors.d.ts.map +1 -1
  13. package/dist/commonjs/errors.js +25 -5
  14. package/dist/commonjs/errors.js.map +1 -1
  15. package/dist/commonjs/index.d.ts +15 -5
  16. package/dist/commonjs/index.d.ts.map +1 -1
  17. package/dist/commonjs/index.js +332 -280
  18. package/dist/commonjs/index.js.map +1 -1
  19. package/dist/commonjs/records.d.ts +44 -0
  20. package/dist/commonjs/records.d.ts.map +1 -0
  21. package/dist/commonjs/records.js +88 -0
  22. package/dist/commonjs/records.js.map +1 -0
  23. package/dist/commonjs/search.d.ts +8 -0
  24. package/dist/commonjs/search.d.ts.map +1 -0
  25. package/dist/commonjs/search.js +56 -0
  26. package/dist/commonjs/search.js.map +1 -0
  27. package/dist/commonjs/types.d.ts +84 -252
  28. package/dist/commonjs/types.d.ts.map +1 -1
  29. package/dist/esm/chunking.d.ts +1 -2
  30. package/dist/esm/chunking.d.ts.map +1 -1
  31. package/dist/esm/chunking.js +2 -4
  32. package/dist/esm/chunking.js.map +1 -1
  33. package/dist/esm/config.d.ts +17 -0
  34. package/dist/esm/config.d.ts.map +1 -0
  35. package/dist/esm/config.js +107 -0
  36. package/dist/esm/config.js.map +1 -0
  37. package/dist/esm/errors.d.ts +10 -6
  38. package/dist/esm/errors.d.ts.map +1 -1
  39. package/dist/esm/errors.js +22 -4
  40. package/dist/esm/errors.js.map +1 -1
  41. package/dist/esm/index.d.ts +15 -5
  42. package/dist/esm/index.d.ts.map +1 -1
  43. package/dist/esm/index.js +332 -278
  44. package/dist/esm/index.js.map +1 -1
  45. package/dist/esm/records.d.ts +44 -0
  46. package/dist/esm/records.d.ts.map +1 -0
  47. package/dist/esm/records.js +77 -0
  48. package/dist/esm/records.js.map +1 -0
  49. package/dist/esm/search.d.ts +8 -0
  50. package/dist/esm/search.d.ts.map +1 -0
  51. package/dist/esm/search.js +52 -0
  52. package/dist/esm/search.js.map +1 -0
  53. package/dist/esm/types.d.ts +84 -252
  54. package/dist/esm/types.d.ts.map +1 -1
  55. package/package.json +7 -42
  56. package/src/chunking.ts +2 -3
  57. package/src/config.ts +121 -0
  58. package/src/errors.ts +20 -6
  59. package/src/index.ts +352 -260
  60. package/src/records.ts +83 -0
  61. package/src/search.ts +57 -0
  62. package/src/types.ts +74 -253
  63. package/dist/commonjs/filter.d.ts +0 -30
  64. package/dist/commonjs/filter.d.ts.map +0 -1
  65. package/dist/commonjs/filter.js +0 -224
  66. package/dist/commonjs/filter.js.map +0 -1
  67. package/dist/commonjs/recall-input.d.ts +0 -8
  68. package/dist/commonjs/recall-input.d.ts.map +0 -1
  69. package/dist/commonjs/recall-input.js +0 -45
  70. package/dist/commonjs/recall-input.js.map +0 -1
  71. package/dist/commonjs/retriever/index.d.ts +0 -7
  72. package/dist/commonjs/retriever/index.d.ts.map +0 -1
  73. package/dist/commonjs/retriever/index.js +0 -167
  74. package/dist/commonjs/retriever/index.js.map +0 -1
  75. package/dist/commonjs/retriever/scope.d.ts +0 -5
  76. package/dist/commonjs/retriever/scope.d.ts.map +0 -1
  77. package/dist/commonjs/retriever/scope.js +0 -69
  78. package/dist/commonjs/retriever/scope.js.map +0 -1
  79. package/dist/commonjs/retriever/types.d.ts +0 -89
  80. package/dist/commonjs/retriever/types.d.ts.map +0 -1
  81. package/dist/commonjs/retriever/types.js +0 -3
  82. package/dist/commonjs/retriever/types.js.map +0 -1
  83. package/dist/commonjs/storage.d.ts +0 -274
  84. package/dist/commonjs/storage.d.ts.map +0 -1
  85. package/dist/commonjs/storage.js +0 -861
  86. package/dist/commonjs/storage.js.map +0 -1
  87. package/dist/commonjs/tools/definitions.d.ts +0 -5
  88. package/dist/commonjs/tools/definitions.d.ts.map +0 -1
  89. package/dist/commonjs/tools/definitions.js +0 -147
  90. package/dist/commonjs/tools/definitions.js.map +0 -1
  91. package/dist/commonjs/tools/evidence-ids.d.ts +0 -7
  92. package/dist/commonjs/tools/evidence-ids.d.ts.map +0 -1
  93. package/dist/commonjs/tools/evidence-ids.js +0 -71
  94. package/dist/commonjs/tools/evidence-ids.js.map +0 -1
  95. package/dist/commonjs/tools/factory.d.ts +0 -7
  96. package/dist/commonjs/tools/factory.d.ts.map +0 -1
  97. package/dist/commonjs/tools/factory.js +0 -169
  98. package/dist/commonjs/tools/factory.js.map +0 -1
  99. package/dist/commonjs/tools/index.d.ts +0 -6
  100. package/dist/commonjs/tools/index.d.ts.map +0 -1
  101. package/dist/commonjs/tools/index.js +0 -25
  102. package/dist/commonjs/tools/index.js.map +0 -1
  103. package/dist/commonjs/tools/internal-types.d.ts +0 -18
  104. package/dist/commonjs/tools/internal-types.d.ts.map +0 -1
  105. package/dist/commonjs/tools/internal-types.js +0 -3
  106. package/dist/commonjs/tools/internal-types.js.map +0 -1
  107. package/dist/commonjs/tools/short-ids.d.ts +0 -8
  108. package/dist/commonjs/tools/short-ids.d.ts.map +0 -1
  109. package/dist/commonjs/tools/short-ids.js +0 -81
  110. package/dist/commonjs/tools/short-ids.js.map +0 -1
  111. package/dist/commonjs/tools/types.d.ts +0 -53
  112. package/dist/commonjs/tools/types.d.ts.map +0 -1
  113. package/dist/commonjs/tools/types.js +0 -3
  114. package/dist/commonjs/tools/types.js.map +0 -1
  115. package/dist/commonjs/values.d.ts +0 -17
  116. package/dist/commonjs/values.d.ts.map +0 -1
  117. package/dist/commonjs/values.js +0 -90
  118. package/dist/commonjs/values.js.map +0 -1
  119. package/dist/esm/filter.d.ts +0 -30
  120. package/dist/esm/filter.d.ts.map +0 -1
  121. package/dist/esm/filter.js +0 -209
  122. package/dist/esm/filter.js.map +0 -1
  123. package/dist/esm/recall-input.d.ts +0 -8
  124. package/dist/esm/recall-input.d.ts.map +0 -1
  125. package/dist/esm/recall-input.js +0 -40
  126. package/dist/esm/recall-input.js.map +0 -1
  127. package/dist/esm/retriever/index.d.ts +0 -7
  128. package/dist/esm/retriever/index.d.ts.map +0 -1
  129. package/dist/esm/retriever/index.js +0 -163
  130. package/dist/esm/retriever/index.js.map +0 -1
  131. package/dist/esm/retriever/scope.d.ts +0 -5
  132. package/dist/esm/retriever/scope.d.ts.map +0 -1
  133. package/dist/esm/retriever/scope.js +0 -66
  134. package/dist/esm/retriever/scope.js.map +0 -1
  135. package/dist/esm/retriever/types.d.ts +0 -89
  136. package/dist/esm/retriever/types.d.ts.map +0 -1
  137. package/dist/esm/retriever/types.js +0 -2
  138. package/dist/esm/retriever/types.js.map +0 -1
  139. package/dist/esm/storage.d.ts +0 -274
  140. package/dist/esm/storage.d.ts.map +0 -1
  141. package/dist/esm/storage.js +0 -845
  142. package/dist/esm/storage.js.map +0 -1
  143. package/dist/esm/tools/definitions.d.ts +0 -5
  144. package/dist/esm/tools/definitions.d.ts.map +0 -1
  145. package/dist/esm/tools/definitions.js +0 -144
  146. package/dist/esm/tools/definitions.js.map +0 -1
  147. package/dist/esm/tools/evidence-ids.d.ts +0 -7
  148. package/dist/esm/tools/evidence-ids.d.ts.map +0 -1
  149. package/dist/esm/tools/evidence-ids.js +0 -68
  150. package/dist/esm/tools/evidence-ids.js.map +0 -1
  151. package/dist/esm/tools/factory.d.ts +0 -7
  152. package/dist/esm/tools/factory.d.ts.map +0 -1
  153. package/dist/esm/tools/factory.js +0 -166
  154. package/dist/esm/tools/factory.js.map +0 -1
  155. package/dist/esm/tools/index.d.ts +0 -6
  156. package/dist/esm/tools/index.d.ts.map +0 -1
  157. package/dist/esm/tools/index.js +0 -22
  158. package/dist/esm/tools/index.js.map +0 -1
  159. package/dist/esm/tools/internal-types.d.ts +0 -18
  160. package/dist/esm/tools/internal-types.d.ts.map +0 -1
  161. package/dist/esm/tools/internal-types.js +0 -2
  162. package/dist/esm/tools/internal-types.js.map +0 -1
  163. package/dist/esm/tools/short-ids.d.ts +0 -8
  164. package/dist/esm/tools/short-ids.d.ts.map +0 -1
  165. package/dist/esm/tools/short-ids.js +0 -78
  166. package/dist/esm/tools/short-ids.js.map +0 -1
  167. package/dist/esm/tools/types.d.ts +0 -53
  168. package/dist/esm/tools/types.d.ts.map +0 -1
  169. package/dist/esm/tools/types.js +0 -2
  170. package/dist/esm/tools/types.js.map +0 -1
  171. package/dist/esm/values.d.ts +0 -17
  172. package/dist/esm/values.d.ts.map +0 -1
  173. package/dist/esm/values.js +0 -87
  174. package/dist/esm/values.js.map +0 -1
  175. package/docs/tools.md +0 -264
  176. package/src/filter.ts +0 -189
  177. package/src/recall-input.ts +0 -46
  178. package/src/retriever/index.ts +0 -142
  179. package/src/retriever/scope.ts +0 -56
  180. package/src/retriever/types.ts +0 -68
  181. package/src/storage.ts +0 -765
  182. package/src/tools/definitions.ts +0 -175
  183. package/src/tools/evidence-ids.ts +0 -57
  184. package/src/tools/factory.ts +0 -153
  185. package/src/tools/index.ts +0 -25
  186. package/src/tools/internal-types.ts +0 -12
  187. package/src/tools/short-ids.ts +0 -58
  188. package/src/tools/types.ts +0 -48
  189. package/src/values.ts +0 -103
package/src/records.ts ADDED
@@ -0,0 +1,83 @@
1
+ import { MemDriveError, requireInput } from './errors.js';
2
+ import { isObject, keysOnly, metadataValues, validDate } from './config.js';
3
+ import { chunkSpans } from './chunking.js';
4
+ import type { CreateFileRequest, DriveFile, MetadataSchema } from './types.js';
5
+
6
+ const encoder = new TextEncoder();
7
+ export const bytes = (value: unknown) => encoder.encode(JSON.stringify(value)).length;
8
+ const fileIdPattern = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
9
+ export const validFileId = (value: unknown): value is string => typeof value === 'string' && fileIdPattern.test(value);
10
+ export const validPath = (value: unknown): value is string => typeof value === 'string' && value.startsWith('/')
11
+ && !/[\\\p{Cc}\p{Surrogate}]/u.test(value) && encoder.encode(value).length <= 4096
12
+ && value.slice(1).split('/').every(part => part !== '' && part !== '.' && part !== '..');
13
+ export const originalId = (fileId: string) => `file:${fileId}`;
14
+
15
+ export function requirePath(path: unknown): asserts path is string {
16
+ requireInput(validPath(path), 'File path must be absolute, at most 4096 UTF-8 bytes, and contain valid nonempty components.');
17
+ }
18
+
19
+ export function resultDocs(response: unknown, collection: string, maximum: number): Record<string, unknown>[] {
20
+ if (!isObject(response) || response.isDocsInline === false || !Array.isArray(response.docs) || response.docs.length > maximum)
21
+ throw new MemDriveError('INVALID_RESULT', 'Read returned an invalid or ambiguous document envelope.');
22
+ return response.docs.map(hit => {
23
+ if (!isObject(hit) || hit.collection !== collection || !isObject(hit.doc))
24
+ throw new MemDriveError('INVALID_RESULT', 'Read returned an invalid document.');
25
+ return hit.doc;
26
+ });
27
+ }
28
+
29
+ export function decodeFile(doc: Record<string, unknown>, schema: MetadataSchema): DriveFile {
30
+ if (doc.kind !== 'file' || !validFileId(doc.fileId) || doc.id !== originalId(doc.fileId)
31
+ || !validPath(doc.path) || typeof doc.body !== 'string' || /\p{Surrogate}/u.test(doc.body) || !validDate(doc.modifiedAt))
32
+ throw new MemDriveError('INVALID_RESULT', 'Read returned an invalid file original.');
33
+ try {
34
+ const metadata = metadataValues(doc.metadata, schema);
35
+ return { fileId: doc.fileId, path: doc.path, modifiedAt: doc.modifiedAt, content: doc.body,
36
+ ...(metadata === undefined ? {} : { metadata }) };
37
+ } catch (cause) {
38
+ throw new MemDriveError('INVALID_RESULT', 'Read returned invalid file metadata.', { cause });
39
+ }
40
+ }
41
+
42
+ async function contentHash(content: string): Promise<string> {
43
+ const hash = new Uint8Array(await globalThis.crypto.subtle.digest('SHA-256', encoder.encode(content)));
44
+ return [...hash].map(byte => byte.toString(16).padStart(2, '0')).join('');
45
+ }
46
+
47
+ export async function fileRecords(request: CreateFileRequest, schema: MetadataSchema) {
48
+ requireInput(keysOnly(request, ['op', 'path', 'content', 'metadata']) && request.op === 'create',
49
+ 'Write requires a create operation with path and original text.');
50
+ const { path, content } = request;
51
+ requirePath(path);
52
+ requireInput(typeof content === 'string' && !/\p{Surrogate}/u.test(content), 'File content must be UTF-8 text.');
53
+ const metadata = metadataValues(request.metadata, schema);
54
+ const fileId = globalThis.crypto.randomUUID();
55
+ const modifiedAt = new Date().toISOString();
56
+ const parts = path.slice(1).split('/');
57
+ const fileName = parts.pop()!;
58
+ const ancestorPaths = ['/'];
59
+ let ancestor = '/';
60
+ for (const part of parts) {
61
+ ancestor += part + '/';
62
+ ancestorPaths.push(ancestor);
63
+ }
64
+ const shared = { fileId, path, fileName, ancestorPaths, modifiedAt, ...(metadata === undefined ? {} : { metadata }) };
65
+ const original = { id: originalId(fileId), kind: 'file', ...shared, body: content, changed: true };
66
+ const originalBytes = bytes(original);
67
+ requireInput(originalBytes <= 5 * 1024 * 1024, 'The file original exceeds the native 5 MiB document limit.');
68
+ const spans = chunkSpans(content).filter(span => span.text.trim());
69
+ const occurrences = new Map<string, number>();
70
+ const hashes = await Promise.all(spans.map(span => contentHash(span.text)));
71
+ const chunks = spans.map(({ text, start, end }, index) => {
72
+ const hash = hashes[index]!;
73
+ const occurrence = occurrences.get(hash) ?? 0;
74
+ occurrences.set(hash, occurrence + 1);
75
+ return { id: `chunk:${fileId}:${hash}:${occurrence}`, kind: 'chunk', ...shared, content: text, start, end };
76
+ });
77
+ const file: DriveFile = { fileId, path, modifiedAt, content, ...(metadata === undefined ? {} : { metadata }) };
78
+ // Large files stage their chunks before making the original visible.
79
+ const chunkRecords = chunks.map(doc => ({ doc, inputBytes: encoder.encode(doc.content).length, wireBytes: bytes(doc) + 1 }));
80
+ requireInput(chunkRecords.every(record => record.wireBytes - 1 <= 5 * 1024 * 1024), 'A chunk exceeds the native 5 MiB document limit.');
81
+ const docs = [...chunkRecords, { doc: original, inputBytes: 0, wireBytes: originalBytes + 1 }];
82
+ return { file, docs };
83
+ }
package/src/search.ts ADDED
@@ -0,0 +1,57 @@
1
+ import { MemDriveError, requireInput } from './errors.js';
2
+ import { keysOnly } from './config.js';
3
+ import type { ConnectionOptions, MetadataSchema, SearchFilesRequest } from './types.js';
4
+
5
+ type NativeQuery = ReturnType<ConnectionOptions['client']['collection']>['query'];
6
+ type QueryCollectionInput = Parameters<NativeQuery>[0];
7
+ type QueryCollectionResponse = Awaited<ReturnType<NativeQuery>>;
8
+
9
+ const encoder = new TextEncoder();
10
+ const reranker = { provider: 'typesafe', model: 'jev-1.13.0' } as const;
11
+ const usableQuery = (value: unknown): value is string => typeof value === 'string' && value.trim().length > 0
12
+ && !/\p{Surrogate}/u.test(value) && encoder.encode(value).length <= 8192;
13
+
14
+ export function searchQuery(request: SearchFilesRequest, schema: MetadataSchema): QueryCollectionInput {
15
+ requireInput(keysOnly(request, ['query', 'embeddingQuery', 'limit', 'rerank']) && usableQuery(request.query),
16
+ 'Search requires a nonblank query of at most 8192 UTF-8 bytes.');
17
+ const { query, embeddingQuery = query, limit = 10, rerank = true } = request;
18
+ requireInput(usableQuery(embeddingQuery), 'Embedding query must be nonblank text of at most 8192 UTF-8 bytes.');
19
+ requireInput(Number.isSafeInteger(limit) && limit >= 1 && limit <= 1000, 'Search limit must be 1–1000.');
20
+ requireInput(typeof rerank === 'boolean' || keysOnly(rerank, ['fields', 'onFailure']), 'Invalid reranking options.');
21
+ const fields = typeof rerank === 'boolean' ? ['content'] : rerank.fields === undefined ? ['content'] : rerank.fields;
22
+ const onFailure = typeof rerank === 'boolean' ? 'error' : rerank.onFailure === undefined ? 'error' : rerank.onFailure;
23
+ requireInput(Array.isArray(fields) && fields.length >= 1 && fields.length <= 8 && new Set(fields).size === fields.length
24
+ && [...fields].every(field => field === 'content' || (typeof field === 'string' && field.startsWith('metadata.')
25
+ && Object.hasOwn(schema, field.slice(9)) && schema[field.slice(9)]?.type === 'string')),
26
+ 'Reranking fields must be distinct content or declared string metadata paths.');
27
+ requireInput(onFailure === 'error' || onFailure === 'returnOriginal', 'Invalid reranking failure policy.');
28
+ const candidates = Math.max(200, limit);
29
+ const filter = { queryString: { query: '"chunk"', defaultField: 'kind' } };
30
+ return {
31
+ size: limit,
32
+ query: { bayesian: [
33
+ { knn: { field: 'embedding', queryText: embeddingQuery, k: candidates, filter } },
34
+ { bool: [
35
+ { ...filter, occur: 'filter' },
36
+ { queryString: { query, defaultField: 'content', skipSyntax: true }, occur: 'must' },
37
+ ] },
38
+ ] },
39
+ ...(rerank === false ? { candidateSize: candidates }
40
+ : { rerank: { ...reranker, queryText: embeddingQuery, fields: [...fields], candidateSize: candidates, onFailure } }),
41
+ };
42
+ }
43
+
44
+ export function validateReranking(response: QueryCollectionResponse, body: QueryCollectionInput): boolean {
45
+ if (!body.rerank) return false;
46
+ const result = response.rerank;
47
+ if (result?.provider !== reranker.provider || result.model !== reranker.model)
48
+ throw new MemDriveError('RERANK_FAILED', 'Native reranking was not confirmed.');
49
+ const validCount = Number.isSafeInteger(result.candidateCount) && result.candidateCount >= response.docs.length
50
+ && result.candidateCount <= body.rerank.candidateSize!;
51
+ if (result.status === 'applied' && validCount && result.candidateCount === result.scoredCount
52
+ && response.docs.every(hit => typeof hit?.score === 'number' && Number.isFinite(hit.score) && hit.score >= 0 && hit.score <= 1)) return true;
53
+ if (result.status === 'skipped' && result.reason === 'noCandidates' && result.candidateCount === 0
54
+ && result.scoredCount === 0 && response.docs.length === 0) return false;
55
+ if (body.rerank.onFailure === 'returnOriginal' && result.status === 'fallback' && validCount && result.scoredCount === 0) return false;
56
+ throw new MemDriveError('RERANK_FAILED', 'Native reranking was not confirmed.');
57
+ }
package/src/types.ts CHANGED
@@ -1,273 +1,94 @@
1
1
  import type { LambdaDBClient } from '@functional-systems/lambdadb';
2
- export type MetadataType = 'string' | 'string[]' | 'number' | 'boolean' | 'datetime';
3
- export type MetadataValue = string | string[] | number | boolean;
4
- export type Metadata = Record<string, MetadataValue>;
5
- export type MetadataSchema = Record<string, {
6
- type: MetadataType;
7
- description?: string | undefined;
8
- }>;
2
+
3
+ export interface ConnectionOptions {
4
+ client: LambdaDBClient;
5
+ name: string;
6
+ }
7
+
9
8
  export interface RequestOptions {
10
9
  signal?: AbortSignal | undefined;
11
10
  }
12
- export interface TextSearch {
13
- analyzers: string[];
11
+
12
+ export interface OpenFileStoreOptions extends ConnectionOptions {
13
+ at?: { fork: string } | undefined;
14
14
  }
15
- export interface ManagedEmbedding {
16
- mode: 'managed';
17
- provider: 'openai';
18
- model: 'text-embedding-3-small';
19
- dimensions: 1536;
20
- similarity: 'cosine';
21
- }
22
- export interface ExternalEmbedding {
23
- mode: 'external';
24
- dimensions: number;
25
- similarity: 'cosine';
26
- identity: string;
27
- }
28
- export type EmbeddingOptions = {
29
- mode: 'managed';
30
- } | ExternalEmbedding;
31
- export type EmbeddingConfig = ManagedEmbedding | ExternalEmbedding;
32
- export interface MemAttributes {
33
- timestamp?: string | undefined;
34
- metadata?: Metadata | undefined;
35
- }
36
- export interface TextMemInput extends MemAttributes {
37
- id?: string | undefined;
38
- content: string;
39
- embedding?: never;
40
- }
41
- export interface VectorMemInput extends MemAttributes {
42
- id?: string | undefined;
43
- embedding: number[];
44
- content?: never;
45
- }
46
- export type MemInput = TextMemInput | VectorMemInput;
47
- export interface TextMem extends MemAttributes {
48
- id: string;
15
+
16
+ export interface DriveFile {
17
+ fileId: string;
18
+ modifiedAt: string;
19
+ path: string;
49
20
  content: string;
21
+ metadata?: Metadata | undefined;
50
22
  }
51
- /** External-vector reads return source references and metadata, not vectors. */
52
- export interface VectorMem extends MemAttributes {
53
- id: string;
54
- content?: never;
55
- }
56
- export type Mem = TextMem | VectorMem;
57
- export interface TextMatch extends MemAttributes {
58
- kind: 'mem' | 'chunk';
59
- memId: string;
23
+
24
+ export type ReadFileRequest = { fileId: string; path?: never } | { path: string; fileId?: never };
25
+ export interface CreateFileRequest {
26
+ op: 'create';
27
+ path: string;
60
28
  content: string;
61
- start: number;
62
- end: number;
63
- }
64
- export interface VectorMatch extends MemAttributes {
65
- kind: 'mem';
66
- memId: string;
67
- content?: never;
68
- start?: never;
69
- end?: never;
70
- }
71
- export type Match = TextMatch | VectorMatch;
72
- export type FilterScalar = string | number | boolean;
73
- export type Filter = {
74
- and: Filter[];
75
- or?: never;
76
- field?: never;
77
- op?: never;
78
- value?: never;
79
- } | {
80
- or: Filter[];
81
- and?: never;
82
- field?: never;
83
- op?: never;
84
- value?: never;
85
- } | {
86
- field: string;
87
- op: 'eq' | 'neq' | 'gt' | 'gte' | 'lt' | 'lte';
88
- value: FilterScalar;
89
- and?: never;
90
- or?: never;
91
- } | {
92
- field: string;
93
- op: 'in';
94
- value: FilterScalar[];
95
- and?: never;
96
- or?: never;
97
- } | {
98
- field: string;
99
- op: 'exists';
100
- value: boolean;
101
- and?: never;
102
- or?: never;
103
- };
104
- export interface RecallOptions {
105
- filter?: Filter | undefined;
106
- excludeIds?: string[] | undefined;
107
- limit?: number | undefined;
108
- rerank?: boolean | undefined;
109
- }
110
- export interface TextRecallRequest extends RecallOptions {
111
- query: string;
112
- queryVector?: never;
113
- context?: string | undefined;
114
- recentDialogue?: Array<{
115
- role: 'user' | 'assistant';
116
- content: string;
117
- }> | undefined;
118
- }
119
- export interface VectorRecallRequest extends Omit<RecallOptions, 'rerank'> {
120
- queryVector: number[];
121
- query?: never;
122
- rerank?: false | undefined;
123
- context?: never;
124
- recentDialogue?: never;
125
- }
126
- export type RecallRequest = TextRecallRequest | VectorRecallRequest;
127
- export interface Usage {
128
- elapsedMs: number;
129
- databaseRequests: number;
130
- }
131
- export interface RecallResult {
132
- matches: Match[];
133
- usage: Usage;
134
- }
135
- export interface RecallFailure {
136
- error: {
137
- code: string;
138
- message: string;
139
- status?: number;
140
- };
141
- usage: Usage;
142
- }
143
- export interface BatchRecallResult {
144
- results: Array<RecallResult | RecallFailure>;
145
- }
146
- export interface Observation {
147
- type: 'retrieved';
148
- /** Diagnostic physical evidence IDs, not canonical memId values. */
149
- ids: string[];
150
- matches: Match[];
151
- requestIndex?: number | undefined;
152
- }
153
- export type RecordEvent = Record<string, unknown> & {
154
- phase: string;
155
- requestIndex?: number | undefined;
156
- };
157
- export interface RuntimeOptions {
158
- maxConcurrentRequests?: number | undefined;
159
- observe?: ((event: Observation) => void | Promise<void>) | undefined;
160
- record?: ((event: RecordEvent) => void | Promise<void>) | undefined;
161
- }
162
- export interface ConnectionOptions {
163
- client: LambdaDBClient;
164
- name: string;
29
+ metadata?: Metadata | undefined;
165
30
  }
166
- export interface CreateOptions extends ConnectionOptions, RuntimeOptions {
167
- metadataSchema: MetadataSchema;
168
- textSearch?: TextSearch | undefined;
169
- embedding?: EmbeddingOptions | undefined;
31
+
32
+ interface FileView {
33
+ readonly name: string;
34
+ describe(options?: RequestOptions): Promise<FileStoreDescription>;
35
+ search(request: SearchFilesRequest, options?: RequestOptions): Promise<SearchFilesResult>;
36
+ read(request: ReadFileRequest, options?: RequestOptions): Promise<DriveFile | null>;
37
+ read(request: { paths: string[] }, options?: RequestOptions): Promise<DriveFile[]>;
38
+ }
39
+
40
+ export interface MainFileDrive extends FileView {
41
+ readonly at: { readonly kind: 'main' };
42
+ fork(request: { name: string }, options?: RequestOptions): Promise<WorkingFileDrive>;
43
+ }
44
+
45
+ export interface WorkingFileDrive extends FileView {
46
+ readonly at: { readonly kind: 'fork'; readonly name: string };
47
+ write(request: CreateFileRequest, options?: RequestOptions): Promise<DriveFile>;
48
+ write(request: CreateFileRequest[], options?: RequestOptions): Promise<DriveFile[]>;
49
+ }
50
+
51
+ export type FileDrive = MainFileDrive | WorkingFileDrive;
52
+
53
+ export type MetadataType = 'string' | 'number' | 'boolean' | 'datetime';
54
+ export type Metadata = Record<string, string | number | boolean>;
55
+ export type MetadataSchema = Record<string, { type: MetadataType; description?: string | undefined }>;
56
+
57
+ export interface CreateFileStoreOptions extends ConnectionOptions {
170
58
  labels?: Record<string, string> | undefined;
171
- description?: string | undefined;
172
- }
173
- export interface OpenOptions extends ConnectionOptions, RuntimeOptions {
174
- branch?: string | undefined;
59
+ metadataSchema?: MetadataSchema | undefined;
60
+ textSearch?: { analyzers: string[] } | undefined;
175
61
  }
176
- export interface DriveDescriptor {
62
+
63
+ export interface FileStoreDescription {
177
64
  name: string;
178
- description?: string | undefined;
179
65
  labels: Record<string, string>;
180
66
  metadataSchema: MetadataSchema;
181
- embedding: EmbeddingConfig;
182
- textSearch?: TextSearch | undefined;
183
- }
184
- export interface CatalogRequest {
185
- client: LambdaDBClient;
186
- labels?: Record<string, string> | undefined;
187
- limit?: number | undefined;
188
- cursor?: string | undefined;
189
- }
190
- export interface CatalogResult {
191
- drives: DriveDescriptor[];
192
- nextCursor?: string | undefined;
67
+ textSearch: { analyzers: string[] };
68
+ embedding: {
69
+ mode: 'managed';
70
+ provider: 'openai';
71
+ model: 'text-embedding-3-small';
72
+ dimensions: 1536;
73
+ similarity: 'cosine';
74
+ };
193
75
  }
194
- export interface ListRequest {
76
+
77
+ export interface SearchFilesRequest {
78
+ query: string;
79
+ embeddingQuery?: string | undefined;
195
80
  limit?: number | undefined;
196
- cursor?: string | undefined;
197
- }
198
- export interface ListResult {
199
- mems: Mem[];
200
- nextCursor?: string | undefined;
201
- }
202
- export type ForgetRequest = {
203
- ids: string[];
204
- filter?: never;
205
- } | {
206
- filter: Filter;
207
- ids?: never;
208
- };
209
- export type ConfigureRequest = {
210
- metadataSchema: MetadataSchema;
211
- description?: string | undefined;
212
- } | {
213
- description: string;
214
- metadataSchema?: MetadataSchema | undefined;
215
- };
216
- export interface BranchRequest {
217
- name: string;
218
- source?: {
219
- branch: string;
81
+ rerank?: boolean | {
82
+ fields?: string[] | undefined;
83
+ onFailure?: 'error' | 'returnOriginal' | undefined;
220
84
  } | undefined;
221
85
  }
222
- export type Branch = Awaited<ReturnType<ReturnType<LambdaDBClient['collection']>['branches']['create']>>['branch'];
223
- /** Counts distinct keyword values across original memories, not chunks. */
224
- export interface CountValuesRequest {
225
- /** One to five distinct keyword fields, matching the native facet limit. */
226
- fields: string[];
227
- filter?: Filter | undefined;
228
- excludeIds?: string[] | undefined;
229
- /** Values per field: defaults to 10, maximum 100. */
230
- limit?: number | undefined;
231
- }
232
- export interface CountValuesResult {
233
- values: Record<string, {
234
- buckets: Array<{ value: string; count: number }>;
235
- /** Null when the native bucket limit prevents determining completeness. */
236
- hasMore: boolean | null;
237
- }>;
238
- /** Time the cached result was fetched, in ISO 8601 UTC. */
239
- asOf: string;
240
- }
241
- export interface DriveData {
242
- readonly embedding: Readonly<EmbeddingConfig>;
243
- remember(request: {
244
- mems: MemInput[];
245
- }, options?: RequestOptions): Promise<{
246
- ids: string[];
247
- }>;
248
- read(request: {
249
- ids: string[];
250
- }, options?: RequestOptions): Promise<{
251
- mems: Mem[];
252
- }>;
253
- list(request?: ListRequest, options?: RequestOptions): Promise<ListResult>;
254
- countValues(request: CountValuesRequest, options?: RequestOptions): Promise<CountValuesResult>;
255
- forget(request: ForgetRequest, options?: RequestOptions): Promise<void>;
256
- recall(request: RecallRequest, options?: RequestOptions): Promise<RecallResult>;
257
- recall(request: {
258
- requests: RecallRequest[];
259
- }, options?: RequestOptions): Promise<BatchRecallResult>;
86
+
87
+ export interface FileSearchHit extends DriveFile {
88
+ /** Verified UTF-16 offsets into the file's original text. */
89
+ range?: { start: number; end: number } | undefined;
260
90
  }
261
- export interface MemDrive extends DriveData {
262
- at(reference: {
263
- branch: string;
264
- }): DriveData;
265
- readonly branches: {
266
- create(request: BranchRequest, options?: RequestOptions): Promise<Branch>;
267
- list(options?: RequestOptions): Promise<Branch[]>;
268
- delete(name: string, options?: RequestOptions): Promise<void>;
269
- };
270
- configure(request: ConfigureRequest, options?: RequestOptions): Promise<void>;
271
- describe(options?: RequestOptions): Promise<DriveDescriptor>;
272
- destroy(options?: RequestOptions): Promise<void>;
91
+ export interface SearchFilesResult {
92
+ hits: FileSearchHit[];
93
+ reranked: boolean;
273
94
  }
@@ -1,30 +0,0 @@
1
- import type { Filter, Metadata, MetadataSchema, MetadataType } from './types.js';
2
- export declare const isObject: (value: unknown) => value is Record<string, unknown>;
3
- export declare const matchAll: () => {
4
- queryString: {
5
- query: string;
6
- };
7
- };
8
- export declare function dateValue(value: unknown): bigint | undefined;
9
- export declare function validateDescription(description: unknown): string | undefined;
10
- export declare function validateMetadataSchema(schema: unknown): MetadataSchema;
11
- export declare function fieldType(field: unknown, schema: MetadataSchema): MetadataType | undefined;
12
- export declare function validScalar(value: unknown, type: MetadataType | undefined): boolean;
13
- export declare function validateFilter(filter: unknown, schema: MetadataSchema): Filter | undefined;
14
- export interface NativeFilter {
15
- queryString?: {
16
- query: string;
17
- defaultField?: string;
18
- };
19
- bool?: NativeFilter[];
20
- occur?: string | undefined;
21
- }
22
- export declare function compileFilter(filter: unknown, schema: MetadataSchema): NativeFilter;
23
- export declare function buildFilter(filters?: unknown[], excludeIds?: string[], schema?: MetadataSchema): NativeFilter;
24
- export declare function buildSearchFilter(filters?: unknown[], excludeIds?: string[], seenChunkIds?: string[], schema?: MetadataSchema): NativeFilter;
25
- export declare function matchesFilter(mem: {
26
- id: string;
27
- timestamp?: string | undefined;
28
- metadata?: Metadata | undefined;
29
- }, filter: Filter | undefined, schema: MetadataSchema): boolean;
30
- //# sourceMappingURL=filter.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"filter.d.ts","sourceRoot":"","sources":["../../src/filter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAgB,QAAQ,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAI/F,eAAO,MAAM,QAAQ,GAAI,OAAO,OAAO,KAAG,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,CACsB,CAAC;AAGhG,eAAO,MAAM,QAAQ;;;;CAA4C,CAAC;AAIlE,wBAAgB,SAAS,CAAC,KAAK,EAAE,OAAO,sBAavC;AAED,wBAAgB,mBAAmB,CAAC,WAAW,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAI5E;AAED,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,OAAO,GAAG,cAAc,CAUtE;AAED,wBAAgB,SAAS,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,EAAE,cAAc,GAAG,YAAY,GAAG,SAAS,CAQ1F;AAED,wBAAgB,WAAW,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY,GAAG,SAAS,WAKzE;AAED,wBAAgB,cAAc,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,cAAc,GAAG,MAAM,GAAG,SAAS,CA4B1F;AAED,MAAM,WAAW,YAAY;IAAG,WAAW,CAAC,EAAE;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,YAAY,CAAC,EAAE,MAAM,CAAA;KAAC,CAAC;IAAC,IAAI,CAAC,EAAE,YAAY,EAAE,CAAC;IAAC,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;CAAE;AAgDzI,wBAAgB,aAAa,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,cAAc,GAAG,YAAY,CAEnF;AAED,wBAAgB,WAAW,CAAC,OAAO,GAAE,OAAO,EAAO,EAAE,UAAU,GAAE,MAAM,EAAO,EAAE,MAAM,GAAE,cAAmB,GAAG,YAAY,CAQzH;AAED,wBAAgB,iBAAiB,CAAC,OAAO,GAAE,OAAO,EAAO,EAAE,UAAU,GAAE,MAAM,EAAO,EAAE,YAAY,GAAE,MAAM,EAAO,EAAE,MAAM,GAAE,cAAmB,GAAG,YAAY,CAO5J;AAED,wBAAgB,aAAa,CAAC,GAAG,EAAE;IAAC,EAAE,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAAC,QAAQ,CAAC,EAAE,QAAQ,GAAG,SAAS,CAAA;CAAC,EAAE,MAAM,EAAE,MAAM,GAAG,SAAS,EAAE,MAAM,EAAE,cAAc,GAAG,OAAO,CAyB7K"}