@functional-systems/memdrive 0.1.1-dev.2 → 0.1.1-dev.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/README.md +20 -279
- package/dist/commonjs/chunking.d.ts +1 -2
- package/dist/commonjs/chunking.d.ts.map +1 -1
- package/dist/commonjs/chunking.js +2 -5
- package/dist/commonjs/chunking.js.map +1 -1
- package/dist/commonjs/config.d.ts +17 -0
- package/dist/commonjs/config.d.ts.map +1 -0
- package/dist/commonjs/config.js +116 -0
- package/dist/commonjs/config.js.map +1 -0
- package/dist/commonjs/errors.d.ts +10 -6
- package/dist/commonjs/errors.d.ts.map +1 -1
- package/dist/commonjs/errors.js +25 -5
- package/dist/commonjs/errors.js.map +1 -1
- package/dist/commonjs/index.d.ts +15 -5
- package/dist/commonjs/index.d.ts.map +1 -1
- package/dist/commonjs/index.js +332 -280
- package/dist/commonjs/index.js.map +1 -1
- package/dist/commonjs/records.d.ts +44 -0
- package/dist/commonjs/records.d.ts.map +1 -0
- package/dist/commonjs/records.js +88 -0
- package/dist/commonjs/records.js.map +1 -0
- package/dist/commonjs/search.d.ts +8 -0
- package/dist/commonjs/search.d.ts.map +1 -0
- package/dist/commonjs/search.js +56 -0
- package/dist/commonjs/search.js.map +1 -0
- package/dist/commonjs/types.d.ts +84 -252
- package/dist/commonjs/types.d.ts.map +1 -1
- package/dist/esm/chunking.d.ts +1 -2
- package/dist/esm/chunking.d.ts.map +1 -1
- package/dist/esm/chunking.js +2 -4
- package/dist/esm/chunking.js.map +1 -1
- package/dist/esm/config.d.ts +17 -0
- package/dist/esm/config.d.ts.map +1 -0
- package/dist/esm/config.js +107 -0
- package/dist/esm/config.js.map +1 -0
- package/dist/esm/errors.d.ts +10 -6
- package/dist/esm/errors.d.ts.map +1 -1
- package/dist/esm/errors.js +22 -4
- package/dist/esm/errors.js.map +1 -1
- package/dist/esm/index.d.ts +15 -5
- package/dist/esm/index.d.ts.map +1 -1
- package/dist/esm/index.js +332 -278
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/records.d.ts +44 -0
- package/dist/esm/records.d.ts.map +1 -0
- package/dist/esm/records.js +77 -0
- package/dist/esm/records.js.map +1 -0
- package/dist/esm/search.d.ts +8 -0
- package/dist/esm/search.d.ts.map +1 -0
- package/dist/esm/search.js +52 -0
- package/dist/esm/search.js.map +1 -0
- package/dist/esm/types.d.ts +84 -252
- package/dist/esm/types.d.ts.map +1 -1
- package/package.json +7 -42
- package/src/chunking.ts +2 -3
- package/src/config.ts +121 -0
- package/src/errors.ts +20 -6
- package/src/index.ts +352 -260
- package/src/records.ts +83 -0
- package/src/search.ts +57 -0
- package/src/types.ts +74 -253
- package/dist/commonjs/filter.d.ts +0 -30
- package/dist/commonjs/filter.d.ts.map +0 -1
- package/dist/commonjs/filter.js +0 -224
- package/dist/commonjs/filter.js.map +0 -1
- package/dist/commonjs/recall-input.d.ts +0 -8
- package/dist/commonjs/recall-input.d.ts.map +0 -1
- package/dist/commonjs/recall-input.js +0 -45
- package/dist/commonjs/recall-input.js.map +0 -1
- package/dist/commonjs/retriever/index.d.ts +0 -7
- package/dist/commonjs/retriever/index.d.ts.map +0 -1
- package/dist/commonjs/retriever/index.js +0 -167
- package/dist/commonjs/retriever/index.js.map +0 -1
- package/dist/commonjs/retriever/scope.d.ts +0 -5
- package/dist/commonjs/retriever/scope.d.ts.map +0 -1
- package/dist/commonjs/retriever/scope.js +0 -69
- package/dist/commonjs/retriever/scope.js.map +0 -1
- package/dist/commonjs/retriever/types.d.ts +0 -89
- package/dist/commonjs/retriever/types.d.ts.map +0 -1
- package/dist/commonjs/retriever/types.js +0 -3
- package/dist/commonjs/retriever/types.js.map +0 -1
- package/dist/commonjs/storage.d.ts +0 -274
- package/dist/commonjs/storage.d.ts.map +0 -1
- package/dist/commonjs/storage.js +0 -861
- package/dist/commonjs/storage.js.map +0 -1
- package/dist/commonjs/tools/definitions.d.ts +0 -5
- package/dist/commonjs/tools/definitions.d.ts.map +0 -1
- package/dist/commonjs/tools/definitions.js +0 -147
- package/dist/commonjs/tools/definitions.js.map +0 -1
- package/dist/commonjs/tools/evidence-ids.d.ts +0 -7
- package/dist/commonjs/tools/evidence-ids.d.ts.map +0 -1
- package/dist/commonjs/tools/evidence-ids.js +0 -71
- package/dist/commonjs/tools/evidence-ids.js.map +0 -1
- package/dist/commonjs/tools/factory.d.ts +0 -7
- package/dist/commonjs/tools/factory.d.ts.map +0 -1
- package/dist/commonjs/tools/factory.js +0 -169
- package/dist/commonjs/tools/factory.js.map +0 -1
- package/dist/commonjs/tools/index.d.ts +0 -6
- package/dist/commonjs/tools/index.d.ts.map +0 -1
- package/dist/commonjs/tools/index.js +0 -25
- package/dist/commonjs/tools/index.js.map +0 -1
- package/dist/commonjs/tools/internal-types.d.ts +0 -18
- package/dist/commonjs/tools/internal-types.d.ts.map +0 -1
- package/dist/commonjs/tools/internal-types.js +0 -3
- package/dist/commonjs/tools/internal-types.js.map +0 -1
- package/dist/commonjs/tools/short-ids.d.ts +0 -8
- package/dist/commonjs/tools/short-ids.d.ts.map +0 -1
- package/dist/commonjs/tools/short-ids.js +0 -81
- package/dist/commonjs/tools/short-ids.js.map +0 -1
- package/dist/commonjs/tools/types.d.ts +0 -53
- package/dist/commonjs/tools/types.d.ts.map +0 -1
- package/dist/commonjs/tools/types.js +0 -3
- package/dist/commonjs/tools/types.js.map +0 -1
- package/dist/commonjs/values.d.ts +0 -17
- package/dist/commonjs/values.d.ts.map +0 -1
- package/dist/commonjs/values.js +0 -90
- package/dist/commonjs/values.js.map +0 -1
- package/dist/esm/filter.d.ts +0 -30
- package/dist/esm/filter.d.ts.map +0 -1
- package/dist/esm/filter.js +0 -209
- package/dist/esm/filter.js.map +0 -1
- package/dist/esm/recall-input.d.ts +0 -8
- package/dist/esm/recall-input.d.ts.map +0 -1
- package/dist/esm/recall-input.js +0 -40
- package/dist/esm/recall-input.js.map +0 -1
- package/dist/esm/retriever/index.d.ts +0 -7
- package/dist/esm/retriever/index.d.ts.map +0 -1
- package/dist/esm/retriever/index.js +0 -163
- package/dist/esm/retriever/index.js.map +0 -1
- package/dist/esm/retriever/scope.d.ts +0 -5
- package/dist/esm/retriever/scope.d.ts.map +0 -1
- package/dist/esm/retriever/scope.js +0 -66
- package/dist/esm/retriever/scope.js.map +0 -1
- package/dist/esm/retriever/types.d.ts +0 -89
- package/dist/esm/retriever/types.d.ts.map +0 -1
- package/dist/esm/retriever/types.js +0 -2
- package/dist/esm/retriever/types.js.map +0 -1
- package/dist/esm/storage.d.ts +0 -274
- package/dist/esm/storage.d.ts.map +0 -1
- package/dist/esm/storage.js +0 -845
- package/dist/esm/storage.js.map +0 -1
- package/dist/esm/tools/definitions.d.ts +0 -5
- package/dist/esm/tools/definitions.d.ts.map +0 -1
- package/dist/esm/tools/definitions.js +0 -144
- package/dist/esm/tools/definitions.js.map +0 -1
- package/dist/esm/tools/evidence-ids.d.ts +0 -7
- package/dist/esm/tools/evidence-ids.d.ts.map +0 -1
- package/dist/esm/tools/evidence-ids.js +0 -68
- package/dist/esm/tools/evidence-ids.js.map +0 -1
- package/dist/esm/tools/factory.d.ts +0 -7
- package/dist/esm/tools/factory.d.ts.map +0 -1
- package/dist/esm/tools/factory.js +0 -166
- package/dist/esm/tools/factory.js.map +0 -1
- package/dist/esm/tools/index.d.ts +0 -6
- package/dist/esm/tools/index.d.ts.map +0 -1
- package/dist/esm/tools/index.js +0 -22
- package/dist/esm/tools/index.js.map +0 -1
- package/dist/esm/tools/internal-types.d.ts +0 -18
- package/dist/esm/tools/internal-types.d.ts.map +0 -1
- package/dist/esm/tools/internal-types.js +0 -2
- package/dist/esm/tools/internal-types.js.map +0 -1
- package/dist/esm/tools/short-ids.d.ts +0 -8
- package/dist/esm/tools/short-ids.d.ts.map +0 -1
- package/dist/esm/tools/short-ids.js +0 -78
- package/dist/esm/tools/short-ids.js.map +0 -1
- package/dist/esm/tools/types.d.ts +0 -53
- package/dist/esm/tools/types.d.ts.map +0 -1
- package/dist/esm/tools/types.js +0 -2
- package/dist/esm/tools/types.js.map +0 -1
- package/dist/esm/values.d.ts +0 -17
- package/dist/esm/values.d.ts.map +0 -1
- package/dist/esm/values.js +0 -87
- package/dist/esm/values.js.map +0 -1
- package/docs/tools.md +0 -264
- package/src/filter.ts +0 -189
- package/src/recall-input.ts +0 -46
- package/src/retriever/index.ts +0 -142
- package/src/retriever/scope.ts +0 -56
- package/src/retriever/types.ts +0 -68
- package/src/storage.ts +0 -765
- package/src/tools/definitions.ts +0 -175
- package/src/tools/evidence-ids.ts +0 -57
- package/src/tools/factory.ts +0 -153
- package/src/tools/index.ts +0 -25
- package/src/tools/internal-types.ts +0 -12
- package/src/tools/short-ids.ts +0 -58
- package/src/tools/types.ts +0 -48
- package/src/values.ts +0 -103
package/dist/esm/types.d.ts
CHANGED
|
@@ -1,277 +1,109 @@
|
|
|
1
1
|
import type { LambdaDBClient } from '@functional-systems/lambdadb';
|
|
2
|
-
export
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
type: MetadataType;
|
|
7
|
-
description?: string | undefined;
|
|
8
|
-
}>;
|
|
2
|
+
export interface ConnectionOptions {
|
|
3
|
+
client: LambdaDBClient;
|
|
4
|
+
name: string;
|
|
5
|
+
}
|
|
9
6
|
export interface RequestOptions {
|
|
10
7
|
signal?: AbortSignal | undefined;
|
|
11
8
|
}
|
|
12
|
-
export interface
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
mode: 'managed';
|
|
17
|
-
provider: 'openai';
|
|
18
|
-
model: 'text-embedding-3-small';
|
|
19
|
-
dimensions: 1536;
|
|
20
|
-
similarity: 'cosine';
|
|
21
|
-
}
|
|
22
|
-
export interface ExternalEmbedding {
|
|
23
|
-
mode: 'external';
|
|
24
|
-
dimensions: number;
|
|
25
|
-
similarity: 'cosine';
|
|
26
|
-
identity: string;
|
|
27
|
-
}
|
|
28
|
-
export type EmbeddingOptions = {
|
|
29
|
-
mode: 'managed';
|
|
30
|
-
} | ExternalEmbedding;
|
|
31
|
-
export type EmbeddingConfig = ManagedEmbedding | ExternalEmbedding;
|
|
32
|
-
export interface MemAttributes {
|
|
33
|
-
timestamp?: string | undefined;
|
|
34
|
-
metadata?: Metadata | undefined;
|
|
35
|
-
}
|
|
36
|
-
export interface TextMemInput extends MemAttributes {
|
|
37
|
-
id?: string | undefined;
|
|
38
|
-
content: string;
|
|
39
|
-
embedding?: never;
|
|
40
|
-
}
|
|
41
|
-
export interface VectorMemInput extends MemAttributes {
|
|
42
|
-
id?: string | undefined;
|
|
43
|
-
embedding: number[];
|
|
44
|
-
content?: never;
|
|
45
|
-
}
|
|
46
|
-
export type MemInput = TextMemInput | VectorMemInput;
|
|
47
|
-
export interface TextMem extends MemAttributes {
|
|
48
|
-
id: string;
|
|
49
|
-
content: string;
|
|
50
|
-
}
|
|
51
|
-
/** External-vector reads return source references and metadata, not vectors. */
|
|
52
|
-
export interface VectorMem extends MemAttributes {
|
|
53
|
-
id: string;
|
|
54
|
-
content?: never;
|
|
9
|
+
export interface OpenFileStoreOptions extends ConnectionOptions {
|
|
10
|
+
at?: {
|
|
11
|
+
fork: string;
|
|
12
|
+
} | undefined;
|
|
55
13
|
}
|
|
56
|
-
export
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
14
|
+
export interface DriveFile {
|
|
15
|
+
fileId: string;
|
|
16
|
+
modifiedAt: string;
|
|
17
|
+
path: string;
|
|
60
18
|
content: string;
|
|
61
|
-
|
|
62
|
-
end: number;
|
|
63
|
-
}
|
|
64
|
-
export interface VectorMatch extends MemAttributes {
|
|
65
|
-
kind: 'mem';
|
|
66
|
-
memId: string;
|
|
67
|
-
content?: never;
|
|
68
|
-
start?: never;
|
|
69
|
-
end?: never;
|
|
19
|
+
metadata?: Metadata | undefined;
|
|
70
20
|
}
|
|
71
|
-
export type
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
and: Filter[];
|
|
75
|
-
or?: never;
|
|
76
|
-
field?: never;
|
|
77
|
-
op?: never;
|
|
78
|
-
value?: never;
|
|
79
|
-
} | {
|
|
80
|
-
or: Filter[];
|
|
81
|
-
and?: never;
|
|
82
|
-
field?: never;
|
|
83
|
-
op?: never;
|
|
84
|
-
value?: never;
|
|
85
|
-
} | {
|
|
86
|
-
field: string;
|
|
87
|
-
op: 'eq' | 'neq' | 'gt' | 'gte' | 'lt' | 'lte';
|
|
88
|
-
value: FilterScalar;
|
|
89
|
-
and?: never;
|
|
90
|
-
or?: never;
|
|
91
|
-
} | {
|
|
92
|
-
field: string;
|
|
93
|
-
op: 'in';
|
|
94
|
-
value: FilterScalar[];
|
|
95
|
-
and?: never;
|
|
96
|
-
or?: never;
|
|
21
|
+
export type ReadFileRequest = {
|
|
22
|
+
fileId: string;
|
|
23
|
+
path?: never;
|
|
97
24
|
} | {
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
value: boolean;
|
|
101
|
-
and?: never;
|
|
102
|
-
or?: never;
|
|
25
|
+
path: string;
|
|
26
|
+
fileId?: never;
|
|
103
27
|
};
|
|
104
|
-
export interface
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
}
|
|
110
|
-
export interface TextRecallRequest extends RecallOptions {
|
|
111
|
-
query: string;
|
|
112
|
-
queryVector?: never;
|
|
113
|
-
context?: string | undefined;
|
|
114
|
-
recentDialogue?: Array<{
|
|
115
|
-
role: 'user' | 'assistant';
|
|
116
|
-
content: string;
|
|
117
|
-
}> | undefined;
|
|
118
|
-
}
|
|
119
|
-
export interface VectorRecallRequest extends Omit<RecallOptions, 'rerank'> {
|
|
120
|
-
queryVector: number[];
|
|
121
|
-
query?: never;
|
|
122
|
-
rerank?: false | undefined;
|
|
123
|
-
context?: never;
|
|
124
|
-
recentDialogue?: never;
|
|
125
|
-
}
|
|
126
|
-
export type RecallRequest = TextRecallRequest | VectorRecallRequest;
|
|
127
|
-
export interface Usage {
|
|
128
|
-
elapsedMs: number;
|
|
129
|
-
databaseRequests: number;
|
|
28
|
+
export interface CreateFileRequest {
|
|
29
|
+
op: 'create';
|
|
30
|
+
path: string;
|
|
31
|
+
content: string;
|
|
32
|
+
metadata?: Metadata | undefined;
|
|
130
33
|
}
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
34
|
+
interface FileView {
|
|
35
|
+
readonly name: string;
|
|
36
|
+
describe(options?: RequestOptions): Promise<FileStoreDescription>;
|
|
37
|
+
search(request: SearchFilesRequest, options?: RequestOptions): Promise<SearchFilesResult>;
|
|
38
|
+
read(request: ReadFileRequest, options?: RequestOptions): Promise<DriveFile | null>;
|
|
39
|
+
read(request: {
|
|
40
|
+
paths: string[];
|
|
41
|
+
}, options?: RequestOptions): Promise<DriveFile[]>;
|
|
134
42
|
}
|
|
135
|
-
export interface
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
message: string;
|
|
139
|
-
status?: number;
|
|
43
|
+
export interface MainFileDrive extends FileView {
|
|
44
|
+
readonly at: {
|
|
45
|
+
readonly kind: 'main';
|
|
140
46
|
};
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
}
|
|
153
|
-
export type RecordEvent = Record<string, unknown> & {
|
|
154
|
-
phase: string;
|
|
155
|
-
requestIndex?: number | undefined;
|
|
156
|
-
};
|
|
157
|
-
export interface RuntimeOptions {
|
|
158
|
-
maxConcurrentRequests?: number | undefined;
|
|
159
|
-
observe?: ((event: Observation) => void | Promise<void>) | undefined;
|
|
160
|
-
record?: ((event: RecordEvent) => void | Promise<void>) | undefined;
|
|
161
|
-
}
|
|
162
|
-
export interface ConnectionOptions {
|
|
163
|
-
client: LambdaDBClient;
|
|
164
|
-
name: string;
|
|
47
|
+
fork(request: {
|
|
48
|
+
name: string;
|
|
49
|
+
}, options?: RequestOptions): Promise<WorkingFileDrive>;
|
|
50
|
+
}
|
|
51
|
+
export interface WorkingFileDrive extends FileView {
|
|
52
|
+
readonly at: {
|
|
53
|
+
readonly kind: 'fork';
|
|
54
|
+
readonly name: string;
|
|
55
|
+
};
|
|
56
|
+
write(request: CreateFileRequest, options?: RequestOptions): Promise<DriveFile>;
|
|
57
|
+
write(request: CreateFileRequest[], options?: RequestOptions): Promise<DriveFile[]>;
|
|
165
58
|
}
|
|
166
|
-
export
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
59
|
+
export type FileDrive = MainFileDrive | WorkingFileDrive;
|
|
60
|
+
export type MetadataType = 'string' | 'number' | 'boolean' | 'datetime';
|
|
61
|
+
export type Metadata = Record<string, string | number | boolean>;
|
|
62
|
+
export type MetadataSchema = Record<string, {
|
|
63
|
+
type: MetadataType;
|
|
171
64
|
description?: string | undefined;
|
|
65
|
+
}>;
|
|
66
|
+
export interface CreateFileStoreOptions extends ConnectionOptions {
|
|
67
|
+
labels?: Record<string, string> | undefined;
|
|
68
|
+
metadataSchema?: MetadataSchema | undefined;
|
|
69
|
+
textSearch?: {
|
|
70
|
+
analyzers: string[];
|
|
71
|
+
} | undefined;
|
|
172
72
|
}
|
|
173
|
-
export interface
|
|
174
|
-
branch?: string | undefined;
|
|
175
|
-
}
|
|
176
|
-
export interface DriveDescriptor {
|
|
73
|
+
export interface FileStoreDescription {
|
|
177
74
|
name: string;
|
|
178
|
-
description?: string | undefined;
|
|
179
75
|
labels: Record<string, string>;
|
|
180
76
|
metadataSchema: MetadataSchema;
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
}
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
drives: DriveDescriptor[];
|
|
192
|
-
nextCursor?: string | undefined;
|
|
77
|
+
textSearch: {
|
|
78
|
+
analyzers: string[];
|
|
79
|
+
};
|
|
80
|
+
embedding: {
|
|
81
|
+
mode: 'managed';
|
|
82
|
+
provider: 'openai';
|
|
83
|
+
model: 'text-embedding-3-small';
|
|
84
|
+
dimensions: 1536;
|
|
85
|
+
similarity: 'cosine';
|
|
86
|
+
};
|
|
193
87
|
}
|
|
194
|
-
export interface
|
|
88
|
+
export interface SearchFilesRequest {
|
|
89
|
+
query: string;
|
|
90
|
+
embeddingQuery?: string | undefined;
|
|
195
91
|
limit?: number | undefined;
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
mems: Mem[];
|
|
200
|
-
nextCursor?: string | undefined;
|
|
201
|
-
}
|
|
202
|
-
export type ForgetRequest = {
|
|
203
|
-
ids: string[];
|
|
204
|
-
filter?: never;
|
|
205
|
-
} | {
|
|
206
|
-
filter: Filter;
|
|
207
|
-
ids?: never;
|
|
208
|
-
};
|
|
209
|
-
export type ConfigureRequest = {
|
|
210
|
-
metadataSchema: MetadataSchema;
|
|
211
|
-
description?: string | undefined;
|
|
212
|
-
} | {
|
|
213
|
-
description: string;
|
|
214
|
-
metadataSchema?: MetadataSchema | undefined;
|
|
215
|
-
};
|
|
216
|
-
export interface BranchRequest {
|
|
217
|
-
name: string;
|
|
218
|
-
source?: {
|
|
219
|
-
branch: string;
|
|
92
|
+
rerank?: boolean | {
|
|
93
|
+
fields?: string[] | undefined;
|
|
94
|
+
onFailure?: 'error' | 'returnOriginal' | undefined;
|
|
220
95
|
} | undefined;
|
|
221
96
|
}
|
|
222
|
-
export
|
|
223
|
-
/**
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
excludeIds?: string[] | undefined;
|
|
229
|
-
/** Values per field: defaults to 10, maximum 100. */
|
|
230
|
-
limit?: number | undefined;
|
|
231
|
-
}
|
|
232
|
-
export interface CountValuesResult {
|
|
233
|
-
values: Record<string, {
|
|
234
|
-
buckets: Array<{
|
|
235
|
-
value: string;
|
|
236
|
-
count: number;
|
|
237
|
-
}>;
|
|
238
|
-
/** Null when the native bucket limit prevents determining completeness. */
|
|
239
|
-
hasMore: boolean | null;
|
|
240
|
-
}>;
|
|
241
|
-
/** Time the cached result was fetched, in ISO 8601 UTC. */
|
|
242
|
-
asOf: string;
|
|
243
|
-
}
|
|
244
|
-
export interface DriveData {
|
|
245
|
-
readonly embedding: Readonly<EmbeddingConfig>;
|
|
246
|
-
remember(request: {
|
|
247
|
-
mems: MemInput[];
|
|
248
|
-
}, options?: RequestOptions): Promise<{
|
|
249
|
-
ids: string[];
|
|
250
|
-
}>;
|
|
251
|
-
read(request: {
|
|
252
|
-
ids: string[];
|
|
253
|
-
}, options?: RequestOptions): Promise<{
|
|
254
|
-
mems: Mem[];
|
|
255
|
-
}>;
|
|
256
|
-
list(request?: ListRequest, options?: RequestOptions): Promise<ListResult>;
|
|
257
|
-
countValues(request: CountValuesRequest, options?: RequestOptions): Promise<CountValuesResult>;
|
|
258
|
-
forget(request: ForgetRequest, options?: RequestOptions): Promise<void>;
|
|
259
|
-
recall(request: RecallRequest, options?: RequestOptions): Promise<RecallResult>;
|
|
260
|
-
recall(request: {
|
|
261
|
-
requests: RecallRequest[];
|
|
262
|
-
}, options?: RequestOptions): Promise<BatchRecallResult>;
|
|
97
|
+
export interface FileSearchHit extends DriveFile {
|
|
98
|
+
/** Verified UTF-16 offsets into the file's original text. */
|
|
99
|
+
range?: {
|
|
100
|
+
start: number;
|
|
101
|
+
end: number;
|
|
102
|
+
} | undefined;
|
|
263
103
|
}
|
|
264
|
-
export interface
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
}): DriveData;
|
|
268
|
-
readonly branches: {
|
|
269
|
-
create(request: BranchRequest, options?: RequestOptions): Promise<Branch>;
|
|
270
|
-
list(options?: RequestOptions): Promise<Branch[]>;
|
|
271
|
-
delete(name: string, options?: RequestOptions): Promise<void>;
|
|
272
|
-
};
|
|
273
|
-
configure(request: ConfigureRequest, options?: RequestOptions): Promise<void>;
|
|
274
|
-
describe(options?: RequestOptions): Promise<DriveDescriptor>;
|
|
275
|
-
destroy(options?: RequestOptions): Promise<void>;
|
|
104
|
+
export interface SearchFilesResult {
|
|
105
|
+
hits: FileSearchHit[];
|
|
106
|
+
reranked: boolean;
|
|
276
107
|
}
|
|
108
|
+
export {};
|
|
277
109
|
//# sourceMappingURL=types.d.ts.map
|
package/dist/esm/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAEnE,MAAM,WAAW,iBAAiB;IAC9B,MAAM,EAAE,cAAc,CAAC;IACvB,IAAI,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,cAAc;IAC3B,MAAM,CAAC,EAAE,WAAW,GAAG,SAAS,CAAC;CACpC;AAED,MAAM,WAAW,oBAAqB,SAAQ,iBAAiB;IAC3D,EAAE,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAAC;CACrC;AAED,MAAM,WAAW,SAAS;IACtB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,QAAQ,GAAG,SAAS,CAAC;CACnC;AAED,MAAM,MAAM,eAAe,GAAG;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,KAAK,CAAA;CAAE,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,KAAK,CAAA;CAAE,CAAC;AAClG,MAAM,WAAW,iBAAiB;IAC9B,EAAE,EAAE,QAAQ,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,QAAQ,GAAG,SAAS,CAAC;CACnC;AAED,UAAU,QAAQ;IACd,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAClE,MAAM,CAAC,OAAO,EAAE,kBAAkB,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;IAC1F,IAAI,CAAC,OAAO,EAAE,eAAe,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC;IACpF,IAAI,CAAC,OAAO,EAAE;QAAE,KAAK,EAAE,MAAM,EAAE,CAAA;KAAE,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;CACtF;AAED,MAAM,WAAW,aAAc,SAAQ,QAAQ;IAC3C,QAAQ,CAAC,EAAE,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IACvC,IAAI,CAAC,OAAO,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;CACxF;AAED,MAAM,WAAW,gBAAiB,SAAQ,QAAQ;IAC9C,QAAQ,CAAC,EAAE,EAAE;QAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IAC9D,KAAK,CAAC,OAAO,EAAE,iBAAiB,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;IAChF,KAAK,CAAC,OAAO,EAAE,iBAAiB,EAAE,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;CACvF;AAED,MAAM,MAAM,SAAS,GAAG,aAAa,GAAG,gBAAgB,CAAC;AAEzD,MAAM,MAAM,YAAY,GAAG,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,UAAU,CAAC;AACxE,MAAM,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC,CAAC;AACjE,MAAM,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,EAAE;IAAE,IAAI,EAAE,YAAY,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;CAAE,CAAC,CAAC;AAEtG,MAAM,WAAW,sBAAuB,SAAQ,iBAAiB;IAC7D,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,SAAS,CAAC;IAC5C,cAAc,CAAC,EAAE,cAAc,GAAG,SAAS,CAAC;IAC5C,UAAU,CAAC,EAAE;QAAE,SAAS,EAAE,MAAM,EAAE,CAAA;KAAE,GAAG,SAAS,CAAC;CACpD;AAED,MAAM,WAAW,oBAAoB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC/B,cAAc,EAAE,cAAc,CAAC;IAC/B,UAAU,EAAE;QAAE,SAAS,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;IACpC,SAAS,EAAE;QACP,IAAI,EAAE,SAAS,CAAC;QAChB,QAAQ,EAAE,QAAQ,CAAC;QACnB,KAAK,EAAE,wBAAwB,CAAC;QAChC,UAAU,EAAE,IAAI,CAAC;QACjB,UAAU,EAAE,QAAQ,CAAC;KACxB,CAAC;CACL;AAED,MAAM,WAAW,kBAAkB;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACpC,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,MAAM,CAAC,EAAE,OAAO,GAAG;QACf,MAAM,CAAC,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;QAC9B,SAAS,CAAC,EAAE,OAAO,GAAG,gBAAgB,GAAG,SAAS,CAAC;KACtD,GAAG,SAAS,CAAC;CACjB;AAED,MAAM,WAAW,aAAc,SAAQ,SAAS;IAC5C,6DAA6D;IAC7D,KAAK,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAAC;CACtD;AACD,MAAM,WAAW,iBAAiB;IAC9B,IAAI,EAAE,aAAa,EAAE,CAAC;IACtB,QAAQ,EAAE,OAAO,CAAC;CACrB"}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@functional-systems/memdrive",
|
|
3
|
-
"version": "0.1.1-dev.
|
|
4
|
-
"description": "
|
|
3
|
+
"version": "0.1.1-dev.4",
|
|
4
|
+
"description": "Branch-scoped file storage backed by LambdaDB.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"engines": {
|
|
7
7
|
"node": ">=20.12"
|
|
@@ -13,8 +13,7 @@
|
|
|
13
13
|
"README.md",
|
|
14
14
|
"CHANGELOG.md",
|
|
15
15
|
"LICENSE",
|
|
16
|
-
"NOTICE"
|
|
17
|
-
"docs/tools.md"
|
|
16
|
+
"NOTICE"
|
|
18
17
|
],
|
|
19
18
|
"main": "./dist/commonjs/index.js",
|
|
20
19
|
"module": "./dist/esm/index.js",
|
|
@@ -22,10 +21,7 @@
|
|
|
22
21
|
"tshy": {
|
|
23
22
|
"exports": {
|
|
24
23
|
".": "./src/index.ts",
|
|
25
|
-
"./
|
|
26
|
-
"./package.json": "./package.json",
|
|
27
|
-
"./tools": "./src/tools/index.ts",
|
|
28
|
-
"./retriever": "./src/retriever/index.ts"
|
|
24
|
+
"./package.json": "./package.json"
|
|
29
25
|
}
|
|
30
26
|
},
|
|
31
27
|
"scripts": {
|
|
@@ -38,14 +34,13 @@
|
|
|
38
34
|
"check:package": "npm run build && node scripts/check-package.mjs",
|
|
39
35
|
"prepublishOnly": "node scripts/check-publish.mjs",
|
|
40
36
|
"test:live": "npm run build && MEMDRIVE_RUN_LIVE_TESTS=1 node --env-file-if-exists=.env.local --test --test-concurrency=1 test/integration/*-live.test.mjs",
|
|
41
|
-
"benchmark:longmemeval": "node benchmark/longmemeval/run.mjs"
|
|
37
|
+
"benchmark:longmemeval": "npm run build && node benchmark/longmemeval/run.mjs"
|
|
42
38
|
},
|
|
43
39
|
"dependencies": {
|
|
44
40
|
"@functional-systems/lambdadb": "0.12.0"
|
|
45
41
|
},
|
|
46
42
|
"devDependencies": {
|
|
47
43
|
"@eslint/js": "9.32.0",
|
|
48
|
-
"ajv": "8.20.0",
|
|
49
44
|
"esbuild": "0.28.2",
|
|
50
45
|
"eslint": "9.32.0",
|
|
51
46
|
"globals": "15.15.0",
|
|
@@ -66,37 +61,7 @@
|
|
|
66
61
|
"default": "./dist/commonjs/index.js"
|
|
67
62
|
}
|
|
68
63
|
},
|
|
69
|
-
"./
|
|
70
|
-
"import": {
|
|
71
|
-
"types": "./dist/esm/types.d.ts",
|
|
72
|
-
"default": "./dist/esm/types.js"
|
|
73
|
-
},
|
|
74
|
-
"require": {
|
|
75
|
-
"types": "./dist/commonjs/types.d.ts",
|
|
76
|
-
"default": "./dist/commonjs/types.js"
|
|
77
|
-
}
|
|
78
|
-
},
|
|
79
|
-
"./package.json": "./package.json",
|
|
80
|
-
"./tools": {
|
|
81
|
-
"import": {
|
|
82
|
-
"types": "./dist/esm/tools/index.d.ts",
|
|
83
|
-
"default": "./dist/esm/tools/index.js"
|
|
84
|
-
},
|
|
85
|
-
"require": {
|
|
86
|
-
"types": "./dist/commonjs/tools/index.d.ts",
|
|
87
|
-
"default": "./dist/commonjs/tools/index.js"
|
|
88
|
-
}
|
|
89
|
-
},
|
|
90
|
-
"./retriever": {
|
|
91
|
-
"import": {
|
|
92
|
-
"types": "./dist/esm/retriever/index.d.ts",
|
|
93
|
-
"default": "./dist/esm/retriever/index.js"
|
|
94
|
-
},
|
|
95
|
-
"require": {
|
|
96
|
-
"types": "./dist/commonjs/retriever/index.d.ts",
|
|
97
|
-
"default": "./dist/commonjs/retriever/index.js"
|
|
98
|
-
}
|
|
99
|
-
}
|
|
64
|
+
"./package.json": "./package.json"
|
|
100
65
|
},
|
|
101
66
|
"author": "Functional Systems, Inc.",
|
|
102
67
|
"homepage": "https://github.com/lambdadb/memdrive#readme",
|
|
@@ -110,5 +75,5 @@
|
|
|
110
75
|
"publishConfig": {
|
|
111
76
|
"access": "public"
|
|
112
77
|
},
|
|
113
|
-
"memdriveSourceCommit": "
|
|
78
|
+
"memdriveSourceCommit": "d2709491579dbe4eec261b055b020f9a814055e1"
|
|
114
79
|
}
|
package/src/chunking.ts
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
1
|
// Exact source partitions: UTF-16 offsets for slice(), a Unicode-code-point cap.
|
|
2
2
|
// English sentence detection is deliberate; other languages are not optimized.
|
|
3
|
-
export
|
|
4
|
-
|
|
5
|
-
if (!Number.isInteger(limit) || limit < 1) throw new RangeError('Chunk limit must be a positive integer.');
|
|
3
|
+
export function chunkSpans(text: string): Array<{start: number; end: number; text: string}> {
|
|
4
|
+
const limit = 800;
|
|
6
5
|
const offsets = [0];
|
|
7
6
|
for (const char of text) offsets.push(offsets.at(-1)! + char.length);
|
|
8
7
|
const paragraphs = [...text.matchAll(/\r?\n[\t ]*\r?\n(?:\s*\r?\n)*/gu)].map(m => m.index + m[0].length);
|
package/src/config.ts
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import { Analyzer, type IndexConfigsUnion } from '@functional-systems/lambdadb/models/indexconfigsunion.js';
|
|
2
|
+
import type { CreateFileStoreOptions, FileStoreDescription, Metadata, MetadataSchema } from './types.js';
|
|
3
|
+
import { MemDriveError, requireInput } from './errors.js';
|
|
4
|
+
|
|
5
|
+
export const isObject = (value: unknown): value is Record<string, unknown> => value !== null && typeof value === 'object'
|
|
6
|
+
&& !Array.isArray(value) && [Object.prototype, null].includes(Object.getPrototypeOf(value));
|
|
7
|
+
export const keysOnly = (value: unknown, keys: string[]): value is Record<string, unknown> =>
|
|
8
|
+
isObject(value) && Object.keys(value).every(key => keys.includes(key));
|
|
9
|
+
|
|
10
|
+
export const formatTag = 'memdrive.format';
|
|
11
|
+
export const format = 'files-v0';
|
|
12
|
+
const managedEmbedding = { mode: 'managed', provider: 'openai', model: 'text-embedding-3-small',
|
|
13
|
+
dimensions: 1536, similarity: 'cosine' } as const;
|
|
14
|
+
const fileIndexes = {
|
|
15
|
+
kind: { type: 'keyword' },
|
|
16
|
+
path: { type: 'keyword' },
|
|
17
|
+
fileName: { type: 'keyword' },
|
|
18
|
+
ancestorPaths: { type: 'keyword' },
|
|
19
|
+
changed: { type: 'boolean' },
|
|
20
|
+
modifiedAt: { type: 'datetime' },
|
|
21
|
+
} as const;
|
|
22
|
+
const indexTypes = { string: 'keyword', number: 'double', boolean: 'boolean', datetime: 'datetime' } as const;
|
|
23
|
+
const supportedAnalyzers: ReadonlySet<unknown> = new Set(Object.values(Analyzer));
|
|
24
|
+
|
|
25
|
+
export const validDate = (value: unknown): value is string => typeof value === 'string'
|
|
26
|
+
&& /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/.test(value)
|
|
27
|
+
&& !Number.isNaN(Date.parse(value)) && new Date(value).toISOString() === value;
|
|
28
|
+
|
|
29
|
+
function labelEntries(value: unknown = {}): Record<string, string> {
|
|
30
|
+
// Match the native tag restrictions, including Java's definition of whitespace.
|
|
31
|
+
// eslint-disable-next-line no-control-regex
|
|
32
|
+
const blank = /^[\u0009-\u000d\u001c-\u0020\u1680\u2000-\u2006\u2008-\u200a\u2028\u2029\u205f\u3000]*$/u;
|
|
33
|
+
requireInput(isObject(value) && Object.keys(value).length <= 4 && Object.entries(value).every(([key, item]) =>
|
|
34
|
+
/^[A-Za-z0-9_.-]{1,63}$/.test(key) && !key.startsWith('memdrive.') && typeof item === 'string'
|
|
35
|
+
&& item.length <= 127 && !/[:#,]/.test(item) && !blank.test(item)), 'Labels must use native-safe names and values. memdrive.* names are reserved.');
|
|
36
|
+
return { ...value } as Record<string, string>;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function schemaEntries(value: unknown = {}): MetadataSchema {
|
|
40
|
+
requireInput(isObject(value) && Object.entries(value).every(([key, entry]) =>
|
|
41
|
+
/^[A-Za-z_][A-Za-z0-9_]*$/.test(key) && !['id', 'constructor', 'prototype', '__proto__'].includes(key)
|
|
42
|
+
&& keysOnly(entry, ['type', 'description']) && typeof entry.type === 'string' && Object.hasOwn(indexTypes, entry.type)
|
|
43
|
+
&& (entry.description === undefined || (typeof entry.description === 'string' && entry.description.length <= 255))),
|
|
44
|
+
'Metadata schema requires named scalar types and optional field descriptions.');
|
|
45
|
+
return structuredClone(value) as MetadataSchema;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function textAnalyzers(value: unknown = { analyzers: ['standard'] }): Analyzer[] {
|
|
49
|
+
requireInput(keysOnly(value, ['analyzers']) && Array.isArray(value.analyzers) && value.analyzers.length > 0
|
|
50
|
+
&& new Set(value.analyzers).size === value.analyzers.length && [...value.analyzers].every(item => supportedAnalyzers.has(item)),
|
|
51
|
+
'Set a nonempty array of distinct supported text analyzers.');
|
|
52
|
+
return [...value.analyzers] as Analyzer[];
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export function metadataValues(value: unknown, schema: MetadataSchema): Metadata | undefined {
|
|
56
|
+
if (value === undefined) return undefined;
|
|
57
|
+
requireInput(isObject(value) && Object.entries(value).every(([key, item]) => {
|
|
58
|
+
if (!Object.hasOwn(schema, key)) return false;
|
|
59
|
+
switch (schema[key]!.type) {
|
|
60
|
+
case 'string': return typeof item === 'string' && item.length <= 4096 && !/\p{Surrogate}/u.test(item);
|
|
61
|
+
case 'number': return typeof item === 'number' && Number.isFinite(item);
|
|
62
|
+
case 'boolean': return typeof item === 'boolean';
|
|
63
|
+
case 'datetime': return validDate(item);
|
|
64
|
+
}
|
|
65
|
+
}), 'Metadata values must match their declared scalar types. Datetimes use ISO UTC with milliseconds.');
|
|
66
|
+
return { ...value } as Metadata;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export function storeConfiguration(options: CreateFileStoreOptions) {
|
|
70
|
+
const name = options.name;
|
|
71
|
+
const metadataSchema = schemaEntries(options.metadataSchema);
|
|
72
|
+
const labels = labelEntries(options.labels);
|
|
73
|
+
const analyzers = textAnalyzers(options.textSearch);
|
|
74
|
+
const indexConfigs: Record<string, IndexConfigsUnion> = {
|
|
75
|
+
...fileIndexes,
|
|
76
|
+
body: { type: 'text', analyzers },
|
|
77
|
+
content: { type: 'text', analyzers },
|
|
78
|
+
embedding: { type: 'vector', managedEmbedding: true, embedding: {
|
|
79
|
+
provider: managedEmbedding.provider, model: managedEmbedding.model, dimensions: managedEmbedding.dimensions,
|
|
80
|
+
similarity: managedEmbedding.similarity, sourceField: 'content',
|
|
81
|
+
} },
|
|
82
|
+
};
|
|
83
|
+
if (Object.keys(metadataSchema).length) indexConfigs.metadata = { type: 'object', objectIndexConfigs:
|
|
84
|
+
Object.fromEntries(Object.entries(metadataSchema).map(([key, value]) => [key, { ...value, type: indexTypes[value.type] }])) };
|
|
85
|
+
const description: FileStoreDescription = { name, labels, metadataSchema, textSearch: { analyzers }, embedding: { ...managedEmbedding } };
|
|
86
|
+
return { indexConfigs, tags: { ...labels, [formatTag]: format }, description };
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
export function parseStore(collection: unknown, name: string): FileStoreDescription {
|
|
90
|
+
try {
|
|
91
|
+
requireInput(isObject(collection) && collection.collectionName === name && collection.defaultBranchName === 'main'
|
|
92
|
+
&& isObject(collection.tags) && collection.tags[formatTag] === format && isObject(collection.indexConfigs),
|
|
93
|
+
'Collection format or main branch is incompatible.');
|
|
94
|
+
const { tags, indexConfigs } = collection;
|
|
95
|
+
const labels = labelEntries(Object.fromEntries(Object.entries(tags).filter(([key]) => key !== formatTag)));
|
|
96
|
+
requireInput(Object.entries(fileIndexes).every(([field, config]) => isObject(indexConfigs[field])
|
|
97
|
+
&& indexConfigs[field].type === config.type), 'Required file indexes are missing.');
|
|
98
|
+
const { embedding, content, body, metadata } = indexConfigs;
|
|
99
|
+
requireInput(isObject(embedding) && embedding.type === 'vector' && embedding.managedEmbedding !== false && isObject(embedding.embedding)
|
|
100
|
+
&& embedding.embedding.provider === managedEmbedding.provider && embedding.embedding.model === managedEmbedding.model
|
|
101
|
+
&& embedding.embedding.dimensions === managedEmbedding.dimensions && embedding.embedding.similarity === managedEmbedding.similarity
|
|
102
|
+
&& embedding.embedding.sourceField === 'content', 'Managed chunk embedding index is incompatible.');
|
|
103
|
+
requireInput(isObject(content) && content.type === 'text' && isObject(body) && body.type === 'text', 'Text indexes are missing.');
|
|
104
|
+
const analyzers = textAnalyzers({ analyzers: content.analyzers ?? ['standard'] });
|
|
105
|
+
requireInput(JSON.stringify(analyzers) === JSON.stringify(textAnalyzers({ analyzers: body.analyzers ?? ['standard'] })),
|
|
106
|
+
'Original and chunk analyzers must match.');
|
|
107
|
+
let metadataSchema: MetadataSchema = {};
|
|
108
|
+
if (metadata !== undefined) {
|
|
109
|
+
requireInput(isObject(metadata) && metadata.type === 'object' && isObject(metadata.objectIndexConfigs),
|
|
110
|
+
'Metadata index is incompatible.');
|
|
111
|
+
metadataSchema = schemaEntries(Object.fromEntries(Object.entries(metadata.objectIndexConfigs).map(([key, value]) => {
|
|
112
|
+
requireInput(isObject(value), 'Invalid metadata field index.');
|
|
113
|
+
return [key, { type: Object.entries(indexTypes).find(([, native]) => native === value.type)?.[0],
|
|
114
|
+
...(value.description === undefined ? {} : { description: value.description }) }];
|
|
115
|
+
})));
|
|
116
|
+
}
|
|
117
|
+
return { name, labels, metadataSchema, textSearch: { analyzers }, embedding: { ...managedEmbedding } };
|
|
118
|
+
} catch (cause) {
|
|
119
|
+
throw new MemDriveError('INCOMPATIBLE_STORE', 'The collection is not a compatible MemDrive file store.', { cause });
|
|
120
|
+
}
|
|
121
|
+
}
|