@nacre.work/core 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/audit.d.ts +41 -0
- package/dist/audit.d.ts.map +1 -0
- package/dist/audit.js +18 -0
- package/dist/audit.js.map +1 -0
- package/dist/authz/cache.d.ts +20 -0
- package/dist/authz/cache.d.ts.map +1 -1
- package/dist/authz/cache.js +29 -0
- package/dist/authz/cache.js.map +1 -1
- package/dist/authz/index.d.ts +1 -1
- package/dist/authz/index.d.ts.map +1 -1
- package/dist/authz/index.js +1 -1
- package/dist/authz/index.js.map +1 -1
- package/dist/config.d.ts +22 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +108 -4
- package/dist/config.js.map +1 -1
- package/dist/index.d.ts +7 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -1
- package/dist/index.js.map +1 -1
- package/dist/lifecycle.d.ts.map +1 -1
- package/dist/lifecycle.js +13 -26
- package/dist/lifecycle.js.map +1 -1
- package/dist/logging.d.ts +46 -0
- package/dist/logging.d.ts.map +1 -0
- package/dist/logging.js +110 -0
- package/dist/logging.js.map +1 -0
- package/dist/metrics.d.ts +1 -0
- package/dist/metrics.d.ts.map +1 -1
- package/dist/metrics.js +7 -0
- package/dist/metrics.js.map +1 -1
- package/dist/migrations/0014_retired_collections.sql +68 -0
- package/dist/observability.d.ts.map +1 -1
- package/dist/observability.js +19 -0
- package/dist/observability.js.map +1 -1
- package/dist/s3.d.ts +111 -0
- package/dist/s3.d.ts.map +1 -0
- package/dist/s3.js +242 -0
- package/dist/s3.js.map +1 -0
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/dist/vector/search.d.ts +66 -0
- package/dist/vector/search.d.ts.map +1 -1
- package/dist/vector/search.js +154 -0
- package/dist/vector/search.js.map +1 -1
- package/package.json +1 -1
package/dist/vector/search.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { QdrantClient } from '@qdrant/js-client-rest';
|
|
1
2
|
import { type Narrowing, type QueryablePlan } from '../authz/filter.js';
|
|
2
3
|
import { type Branch } from './query.js';
|
|
3
4
|
/**
|
|
@@ -45,6 +46,57 @@ export interface Hit {
|
|
|
45
46
|
readonly score: number;
|
|
46
47
|
readonly payload: Record<string, unknown>;
|
|
47
48
|
}
|
|
49
|
+
/**
|
|
50
|
+
* How many metadata keys one collection will have indexes built for.
|
|
51
|
+
*
|
|
52
|
+
* The keys are the caller's, so unlike `PAYLOAD_INDEXES` the set is not known
|
|
53
|
+
* in advance and has no natural end: `parseMetadata` bounds a *document* to 32
|
|
54
|
+
* keys and says nothing about how many distinct ones an organization uses, so
|
|
55
|
+
* a caller putting an identifier in a key name would otherwise ask for an index
|
|
56
|
+
* per document.
|
|
57
|
+
*
|
|
58
|
+
* A constant rather than a `NACRE_` variable. Every configuration variable is a
|
|
59
|
+
* thing an operator has to understand and a thing `docs/config.md` has to
|
|
60
|
+
* explain, and the failure this bounds — a slower filter past the sixty-fourth
|
|
61
|
+
* distinct key — needs no per-deployment answer.
|
|
62
|
+
*/
|
|
63
|
+
export declare const METADATA_INDEX_LIMIT = 64;
|
|
64
|
+
/**
|
|
65
|
+
* Payload indexes for the keys a caller filters on.
|
|
66
|
+
*
|
|
67
|
+
* `PAYLOAD_INDEXES` covers the fields the permission filter uses, which are
|
|
68
|
+
* fixed. A metadata filter reaches `meta.<key>`, and those fields had no index
|
|
69
|
+
* at all — so the narrowing that `filters` performs was evaluated by scanning,
|
|
70
|
+
* which is the failure the comment on `PAYLOAD_INDEXES` describes: it does not
|
|
71
|
+
* fail a test, it gets slow at a customer's volume.
|
|
72
|
+
*
|
|
73
|
+
* Built when metadata is written, because that is the only moment the keys are
|
|
74
|
+
* known. Three properties make that safe, and each was checked against a real
|
|
75
|
+
* Qdrant rather than read:
|
|
76
|
+
*
|
|
77
|
+
* - **A missing index is slow, never wrong.** A filter on an unindexed field —
|
|
78
|
+
* and on one indexed under the wrong type — still returns exactly the right
|
|
79
|
+
* points. Qdrant falls back to a scan. So this is an optimization end to end,
|
|
80
|
+
* and everything below may fail without changing an answer.
|
|
81
|
+
* - **Creating one is idempotent**, and it back-fills: an index added after the
|
|
82
|
+
* points covers them, so nothing has to be rebuilt or ordered.
|
|
83
|
+
* - **`keyword` is safe for every value type.** A number or a boolean under a
|
|
84
|
+
* keyword index simply indexes nothing and still matches, and strings and
|
|
85
|
+
* lists of strings — the case `filters` exists for, "only documents from this
|
|
86
|
+
* source" — index properly.
|
|
87
|
+
*
|
|
88
|
+
* Which is why a failure here warns and returns. This is the opposite of
|
|
89
|
+
* invariant I3 and deliberately so: nothing about an index decides who sees
|
|
90
|
+
* what, and failing an ingest because an index could not be built would trade a
|
|
91
|
+
* slow query for a document that never arrives.
|
|
92
|
+
*/
|
|
93
|
+
export declare class MetadataIndexer {
|
|
94
|
+
#private;
|
|
95
|
+
constructor(client: QdrantClient, limit?: number);
|
|
96
|
+
ensure(collection: string, keys: Iterable<string>): Promise<void>;
|
|
97
|
+
/** The `meta.` indexes a collection already has, for carrying across a copy. */
|
|
98
|
+
static fieldsOf(client: QdrantClient, collection: string): Promise<readonly string[]>;
|
|
99
|
+
}
|
|
48
100
|
/**
|
|
49
101
|
* The vector store.
|
|
50
102
|
*
|
|
@@ -101,6 +153,20 @@ export declare class VectorStore {
|
|
|
101
153
|
* argument the ACL retag path is built on.
|
|
102
154
|
*/
|
|
103
155
|
setMetadata(collection: string, documentId: string, metadata: Record<string, unknown>): Promise<void>;
|
|
156
|
+
/**
|
|
157
|
+
* Remove a collection.
|
|
158
|
+
*
|
|
159
|
+
* Tolerates one that is not there. The caller is the retire sweep, which
|
|
160
|
+
* drops before it forgets the row — so a pass that dropped and then failed to
|
|
161
|
+
* forget comes back to a name whose collection is already gone, and that is a
|
|
162
|
+
* completed job rather than an error. The alternative order loses the only
|
|
163
|
+
* record that the collection exists.
|
|
164
|
+
*
|
|
165
|
+
* There is no confirmation and no dry run because there is no ambiguity at
|
|
166
|
+
* this level: by the time a name reaches here it has been through the
|
|
167
|
+
* rollback window and been checked against every organization's pointer.
|
|
168
|
+
*/
|
|
169
|
+
dropCollection(collection: string): Promise<void>;
|
|
104
170
|
/**
|
|
105
171
|
* The named vectors a collection has, and their widths.
|
|
106
172
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../vector/search.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../vector/search.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAA;AAErD,OAAO,EAAgC,KAAK,SAAS,EAAE,KAAK,aAAa,EAAE,MAAM,oBAAoB,CAAA;AAErG,OAAO,EAAqF,KAAK,MAAM,EAAE,MAAM,YAAY,CAAA;AAE3H;;;;;;;;GAQG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAGpD;AAED,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;IACpB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CACzB;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB;;;;;;;;;OASG;IACH,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,2EAA2E;IAC3E,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAA;IAC5B,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAA;IACpC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB;;;;;;OAMG;IACH,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,CAAA;CAC5B;AAED,MAAM,WAAW,GAAG;IAClB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAC1C;AAED;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,oBAAoB,KAAK,CAAA;AAEtC;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,qBAAa,eAAe;;gBAOd,MAAM,EAAE,YAAY,EAAE,KAAK,GAAE,MAA6B;IAwBhE,MAAM,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAqCvE,gFAAgF;WACnE,QAAQ,CAAC,MAAM,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,MAAM,EAAE,CAAC;CAK5F;AAED;;;;;;;;;;;GAWG;AACH,qBAAa,WAAW;;gBAIV,OAAO,EAAE,kBAAkB;IAYvC;;;;;;;OAOG;IACG,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAoB1F;;;;;;;;;;;;OAYG;IACG,SAAS,CAAC,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAQtE;;;;;;;;;;;;;;;;OAgBG;IACG,WAAW,CACf,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAChC,OAAO,CAAC,IAAI,CAAC;IAmBhB;;;;;;;;;;;;OAYG;IACG,cAAc,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAWvD;;;;;;;;OAQG;IACG,SAAS,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAQpE;;;;;;;;;;;;;;;;;;;;OAoBG;IACG,cAAc,CAAC,KAAK,EAAE;QAC1B,IAAI,EAAE,MAAM,CAAA;QACZ,EAAE,EAAE,MAAM,CAAA;QACV,yDAAyD;QACzD,SAAS,EAAE;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAA;QACzC,6CAA6C;QAC7C,KAAK,CAAC,EAAE,MAAM,CAAA;QACd,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,CAAA;KACtC,GAAG,OAAO,CAAC,IAAI,CAAC;IA4EX,MAAM,CAAC,OAAO,EAAE,aAAa,GAAG,OAAO,CAAC,SAAS,GAAG,EAAE,CAAC;IA4B7D;;;;;;;OAOG;IACH,MAAM,CAAC,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,GAAG,EAAE,GAAG,SAAS,GAAG,EAAE;IAYxE;;;;;;OAMG;IACG,KAAK,IAAI,OAAO,CAAC,OAAO,CAAC;IAKzB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAI7B"}
|
package/dist/vector/search.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { QdrantClient } from '@qdrant/js-client-rest';
|
|
2
2
|
import { buildFilter, METADATA_PREFIX } from '../authz/filter.js';
|
|
3
|
+
import { logger } from '../logging.js';
|
|
3
4
|
import { buildHybridQuery, collectionConfig, collectionName, PAYLOAD_INDEXES, vectorParams } from './query.js';
|
|
4
5
|
/**
|
|
5
6
|
* What Qdrant actually said.
|
|
@@ -14,6 +15,118 @@ export function explainQdrant(cause) {
|
|
|
14
15
|
const data = cause?.data;
|
|
15
16
|
return data === undefined ? String(cause) : `${String(cause)} — ${JSON.stringify(data)}`;
|
|
16
17
|
}
|
|
18
|
+
/**
|
|
19
|
+
* How many metadata keys one collection will have indexes built for.
|
|
20
|
+
*
|
|
21
|
+
* The keys are the caller's, so unlike `PAYLOAD_INDEXES` the set is not known
|
|
22
|
+
* in advance and has no natural end: `parseMetadata` bounds a *document* to 32
|
|
23
|
+
* keys and says nothing about how many distinct ones an organization uses, so
|
|
24
|
+
* a caller putting an identifier in a key name would otherwise ask for an index
|
|
25
|
+
* per document.
|
|
26
|
+
*
|
|
27
|
+
* A constant rather than a `NACRE_` variable. Every configuration variable is a
|
|
28
|
+
* thing an operator has to understand and a thing `docs/config.md` has to
|
|
29
|
+
* explain, and the failure this bounds — a slower filter past the sixty-fourth
|
|
30
|
+
* distinct key — needs no per-deployment answer.
|
|
31
|
+
*/
|
|
32
|
+
export const METADATA_INDEX_LIMIT = 64;
|
|
33
|
+
/**
|
|
34
|
+
* Payload indexes for the keys a caller filters on.
|
|
35
|
+
*
|
|
36
|
+
* `PAYLOAD_INDEXES` covers the fields the permission filter uses, which are
|
|
37
|
+
* fixed. A metadata filter reaches `meta.<key>`, and those fields had no index
|
|
38
|
+
* at all — so the narrowing that `filters` performs was evaluated by scanning,
|
|
39
|
+
* which is the failure the comment on `PAYLOAD_INDEXES` describes: it does not
|
|
40
|
+
* fail a test, it gets slow at a customer's volume.
|
|
41
|
+
*
|
|
42
|
+
* Built when metadata is written, because that is the only moment the keys are
|
|
43
|
+
* known. Three properties make that safe, and each was checked against a real
|
|
44
|
+
* Qdrant rather than read:
|
|
45
|
+
*
|
|
46
|
+
* - **A missing index is slow, never wrong.** A filter on an unindexed field —
|
|
47
|
+
* and on one indexed under the wrong type — still returns exactly the right
|
|
48
|
+
* points. Qdrant falls back to a scan. So this is an optimization end to end,
|
|
49
|
+
* and everything below may fail without changing an answer.
|
|
50
|
+
* - **Creating one is idempotent**, and it back-fills: an index added after the
|
|
51
|
+
* points covers them, so nothing has to be rebuilt or ordered.
|
|
52
|
+
* - **`keyword` is safe for every value type.** A number or a boolean under a
|
|
53
|
+
* keyword index simply indexes nothing and still matches, and strings and
|
|
54
|
+
* lists of strings — the case `filters` exists for, "only documents from this
|
|
55
|
+
* source" — index properly.
|
|
56
|
+
*
|
|
57
|
+
* Which is why a failure here warns and returns. This is the opposite of
|
|
58
|
+
* invariant I3 and deliberately so: nothing about an index decides who sees
|
|
59
|
+
* what, and failing an ingest because an index could not be built would trade a
|
|
60
|
+
* slow query for a document that never arrives.
|
|
61
|
+
*/
|
|
62
|
+
export class MetadataIndexer {
|
|
63
|
+
#client;
|
|
64
|
+
#limit;
|
|
65
|
+
/** Per collection, the `meta.` fields known to be indexed. */
|
|
66
|
+
#known = new Map();
|
|
67
|
+
#atLimit = new Set();
|
|
68
|
+
constructor(client, limit = METADATA_INDEX_LIMIT) {
|
|
69
|
+
this.#client = client;
|
|
70
|
+
this.#limit = limit;
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Held on the instance rather than in a module-level map: two stores against
|
|
74
|
+
* different collections must not share it, and a cache that outlives the
|
|
75
|
+
* process that built it is a cache nothing invalidates. Staleness is harmless
|
|
76
|
+
* anyway — the operation it saves is idempotent.
|
|
77
|
+
*/
|
|
78
|
+
async #indexed(collection) {
|
|
79
|
+
const cached = this.#known.get(collection);
|
|
80
|
+
if (cached !== undefined)
|
|
81
|
+
return cached;
|
|
82
|
+
const info = await this.#client.getCollection(collection);
|
|
83
|
+
const schema = (info.payload_schema ?? {});
|
|
84
|
+
const fields = new Set(Object.keys(schema).filter((field) => field.startsWith(`${METADATA_PREFIX}.`)));
|
|
85
|
+
this.#known.set(collection, fields);
|
|
86
|
+
return fields;
|
|
87
|
+
}
|
|
88
|
+
async ensure(collection, keys) {
|
|
89
|
+
const wanted = [...keys];
|
|
90
|
+
if (wanted.length === 0)
|
|
91
|
+
return;
|
|
92
|
+
try {
|
|
93
|
+
const indexed = await this.#indexed(collection);
|
|
94
|
+
for (const key of wanted) {
|
|
95
|
+
const field = `${METADATA_PREFIX}.${key}`;
|
|
96
|
+
if (indexed.has(field))
|
|
97
|
+
continue;
|
|
98
|
+
if (indexed.size >= this.#limit) {
|
|
99
|
+
// Once per collection per process. The filter still answers
|
|
100
|
+
// correctly; it stops being answered from an index.
|
|
101
|
+
if (!this.#atLimit.has(collection)) {
|
|
102
|
+
this.#atLimit.add(collection);
|
|
103
|
+
logger.warn('metadata index limit reached', { collection,
|
|
104
|
+
limit: this.#limit,
|
|
105
|
+
unindexed: field });
|
|
106
|
+
}
|
|
107
|
+
continue;
|
|
108
|
+
}
|
|
109
|
+
await this.#client.createPayloadIndex(collection, {
|
|
110
|
+
field_name: field,
|
|
111
|
+
field_schema: 'keyword',
|
|
112
|
+
wait: true,
|
|
113
|
+
});
|
|
114
|
+
indexed.add(field);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
catch (cause) {
|
|
118
|
+
// Never fatal. See the note on this class.
|
|
119
|
+
logger.warn('metadata index build failed', { collection,
|
|
120
|
+
error: explainQdrant(cause) });
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
/** The `meta.` indexes a collection already has, for carrying across a copy. */
|
|
124
|
+
static async fieldsOf(client, collection) {
|
|
125
|
+
const info = await client.getCollection(collection);
|
|
126
|
+
const schema = (info.payload_schema ?? {});
|
|
127
|
+
return Object.keys(schema).filter((field) => field.startsWith(`${METADATA_PREFIX}.`));
|
|
128
|
+
}
|
|
129
|
+
}
|
|
17
130
|
/**
|
|
18
131
|
* The vector store.
|
|
19
132
|
*
|
|
@@ -28,6 +141,7 @@ export function explainQdrant(cause) {
|
|
|
28
141
|
*/
|
|
29
142
|
export class VectorStore {
|
|
30
143
|
#client;
|
|
144
|
+
#metadataIndexes;
|
|
31
145
|
constructor(options) {
|
|
32
146
|
// exactOptionalPropertyTypes distinguishes "absent" from "present and
|
|
33
147
|
// undefined", and the client's parameters mean it: passing apiKey:
|
|
@@ -35,6 +149,7 @@ export class VectorStore {
|
|
|
35
149
|
this.#client = new QdrantClient(options.apiKey === undefined
|
|
36
150
|
? { url: options.url }
|
|
37
151
|
: { url: options.url, apiKey: options.apiKey });
|
|
152
|
+
this.#metadataIndexes = new MetadataIndexer(this.#client);
|
|
38
153
|
}
|
|
39
154
|
/**
|
|
40
155
|
* Create an organization's first collection, if it is not there.
|
|
@@ -109,6 +224,33 @@ export class VectorStore {
|
|
|
109
224
|
catch (cause) {
|
|
110
225
|
throw new Error(`metadata write to ${collection} rejected: ${explainQdrant(cause)}`, { cause });
|
|
111
226
|
}
|
|
227
|
+
// After the write, not before: an index built for a write that then failed
|
|
228
|
+
// is an index for a key no point carries.
|
|
229
|
+
await this.#metadataIndexes.ensure(collection, Object.keys(metadata));
|
|
230
|
+
}
|
|
231
|
+
/**
|
|
232
|
+
* Remove a collection.
|
|
233
|
+
*
|
|
234
|
+
* Tolerates one that is not there. The caller is the retire sweep, which
|
|
235
|
+
* drops before it forgets the row — so a pass that dropped and then failed to
|
|
236
|
+
* forget comes back to a name whose collection is already gone, and that is a
|
|
237
|
+
* completed job rather than an error. The alternative order loses the only
|
|
238
|
+
* record that the collection exists.
|
|
239
|
+
*
|
|
240
|
+
* There is no confirmation and no dry run because there is no ambiguity at
|
|
241
|
+
* this level: by the time a name reaches here it has been through the
|
|
242
|
+
* rollback window and been checked against every organization's pointer.
|
|
243
|
+
*/
|
|
244
|
+
async dropCollection(collection) {
|
|
245
|
+
try {
|
|
246
|
+
await this.#client.deleteCollection(collection);
|
|
247
|
+
}
|
|
248
|
+
catch (cause) {
|
|
249
|
+
const existing = await this.#client.getCollections();
|
|
250
|
+
if (existing.collections.some((c) => c.name === collection)) {
|
|
251
|
+
throw new Error(`deleting ${collection} rejected: ${explainQdrant(cause)}`, { cause });
|
|
252
|
+
}
|
|
253
|
+
}
|
|
112
254
|
}
|
|
113
255
|
/**
|
|
114
256
|
* The named vectors a collection has, and their widths.
|
|
@@ -170,6 +312,18 @@ export class VectorStore {
|
|
|
170
312
|
wait: true,
|
|
171
313
|
});
|
|
172
314
|
}
|
|
315
|
+
// The metadata indexes too, or a reindex quietly undoes them: the new
|
|
316
|
+
// collection is built from `PAYLOAD_INDEXES`, which by construction knows
|
|
317
|
+
// nothing about keys a caller chose, so every metadata filter in that
|
|
318
|
+
// organization would go back to scanning the moment the pointer moved. The
|
|
319
|
+
// source is the only record of which keys are in use.
|
|
320
|
+
for (const field of await MetadataIndexer.fieldsOf(this.#client, input.from)) {
|
|
321
|
+
await this.#client.createPayloadIndex(input.to, {
|
|
322
|
+
field_name: field,
|
|
323
|
+
field_schema: 'keyword',
|
|
324
|
+
wait: true,
|
|
325
|
+
});
|
|
326
|
+
}
|
|
173
327
|
const batch = input.batch ?? 256;
|
|
174
328
|
let offset = undefined;
|
|
175
329
|
let copied = 0;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"search.js","sourceRoot":"","sources":["../../vector/search.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAA;AAErD,OAAO,EAAE,WAAW,EAAE,eAAe,EAAsC,MAAM,oBAAoB,CAAA;AACrG,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,cAAc,EAAE,eAAe,EAAE,YAAY,EAAe,MAAM,YAAY,CAAA;AAE3H;;;;;;;;GAQG;AACH,MAAM,UAAU,aAAa,CAAC,KAAc;IAC1C,MAAM,IAAI,GAAI,KAAmC,EAAE,IAAI,CAAA;IACvD,OAAO,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAA;AAC1F,CAAC;AAwCD;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,WAAW;IACb,OAAO,CAAc;
|
|
1
|
+
{"version":3,"file":"search.js","sourceRoot":"","sources":["../../vector/search.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAA;AAErD,OAAO,EAAE,WAAW,EAAE,eAAe,EAAsC,MAAM,oBAAoB,CAAA;AACrG,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAA;AACtC,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,cAAc,EAAE,eAAe,EAAE,YAAY,EAAe,MAAM,YAAY,CAAA;AAE3H;;;;;;;;GAQG;AACH,MAAM,UAAU,aAAa,CAAC,KAAc;IAC1C,MAAM,IAAI,GAAI,KAAmC,EAAE,IAAI,CAAA;IACvD,OAAO,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAA;AAC1F,CAAC;AAwCD;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,EAAE,CAAA;AAEtC;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,MAAM,OAAO,eAAe;IACjB,OAAO,CAAc;IACrB,MAAM,CAAQ;IACvB,8DAA8D;IACrD,MAAM,GAAG,IAAI,GAAG,EAAuB,CAAA;IACvC,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAA;IAErC,YAAY,MAAoB,EAAE,QAAgB,oBAAoB;QACpE,IAAI,CAAC,OAAO,GAAG,MAAM,CAAA;QACrB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAA;IACrB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,QAAQ,CAAC,UAAkB;QAC/B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;QAC1C,IAAI,MAAM,KAAK,SAAS;YAAE,OAAO,MAAM,CAAA;QAEvC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,CAAA;QACzD,MAAM,MAAM,GAAG,CAAC,IAAI,CAAC,cAAc,IAAI,EAAE,CAA4B,CAAA;QACrE,MAAM,MAAM,GAAG,IAAI,GAAG,CACpB,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,eAAe,GAAG,CAAC,CAAC,CAC/E,CAAA;QACD,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,CAAA;QACnC,OAAO,MAAM,CAAA;IACf,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,UAAkB,EAAE,IAAsB;QACrD,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,CAAA;QACxB,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAM;QAE/B,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAA;YAE/C,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;gBACzB,MAAM,KAAK,GAAG,GAAG,eAAe,IAAI,GAAG,EAAE,CAAA;gBACzC,IAAI,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC;oBAAE,SAAQ;gBAEhC,IAAI,OAAO,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;oBAChC,4DAA4D;oBAC5D,oDAAoD;oBACpD,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;wBACnC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,CAAA;wBAC7B,MAAM,CAAC,IAAI,CAAC,8BAA8B,EAAE,EAAE,UAAU;4BACpD,KAAK,EAAE,IAAI,CAAC,MAAM;4BAClB,SAAS,EAAE,KAAK,EAAE,CAAC,CAAA;oBACzB,CAAC;oBACD,SAAQ;gBACV,CAAC;gBAED,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,UAAU,EAAE;oBAChD,UAAU,EAAE,KAAK;oBACjB,YAAY,EAAE,SAAkB;oBAChC,IAAI,EAAE,IAAI;iBACX,CAAC,CAAA;gBACF,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;YACpB,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,2CAA2C;YAC3C,MAAM,CAAC,IAAI,CAAC,6BAA6B,EAAE,EAAE,UAAU;gBACnD,KAAK,EAAE,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QACpC,CAAC;IACH,CAAC;IAED,gFAAgF;IAChF,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAoB,EAAE,UAAkB;QAC5D,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,UAAU,CAAC,CAAA;QACnD,MAAM,MAAM,GAAG,CAAC,IAAI,CAAC,cAAc,IAAI,EAAE,CAA4B,CAAA;QACrE,OAAO,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,eAAe,GAAG,CAAC,CAAC,CAAA;IACvF,CAAC;CACF;AAED;;;;;;;;;;;GAWG;AACH,MAAM,OAAO,WAAW;IACb,OAAO,CAAc;IACrB,gBAAgB,CAAiB;IAE1C,YAAY,OAA2B;QACrC,sEAAsE;QACtE,mEAAmE;QACnE,4CAA4C;QAC5C,IAAI,CAAC,OAAO,GAAG,IAAI,YAAY,CAC7B,OAAO,CAAC,MAAM,KAAK,SAAS;YAC1B,CAAC,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE;YACtB,CAAC,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CACjD,CAAA;QACD,IAAI,CAAC,gBAAgB,GAAG,IAAI,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IAC3D,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,gBAAgB,CAAC,OAAe,EAAE,UAAkB,EAAE,IAAY;QACtE,MAAM,IAAI,GAAG,cAAc,CAAC,OAAO,CAAC,CAAA;QACpC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAA;QACpD,IAAI,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC;YAAE,OAAO,IAAI,CAAA;QAElE,MAAM,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,IAAI,EAAE,gBAAgB,CAAC,UAAU,EAAE,IAAI,CAAU,CAAC,CAAA;QAEtF,4EAA4E;QAC5E,yEAAyE;QACzE,KAAK,MAAM,KAAK,IAAI,eAAe,EAAE,CAAC;YACpC,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,IAAI,EAAE;gBAC1C,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,YAAY,EAAE,KAAK,CAAC,YAAqB;gBACzC,IAAI,EAAE,IAAI;aACX,CAAC,CAAA;QACJ,CAAC;QAED,OAAO,IAAI,CAAA;IACb,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,SAAS,CAAC,UAAkB,EAAE,UAAkB;QACpD,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,UAAU,EAAE;YACxC,IAAI,EAAE,IAAI;YACV,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE;YAC1B,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE,CAAC,EAAE;SAC3D,CAAC,CAAA;IACb,CAAC;IAED;;;;;;;;;;;;;;;;OAgBG;IACH,KAAK,CAAC,WAAW,CACf,UAAkB,EAClB,UAAkB,EAClB,QAAiC;QAEjC,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,UAAU,EAAE;gBACxC,IAAI,EAAE,IAAI;gBACV,OAAO,EAAE,EAAE,CAAC,eAAe,CAAC,EAAE,QAAQ,EAAE;gBACxC,MAAM,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE,CAAC,EAAE;aAC3D,CAAC,CAAA;QACb,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,qBAAqB,UAAU,cAAc,aAAa,CAAC,KAAK,CAAC,EAAE,EACnE,EAAE,KAAK,EAAE,CACV,CAAA;QACH,CAAC;QAED,2EAA2E;QAC3E,0CAA0C;QAC1C,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAA;IACvE,CAAC;IAED;;;;;;;;;;;;OAYG;IACH,KAAK,CAAC,cAAc,CAAC,UAAkB;QACrC,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAA;QACjD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAA;YACpD,IAAI,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,EAAE,CAAC;gBAC5D,MAAM,IAAI,KAAK,CAAC,YAAY,UAAU,cAAc,aAAa,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAAA;YACxF,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,SAAS,CAAC,UAAkB;QAChC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,UAAU,CAAC,CAAA;QACzD,MAAM,OAAO,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,IAAI,EAAE,CAAsC,CAAA;QACzF,OAAO,MAAM,CAAC,WAAW,CACvB,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,CACxE,CAAA;IACH,CAAC;IAED;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,KAAK,CAAC,cAAc,CAAC,KAQpB;QACC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QACjD,IAAI,KAAK,CAAC,SAAS,CAAC,IAAI,IAAI,QAAQ,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CACb,GAAG,KAAK,CAAC,IAAI,gCAAgC,KAAK,CAAC,SAAS,CAAC,IAAI,uBAAuB,CACzF,CAAA;QACH,CAAC;QAED,MAAM,OAAO,GAA4B,EAAE,CAAA;QAC3C,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;YACpD,OAAO,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,IAAI,CAAC,CAAA;QACpC,CAAC;QACD,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;QAElE,mDAAmD;QACnD,MAAM,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAA;QACpE,MAAM,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,KAAK,CAAC,EAAE,EAAE;YAC5C,OAAO,EAAE,OAAgB;YACzB,cAAc,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;YAC5B,iBAAiB,EAAE,EAAE,sBAAsB,EAAE,CAAC,EAAE;YAChD,eAAe,EAAE,IAAI;SACb,CAAC,CAAA;QAEX,KAAK,MAAM,KAAK,IAAI,eAAe,EAAE,CAAC;YACpC,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,KAAK,CAAC,EAAE,EAAE;gBAC9C,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,YAAY,EAAE,KAAK,CAAC,YAAqB;gBACzC,IAAI,EAAE,IAAI;aACX,CAAC,CAAA;QACJ,CAAC;QAED,sEAAsE;QACtE,0EAA0E;QAC1E,sEAAsE;QACtE,2EAA2E;QAC3E,sDAAsD;QACtD,KAAK,MAAM,KAAK,IAAI,MAAM,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YAC7E,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB,CAAC,KAAK,CAAC,EAAE,EAAE;gBAC9C,UAAU,EAAE,KAAK;gBACjB,YAAY,EAAE,SAAkB;gBAChC,IAAI,EAAE,IAAI;aACX,CAAC,CAAA;QACJ,CAAC;QAED,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,IAAI,GAAG,CAAA;QAChC,IAAI,MAAM,GAAY,SAAS,CAAA;QAC/B,IAAI,MAAM,GAAG,CAAC,CAAA;QAEd,GAAG,CAAC;YACF,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE;gBACjD,KAAK,EAAE,KAAK;gBACZ,YAAY,EAAE,IAAI;gBAClB,WAAW,EAAE,IAAI;gBACjB,GAAG,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC;aACtD,CAAC,CAAA;YAEX,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,EAAE,CAAA;YAChC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtB,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,EAAE;oBAClC,IAAI,EAAE,IAAI;oBACV,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;wBACzB,EAAE,EAAE,CAAC,CAAC,EAAE;wBACR,+DAA+D;wBAC/D,sCAAsC;wBACtC,MAAM,EAAE,CAAC,CAAC,MAAe;wBACzB,OAAO,EAAE,CAAC,CAAC,OAAO,IAAI,EAAE;qBACzB,CAAC,CAAC;iBACK,CAAC,CAAA;gBACX,MAAM,IAAI,MAAM,CAAC,MAAM,CAAA;gBACvB,KAAK,CAAC,UAAU,EAAE,CAAC,MAAM,CAAC,CAAA;YAC5B,CAAC;YAED,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAA;QAChC,CAAC,QAAQ,MAAM,KAAK,SAAS,IAAI,MAAM,KAAK,IAAI,EAAC;IACnD,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,OAAsB;QACjC,MAAM,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;QACvE,MAAM,KAAK,GAAG,gBAAgB,CAAC;YAC7B,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,MAAM;YACN,KAAK,EAAE,OAAO,CAAC,IAAI;SACpB,CAAC,CAAA;QAEF,IAAI,MAAM,CAAA;QACV,IAAI,CAAC;YACH,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,EAAE,KAAc,CAAC,CAAA;QACvE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,wEAAwE;YACxE,sEAAsE;YACtE,MAAM,IAAI,KAAK,CACb,iBAAiB,OAAO,CAAC,UAAU,SAAS;gBAC1C,GAAG,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,aAAa,CAAC,KAAK,CAAC,EAAE,EACtF,EAAE,KAAK,EAAE,CACV,CAAA;QACH,CAAC;QAED,OAAO,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC/B,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;YAChB,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,OAAO,EAAE,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAA4B;SACtD,CAAC,CAAC,CAAA;IACL,CAAC;IAED;;;;;;;OAOG;IACH,MAAM,CAAC,YAAY,CAAC,KAAa,EAAE,IAAoB;QACrD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC;gBACjC,MAAM,IAAI,KAAK,CACb,yCAAyC,GAAG,CAAC,EAAE,kBAAkB;oBAC/D,GAAG,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,4BAA4B,KAAK,EAAE,CACnE,CAAA;YACH,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAA;IACb,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAA;QACnC,OAAO,IAAI,CAAA;IACb,CAAC;IAED,KAAK,CAAC,KAAK;QACT,0EAA0E;QAC1E,0BAA0B;IAC5B,CAAC;CACF"}
|