@equinor/fusion-framework-cli-plugin-ai-index 2.0.1 ā 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +52 -0
- package/dist/esm/bin/apply-metadata.js +15 -5
- package/dist/esm/bin/apply-metadata.js.map +1 -1
- package/dist/esm/bin/apply-schema.js +64 -0
- package/dist/esm/bin/apply-schema.js.map +1 -0
- package/dist/esm/bin/apply-schema.test.js +143 -0
- package/dist/esm/bin/apply-schema.test.js.map +1 -0
- package/dist/esm/bin/delete-removed-files.js +1 -1
- package/dist/esm/bin/delete-removed-files.js.map +1 -1
- package/dist/esm/bin/embed.js +188 -47
- package/dist/esm/bin/embed.js.map +1 -1
- package/dist/esm/create-command.js +186 -0
- package/dist/esm/create-command.js.map +1 -0
- package/dist/esm/delete-command.js +14 -2
- package/dist/esm/delete-command.js.map +1 -1
- package/dist/esm/delete-command.options.js +7 -31
- package/dist/esm/delete-command.options.js.map +1 -1
- package/dist/esm/delete-index-command.js +94 -0
- package/dist/esm/delete-index-command.js.map +1 -0
- package/dist/esm/embed-command.js +30 -0
- package/dist/esm/embed-command.js.map +1 -0
- package/dist/esm/embeddings-command.js +14 -17
- package/dist/esm/embeddings-command.js.map +1 -1
- package/dist/esm/embeddings-command.options.js +12 -43
- package/dist/esm/embeddings-command.options.js.map +1 -1
- package/dist/esm/index.js +12 -3
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/schema.js +41 -0
- package/dist/esm/schema.js.map +1 -0
- package/dist/esm/search-command.js +17 -5
- package/dist/esm/search-command.js.map +1 -1
- package/dist/esm/utils/embedding-dimensions.js +37 -0
- package/dist/esm/utils/embedding-dimensions.js.map +1 -0
- package/dist/esm/utils/zod-to-azure-fields.js +120 -0
- package/dist/esm/utils/zod-to-azure-fields.js.map +1 -0
- package/dist/esm/utils/zod-to-azure-fields.test.js +112 -0
- package/dist/esm/utils/zod-to-azure-fields.test.js.map +1 -0
- package/dist/esm/version.js +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/dist/types/bin/apply-metadata.d.ts +2 -1
- package/dist/types/bin/apply-schema.d.ts +22 -0
- package/dist/types/bin/apply-schema.test.d.ts +1 -0
- package/dist/types/config.d.ts +14 -0
- package/dist/types/create-command.d.ts +6 -0
- package/dist/types/delete-command.options.d.ts +9 -23
- package/dist/types/delete-index-command.d.ts +6 -0
- package/dist/types/embed-command.d.ts +12 -0
- package/dist/types/embeddings-command.options.d.ts +9 -28
- package/dist/types/index.d.ts +1 -0
- package/dist/types/schema.d.ts +137 -0
- package/dist/types/utils/embedding-dimensions.d.ts +13 -0
- package/dist/types/utils/zod-to-azure-fields.d.ts +61 -0
- package/dist/types/utils/zod-to-azure-fields.test.d.ts +1 -0
- package/dist/types/version.d.ts +1 -1
- package/package.json +6 -6
- package/src/bin/apply-metadata.ts +20 -4
- package/src/bin/apply-schema.test.ts +170 -0
- package/src/bin/apply-schema.ts +86 -0
- package/src/bin/delete-removed-files.ts +1 -1
- package/src/bin/embed.ts +248 -76
- package/src/config.ts +15 -0
- package/src/create-command.ts +218 -0
- package/src/delete-command.options.ts +7 -37
- package/src/delete-command.ts +19 -2
- package/src/delete-index-command.ts +121 -0
- package/src/embed-command.ts +44 -0
- package/src/embeddings-command.options.ts +12 -50
- package/src/embeddings-command.ts +18 -18
- package/src/index.ts +12 -3
- package/src/schema.ts +149 -0
- package/src/search-command.ts +22 -5
- package/src/utils/embedding-dimensions.ts +39 -0
- package/src/utils/zod-to-azure-fields.test.ts +136 -0
- package/src/utils/zod-to-azure-fields.ts +177 -0
- package/src/version.ts +1 -1
package/dist/esm/bin/embed.js
CHANGED
|
@@ -1,22 +1,117 @@
|
|
|
1
1
|
import { globbyStream } from 'globby';
|
|
2
2
|
import { relative } from 'node:path';
|
|
3
3
|
import multimatch from 'multimatch';
|
|
4
|
-
import {
|
|
5
|
-
import { concatMap, filter, map, mergeMap, retry, shareReplay,
|
|
4
|
+
import { from, merge, timer } from 'rxjs';
|
|
5
|
+
import { bufferCount, bufferTime, concatMap, filter, finalize, map, mergeMap, retry, shareReplay, tap, } from 'rxjs/operators';
|
|
6
6
|
import { isMarkdownFile, parseMarkdownFile } from '../utils/markdown/index.js';
|
|
7
7
|
import { getFileStatus, resolveProjectRoot } from '../utils/git/index.js';
|
|
8
8
|
import { isTypescriptFile, parseTsDocFromFileSync } from '../utils/ts-doc/index.js';
|
|
9
9
|
import { getDiff } from './get-diff.js';
|
|
10
10
|
import { createDeleteRemovedFilesStream } from './delete-removed-files.js';
|
|
11
11
|
import { applyMetadata } from './apply-metadata.js';
|
|
12
|
+
import { applySchema } from './apply-schema.js';
|
|
12
13
|
import { readFileSync } from 'node:fs';
|
|
13
14
|
import { generateChunkId } from '../utils/generate-chunk-id.js';
|
|
15
|
+
/** Braille spinner frames (same as ora's default). */
|
|
16
|
+
const SPINNER_FRAMES = ['ā ', 'ā ', 'ā ¹', 'ā ø', 'ā ¼', 'ā “', 'ā ¦', 'ā §', 'ā ', 'ā '];
|
|
17
|
+
/**
|
|
18
|
+
* Manages a fixed block of sticky progress lines with per-line spinners.
|
|
19
|
+
* Each line can be updated independently without overwriting the others.
|
|
20
|
+
* @internal
|
|
21
|
+
*/
|
|
22
|
+
class ProgressDisplay {
|
|
23
|
+
lines = [];
|
|
24
|
+
spinning = [];
|
|
25
|
+
started = false;
|
|
26
|
+
frame = 0;
|
|
27
|
+
timer;
|
|
28
|
+
/** Register the line labels up front and print empty placeholders. */
|
|
29
|
+
start(count) {
|
|
30
|
+
this.lines = new Array(count).fill('');
|
|
31
|
+
this.spinning = new Array(count).fill(false);
|
|
32
|
+
// Print placeholder lines so the cursor block exists
|
|
33
|
+
for (let i = 0; i < count; i++) {
|
|
34
|
+
process.stdout.write('\n');
|
|
35
|
+
}
|
|
36
|
+
this.started = true;
|
|
37
|
+
// Tick spinner at 80ms (same cadence as ora)
|
|
38
|
+
this.timer = setInterval(() => this.tick(), 80);
|
|
39
|
+
}
|
|
40
|
+
/** Update a specific line (0-indexed) without touching the others. */
|
|
41
|
+
update(line, message) {
|
|
42
|
+
if (!this.started)
|
|
43
|
+
return;
|
|
44
|
+
this.lines[line] = message;
|
|
45
|
+
this.spinning[line] = true;
|
|
46
|
+
this.render(line);
|
|
47
|
+
}
|
|
48
|
+
/** Mark a line as completed ā stops its spinner and shows a checkmark. */
|
|
49
|
+
succeed(line, message) {
|
|
50
|
+
if (!this.started)
|
|
51
|
+
return;
|
|
52
|
+
this.lines[line] = `ā
${message}`;
|
|
53
|
+
this.spinning[line] = false;
|
|
54
|
+
this.render(line);
|
|
55
|
+
}
|
|
56
|
+
/** Clear all progress lines and leave the cursor on a clean line. */
|
|
57
|
+
clear() {
|
|
58
|
+
if (!this.started)
|
|
59
|
+
return;
|
|
60
|
+
if (this.timer)
|
|
61
|
+
clearInterval(this.timer);
|
|
62
|
+
// Move up to the first progress line and clear each one
|
|
63
|
+
for (let i = 0; i < this.lines.length; i++) {
|
|
64
|
+
const linesUp = this.lines.length - i;
|
|
65
|
+
process.stdout.write(`\x1b[${linesUp}A\x1b[2K\r\x1b[${linesUp}B\r`);
|
|
66
|
+
}
|
|
67
|
+
// Move cursor up past the now-empty block
|
|
68
|
+
process.stdout.write(`\x1b[${this.lines.length}A\r`);
|
|
69
|
+
this.started = false;
|
|
70
|
+
}
|
|
71
|
+
/** Advance the spinner frame and re-render all spinning lines. */
|
|
72
|
+
tick() {
|
|
73
|
+
this.frame = (this.frame + 1) % SPINNER_FRAMES.length;
|
|
74
|
+
for (let i = 0; i < this.lines.length; i++) {
|
|
75
|
+
if (this.spinning[i] && this.lines[i]) {
|
|
76
|
+
this.render(i);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
/** Render a single line at its position. */
|
|
81
|
+
render(line) {
|
|
82
|
+
const linesUp = this.lines.length - line;
|
|
83
|
+
const prefix = this.spinning[line] ? SPINNER_FRAMES[this.frame] : '';
|
|
84
|
+
const text = this.spinning[line] ? `${prefix} ${this.lines[line]}` : this.lines[line];
|
|
85
|
+
process.stdout.write(`\x1b[${linesUp}A\x1b[2K\r${text}\x1b[${linesUp}B\r`);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
/** Progress line indices */
|
|
89
|
+
const LINE_PARSE = 0;
|
|
90
|
+
const LINE_META = 1;
|
|
91
|
+
const LINE_EMBED = 2;
|
|
92
|
+
const LINE_INDEX = 3;
|
|
14
93
|
/**
|
|
15
94
|
* Default directories to skip before expensive git operations.
|
|
16
95
|
* These are common build artifacts and dependencies that should be ignored.
|
|
17
96
|
* @internal
|
|
18
97
|
*/
|
|
19
98
|
const defaultIgnore = ['node_modules', '**/node_modules/**', 'dist', '**/dist/**', '.git'];
|
|
99
|
+
/** Concurrency limit for git subprocess operations (status, log, etc.). */
|
|
100
|
+
const GIT_CONCURRENCY = 20;
|
|
101
|
+
/** Maximum parallel upsert requests to the vector store. */
|
|
102
|
+
const UPSERT_CONCURRENCY = 10;
|
|
103
|
+
/** Number of texts to embed per API request. */
|
|
104
|
+
const EMBED_BATCH_SIZE = 20;
|
|
105
|
+
/** Number of concurrent batch requests in flight. */
|
|
106
|
+
const EMBED_BATCH_CONCURRENCY = 4;
|
|
107
|
+
/**
|
|
108
|
+
* Maximum time (ms) to wait before flushing a partial embedding batch.
|
|
109
|
+
* Without this, `bufferCount` waits indefinitely for a full batch, which
|
|
110
|
+
* starves `mergeMap` concurrency when upstream document throughput is slow.
|
|
111
|
+
*/
|
|
112
|
+
const EMBED_BUFFER_FLUSH_MS = 250;
|
|
113
|
+
/** Maximum retry attempts for transient / rate-limit errors per chunk. */
|
|
114
|
+
const MAX_RETRIES = 4;
|
|
20
115
|
/**
|
|
21
116
|
* Main entry point for the embeddings bin.
|
|
22
117
|
* Orchestrates the entire embeddings generation pipeline.
|
|
@@ -24,9 +119,10 @@ const defaultIgnore = ['node_modules', '**/node_modules/**', 'dist', '**/dist/**
|
|
|
24
119
|
*/
|
|
25
120
|
export async function embed(binOptions) {
|
|
26
121
|
const { framework, options, config, filePatterns } = binOptions;
|
|
27
|
-
console.log(`š Index: ${options.
|
|
122
|
+
console.log(`š Index: ${options.indexName}`);
|
|
123
|
+
const progress = new ProgressDisplay();
|
|
28
124
|
// Handle clean operation (destructive - deletes all existing documents)
|
|
29
|
-
const vectorStoreService = framework.ai.
|
|
125
|
+
const vectorStoreService = framework.ai.useIndex(options.indexName);
|
|
30
126
|
if (options.clean && !options.dryRun) {
|
|
31
127
|
console.log('š§¹ Cleaning vector store: deleting all existing documents...');
|
|
32
128
|
// OData filter: delete all documents with non-empty source (all indexed docs)
|
|
@@ -55,8 +151,8 @@ export async function embed(binOptions) {
|
|
|
55
151
|
gitignore,
|
|
56
152
|
absolute: true,
|
|
57
153
|
})).pipe(
|
|
58
|
-
// Get git status concurrently
|
|
59
|
-
mergeMap((path) => getFileStatus(path)), concatMap((files) => from(files)),
|
|
154
|
+
// Get git status concurrently (capped to avoid spawning too many git processes)
|
|
155
|
+
mergeMap((path) => getFileStatus(path), GIT_CONCURRENCY), concatMap((files) => from(files)),
|
|
60
156
|
// Share stream for multiple subscribers (removedFiles$ and indexFiles$)
|
|
61
157
|
shareReplay({ refCount: true }));
|
|
62
158
|
})();
|
|
@@ -68,6 +164,7 @@ export async function embed(binOptions) {
|
|
|
68
164
|
'**/*.mdx',
|
|
69
165
|
];
|
|
70
166
|
// Process files: enrich with metadata and filter by allowed patterns
|
|
167
|
+
let fileCount = 0;
|
|
71
168
|
const processedFiles$ = files$.pipe(map((file) => {
|
|
72
169
|
const { filepath, status } = file;
|
|
73
170
|
const projectRoot = resolveProjectRoot(filepath);
|
|
@@ -81,6 +178,10 @@ export async function embed(binOptions) {
|
|
|
81
178
|
}), filter((file) => {
|
|
82
179
|
const matches = multimatch(file.relativePath, allowedFilePatterns);
|
|
83
180
|
return matches.length > 0;
|
|
181
|
+
}), tap((file) => {
|
|
182
|
+
fileCount++;
|
|
183
|
+
const label = file.status === 'removed' ? 'šļø' : 'š';
|
|
184
|
+
progress.update(LINE_PARSE, `${label} Parsing [${fileCount}] ${file.relativePath}`);
|
|
84
185
|
}),
|
|
85
186
|
// Share for multiple subscribers (removedFiles$, markdown$, typescript$)
|
|
86
187
|
shareReplay({ refCount: true }));
|
|
@@ -99,6 +200,7 @@ export async function embed(binOptions) {
|
|
|
99
200
|
}
|
|
100
201
|
return false;
|
|
101
202
|
};
|
|
203
|
+
let docCount = 0;
|
|
102
204
|
const rawFiles$ = indexFiles$.pipe(filter(isRawFile), map((file) => {
|
|
103
205
|
const document = {
|
|
104
206
|
id: generateChunkId(file.relativePath),
|
|
@@ -108,81 +210,107 @@ export async function embed(binOptions) {
|
|
|
108
210
|
type: 'raw',
|
|
109
211
|
},
|
|
110
212
|
};
|
|
213
|
+
docCount++;
|
|
214
|
+
progress.update(LINE_PARSE, `š Parsing [${docCount}] ${file.relativePath}`);
|
|
111
215
|
return { status: file.status, documents: [document] };
|
|
112
216
|
}));
|
|
113
217
|
const markdown$ = indexFiles$.pipe(filter((x) => !isRawFile(x)), filter((file) => isMarkdownFile(file.path)), mergeMap(async (file) => {
|
|
114
218
|
const documents = await parseMarkdownFile(file);
|
|
219
|
+
docCount++;
|
|
220
|
+
progress.update(LINE_PARSE, `š Parsing [${docCount}] ${file.relativePath}`);
|
|
115
221
|
return { status: file.status, documents };
|
|
116
222
|
}));
|
|
117
223
|
const typescript$ = indexFiles$.pipe(filter((x) => !isRawFile(x)), filter((file) => isTypescriptFile(file.path)), map((file) => {
|
|
118
224
|
const documents = parseTsDocFromFileSync(file);
|
|
225
|
+
docCount++;
|
|
226
|
+
progress.update(LINE_PARSE, `š Parsing [${docCount}] ${file.relativePath}`);
|
|
119
227
|
return { status: file.status, documents };
|
|
120
228
|
}));
|
|
229
|
+
// Merge parsed streams and signal when all parsing is done
|
|
230
|
+
const parsed$ = merge(rawFiles$, markdown$, typescript$).pipe(finalize(() => {
|
|
231
|
+
progress.succeed(LINE_PARSE, `š Parsed ${docCount} files`);
|
|
232
|
+
}));
|
|
121
233
|
// Apply metadata to documents
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
const
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
234
|
+
let metadataCount = 0;
|
|
235
|
+
let metadataDone = false;
|
|
236
|
+
const applyMetadata$ = applyMetadata(parsed$, config.index, (source) => {
|
|
237
|
+
metadataCount++;
|
|
238
|
+
progress.update(LINE_META, `š·ļø Metadata [${metadataCount}] ${source}`);
|
|
239
|
+
}).pipe(finalize(() => {
|
|
240
|
+
metadataDone = true;
|
|
241
|
+
progress.succeed(LINE_META, `š·ļø Metadata ${metadataCount} documents`);
|
|
242
|
+
}));
|
|
243
|
+
// Resolve promoted schema fields (if schema is configured) ā runs after
|
|
244
|
+
// metadata enrichment so the resolver has access to git, package, and
|
|
245
|
+
// custom attributes from attributeProcessor
|
|
246
|
+
const applySchema$ = applySchema(applyMetadata$, config.index?.schema);
|
|
247
|
+
// Generate embeddings in batches with retry on rate-limit (429) errors
|
|
248
|
+
const embeddingService = framework.ai.useEmbed(options.embedModel);
|
|
249
|
+
let embeddedCount = 0;
|
|
250
|
+
let embeddingDone = false;
|
|
251
|
+
const applyEmbedding$ = applySchema$.pipe(
|
|
252
|
+
// Flatten all documents from file-level batches, then re-batch for the API
|
|
253
|
+
concatMap((documents) => from(documents)),
|
|
254
|
+
// Flush when EMBED_BATCH_SIZE docs accumulate OR after EMBED_BUFFER_FLUSH_MS,
|
|
255
|
+
// whichever comes first ā prevents upstream starvation from blocking concurrency
|
|
256
|
+
bufferTime(EMBED_BUFFER_FLUSH_MS, null, EMBED_BATCH_SIZE), filter((batch) => batch.length > 0), mergeMap((batch) => from(embeddingService.embedDocuments(batch.map((d) => d.pageContent))).pipe(retry({
|
|
132
257
|
count: MAX_RETRIES,
|
|
133
258
|
delay: (error, retryIndex) => {
|
|
134
|
-
//
|
|
259
|
+
// Auth errors are terminal ā abort immediately with actionable message
|
|
260
|
+
if (error?.name === 'NoAccountsError') {
|
|
261
|
+
console.error('\nš Authentication expired. Run `ffc auth login` then retry with `--diff`.');
|
|
262
|
+
throw error;
|
|
263
|
+
}
|
|
135
264
|
const retryAfterSec = error?.response?.headers?.get?.('retry-after') ??
|
|
136
265
|
error?.responseHeaders?.['retry-after'];
|
|
137
266
|
const retryAfterMs = retryAfterSec ? Number(retryAfterSec) * 1000 : 0;
|
|
138
|
-
// Exponential backoff: 2s, 4s, 8s, 16s ā or Retry-After if larger
|
|
139
267
|
const backoffMs = 2 ** retryIndex * 1000;
|
|
140
268
|
const delayMs = Math.max(backoffMs, retryAfterMs);
|
|
141
|
-
console.warn(
|
|
269
|
+
console.warn(`\nā³ Retry ${retryIndex}/${MAX_RETRIES} for batch of ${batch.length} in ${delayMs}ms`);
|
|
142
270
|
return timer(delayMs);
|
|
143
271
|
},
|
|
144
|
-
}), map((
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
272
|
+
}), map((allEmbeddings) => {
|
|
273
|
+
return batch.map((document, i) => {
|
|
274
|
+
embeddedCount++;
|
|
275
|
+
const total = metadataDone ? metadataCount : 0;
|
|
276
|
+
const pct = total > 0 ? ` ${Math.round((embeddedCount / total) * 100)}%` : '';
|
|
277
|
+
const denominator = total > 0 ? `/${total}` : '';
|
|
278
|
+
progress.update(LINE_EMBED, `š§ Embedding [${embeddedCount}${denominator}]${pct} ā ${document.metadata.source}`);
|
|
279
|
+
const metadata = { ...document.metadata, embedding: allEmbeddings[i] };
|
|
280
|
+
return { ...document, metadata };
|
|
281
|
+
});
|
|
282
|
+
})), EMBED_BATCH_CONCURRENCY), finalize(() => {
|
|
283
|
+
embeddingDone = true;
|
|
284
|
+
progress.succeed(LINE_EMBED, `š§ Embedded ${embeddedCount} documents`);
|
|
285
|
+
}));
|
|
286
|
+
// Update vector store ā batch documents and upsert concurrently
|
|
287
|
+
const upsert$ = applyEmbedding$.pipe(
|
|
288
|
+
// Flatten file-level batches, then re-batch into groups of 20 for bulk upsert
|
|
289
|
+
concatMap((documents) => from(documents)), bufferCount(20), mergeMap(async (documents) => {
|
|
290
|
+
const vectorStoreService = framework.ai.useIndex(options.indexName);
|
|
152
291
|
if (documents.length === 0) {
|
|
153
292
|
return undefined;
|
|
154
293
|
}
|
|
155
|
-
for (const document of documents) {
|
|
156
|
-
console.log(`Adding entry [${document.id}] to vector store`, document.metadata.source);
|
|
157
|
-
}
|
|
158
294
|
if (!options.dryRun) {
|
|
159
|
-
// For multiple chunks from same file, delete existing chunks first
|
|
160
|
-
if (documents.length > 1) {
|
|
161
|
-
const sources = documents
|
|
162
|
-
.map((document) => document.metadata.source)
|
|
163
|
-
.reduce((acc, source) => acc.add(source), new Set());
|
|
164
|
-
const filterExpression = Array.from(sources)
|
|
165
|
-
.map((source) => `metadata/source eq '${source}'`)
|
|
166
|
-
.join(' or ');
|
|
167
|
-
// Fire-and-forget deletion (not awaited) - brief gap before new docs are indexed
|
|
168
|
-
vectorStoreService.deleteDocuments({ filter: { filterExpression } });
|
|
169
|
-
}
|
|
170
295
|
await vectorStoreService.addDocuments(documents);
|
|
171
296
|
}
|
|
172
297
|
return {
|
|
173
298
|
status: 'added',
|
|
174
299
|
documents,
|
|
175
300
|
};
|
|
176
|
-
}), filter((result) => Boolean(result)));
|
|
301
|
+
}, UPSERT_CONCURRENCY), filter((result) => Boolean(result)));
|
|
177
302
|
// Execute pipeline
|
|
178
303
|
// Track indexing results for reporting: deleted file paths and added document IDs
|
|
304
|
+
let indexedCount = 0;
|
|
179
305
|
const indexingResults = {
|
|
180
306
|
deleted: [],
|
|
181
307
|
added: [],
|
|
182
308
|
};
|
|
183
|
-
// Execute pipeline:
|
|
184
|
-
//
|
|
185
|
-
|
|
309
|
+
// Execute pipeline: merge runs deletions and additions concurrently so
|
|
310
|
+
// the embedding pipeline can start as soon as metadata-enriched documents
|
|
311
|
+
// are available, without waiting for all file discovery to complete.
|
|
312
|
+
progress.start(4);
|
|
313
|
+
merge(delete$, upsert$).subscribe({
|
|
186
314
|
next: (result) => {
|
|
187
315
|
// Track deleted files by relative path
|
|
188
316
|
if (result.status === 'deleted') {
|
|
@@ -194,6 +322,12 @@ export async function embed(binOptions) {
|
|
|
194
322
|
source: document.metadata.source,
|
|
195
323
|
id: document.id,
|
|
196
324
|
})));
|
|
325
|
+
indexedCount += result.documents.length;
|
|
326
|
+
// Use embeddedCount as denominator ā only show % once embedding is done
|
|
327
|
+
const total = embeddingDone ? embeddedCount : 0;
|
|
328
|
+
const pct = total > 0 ? ` ${Math.round((indexedCount / total) * 100)}%` : '';
|
|
329
|
+
const denominator = total > 0 ? `/${total}` : '';
|
|
330
|
+
progress.update(LINE_INDEX, `š¤ Indexed [${indexedCount}${denominator}]${pct}`);
|
|
197
331
|
}
|
|
198
332
|
},
|
|
199
333
|
error: (error) => {
|
|
@@ -201,8 +335,15 @@ export async function embed(binOptions) {
|
|
|
201
335
|
process.exit(1);
|
|
202
336
|
},
|
|
203
337
|
complete: () => {
|
|
204
|
-
//
|
|
205
|
-
|
|
338
|
+
// Clear the progress block before final output
|
|
339
|
+
progress.clear();
|
|
340
|
+
// Pipeline completed - log summary
|
|
341
|
+
if (indexingResults.deleted.length > 0) {
|
|
342
|
+
console.log(`šļø Deleted: ${indexingResults.deleted.length} files`);
|
|
343
|
+
}
|
|
344
|
+
if (indexingResults.added.length > 0) {
|
|
345
|
+
console.log(`š„ Indexed: ${indexingResults.added.length} documents`);
|
|
346
|
+
}
|
|
206
347
|
console.log('ā
Embeddings generation completed!');
|
|
207
348
|
process.exit(0);
|
|
208
349
|
},
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"embed.js","sourceRoot":"","sources":["../../../src/bin/embed.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,UAAU,MAAM,YAAY,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,MAAM,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AAE/F,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;AAC/E,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAC1E,OAAO,EAAE,gBAAgB,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAEpF,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AACxC,OAAO,EAAE,8BAA8B,EAAE,MAAM,2BAA2B,CAAC;AAC3E,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAQpD,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAEhE;;;;GAIG;AACH,MAAM,aAAa,GAAG,CAAC,cAAc,EAAE,oBAAoB,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;AAE3F;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,KAAK,CAAC,UAAgC;IAC1D,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,GAAG,UAAU,CAAC;IAEhE,OAAO,CAAC,GAAG,CAAC,aAAa,OAAO,CAAC,oBAAoB,EAAE,CAAC,CAAC;IAEzD,wEAAwE;IACxE,MAAM,kBAAkB,GAAG,SAAS,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,EAAE,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAC3F,IAAI,OAAO,CAAC,KAAK,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QACrC,OAAO,CAAC,GAAG,CAAC,8DAA8D,CAAC,CAAC;QAC5E,8EAA8E;QAC9E,MAAM,kBAAkB,CAAC,eAAe,CAAC;YACvC,MAAM,EAAE,EAAE,gBAAgB,EAAE,uBAAuB,EAAE;SACtD,CAAC,CAAC;QACH,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC;IACrD,CAAC;IAED,+CAA+C;IAC/C,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAEhE,0EAA0E;IAC1E,MAAM,MAAM,GAAG,CAAC,GAAG,EAAE;QACnB,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACjB,OAAO,IAAI,CAAC,YAAY,CAAC,CAAC;QAC5B,CAAC;QAED,uDAAuD;QACvD,8FAA8F;QAC9F,wFAAwF;QACxF,oDAAoD;QACpD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,EAAE,MAAM,IAAI,aAAa,CAAC;QAErD,gFAAgF;QAChF,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,EAAE,SAAS,IAAI,IAAI,CAAC;QAElD,OAAO,IAAI,CACT,YAAY,CAAC,YAAY,EAAE;YACzB,MAAM;YACN,SAAS,EAAE,IAAI;YACf,SAAS;YACT,QAAQ,EAAE,IAAI;SACf,CAAC,CACH,CAAC,IAAI;QACJ,0DAA0D;QAC1D,QAAQ,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,EACvC,SAAS,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACjC,wEAAwE;QACxE,WAAW,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAChC,CAAC;IACJ,CAAC,CAAC,EAAE,CAAC;IAEL,qEAAqE;IACrE,MAAM,mBAAmB,GAAG,MAAM,CAAC,KAAK,EAAE,QAAQ,IAAI;QACpD,SAAS;QACT,UAAU;QACV,SAAS;QACT,UAAU;KACX,CAAC;IAEF,qEAAqE;IACrE,MAAM,eAAe,GAAG,MAAM,CAAC,IAAI,CACjC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACX,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;QAClC,MAAM,WAAW,GAAG,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QACjD,MAAM,YAAY,GAAG,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;QAE9E,OAAO;YACL,IAAI,EAAE,QAAQ;YACd,MAAM;YACN,WAAW;YACX,YAAY;SACb,CAAC;IACJ,CAAC,CAAC,EACF,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;QACd,MAAM,OAAO,GAAG,UAAU,CAAC,IAAI,CAAC,YAAY,EAAE,mBAAmB,CAAC,CAAC;QACnE,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;IAC5B,CAAC,CAAC;IACF,yEAAyE;IACzE,WAAW,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAChC,CAAC;IAEF,sEAAsE;IACtE,MAAM,aAAa,GAAG,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC;IAExF,4BAA4B;IAC5B,MAAM,OAAO,GAAG,8BAA8B,CAAC,aAAa,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IAElF,kCAAkC;IAClC,MAAM,WAAW,GAAG,eAAe,CAAC,IAAI,CACtC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,KAAK,IAAI,IAAI,CAAC,MAAM,KAAK,UAAU,CAAC;IACrE,gDAAgD;IAChD,WAAW,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAChC,CAAC;IAEF,MAAM,SAAS,GAAG,CAAC,IAAmB,EAAW,EAAE;QACjD,MAAM,OAAO,GAAG,UAAU,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,KAAK,EAAE,WAAW,IAAI,EAAE,CAAC,CAAC;QAC/E,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC,CAAC;IAEF,MAAM,SAAS,GAAG,WAAW,CAAC,IAAI,CAChC,MAAM,CAAC,SAAS,CAAC,EACjB,GAAG,CAAC,CAAC,IAAI,EAAiB,EAAE;QAC1B,MAAM,QAAQ,GAAwB;YACpC,EAAE,EAAE,eAAe,CAAC,IAAI,CAAC,YAAY,CAAC;YACtC,WAAW,EAAE,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC;YAC5C,QAAQ,EAAE;gBACR,MAAM,EAAE,IAAI,CAAC,YAAY;gBACzB,IAAI,EAAE,KAAK;aACZ;SACF,CAAC;QACF,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC;IACxD,CAAC,CAAC,CACH,CAAC;IAEF,MAAM,SAAS,GAAG,WAAW,CAAC,IAAI,CAChC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAC5B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAC3C,QAAQ,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;QACtB,MAAM,SAAS,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,CAAC;QAChD,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC;IAC5C,CAAC,CAAC,CACH,CAAC;IAEF,MAAM,WAAW,GAAG,WAAW,CAAC,IAAI,CAClC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAC5B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAC7C,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACX,MAAM,SAAS,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC;QAC/C,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC;IAC5C,CAAC,CAAC,CACH,CAAC;IAEF,8BAA8B;IAC9B,MAAM,cAAc,GAAG,aAAa,CAAC,KAAK,CAAC,SAAS,EAAE,SAAS,EAAE,WAAW,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;IAE7F,kFAAkF;IAClF,MAAM,gBAAgB,GAAG,SAAS,CAAC,EAAE,CAAC,UAAU,CAAC,YAAY,EAAE,OAAO,CAAC,yBAAyB,CAAC,CAAC;IAElG,oFAAoF;IACpF,MAAM,qBAAqB,GAAG,CAAC,CAAC;IAEhC,0EAA0E;IAC1E,MAAM,WAAW,GAAG,CAAC,CAAC;IAEtB,MAAM,eAAe,GAAG,cAAc,CAAC,IAAI,CACzC,QAAQ,CAAC,CAAC,SAAS,EAAE,EAAE,CACrB,IAAI,CAAC,SAAS,CAAC,CAAC,IAAI;IAClB,4DAA4D;IAC5D,QAAQ,CACN,CAAC,QAAQ,EAAE,EAAE,CACX,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAC1D,KAAK,CAAC;QACJ,KAAK,EAAE,WAAW;QAClB,KAAK,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE;YAC3B,gEAAgE;YAChE,MAAM,aAAa,GACjB,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,aAAa,CAAC;gBAC9C,KAAK,EAAE,eAAe,EAAE,CAAC,aAAa,CAAC,CAAC;YAC1C,MAAM,YAAY,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YAEtE,kEAAkE;YAClE,MAAM,SAAS,GAAG,CAAC,IAAI,UAAU,GAAG,IAAI,CAAC;YACzC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;YAElD,OAAO,CAAC,IAAI,CACV,WAAW,UAAU,IAAI,WAAW,SAAS,QAAQ,CAAC,QAAQ,CAAC,MAAM,QAAQ,OAAO,IAAI,CACzF,CAAC;YACF,OAAO,KAAK,CAAC,OAAO,CAAC,CAAC;QACxB,CAAC;KACF,CAAC,EACF,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE;QACjB,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAAE,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC5D,MAAM,QAAQ,GAAG,EAAE,GAAG,QAAQ,CAAC,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,CAAC;QACjE,OAAO,EAAE,GAAG,QAAQ,EAAE,QAAQ,EAAE,CAAC;IACnC,CAAC,CAAC,CACH,EACH,qBAAqB,CACtB,EACD,OAAO,EAAE,CACV,CACF,CACF,CAAC;IAEF,sBAAsB;IACtB,MAAM,OAAO,GAAG,eAAe,CAAC,IAAI,CAClC,QAAQ,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE;QAC3B,MAAM,kBAAkB,GAAG,SAAS,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,EAAE,OAAO,CAAC,oBAAoB,CAAC,CAAC;QAC3F,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,OAAO,CAAC,GAAG,CAAC,iBAAiB,QAAQ,CAAC,EAAE,mBAAmB,EAAE,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACzF,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;YACpB,mEAAmE;YACnE,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACzB,MAAM,OAAO,GAAG,SAAS;qBACtB,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC;qBAC3C,MAAM,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,IAAI,GAAG,EAAU,CAAC,CAAC;gBAE/D,MAAM,gBAAgB,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC;qBACzC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,uBAAuB,MAAM,GAAG,CAAC;qBACjD,IAAI,CAAC,MAAM,CAAC,CAAC;gBAEhB,iFAAiF;gBACjF,kBAAkB,CAAC,eAAe,CAAC,EAAE,MAAM,EAAE,EAAE,gBAAgB,EAAE,EAAE,CAAC,CAAC;YACvE,CAAC;YACD,MAAM,kBAAkB,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;QACnD,CAAC;QACD,OAAO;YACL,MAAM,EAAE,OAAO;YACf,SAAS;SACiB,CAAC;IAC/B,CAAC,CAAC,EACF,MAAM,CAAC,CAAC,MAAM,EAAqC,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CACvE,CAAC;IAEF,mBAAmB;IACnB,kFAAkF;IAClF,MAAM,eAAe,GAAmE;QACtF,OAAO,EAAE,EAAE;QACX,KAAK,EAAE,EAAE;KACV,CAAC;IAEF,qEAAqE;IACrE,wEAAwE;IACxE,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,SAAS,CAAC;QACjC,IAAI,EAAE,CAAC,MAAM,EAAE,EAAE;YACf,uCAAuC;YACvC,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;gBAChC,eAAe,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;YACjF,CAAC;YACD,+EAA+E;iBAC1E,IAAI,MAAM,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;gBACnC,eAAe,CAAC,KAAK,CAAC,IAAI,CACxB,GAAG,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;oBACrC,MAAM,EAAE,QAAQ,CAAC,QAAQ,CAAC,MAAM;oBAChC,EAAE,EAAE,QAAQ,CAAC,EAAE;iBAChB,CAAC,CAAC,CACJ,CAAC;YACJ,CAAC;QACH,CAAC;QACD,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE;YACf,OAAO,CAAC,KAAK,CAAC,YAAY,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;YACtF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QACD,QAAQ,EAAE,GAAG,EAAE;YACb,4CAA4C;YAC5C,OAAO,CAAC,GAAG,CAAC,uBAAuB,EAAE,eAAe,CAAC,CAAC;YACtD,OAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;YAClD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;KACF,CAAC,CAAC;AACL,CAAC"}
|
|
1
|
+
{"version":3,"file":"embed.js","sourceRoot":"","sources":["../../../src/bin/embed.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACrC,OAAO,UAAU,MAAM,YAAY,CAAC;AACpC,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,MAAM,CAAC;AAC1C,OAAO,EACL,WAAW,EACX,UAAU,EACV,SAAS,EACT,MAAM,EACN,QAAQ,EACR,GAAG,EACH,QAAQ,EACR,KAAK,EACL,WAAW,EACX,GAAG,GACJ,MAAM,gBAAgB,CAAC;AAExB,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;AAC/E,OAAO,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAC1E,OAAO,EAAE,gBAAgB,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAEpF,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AACxC,OAAO,EAAE,8BAA8B,EAAE,MAAM,2BAA2B,CAAC;AAC3E,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAQhD,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,eAAe,EAAE,MAAM,+BAA+B,CAAC;AAEhE,sDAAsD;AACtD,MAAM,cAAc,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;AAE1E;;;;GAIG;AACH,MAAM,eAAe;IACX,KAAK,GAAa,EAAE,CAAC;IACrB,QAAQ,GAAc,EAAE,CAAC;IACzB,OAAO,GAAG,KAAK,CAAC;IAChB,KAAK,GAAG,CAAC,CAAC;IACV,KAAK,CAA6C;IAE1D,sEAAsE;IACtE,KAAK,CAAC,KAAa;QACjB,IAAI,CAAC,KAAK,GAAG,IAAI,KAAK,CAAS,KAAK,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC/C,IAAI,CAAC,QAAQ,GAAG,IAAI,KAAK,CAAU,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACtD,qDAAqD;QACrD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC7B,CAAC;QACD,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,6CAA6C;QAC7C,IAAI,CAAC,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IAClD,CAAC;IAED,sEAAsE;IACtE,MAAM,CAAC,IAAY,EAAE,OAAe;QAClC,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,OAAO;QAC1B,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC;QAC3B,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;QAC3B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC;IAED,0EAA0E;IAC1E,OAAO,CAAC,IAAY,EAAE,OAAe;QACnC,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,OAAO;QAC1B,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,OAAO,EAAE,CAAC;QAClC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;QAC5B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACpB,CAAC;IAED,qEAAqE;IACrE,KAAK;QACH,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,OAAO;QAC1B,IAAI,IAAI,CAAC,KAAK;YAAE,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC1C,wDAAwD;QACxD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;YACtC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,OAAO,kBAAkB,OAAO,KAAK,CAAC,CAAC;QACtE,CAAC;QACD,0CAA0C;QAC1C,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC;QACrD,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;IACvB,CAAC;IAED,kEAAkE;IAC1D,IAAI;QACV,IAAI,CAAC,KAAK,GAAG,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,cAAc,CAAC,MAAM,CAAC;QACtD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3C,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;gBACtC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YACjB,CAAC;QACH,CAAC;IACH,CAAC;IAED,4CAA4C;IACpC,MAAM,CAAC,IAAY;QACzB,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC;QACzC,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACrE,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACtF,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,OAAO,aAAa,IAAI,QAAQ,OAAO,KAAK,CAAC,CAAC;IAC7E,CAAC;CACF;AAED,4BAA4B;AAC5B,MAAM,UAAU,GAAG,CAAC,CAAC;AACrB,MAAM,SAAS,GAAG,CAAC,CAAC;AACpB,MAAM,UAAU,GAAG,CAAC,CAAC;AACrB,MAAM,UAAU,GAAG,CAAC,CAAC;AAErB;;;;GAIG;AACH,MAAM,aAAa,GAAG,CAAC,cAAc,EAAE,oBAAoB,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;AAE3F,2EAA2E;AAC3E,MAAM,eAAe,GAAG,EAAE,CAAC;AAE3B,4DAA4D;AAC5D,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAE9B,gDAAgD;AAChD,MAAM,gBAAgB,GAAG,EAAE,CAAC;AAE5B,qDAAqD;AACrD,MAAM,uBAAuB,GAAG,CAAC,CAAC;AAElC;;;;GAIG;AACH,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAElC,0EAA0E;AAC1E,MAAM,WAAW,GAAG,CAAC,CAAC;AAEtB;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,KAAK,CAAC,UAAgC;IAC1D,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,GAAG,UAAU,CAAC;IAEhE,OAAO,CAAC,GAAG,CAAC,aAAa,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;IAE9C,MAAM,QAAQ,GAAG,IAAI,eAAe,EAAE,CAAC;IAEvC,wEAAwE;IACxE,MAAM,kBAAkB,GAAG,SAAS,CAAC,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACpE,IAAI,OAAO,CAAC,KAAK,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QACrC,OAAO,CAAC,GAAG,CAAC,8DAA8D,CAAC,CAAC;QAC5E,8EAA8E;QAC9E,MAAM,kBAAkB,CAAC,eAAe,CAAC;YACvC,MAAM,EAAE,EAAE,gBAAgB,EAAE,uBAAuB,EAAE;SACtD,CAAC,CAAC;QACH,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC;IACrD,CAAC;IAED,+CAA+C;IAC/C,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAEhE,0EAA0E;IAC1E,MAAM,MAAM,GAAG,CAAC,GAAG,EAAE;QACnB,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACjB,OAAO,IAAI,CAAC,YAAY,CAAC,CAAC;QAC5B,CAAC;QAED,uDAAuD;QACvD,8FAA8F;QAC9F,wFAAwF;QACxF,oDAAoD;QACpD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,EAAE,MAAM,IAAI,aAAa,CAAC;QAErD,gFAAgF;QAChF,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,EAAE,SAAS,IAAI,IAAI,CAAC;QAElD,OAAO,IAAI,CACT,YAAY,CAAC,YAAY,EAAE;YACzB,MAAM;YACN,SAAS,EAAE,IAAI;YACf,SAAS;YACT,QAAQ,EAAE,IAAI;SACf,CAAC,CACH,CAAC,IAAI;QACJ,gFAAgF;QAChF,QAAQ,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,eAAe,CAAC,EACxD,SAAS,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACjC,wEAAwE;QACxE,WAAW,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAChC,CAAC;IACJ,CAAC,CAAC,EAAE,CAAC;IAEL,qEAAqE;IACrE,MAAM,mBAAmB,GAAG,MAAM,CAAC,KAAK,EAAE,QAAQ,IAAI;QACpD,SAAS;QACT,UAAU;QACV,SAAS;QACT,UAAU;KACX,CAAC;IAEF,qEAAqE;IACrE,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,MAAM,eAAe,GAAG,MAAM,CAAC,IAAI,CACjC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACX,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;QAClC,MAAM,WAAW,GAAG,kBAAkB,CAAC,QAAQ,CAAC,CAAC;QACjD,MAAM,YAAY,GAAG,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;QAE9E,OAAO;YACL,IAAI,EAAE,QAAQ;YACd,MAAM;YACN,WAAW;YACX,YAAY;SACb,CAAC;IACJ,CAAC,CAAC,EACF,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;QACd,MAAM,OAAO,GAAG,UAAU,CAAC,IAAI,CAAC,YAAY,EAAE,mBAAmB,CAAC,CAAC;QACnE,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;IAC5B,CAAC,CAAC,EACF,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACX,SAAS,EAAE,CAAC;QACZ,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;QACvD,QAAQ,CAAC,MAAM,CAAC,UAAU,EAAE,GAAG,KAAK,aAAa,SAAS,KAAK,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;IACtF,CAAC,CAAC;IACF,yEAAyE;IACzE,WAAW,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAChC,CAAC;IAEF,sEAAsE;IACtE,MAAM,aAAa,GAAG,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC;IAExF,4BAA4B;IAC5B,MAAM,OAAO,GAAG,8BAA8B,CAAC,aAAa,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IAElF,kCAAkC;IAClC,MAAM,WAAW,GAAG,eAAe,CAAC,IAAI,CACtC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,KAAK,KAAK,IAAI,IAAI,CAAC,MAAM,KAAK,UAAU,CAAC;IACrE,gDAAgD;IAChD,WAAW,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAChC,CAAC;IAEF,MAAM,SAAS,GAAG,CAAC,IAAmB,EAAW,EAAE;QACjD,MAAM,OAAO,GAAG,UAAU,CAAC,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,KAAK,EAAE,WAAW,IAAI,EAAE,CAAC,CAAC;QAC/E,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC,CAAC;IAEF,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,MAAM,SAAS,GAAG,WAAW,CAAC,IAAI,CAChC,MAAM,CAAC,SAAS,CAAC,EACjB,GAAG,CAAC,CAAC,IAAI,EAAiB,EAAE;QAC1B,MAAM,QAAQ,GAAwB;YACpC,EAAE,EAAE,eAAe,CAAC,IAAI,CAAC,YAAY,CAAC;YACtC,WAAW,EAAE,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC;YAC5C,QAAQ,EAAE;gBACR,MAAM,EAAE,IAAI,CAAC,YAAY;gBACzB,IAAI,EAAE,KAAK;aACZ;SACF,CAAC;QACF,QAAQ,EAAE,CAAC;QACX,QAAQ,CAAC,MAAM,CAAC,UAAU,EAAE,eAAe,QAAQ,KAAK,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;QAC7E,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC;IACxD,CAAC,CAAC,CACH,CAAC;IAEF,MAAM,SAAS,GAAG,WAAW,CAAC,IAAI,CAChC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAC5B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAC3C,QAAQ,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;QACtB,MAAM,SAAS,GAAG,MAAM,iBAAiB,CAAC,IAAI,CAAC,CAAC;QAChD,QAAQ,EAAE,CAAC;QACX,QAAQ,CAAC,MAAM,CAAC,UAAU,EAAE,eAAe,QAAQ,KAAK,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;QAC7E,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC;IAC5C,CAAC,CAAC,CACH,CAAC;IAEF,MAAM,WAAW,GAAG,WAAW,CAAC,IAAI,CAClC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAC5B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAC7C,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACX,MAAM,SAAS,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC;QAC/C,QAAQ,EAAE,CAAC;QACX,QAAQ,CAAC,MAAM,CAAC,UAAU,EAAE,eAAe,QAAQ,KAAK,IAAI,CAAC,YAAY,EAAE,CAAC,CAAC;QAC7E,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC;IAC5C,CAAC,CAAC,CACH,CAAC;IAEF,2DAA2D;IAC3D,MAAM,OAAO,GAAG,KAAK,CAAC,SAAS,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC,IAAI,CAC3D,QAAQ,CAAC,GAAG,EAAE;QACZ,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,aAAa,QAAQ,QAAQ,CAAC,CAAC;IAC9D,CAAC,CAAC,CACH,CAAC;IAEF,8BAA8B;IAC9B,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,YAAY,GAAG,KAAK,CAAC;IACzB,MAAM,cAAc,GAAG,aAAa,CAAC,OAAO,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,MAAM,EAAE,EAAE;QACrE,aAAa,EAAE,CAAC;QAChB,QAAQ,CAAC,MAAM,CAAC,SAAS,EAAE,kBAAkB,aAAa,KAAK,MAAM,EAAE,CAAC,CAAC;IAC3E,CAAC,CAAC,CAAC,IAAI,CACL,QAAQ,CAAC,GAAG,EAAE;QACZ,YAAY,GAAG,IAAI,CAAC;QACpB,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,iBAAiB,aAAa,YAAY,CAAC,CAAC;IAC1E,CAAC,CAAC,CACH,CAAC;IAEF,wEAAwE;IACxE,sEAAsE;IACtE,4CAA4C;IAC5C,MAAM,YAAY,GAAG,WAAW,CAAC,cAAc,EAAE,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAEvE,uEAAuE;IACvE,MAAM,gBAAgB,GAAG,SAAS,CAAC,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IAEnE,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,aAAa,GAAG,KAAK,CAAC;IAC1B,MAAM,eAAe,GAAG,YAAY,CAAC,IAAI;IACvC,2EAA2E;IAC3E,SAAS,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACzC,8EAA8E;IAC9E,iFAAiF;IACjF,UAAU,CAAC,qBAAqB,EAAE,IAAI,EAAE,gBAAgB,CAAC,EACzD,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,EACnC,QAAQ,CACN,CAAC,KAAK,EAAE,EAAE,CACR,IAAI,CAAC,gBAAgB,CAAC,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI,CACzE,KAAK,CAAC;QACJ,KAAK,EAAE,WAAW;QAClB,KAAK,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE;YAC3B,uEAAuE;YACvE,IAAI,KAAK,EAAE,IAAI,KAAK,iBAAiB,EAAE,CAAC;gBACtC,OAAO,CAAC,KAAK,CACX,6EAA6E,CAC9E,CAAC;gBACF,MAAM,KAAK,CAAC;YACd,CAAC;YAED,MAAM,aAAa,GACjB,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,aAAa,CAAC;gBAC9C,KAAK,EAAE,eAAe,EAAE,CAAC,aAAa,CAAC,CAAC;YAC1C,MAAM,YAAY,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YAEtE,MAAM,SAAS,GAAG,CAAC,IAAI,UAAU,GAAG,IAAI,CAAC;YACzC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;YAElD,OAAO,CAAC,IAAI,CACV,aAAa,UAAU,IAAI,WAAW,iBAAiB,KAAK,CAAC,MAAM,OAAO,OAAO,IAAI,CACtF,CAAC;YACF,OAAO,KAAK,CAAC,OAAO,CAAC,CAAC;QACxB,CAAC;KACF,CAAC,EACF,GAAG,CAAC,CAAC,aAAa,EAAE,EAAE;QACpB,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,EAAE,EAAE;YAC/B,aAAa,EAAE,CAAC;YAChB,MAAM,KAAK,GAAG,YAAY,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;YAC/C,MAAM,GAAG,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,aAAa,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YAC9E,MAAM,WAAW,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjD,QAAQ,CAAC,MAAM,CACb,UAAU,EACV,iBAAiB,aAAa,GAAG,WAAW,IAAI,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,MAAM,EAAE,CACpF,CAAC;YACF,MAAM,QAAQ,GAAG,EAAE,GAAG,QAAQ,CAAC,QAAQ,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC;YACvE,OAAO,EAAE,GAAG,QAAQ,EAAE,QAAQ,EAAE,CAAC;QACnC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CACH,EACH,uBAAuB,CACxB,EACD,QAAQ,CAAC,GAAG,EAAE;QACZ,aAAa,GAAG,IAAI,CAAC;QACrB,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,eAAe,aAAa,YAAY,CAAC,CAAC;IACzE,CAAC,CAAC,CACH,CAAC;IAEF,gEAAgE;IAChE,MAAM,OAAO,GAAG,eAAe,CAAC,IAAI;IAClC,8EAA8E;IAC9E,SAAS,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EACzC,WAAW,CAAC,EAAE,CAAC,EACf,QAAQ,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE;QAC3B,MAAM,kBAAkB,GAAG,SAAS,CAAC,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACpE,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;YACpB,MAAM,kBAAkB,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;QACnD,CAAC;QACD,OAAO;YACL,MAAM,EAAE,OAAO;YACf,SAAS;SACiB,CAAC;IAC/B,CAAC,EAAE,kBAAkB,CAAC,EACtB,MAAM,CAAC,CAAC,MAAM,EAAqC,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CACvE,CAAC;IAEF,mBAAmB;IACnB,kFAAkF;IAClF,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,MAAM,eAAe,GAAmE;QACtF,OAAO,EAAE,EAAE;QACX,KAAK,EAAE,EAAE;KACV,CAAC;IAEF,uEAAuE;IACvE,0EAA0E;IAC1E,qEAAqE;IACrE,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAClB,KAAK,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,SAAS,CAAC;QAChC,IAAI,EAAE,CAAC,MAAM,EAAE,EAAE;YACf,uCAAuC;YACvC,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;gBAChC,eAAe,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;YACjF,CAAC;YACD,+EAA+E;iBAC1E,IAAI,MAAM,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;gBACnC,eAAe,CAAC,KAAK,CAAC,IAAI,CACxB,GAAG,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;oBACrC,MAAM,EAAE,QAAQ,CAAC,QAAQ,CAAC,MAAM;oBAChC,EAAE,EAAE,QAAQ,CAAC,EAAE;iBAChB,CAAC,CAAC,CACJ,CAAC;gBACF,YAAY,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC;gBACxC,wEAAwE;gBACxE,MAAM,KAAK,GAAG,aAAa,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;gBAChD,MAAM,GAAG,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,YAAY,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC7E,MAAM,WAAW,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACjD,QAAQ,CAAC,MAAM,CAAC,UAAU,EAAE,eAAe,YAAY,GAAG,WAAW,IAAI,GAAG,EAAE,CAAC,CAAC;YAClF,CAAC;QACH,CAAC;QACD,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE;YACf,OAAO,CAAC,KAAK,CAAC,YAAY,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;YACtF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QACD,QAAQ,EAAE,GAAG,EAAE;YACb,+CAA+C;YAC/C,QAAQ,CAAC,KAAK,EAAE,CAAC;YACjB,mCAAmC;YACnC,IAAI,eAAe,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvC,OAAO,CAAC,GAAG,CAAC,iBAAiB,eAAe,CAAC,OAAO,CAAC,MAAM,QAAQ,CAAC,CAAC;YACvE,CAAC;YACD,IAAI,eAAe,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACrC,OAAO,CAAC,GAAG,CAAC,eAAe,eAAe,CAAC,KAAK,CAAC,MAAM,YAAY,CAAC,CAAC;YACvE,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;YAClD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;KACF,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
import { createCommand, createOption } from 'commander';
|
|
2
|
+
import { loadFusionAIConfig, setupFramework } from '@equinor/fusion-framework-cli-plugin-ai-base';
|
|
3
|
+
import { withOptions as withAiOptions, } from '@equinor/fusion-framework-cli-plugin-ai-base/command-options';
|
|
4
|
+
import { zodToAzureFields } from './utils/zod-to-azure-fields.js';
|
|
5
|
+
import { resolveEmbeddingDimensions } from './utils/embedding-dimensions.js';
|
|
6
|
+
/**
|
|
7
|
+
* CLI command: `ai index create`
|
|
8
|
+
*
|
|
9
|
+
* Generate and preview the Azure AI Search index schema derived from the
|
|
10
|
+
* Zod-based schema definition in the fusion-ai config.
|
|
11
|
+
*
|
|
12
|
+
* Reads the `index.schema` configuration, converts the Zod shape into
|
|
13
|
+
* Azure AI Search field definitions, and combines them with the base
|
|
14
|
+
* fields (`id`, `content`, `content_vector`, `metadata`) to produce
|
|
15
|
+
* the full index schema.
|
|
16
|
+
*
|
|
17
|
+
* Usage:
|
|
18
|
+
* $ ffc ai index create [options]
|
|
19
|
+
*
|
|
20
|
+
* Options:
|
|
21
|
+
* --config <config> Path to a config file (default: fusion-ai.config)
|
|
22
|
+
* --dry-run Preview the schema without creating the index
|
|
23
|
+
*
|
|
24
|
+
* Examples:
|
|
25
|
+
* $ ffc ai index create --dry-run
|
|
26
|
+
* $ ffc ai index create --config fusion-ai.config.eds.ts --dry-run
|
|
27
|
+
*/
|
|
28
|
+
const _command = createCommand('create')
|
|
29
|
+
.description('Create an Azure AI Search index from the config schema definition')
|
|
30
|
+
.addOption(createOption('--config <config>', 'Path to a config file').default('fusion-ai.config'))
|
|
31
|
+
.addOption(createOption('--dry-run', 'Preview the index schema without creating it').default(false))
|
|
32
|
+
.action(async function (commandOptions) {
|
|
33
|
+
const config = await loadFusionAIConfig(commandOptions.config, {
|
|
34
|
+
baseDir: process.cwd(),
|
|
35
|
+
});
|
|
36
|
+
const indexConfig = config.index;
|
|
37
|
+
if (!indexConfig?.schema) {
|
|
38
|
+
console.error('ā No schema defined in config. Add a `schema` property to `index` using defineIndexSchema().');
|
|
39
|
+
process.exit(1);
|
|
40
|
+
}
|
|
41
|
+
// Resolve index name from config
|
|
42
|
+
const indexName = indexConfig.name;
|
|
43
|
+
if (!indexName) {
|
|
44
|
+
console.error('ā Index name is required. Set `name` in the index config.');
|
|
45
|
+
process.exit(1);
|
|
46
|
+
}
|
|
47
|
+
// Convert Zod shape to Azure AI Search field definitions
|
|
48
|
+
const schemaFields = zodToAzureFields(indexConfig.schema.shape);
|
|
49
|
+
// Guard against schema fields that collide with reserved base-schema names
|
|
50
|
+
const reservedFieldNames = ['id', 'content', 'content_vector', 'metadata'];
|
|
51
|
+
const conflictingSchemaFields = schemaFields
|
|
52
|
+
.map((field) => field.name)
|
|
53
|
+
.filter((name) => reservedFieldNames.includes(name));
|
|
54
|
+
if (conflictingSchemaFields.length > 0) {
|
|
55
|
+
const conflicts = [...new Set(conflictingSchemaFields)].sort().join(', ');
|
|
56
|
+
console.error(`ā Schema fields use reserved names: ${conflicts}. Reserved field names are: ${reservedFieldNames.join(', ')}. Rename these fields in \`index.schema\`.`);
|
|
57
|
+
process.exit(1);
|
|
58
|
+
}
|
|
59
|
+
// Resolve embedding vector dimensions from the model name or explicit config
|
|
60
|
+
const model = indexConfig.model ?? 'text-embedding-3-large';
|
|
61
|
+
let dimensions;
|
|
62
|
+
try {
|
|
63
|
+
dimensions = resolveEmbeddingDimensions(model, indexConfig.embedding?.dimensions);
|
|
64
|
+
}
|
|
65
|
+
catch (error) {
|
|
66
|
+
console.error(`ā ${error instanceof Error ? error.message : String(error)}`);
|
|
67
|
+
process.exit(1);
|
|
68
|
+
}
|
|
69
|
+
// Base Azure AI Search fields that every index requires
|
|
70
|
+
const baseFields = [
|
|
71
|
+
{
|
|
72
|
+
name: 'id',
|
|
73
|
+
type: 'Edm.String',
|
|
74
|
+
key: true,
|
|
75
|
+
filterable: true,
|
|
76
|
+
sortable: false,
|
|
77
|
+
facetable: false,
|
|
78
|
+
searchable: false,
|
|
79
|
+
},
|
|
80
|
+
{
|
|
81
|
+
name: 'content',
|
|
82
|
+
type: 'Edm.String',
|
|
83
|
+
filterable: false,
|
|
84
|
+
sortable: false,
|
|
85
|
+
facetable: false,
|
|
86
|
+
searchable: true,
|
|
87
|
+
},
|
|
88
|
+
{
|
|
89
|
+
name: 'content_vector',
|
|
90
|
+
type: 'Collection(Edm.Single)',
|
|
91
|
+
filterable: false,
|
|
92
|
+
sortable: false,
|
|
93
|
+
facetable: false,
|
|
94
|
+
searchable: true,
|
|
95
|
+
dimensions,
|
|
96
|
+
vectorSearchProfile: 'default-vector-profile',
|
|
97
|
+
},
|
|
98
|
+
{
|
|
99
|
+
name: 'metadata',
|
|
100
|
+
type: 'Edm.ComplexType',
|
|
101
|
+
fields: [
|
|
102
|
+
{
|
|
103
|
+
name: 'source',
|
|
104
|
+
type: 'Edm.String',
|
|
105
|
+
filterable: true,
|
|
106
|
+
sortable: false,
|
|
107
|
+
facetable: false,
|
|
108
|
+
searchable: false,
|
|
109
|
+
},
|
|
110
|
+
{
|
|
111
|
+
name: 'attributes',
|
|
112
|
+
type: 'Collection(Edm.ComplexType)',
|
|
113
|
+
fields: [
|
|
114
|
+
{
|
|
115
|
+
name: 'key',
|
|
116
|
+
type: 'Edm.String',
|
|
117
|
+
filterable: true,
|
|
118
|
+
sortable: false,
|
|
119
|
+
facetable: false,
|
|
120
|
+
searchable: false,
|
|
121
|
+
},
|
|
122
|
+
{
|
|
123
|
+
name: 'value',
|
|
124
|
+
type: 'Edm.String',
|
|
125
|
+
filterable: true,
|
|
126
|
+
sortable: false,
|
|
127
|
+
facetable: true,
|
|
128
|
+
searchable: false,
|
|
129
|
+
},
|
|
130
|
+
],
|
|
131
|
+
},
|
|
132
|
+
],
|
|
133
|
+
},
|
|
134
|
+
];
|
|
135
|
+
const fullSchema = {
|
|
136
|
+
name: indexName,
|
|
137
|
+
fields: [...baseFields, ...schemaFields],
|
|
138
|
+
vectorSearch: {
|
|
139
|
+
algorithms: [{ name: 'default-hnsw', kind: 'hnsw' }],
|
|
140
|
+
profiles: [
|
|
141
|
+
{
|
|
142
|
+
name: 'default-vector-profile',
|
|
143
|
+
algorithm: 'default-hnsw',
|
|
144
|
+
},
|
|
145
|
+
],
|
|
146
|
+
},
|
|
147
|
+
};
|
|
148
|
+
if (commandOptions.dryRun) {
|
|
149
|
+
console.log('š Index schema preview (dry-run):');
|
|
150
|
+
console.log(JSON.stringify(fullSchema, null, 2));
|
|
151
|
+
console.log(`\nā
Schema has ${baseFields.length} base fields + ${schemaFields.length} promoted fields`);
|
|
152
|
+
process.exit(0);
|
|
153
|
+
}
|
|
154
|
+
// Create or update the index via the Fusion AI proxy
|
|
155
|
+
const framework = await setupFramework(commandOptions);
|
|
156
|
+
const service = await framework.serviceDiscovery.resolveService('ai');
|
|
157
|
+
const baseUri = service.uri.replace(/\/+$/, '');
|
|
158
|
+
const scopes = service.scopes ?? service.defaultScopes ?? [];
|
|
159
|
+
const token = await framework.auth.acquireAccessToken({ request: { scopes } });
|
|
160
|
+
if (!token) {
|
|
161
|
+
console.error('ā Failed to acquire access token for the AI service.');
|
|
162
|
+
process.exit(1);
|
|
163
|
+
}
|
|
164
|
+
const url = `${baseUri}/indexes/${encodeURIComponent(indexName)}?api-version=2024-07-01`;
|
|
165
|
+
const response = await fetch(url, {
|
|
166
|
+
method: 'PUT',
|
|
167
|
+
headers: {
|
|
168
|
+
'Content-Type': 'application/json',
|
|
169
|
+
Authorization: `Bearer ${token}`,
|
|
170
|
+
},
|
|
171
|
+
body: JSON.stringify(fullSchema),
|
|
172
|
+
});
|
|
173
|
+
if (!response.ok) {
|
|
174
|
+
const body = await response.text();
|
|
175
|
+
console.error(`ā Index creation failed (${response.status} ${response.statusText})`);
|
|
176
|
+
console.error(body);
|
|
177
|
+
process.exit(1);
|
|
178
|
+
}
|
|
179
|
+
console.log(`ā
Index "${indexName}" created/updated successfully.`);
|
|
180
|
+
});
|
|
181
|
+
/**
|
|
182
|
+
* The `ai index create` command with inherited AI base options for
|
|
183
|
+
* authentication and service discovery.
|
|
184
|
+
*/
|
|
185
|
+
export const createIndexCommand = withAiOptions(_command);
|
|
186
|
+
//# sourceMappingURL=create-command.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"create-command.js","sourceRoot":"","sources":["../../src/create-command.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,aAAa,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAEtE,OAAO,EAAE,kBAAkB,EAAE,cAAc,EAAE,MAAM,8CAA8C,CAAC;AAClG,OAAO,EACL,WAAW,IAAI,aAAa,GAE7B,MAAM,8DAA8D,CAAC;AAGtE,OAAO,EAAE,gBAAgB,EAAE,MAAM,gCAAgC,CAAC;AAClE,OAAO,EAAE,0BAA0B,EAAE,MAAM,iCAAiC,CAAC;AAE7E;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,QAAQ,GAAG,aAAa,CAAC,QAAQ,CAAC;KACrC,WAAW,CAAC,mEAAmE,CAAC;KAChF,SAAS,CAAC,YAAY,CAAC,mBAAmB,EAAE,uBAAuB,CAAC,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;KACjG,SAAS,CACR,YAAY,CAAC,WAAW,EAAE,8CAA8C,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CACzF;KACA,MAAM,CAAC,KAAK,WAEX,cAA+D;IAE/D,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAA0B,cAAc,CAAC,MAAM,EAAE;QACtF,OAAO,EAAE,OAAO,CAAC,GAAG,EAAE;KACvB,CAAC,CAAC;IAEH,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC;IACjC,IAAI,CAAC,WAAW,EAAE,MAAM,EAAE,CAAC;QACzB,OAAO,CAAC,KAAK,CACX,8FAA8F,CAC/F,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,iCAAiC;IACjC,MAAM,SAAS,GAAG,WAAW,CAAC,IAAI,CAAC;IAEnC,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,2DAA2D,CAAC,CAAC;QAC3E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,yDAAyD;IACzD,MAAM,YAAY,GAAG,gBAAgB,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAEhE,2EAA2E;IAC3E,MAAM,kBAAkB,GAAG,CAAC,IAAI,EAAE,SAAS,EAAE,gBAAgB,EAAE,UAAU,CAAU,CAAC;IACpF,MAAM,uBAAuB,GAAG,YAAY;SACzC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC;SAC1B,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,kBAAkB,CAAC,QAAQ,CAAC,IAA2C,CAAC,CAAC,CAAC;IAE9F,IAAI,uBAAuB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvC,MAAM,SAAS,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,uBAAuB,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1E,OAAO,CAAC,KAAK,CACX,uCAAuC,SAAS,+BAA+B,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,4CAA4C,CACzJ,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,6EAA6E;IAC7E,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,IAAI,wBAAwB,CAAC;IAC5D,IAAI,UAAkB,CAAC;IACvB,IAAI,CAAC;QACH,UAAU,GAAG,0BAA0B,CAAC,KAAK,EAAE,WAAW,CAAC,SAAS,EAAE,UAAU,CAAC,CAAC;IACpF,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,KAAK,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC7E,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,wDAAwD;IACxD,MAAM,UAAU,GAAG;QACjB;YACE,IAAI,EAAE,IAAI;YACV,IAAI,EAAE,YAAqB;YAC3B,GAAG,EAAE,IAAI;YACT,UAAU,EAAE,IAAI;YAChB,QAAQ,EAAE,KAAK;YACf,SAAS,EAAE,KAAK;YAChB,UAAU,EAAE,KAAK;SAClB;QACD;YACE,IAAI,EAAE,SAAS;YACf,IAAI,EAAE,YAAqB;YAC3B,UAAU,EAAE,KAAK;YACjB,QAAQ,EAAE,KAAK;YACf,SAAS,EAAE,KAAK;YAChB,UAAU,EAAE,IAAI;SACjB;QACD;YACE,IAAI,EAAE,gBAAgB;YACtB,IAAI,EAAE,wBAAiC;YACvC,UAAU,EAAE,KAAK;YACjB,QAAQ,EAAE,KAAK;YACf,SAAS,EAAE,KAAK;YAChB,UAAU,EAAE,IAAI;YAChB,UAAU;YACV,mBAAmB,EAAE,wBAAwB;SAC9C;QACD;YACE,IAAI,EAAE,UAAU;YAChB,IAAI,EAAE,iBAA0B;YAChC,MAAM,EAAE;gBACN;oBACE,IAAI,EAAE,QAAQ;oBACd,IAAI,EAAE,YAAqB;oBAC3B,UAAU,EAAE,IAAI;oBAChB,QAAQ,EAAE,KAAK;oBACf,SAAS,EAAE,KAAK;oBAChB,UAAU,EAAE,KAAK;iBAClB;gBACD;oBACE,IAAI,EAAE,YAAY;oBAClB,IAAI,EAAE,6BAAsC;oBAC5C,MAAM,EAAE;wBACN;4BACE,IAAI,EAAE,KAAK;4BACX,IAAI,EAAE,YAAqB;4BAC3B,UAAU,EAAE,IAAI;4BAChB,QAAQ,EAAE,KAAK;4BACf,SAAS,EAAE,KAAK;4BAChB,UAAU,EAAE,KAAK;yBAClB;wBACD;4BACE,IAAI,EAAE,OAAO;4BACb,IAAI,EAAE,YAAqB;4BAC3B,UAAU,EAAE,IAAI;4BAChB,QAAQ,EAAE,KAAK;4BACf,SAAS,EAAE,IAAI;4BACf,UAAU,EAAE,KAAK;yBAClB;qBACF;iBACF;aACF;SACF;KACF,CAAC;IAEF,MAAM,UAAU,GAAG;QACjB,IAAI,EAAE,SAAS;QACf,MAAM,EAAE,CAAC,GAAG,UAAU,EAAE,GAAG,YAAY,CAAC;QACxC,YAAY,EAAE;YACZ,UAAU,EAAE,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;YACpD,QAAQ,EAAE;gBACR;oBACE,IAAI,EAAE,wBAAwB;oBAC9B,SAAS,EAAE,cAAc;iBAC1B;aACF;SACF;KACF,CAAC;IAEF,IAAI,cAAc,CAAC,MAAM,EAAE,CAAC;QAC1B,OAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;QAClD,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QACjD,OAAO,CAAC,GAAG,CACT,kBAAkB,UAAU,CAAC,MAAM,kBAAkB,YAAY,CAAC,MAAM,kBAAkB,CAC3F,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,qDAAqD;IACrD,MAAM,SAAS,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,CAAC;IACvD,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,gBAAgB,CAAC,cAAc,CAAC,IAAI,CAAC,CAAC;IACtE,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAChD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,aAAa,IAAI,EAAE,CAAC;IAC7D,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,kBAAkB,CAAC,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;IAE/E,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,CAAC,KAAK,CAAC,sDAAsD,CAAC,CAAC;QACtE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,GAAG,GAAG,GAAG,OAAO,YAAY,kBAAkB,CAAC,SAAS,CAAC,yBAAyB,CAAC;IACzF,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;QAChC,MAAM,EAAE,KAAK;QACb,OAAO,EAAE;YACP,cAAc,EAAE,kBAAkB;YAClC,aAAa,EAAE,UAAU,KAAK,EAAE;SACjC;QACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC;KACjC,CAAC,CAAC;IAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,OAAO,CAAC,KAAK,CAAC,4BAA4B,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,GAAG,CAAC,CAAC;QACrF,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACpB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,YAAY,SAAS,iCAAiC,CAAC,CAAC;AACtE,CAAC,CAAC,CAAC;AAEL;;;GAGG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAY,aAAa,CAAC,QAAQ,CAAC,CAAC"}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { createCommand, createOption } from 'commander';
|
|
2
|
-
import { setupFramework } from '@equinor/fusion-framework-cli-plugin-ai-base';
|
|
2
|
+
import { loadFusionAIConfig, setupFramework } from '@equinor/fusion-framework-cli-plugin-ai-base';
|
|
3
3
|
import { withOptions as withAiOptions } from '@equinor/fusion-framework-cli-plugin-ai-base/command-options';
|
|
4
4
|
import { DeleteOptionsSchema } from './delete-command.options.js';
|
|
5
5
|
/**
|
|
@@ -55,9 +55,21 @@ function buildFilter(sources, rawFilter) {
|
|
|
55
55
|
*/
|
|
56
56
|
const _command = createCommand('remove')
|
|
57
57
|
.description('Remove documents from the search index by source path or OData filter')
|
|
58
|
+
.addOption(createOption('--config <config>', 'Path to a config file').default('fusion-ai.config'))
|
|
58
59
|
.addOption(createOption('--dry-run', 'Preview matching documents without deleting them').default(false))
|
|
59
60
|
.addOption(createOption('--filter <expression>', 'Raw OData filter expression for selecting documents to delete'))
|
|
60
61
|
.argument('[source-paths...]', 'Relative file paths whose indexed chunks should be removed')
|
|
62
|
+
.hook('preAction', async (thisCommand) => {
|
|
63
|
+
const opts = thisCommand.opts();
|
|
64
|
+
const config = await loadFusionAIConfig(opts.config ?? 'fusion-ai.config', { baseDir: process.cwd() });
|
|
65
|
+
const indexConfig = config.index ?? {};
|
|
66
|
+
if (indexConfig.name && !opts.indexName?.trim()) {
|
|
67
|
+
thisCommand.setOptionValue('indexName', indexConfig.name);
|
|
68
|
+
}
|
|
69
|
+
if (indexConfig.model && !opts.embedModel?.trim()) {
|
|
70
|
+
thisCommand.setOptionValue('embedModel', indexConfig.model);
|
|
71
|
+
}
|
|
72
|
+
})
|
|
61
73
|
.action(async (sources, commandOptions) => {
|
|
62
74
|
const options = await DeleteOptionsSchema.parseAsync(commandOptions);
|
|
63
75
|
const filterExpression = buildFilter(sources, options.filter);
|
|
@@ -79,7 +91,7 @@ const _command = createCommand('remove')
|
|
|
79
91
|
return;
|
|
80
92
|
}
|
|
81
93
|
const framework = await setupFramework(options);
|
|
82
|
-
const vectorStoreService = framework.ai.
|
|
94
|
+
const vectorStoreService = framework.ai.useIndex(options.indexName);
|
|
83
95
|
await vectorStoreService.deleteDocuments({
|
|
84
96
|
filter: { filterExpression },
|
|
85
97
|
});
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"delete-command.js","sourceRoot":"","sources":["../../src/delete-command.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAExD,OAAO,EAAE,cAAc,EAAE,MAAM,8CAA8C,CAAC;
|
|
1
|
+
{"version":3,"file":"delete-command.js","sourceRoot":"","sources":["../../src/delete-command.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAExD,OAAO,EAAE,kBAAkB,EAAE,cAAc,EAAE,MAAM,8CAA8C,CAAC;AAClG,OAAO,EAAE,WAAW,IAAI,aAAa,EAAE,MAAM,8DAA8D,CAAC;AAG5G,OAAO,EAAE,mBAAmB,EAAsB,MAAM,6BAA6B,CAAC;AAEtF;;;;;;;;;;;GAWG;AACH,SAAS,WAAW,CAAC,OAAiB,EAAE,SAAkB;IACxD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,uBAAuB,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACtE,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,MAAM,QAAQ,GAAG,aAAa,CAAC,QAAQ,CAAC;KACrC,WAAW,CAAC,uEAAuE,CAAC;KACpF,SAAS,CAAC,YAAY,CAAC,mBAAmB,EAAE,uBAAuB,CAAC,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;KACjG,SAAS,CACR,YAAY,CAAC,WAAW,EAAE,kDAAkD,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAC7F;KACA,SAAS,CACR,YAAY,CACV,uBAAuB,EACvB,+DAA+D,CAChE,CACF;KACA,QAAQ,CAAC,mBAAmB,EAAE,4DAA4D,CAAC;KAC3F,IAAI,CAAC,WAAW,EAAE,KAAK,EAAE,WAAW,EAAE,EAAE;IACvC,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,EAAE,CAAC;IAChC,MAAM,MAAM,GAAG,MAAM,kBAAkB,CACpC,IAAI,CAAC,MAAiB,IAAI,kBAAkB,EAC7C,EAAE,OAAO,EAAE,OAAO,CAAC,GAAG,EAAE,EAAE,CAC3B,CAAC;IACF,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC;IAEvC,IAAI,WAAW,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,EAAE,CAAC;QAChD,WAAW,CAAC,cAAc,CAAC,WAAW,EAAE,WAAW,CAAC,IAAI,CAAC,CAAC;IAC5D,CAAC;IACD,IAAI,WAAW,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,IAAI,EAAE,EAAE,CAAC;QAClD,WAAW,CAAC,cAAc,CAAC,YAAY,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC;IAC9D,CAAC;AACH,CAAC,CAAC;KACD,MAAM,CAAC,KAAK,EAAE,OAAiB,EAAE,cAA6B,EAAE,EAAE;IACjE,MAAM,OAAO,GAAG,MAAM,mBAAmB,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;IACrE,MAAM,gBAAgB,GAAG,WAAW,CAAC,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAE9D,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CACb,0FAA0F,CAC3F,CAAC;IACJ,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,GAAG,CAAC,eAAe,OAAO,CAAC,MAAM,oBAAoB,CAAC,CAAC;QAC/D,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;YACjC,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,aAAa,gBAAgB,EAAE,CAAC,CAAC;IAC/C,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,OAAO,CAAC,GAAG,CAAC,2CAA2C,CAAC,CAAC;QACzD,OAAO,CAAC,GAAG,CAAC,yBAAyB,gBAAgB,EAAE,CAAC,CAAC;QACzD,OAAO;IACT,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC;IAChD,MAAM,kBAAkB,GAAG,SAAS,CAAC,EAAE,CAAC,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACpE,MAAM,kBAAkB,CAAC,eAAe,CAAC;QACvC,MAAM,EAAE,EAAE,gBAAgB,EAAE;KAC7B,CAAC,CAAC;IAEH,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC;AACrD,CAAC,CAAC,CAAC;AAEL;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,aAAa,CAAC,QAAQ,EAAE;IACnD,gBAAgB,EAAE,IAAI;IACtB,aAAa,EAAE,IAAI;CACpB,CAAC,CAAC"}
|