@qvac/core 1.0.0-rc.8 → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +243 -3
- package/README.md +1 -0
- package/dist-lib/lib/api/attachments.d.ts +1 -1
- package/dist-lib/lib/api/attachments.js +11 -0
- package/dist-lib/lib/api/chunks.d.ts +4 -2
- package/dist-lib/lib/api/chunks.js +6 -3
- package/dist-lib/lib/api/devices.js +6 -18
- package/dist-lib/lib/api/execution.d.ts +6 -2
- package/dist-lib/lib/api/execution.js +29 -3
- package/dist-lib/lib/api/host-run-log.d.ts +7 -1
- package/dist-lib/lib/api/host-run-log.js +3 -1
- package/dist-lib/lib/api/index.d.ts +3 -1
- package/dist-lib/lib/api/index.js +17 -6
- package/dist-lib/lib/api/members.js +13 -29
- package/dist-lib/lib/api/mesh.d.ts +2 -1
- package/dist-lib/lib/api/projects.d.ts +1 -1
- package/dist-lib/lib/api/projects.js +9 -5
- package/dist-lib/lib/api/rag-label.d.ts +4 -0
- package/dist-lib/lib/api/rag-label.js +21 -0
- package/dist-lib/lib/api/rag-lexical.d.ts +22 -0
- package/dist-lib/lib/api/rag-lexical.js +1058 -0
- package/dist-lib/lib/api/rag-manifest.d.ts +13 -0
- package/dist-lib/lib/api/rag-manifest.js +46 -0
- package/dist-lib/lib/api/rag-outline.d.ts +36 -0
- package/dist-lib/lib/api/rag-outline.js +426 -0
- package/dist-lib/lib/api/rag-plugin.d.ts +20 -0
- package/dist-lib/lib/api/rag-plugin.js +70 -0
- package/dist-lib/lib/api/rag-synthesis.d.ts +7 -0
- package/dist-lib/lib/api/rag-synthesis.js +102 -0
- package/dist-lib/lib/api/rag-targets.d.ts +23 -0
- package/dist-lib/lib/api/rag-targets.js +215 -0
- package/dist-lib/lib/api/rag-window.d.ts +5 -0
- package/dist-lib/lib/api/rag-window.js +46 -0
- package/dist-lib/lib/api/rag.d.ts +3 -2
- package/dist-lib/lib/api/rag.js +421 -181
- package/dist-lib/lib/api/served.js +8 -14
- package/dist-lib/lib/api/workspace.d.ts +8 -1
- package/dist-lib/lib/api/workspace.js +27 -13
- package/dist-lib/lib/assistant/harness.d.ts +9 -1
- package/dist-lib/lib/assistant/history.d.ts +11 -2
- package/dist-lib/lib/assistant/history.js +74 -4
- package/dist-lib/lib/assistant/media-round.d.ts +61 -0
- package/dist-lib/lib/assistant/media-round.js +111 -0
- package/dist-lib/lib/assistant/message.d.ts +7 -0
- package/dist-lib/lib/assistant/model-round.d.ts +12 -0
- package/dist-lib/lib/assistant/model-round.js +130 -17
- package/dist-lib/lib/assistant/pdf-pages.d.ts +16 -1
- package/dist-lib/lib/assistant/pdf-pages.js +67 -3
- package/dist-lib/lib/assistant/round-host.d.ts +22 -0
- package/dist-lib/lib/assistant/round-host.js +142 -8
- package/dist-lib/lib/assistant/upload.d.ts +12 -0
- package/dist-lib/lib/assistant/upload.js +36 -0
- package/dist-lib/lib/assistant-client.d.ts +1 -0
- package/dist-lib/lib/assistant-client.js +40 -19
- package/dist-lib/lib/assistant-tree.d.ts +2 -0
- package/dist-lib/lib/assistant-tree.js +41 -1
- package/dist-lib/lib/assistant.d.ts +15 -0
- package/dist-lib/lib/assistant.js +474 -89
- package/dist-lib/lib/client.d.ts +1 -0
- package/dist-lib/lib/client.js +4 -1
- package/dist-lib/lib/compaction.js +2 -2
- package/dist-lib/lib/core/agent.js +12 -45
- package/dist-lib/lib/core/approval-rules.js +1 -8
- package/dist-lib/lib/core/archived-mesh.d.ts +2 -1
- package/dist-lib/lib/core/archived-mesh.js +12 -8
- package/dist-lib/lib/core/chat-copy.d.ts +18 -3
- package/dist-lib/lib/core/chat-copy.js +74 -25
- package/dist-lib/lib/core/chat-fork.js +15 -6
- package/dist-lib/lib/core/chat-group.js +16 -24
- package/dist-lib/lib/core/chat-move.d.ts +9 -0
- package/dist-lib/lib/core/chat-move.js +94 -3
- package/dist-lib/lib/core/chat-order.js +12 -18
- package/dist-lib/lib/core/chat.js +38 -31
- package/dist-lib/lib/core/chunk.d.ts +2 -2
- package/dist-lib/lib/core/chunk.js +18 -25
- package/dist-lib/lib/core/collections.d.ts +2 -0
- package/dist-lib/lib/core/collections.js +2 -0
- package/dist-lib/lib/core/contract/activity.d.ts +4 -0
- package/dist-lib/lib/core/contract/activity.js +34 -4
- package/dist-lib/lib/core/contract/chat-group.d.ts +1 -0
- package/dist-lib/lib/core/contract/chat-group.js +21 -1
- package/dist-lib/lib/core/contract/chat.d.ts +9 -0
- package/dist-lib/lib/core/contract/chat.js +127 -5
- package/dist-lib/lib/core/contract/chunks.d.ts +1 -0
- package/dist-lib/lib/core/contract/chunks.js +37 -6
- package/dist-lib/lib/core/contract/features.d.ts +21 -0
- package/dist-lib/lib/core/contract/features.js +22 -0
- package/dist-lib/lib/core/contract/index.js +2 -0
- package/dist-lib/lib/core/contract/indexing.js +4 -2
- package/dist-lib/lib/core/contract/knowledge.js +5 -4
- package/dist-lib/lib/core/contract/members.js +16 -10
- package/dist-lib/lib/core/contract/mesh-kind.d.ts +1 -0
- package/dist-lib/lib/core/contract/mesh-kind.js +3 -1
- package/dist-lib/lib/core/contract/projects.d.ts +5 -0
- package/dist-lib/lib/core/contract/projects.js +13 -5
- package/dist-lib/lib/core/contract/roles.js +32 -4
- package/dist-lib/lib/core/contract/run-requests.d.ts +1 -0
- package/dist-lib/lib/core/contract/run-requests.js +35 -2
- package/dist-lib/lib/core/contract/seed-defaults.js +2 -2
- package/dist-lib/lib/core/contract/tool-configs.js +3 -2
- package/dist-lib/lib/core/contract/workspace.d.ts +2 -0
- package/dist-lib/lib/core/contract/workspace.js +40 -7
- package/dist-lib/lib/core/contract/writers.js +2 -2
- package/dist-lib/lib/core/departures.d.ts +3 -1
- package/dist-lib/lib/core/departures.js +3 -0
- package/dist-lib/lib/core/devices.js +1 -8
- package/dist-lib/lib/core/files/attachments.d.ts +2 -0
- package/dist-lib/lib/core/files/attachments.js +19 -1
- package/dist-lib/lib/core/files/index.d.ts +20 -4
- package/dist-lib/lib/core/files/index.js +132 -21
- package/dist-lib/lib/core/files/processing/document.d.ts +4 -0
- package/dist-lib/lib/core/files/processing/document.js +7 -0
- package/dist-lib/lib/core/files/processing/index.d.ts +13 -0
- package/dist-lib/lib/core/files/processing/index.js +144 -12
- package/dist-lib/lib/core/files/processing/protocol.js +13 -34
- package/dist-lib/lib/core/files/processing/state.d.ts +2 -0
- package/dist-lib/lib/core/files/rag/build.d.ts +19 -2
- package/dist-lib/lib/core/files/rag/build.js +123 -19
- package/dist-lib/lib/core/files/rag/failure.js +2 -1
- package/dist-lib/lib/core/files/rag/index.d.ts +11 -3
- package/dist-lib/lib/core/files/rag/index.js +13 -1
- package/dist-lib/lib/core/files/rag/load.d.ts +6 -3
- package/dist-lib/lib/core/files/rag/load.js +27 -6
- package/dist-lib/lib/core/files/rag/plugin-index.d.ts +32 -0
- package/dist-lib/lib/core/files/rag/plugin-index.js +127 -0
- package/dist-lib/lib/core/files/rag/produce.d.ts +5 -3
- package/dist-lib/lib/core/files/rag/produce.js +104 -16
- package/dist-lib/lib/core/files/rag/retrieval/full-text.d.ts +3 -1
- package/dist-lib/lib/core/files/rag/retrieval/full-text.js +62 -13
- package/dist-lib/lib/core/files/rag/text-cache.d.ts +28 -0
- package/dist-lib/lib/core/files/rag/text-cache.js +141 -0
- package/dist-lib/lib/core/files/records/knowledge.d.ts +1 -3
- package/dist-lib/lib/core/files/records/knowledge.js +7 -40
- package/dist-lib/lib/core/files/records/reclaim.js +5 -10
- package/dist-lib/lib/core/files/upload.js +68 -41
- package/dist-lib/lib/core/fs-drives.js +14 -26
- package/dist-lib/lib/core/invite.d.ts +1 -0
- package/dist-lib/lib/core/invite.js +4 -2
- package/dist-lib/lib/core/local/files.d.ts +1 -1
- package/dist-lib/lib/core/local/files.js +5 -5
- package/dist-lib/lib/core/local/index.d.ts +13 -6
- package/dist-lib/lib/core/local/index.js +103 -69
- package/dist-lib/lib/core/local/run-logs.d.ts +1 -1
- package/dist-lib/lib/core/local/run-logs.js +5 -5
- package/dist-lib/lib/core/mesh-processor.d.ts +19 -0
- package/dist-lib/lib/core/mesh-processor.js +87 -0
- package/dist-lib/lib/core/mesh.d.ts +48 -5
- package/dist-lib/lib/core/mesh.js +318 -129
- package/dist-lib/lib/core/migrations/agent-catalog.js +2 -8
- package/dist-lib/lib/core/migrations/agent-tools-scope.js +5 -11
- package/dist-lib/lib/core/migrations/workspace-from-0-21/index.d.ts +88 -0
- package/dist-lib/lib/core/migrations/workspace-from-0-21/index.js +731 -0
- package/dist-lib/lib/core/migrations/writer-links.d.ts +2 -0
- package/dist-lib/lib/core/migrations/writer-links.js +32 -0
- package/dist-lib/lib/core/project-keys.d.ts +6 -0
- package/dist-lib/lib/core/project-keys.js +11 -0
- package/dist-lib/lib/core/public-mesh.d.ts +3 -0
- package/dist-lib/lib/core/public-mesh.js +10 -8
- package/dist-lib/lib/core/run-requests.d.ts +3 -0
- package/dist-lib/lib/core/run-requests.js +21 -1
- package/dist-lib/lib/core/skill-availability.d.ts +33 -0
- package/dist-lib/lib/core/skill-availability.js +67 -0
- package/dist-lib/lib/core/sync/background.d.ts +0 -1
- package/dist-lib/lib/core/sync/background.js +2 -1
- package/dist-lib/lib/core/sync/changed.d.ts +4 -2
- package/dist-lib/lib/core/sync/changed.js +1 -1
- package/dist-lib/lib/core/sync/index.js +52 -10
- package/dist-lib/lib/core/sync/view-updates.d.ts +18 -12
- package/dist-lib/lib/core/sync/view-updates.js +82 -91
- package/dist-lib/lib/core/sync-status.js +97 -41
- package/dist-lib/lib/core/tool-activity.js +6 -12
- package/dist-lib/lib/core/tool-configs.js +17 -29
- package/dist-lib/lib/core/upgrade-summary.d.ts +19 -0
- package/dist-lib/lib/core/upgrade-summary.js +67 -0
- package/dist-lib/lib/core/view-read.d.ts +4 -0
- package/dist-lib/lib/core/view-read.js +23 -0
- package/dist-lib/lib/core/workspace-upgrade.d.ts +7 -0
- package/dist-lib/lib/core/workspace-upgrade.js +27 -16
- package/dist-lib/lib/core.d.ts +39 -42
- package/dist-lib/lib/core.js +661 -639
- package/dist-lib/lib/errors.d.ts +20 -0
- package/dist-lib/lib/errors.js +20 -0
- package/dist-lib/lib/llm-model-names.d.ts +1 -1
- package/dist-lib/lib/llm-model-names.js +1 -1
- package/dist-lib/lib/mesh-scoped-engine.js +4 -0
- package/dist-lib/lib/model-context.js +1 -2
- package/dist-lib/lib/probe-rounds.d.ts +7 -2
- package/dist-lib/lib/probe-rounds.js +6 -1
- package/dist-lib/lib/run-transcript.d.ts +4 -1
- package/dist-lib/lib/run-transcript.js +15 -6
- package/dist-lib/schema/engine.js +83 -6
- package/dist-lib/schema/indexes.d.ts +3 -0
- package/dist-lib/schema/indexes.js +5 -0
- package/dist-lib/schema/local.js +11 -2
- package/dist-lib/schema/qvac.js +49 -5
- package/dist-lib/spec/bind.js +34 -0
- package/dist-lib/spec/calls.js +5 -0
- package/dist-lib/spec/capabilities.d.ts +17 -0
- package/dist-lib/spec/hrpc/hrpc.json +65 -0
- package/dist-lib/spec/hrpc/index.js +68 -5
- package/dist-lib/spec/hrpc/messages.js +260 -23
- package/dist-lib/spec/hyperschema/index.js +260 -23
- package/dist-lib/spec/hyperschema/schema.json +181 -0
- package/dist-lib/spec/hyperschema/types.d.ts +48 -0
- package/dist-lib/spec/local/hyperdb/messages.js +270 -224
- package/dist-lib/spec/local/hyperschema/index.js +73 -38
- package/dist-lib/spec/local/hyperschema/schema.json +31 -0
- package/dist-lib/spec/local/hyperschema/types.d.ts +8 -0
- package/dist-lib/spec/mesh/hyperdb/db.json +39 -2
- package/dist-lib/spec/mesh/hyperdb/index.js +120 -3
- package/dist-lib/spec/mesh/hyperdb/messages.js +1328 -838
- package/dist-lib/spec/mesh/hyperdispatch/dispatch.json +30 -0
- package/dist-lib/spec/mesh/hyperdispatch/index.js +86 -1
- package/dist-lib/spec/mesh/hyperdispatch/messages.js +682 -287
- package/dist-lib/spec/mesh/hyperschema/index.js +682 -287
- package/dist-lib/spec/mesh/hyperschema/schema.json +201 -0
- package/dist-lib/spec/mesh/hyperschema/types.d.ts +52 -0
- package/package.json +75 -70
- package/tui/README.md +2 -1
- package/tui/app.mjs +28 -7
- package/tui/boot.mjs +5 -1
- package/tui/lib/attachment-files.mjs +82 -0
- package/tui/lib/reopen-meshes.mjs +15 -0
- package/tui/main.mjs +4 -43
- package/worker/assistant-sidecar.mjs +10 -3
- package/worker/stow-entry.mjs +3 -52
|
@@ -17,12 +17,15 @@ export interface FileProcessor {
|
|
|
17
17
|
readonly input: FileInput;
|
|
18
18
|
readonly output: 'blob' | 'inline';
|
|
19
19
|
readonly mediaKind?: string;
|
|
20
|
+
readonly deferChatVectors?: boolean;
|
|
20
21
|
accepts?(output: QvacKnowledgeArtifact, document: FileRecord): boolean;
|
|
21
22
|
applies(document: FileRecord): boolean;
|
|
22
23
|
canProduce(document: FileRecord, host: ProcessorHost): boolean | Promise<boolean>;
|
|
24
|
+
text?(document: FileDocument, signal: AbortSignal): Promise<void>;
|
|
23
25
|
produce(document: FileDocument, signal: AbortSignal): Promise<void>;
|
|
24
26
|
load?(document: FileDocument, signal: AbortSignal): Promise<void>;
|
|
25
27
|
evict?(loaded: LoadedArtifact): Promise<void>;
|
|
28
|
+
allLoaded?(): Promise<void>;
|
|
26
29
|
close(): Promise<boolean>;
|
|
27
30
|
}
|
|
28
31
|
export interface FileProcessingOptions {
|
|
@@ -44,14 +47,18 @@ export declare class FileProcessing {
|
|
|
44
47
|
private readonly _output;
|
|
45
48
|
private readonly _progress?;
|
|
46
49
|
private readonly _lanes;
|
|
50
|
+
private readonly _demanded;
|
|
51
|
+
private readonly _vectorDemanded;
|
|
47
52
|
private readonly _deadlines;
|
|
48
53
|
private readonly _scheduler;
|
|
49
54
|
private _scanning;
|
|
55
|
+
private readonly _reads;
|
|
50
56
|
private _dirty;
|
|
51
57
|
private _paused;
|
|
52
58
|
private _closed;
|
|
53
59
|
private _close;
|
|
54
60
|
private _indexCleared;
|
|
61
|
+
private _stops;
|
|
55
62
|
private readonly _onDirty;
|
|
56
63
|
private readonly _onAll;
|
|
57
64
|
constructor(mesh: Mesh, options: FileProcessingOptions);
|
|
@@ -64,20 +71,26 @@ export declare class FileProcessing {
|
|
|
64
71
|
get indexCleared(): boolean;
|
|
65
72
|
settle(): Promise<void>;
|
|
66
73
|
enqueue(ids: Iterable<string>, options?: EnqueueOptions): Promise<void>;
|
|
74
|
+
enqueueVectors(ids: Iterable<string>, options?: EnqueueOptions): Promise<void>;
|
|
67
75
|
private _enqueue;
|
|
76
|
+
private _entry;
|
|
68
77
|
get active(): boolean;
|
|
69
78
|
pending(id: string): boolean;
|
|
70
79
|
awaitsBytes(id: string): boolean;
|
|
80
|
+
demand(ids: string[]): () => void;
|
|
81
|
+
private _demandVectors;
|
|
71
82
|
prioritize(ids: string[]): void;
|
|
72
83
|
private _shutdown;
|
|
73
84
|
private _stop;
|
|
74
85
|
private _kick;
|
|
86
|
+
private _afterFirstLoad;
|
|
75
87
|
private _scan;
|
|
76
88
|
private _documents;
|
|
77
89
|
private _pump;
|
|
78
90
|
private _forget;
|
|
79
91
|
private _run;
|
|
80
92
|
private _recordFailure;
|
|
93
|
+
private _queueProduce;
|
|
81
94
|
private _queueLoad;
|
|
82
95
|
private _schedule;
|
|
83
96
|
private _retry;
|
|
@@ -5,6 +5,7 @@ import { IndexingSource } from './source.js';
|
|
|
5
5
|
import { fileIds, fileRecord } from '../records/index.js';
|
|
6
6
|
import { FileDocument } from './document.js';
|
|
7
7
|
import { errorMessage } from '../../../guards.js';
|
|
8
|
+
const MAX_JOBS = 2;
|
|
8
9
|
export class FileProcessing {
|
|
9
10
|
_mesh;
|
|
10
11
|
_processor;
|
|
@@ -14,14 +15,18 @@ export class FileProcessing {
|
|
|
14
15
|
_output;
|
|
15
16
|
_progress;
|
|
16
17
|
_lanes = [];
|
|
18
|
+
_demanded = new Map();
|
|
19
|
+
_vectorDemanded = new Map();
|
|
17
20
|
_deadlines = new Map();
|
|
18
21
|
_scheduler = new DeadlineScheduler(() => this._retry());
|
|
19
22
|
_scanning = null;
|
|
23
|
+
_reads = new Set();
|
|
20
24
|
_dirty = false;
|
|
21
25
|
_paused = false;
|
|
22
26
|
_closed = false;
|
|
23
27
|
_close = null;
|
|
24
28
|
_indexCleared = true;
|
|
29
|
+
_stops = 0;
|
|
25
30
|
_onDirty = (ids) => {
|
|
26
31
|
if (ids) {
|
|
27
32
|
void this.enqueue(ids);
|
|
@@ -54,6 +59,9 @@ export class FileProcessing {
|
|
|
54
59
|
ready: (id) => this._schedule(mesh.files.processorState(options.processor.id).artifact(id).knowledgeId ?? id, Date.now())
|
|
55
60
|
}
|
|
56
61
|
});
|
|
62
|
+
if (options.processor.text) {
|
|
63
|
+
this._lanes.push({ stage: 'text', queue: new Map(), jobs: new Set() });
|
|
64
|
+
}
|
|
57
65
|
this._lanes.push({ stage: 'produce', queue: new Map(), jobs: new Set() });
|
|
58
66
|
if (options.processor.load) {
|
|
59
67
|
this._lanes.push({ stage: 'load', queue: new Map(), jobs: new Set() });
|
|
@@ -64,6 +72,7 @@ export class FileProcessing {
|
|
|
64
72
|
this._mesh.autobee.on('move-to', this._onAll);
|
|
65
73
|
this._mesh.autobee.on('writable', this._onAll);
|
|
66
74
|
this._kick();
|
|
75
|
+
void this._afterFirstLoad();
|
|
67
76
|
}
|
|
68
77
|
async suspend() {
|
|
69
78
|
this._paused = true;
|
|
@@ -93,19 +102,46 @@ export class FileProcessing {
|
|
|
93
102
|
if (this._closed || this._paused)
|
|
94
103
|
return;
|
|
95
104
|
const ids = await this._documents();
|
|
96
|
-
|
|
97
|
-
|
|
105
|
+
await this.enqueueVectors(ids, { wait: true });
|
|
106
|
+
const loader = this._lanes.find((entry) => entry.stage === 'load');
|
|
107
|
+
if (loader) {
|
|
108
|
+
await this._enqueue(ids, [loader], { wait: true, direct: true });
|
|
98
109
|
}
|
|
99
110
|
}
|
|
100
111
|
enqueue(ids, options = {}) {
|
|
101
|
-
|
|
112
|
+
const values = [...ids];
|
|
113
|
+
const release = options.wait ? this.demand(values) : null;
|
|
114
|
+
const releaseVectors = options.wait && this._processor.deferChatVectors ? this._demandVectors(values) : null;
|
|
115
|
+
return this._enqueue(values, this._lanes, options).finally(() => {
|
|
116
|
+
release?.();
|
|
117
|
+
releaseVectors?.();
|
|
118
|
+
});
|
|
102
119
|
}
|
|
103
|
-
|
|
120
|
+
enqueueVectors(ids, options = {}) {
|
|
121
|
+
const producer = this._lanes.find((entry) => entry.stage === 'produce');
|
|
122
|
+
if (!producer) {
|
|
123
|
+
return Promise.resolve();
|
|
124
|
+
}
|
|
125
|
+
const values = [...ids];
|
|
126
|
+
const release = this.demand(values);
|
|
127
|
+
const releaseVectors = this._demandVectors(values);
|
|
128
|
+
const queued = this._enqueue(values, [producer], { wait: true }).finally(() => {
|
|
129
|
+
release();
|
|
130
|
+
releaseVectors();
|
|
131
|
+
});
|
|
132
|
+
if (options.wait) {
|
|
133
|
+
return queued;
|
|
134
|
+
}
|
|
135
|
+
void queued.catch(noop);
|
|
136
|
+
return Promise.resolve();
|
|
137
|
+
}
|
|
138
|
+
async _enqueue(ids, lanes, { wait = false, direct = false } = {}) {
|
|
104
139
|
if (this._closed || this._paused)
|
|
105
140
|
return;
|
|
106
141
|
const pending = [];
|
|
142
|
+
const entries = new Set(lanes.map((lane) => (direct ? lane : this._entry(lane))));
|
|
107
143
|
for (const id of ids) {
|
|
108
|
-
for (const lane of
|
|
144
|
+
for (const lane of entries) {
|
|
109
145
|
let wakes = lane.queue.get(id);
|
|
110
146
|
if (!wakes) {
|
|
111
147
|
lane.queue.set(id, (wakes = []));
|
|
@@ -115,11 +151,14 @@ export class FileProcessing {
|
|
|
115
151
|
}
|
|
116
152
|
}
|
|
117
153
|
}
|
|
118
|
-
for (const lane of
|
|
154
|
+
for (const lane of entries) {
|
|
119
155
|
this._pump(lane);
|
|
120
156
|
}
|
|
121
157
|
await Promise.all(pending);
|
|
122
158
|
}
|
|
159
|
+
_entry(lane) {
|
|
160
|
+
return (lane.stage === 'produce' && this._lanes.find((entry) => entry.stage === 'text')) || lane;
|
|
161
|
+
}
|
|
123
162
|
get active() {
|
|
124
163
|
return !this._paused && !this._closed && !this._mesh.closing;
|
|
125
164
|
}
|
|
@@ -134,6 +173,39 @@ export class FileProcessing {
|
|
|
134
173
|
return (this._source.awaitsBytes((key) => key === id) ||
|
|
135
174
|
this._output.awaitsBytes((key) => this._mesh.files.processorState(this._processor.id).artifact(key).knowledgeId === id));
|
|
136
175
|
}
|
|
176
|
+
demand(ids) {
|
|
177
|
+
for (const id of ids) {
|
|
178
|
+
this._demanded.set(id, (this._demanded.get(id) ?? 0) + 1);
|
|
179
|
+
}
|
|
180
|
+
this.prioritize(ids);
|
|
181
|
+
return () => {
|
|
182
|
+
for (const id of ids) {
|
|
183
|
+
const count = (this._demanded.get(id) ?? 1) - 1;
|
|
184
|
+
if (count > 0) {
|
|
185
|
+
this._demanded.set(id, count);
|
|
186
|
+
}
|
|
187
|
+
else {
|
|
188
|
+
this._demanded.delete(id);
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
};
|
|
192
|
+
}
|
|
193
|
+
_demandVectors(ids) {
|
|
194
|
+
for (const id of ids) {
|
|
195
|
+
this._vectorDemanded.set(id, (this._vectorDemanded.get(id) ?? 0) + 1);
|
|
196
|
+
}
|
|
197
|
+
return () => {
|
|
198
|
+
for (const id of ids) {
|
|
199
|
+
const count = (this._vectorDemanded.get(id) ?? 1) - 1;
|
|
200
|
+
if (count > 0) {
|
|
201
|
+
this._vectorDemanded.set(id, count);
|
|
202
|
+
}
|
|
203
|
+
else {
|
|
204
|
+
this._vectorDemanded.delete(id);
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
};
|
|
208
|
+
}
|
|
137
209
|
prioritize(ids) {
|
|
138
210
|
for (const lane of this._lanes) {
|
|
139
211
|
const queued = new Map(lane.queue);
|
|
@@ -164,6 +236,7 @@ export class FileProcessing {
|
|
|
164
236
|
this._indexCleared = await this._processor.close();
|
|
165
237
|
}
|
|
166
238
|
async _stop() {
|
|
239
|
+
this._stops++;
|
|
167
240
|
this._scheduler.clear();
|
|
168
241
|
this._deadlines.clear();
|
|
169
242
|
this._mesh.files.emit('update');
|
|
@@ -173,6 +246,7 @@ export class FileProcessing {
|
|
|
173
246
|
type: this._closed ? 'file-processing-closed' : 'file-processing-suspended'
|
|
174
247
|
});
|
|
175
248
|
}
|
|
249
|
+
await Promise.all([...this._reads].map((snap) => snap.close()));
|
|
176
250
|
await Promise.all([this._scanning, ...jobs.map((job) => job.done)]);
|
|
177
251
|
await Promise.all([this._source.cancel(), this._output.cancel()]);
|
|
178
252
|
for (const lane of this._lanes) {
|
|
@@ -198,6 +272,22 @@ export class FileProcessing {
|
|
|
198
272
|
}
|
|
199
273
|
});
|
|
200
274
|
}
|
|
275
|
+
async _afterFirstLoad() {
|
|
276
|
+
const loader = this._lanes.find((lane) => lane.stage === 'load');
|
|
277
|
+
if (!loader || !this._processor.allLoaded) {
|
|
278
|
+
return;
|
|
279
|
+
}
|
|
280
|
+
const stops = this._stops;
|
|
281
|
+
try {
|
|
282
|
+
await this._enqueue(await this._documents(), [loader], { wait: true });
|
|
283
|
+
if (stops === this._stops && this.active) {
|
|
284
|
+
await this._processor.allLoaded();
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
catch (err) {
|
|
288
|
+
this._mesh.log?.warn('rag', 'file processing first load pass failed', { err });
|
|
289
|
+
}
|
|
290
|
+
}
|
|
201
291
|
async _scan() {
|
|
202
292
|
while (this._dirty && !this._closed && !this._paused) {
|
|
203
293
|
this._dirty = false;
|
|
@@ -213,20 +303,23 @@ export class FileProcessing {
|
|
|
213
303
|
if (this._mesh.closing)
|
|
214
304
|
return [];
|
|
215
305
|
const snap = this._mesh.view.snapshot();
|
|
306
|
+
this._reads.add(snap);
|
|
216
307
|
try {
|
|
217
308
|
return await fileIds(snap, this._processor.input, this._processor.mediaKind);
|
|
218
309
|
}
|
|
219
310
|
finally {
|
|
311
|
+
this._reads.delete(snap);
|
|
220
312
|
await snap.close();
|
|
221
313
|
}
|
|
222
314
|
}
|
|
223
315
|
_pump(lane) {
|
|
224
316
|
while (lane.queue.size && !this._closed && !this._paused && !this._mesh.closing) {
|
|
225
317
|
const busy = [...lane.jobs];
|
|
226
|
-
if (
|
|
318
|
+
if (laneFull(lane, busy)) {
|
|
227
319
|
return;
|
|
228
320
|
}
|
|
229
|
-
const
|
|
321
|
+
const full = busy.length >= MAX_JOBS;
|
|
322
|
+
const next = [...lane.queue].find(([queued]) => !busy.some((job) => job.id === queued) && (!full || this._demanded.has(queued)));
|
|
230
323
|
if (!next) {
|
|
231
324
|
return;
|
|
232
325
|
}
|
|
@@ -244,10 +337,12 @@ export class FileProcessing {
|
|
|
244
337
|
job.done = this._run(lane, { id, wakes, signal: abort.signal, admit }).finally(() => {
|
|
245
338
|
lane.jobs.delete(job);
|
|
246
339
|
this._pump(lane);
|
|
340
|
+
this._mesh.files.emit('update');
|
|
247
341
|
});
|
|
248
342
|
}
|
|
249
343
|
}
|
|
250
344
|
async _forget(id) {
|
|
345
|
+
await this._mesh.files.forgetText(id);
|
|
251
346
|
const loaded = this._mesh.files.processorState(this._processor.id).forget(id)?.loaded;
|
|
252
347
|
if (loaded) {
|
|
253
348
|
await this._processor.evict?.(loaded);
|
|
@@ -255,8 +350,9 @@ export class FileProcessing {
|
|
|
255
350
|
}
|
|
256
351
|
async _run(lane, { id, wakes, signal, admit }) {
|
|
257
352
|
let produced = false;
|
|
353
|
+
let record = null;
|
|
258
354
|
try {
|
|
259
|
-
|
|
355
|
+
record = await fileRecord(this._mesh.view, id, this._processor.input);
|
|
260
356
|
if (!record && lane.stage === 'load') {
|
|
261
357
|
await this._forget(id);
|
|
262
358
|
}
|
|
@@ -267,7 +363,7 @@ export class FileProcessing {
|
|
|
267
363
|
if (!record || !this._processor.applies(record) || signal.aborted) {
|
|
268
364
|
return;
|
|
269
365
|
}
|
|
270
|
-
if (lane.stage
|
|
366
|
+
if (lane.stage !== 'load' &&
|
|
271
367
|
(!this._mesh.writable ||
|
|
272
368
|
!(await this._processor.canProduce(record, {
|
|
273
369
|
mesh: this._mesh,
|
|
@@ -302,8 +398,13 @@ export class FileProcessing {
|
|
|
302
398
|
if (produced && this.active) {
|
|
303
399
|
this._queueLoad(id);
|
|
304
400
|
}
|
|
305
|
-
|
|
306
|
-
|
|
401
|
+
if (lane.stage === 'text' && this.active) {
|
|
402
|
+
this._queueProduce(id, wakes, record);
|
|
403
|
+
}
|
|
404
|
+
else {
|
|
405
|
+
for (const wake of wakes) {
|
|
406
|
+
wake();
|
|
407
|
+
}
|
|
307
408
|
}
|
|
308
409
|
this._mesh.files.emit('update');
|
|
309
410
|
}
|
|
@@ -321,6 +422,26 @@ export class FileProcessing {
|
|
|
321
422
|
}
|
|
322
423
|
this._mesh.log?.warn('rag', 'file processing failed', { id, stage, err });
|
|
323
424
|
}
|
|
425
|
+
_queueProduce(id, wakes, record) {
|
|
426
|
+
const producer = this._lanes.find((entry) => entry.stage === 'produce');
|
|
427
|
+
if (!producer ||
|
|
428
|
+
!record ||
|
|
429
|
+
record.deletedAt ||
|
|
430
|
+
(this._processor.deferChatVectors && record.chatId && !this._vectorDemanded.has(id))) {
|
|
431
|
+
for (const wake of wakes) {
|
|
432
|
+
wake();
|
|
433
|
+
}
|
|
434
|
+
return;
|
|
435
|
+
}
|
|
436
|
+
const queued = producer.queue.get(id);
|
|
437
|
+
if (queued) {
|
|
438
|
+
queued.push(...wakes);
|
|
439
|
+
}
|
|
440
|
+
else {
|
|
441
|
+
producer.queue.set(id, wakes);
|
|
442
|
+
}
|
|
443
|
+
this._pump(producer);
|
|
444
|
+
}
|
|
324
445
|
_queueLoad(id) {
|
|
325
446
|
const loader = this._lanes.find((entry) => entry.stage === 'load');
|
|
326
447
|
if (!loader) {
|
|
@@ -356,3 +477,14 @@ export class FileProcessing {
|
|
|
356
477
|
}
|
|
357
478
|
export { isClaimLive, listIndexingByChat, listIndexingByKnowledge } from './protocol.js';
|
|
358
479
|
export { FileState } from './state.js';
|
|
480
|
+
function noop() { }
|
|
481
|
+
function laneFull(lane, busy) {
|
|
482
|
+
const extracting = busy.some((job) => !job.admitted);
|
|
483
|
+
if (lane.stage === 'text') {
|
|
484
|
+
return busy.length > 0;
|
|
485
|
+
}
|
|
486
|
+
if (lane.stage === 'produce') {
|
|
487
|
+
return extracting || busy.length > MAX_JOBS;
|
|
488
|
+
}
|
|
489
|
+
return extracting;
|
|
490
|
+
}
|
|
@@ -10,14 +10,7 @@ export function isClaimLive(row, at) {
|
|
|
10
10
|
row.expiresAt <= at + INDEXING_MAX_CLOCK_SKEW_MS + INDEXING_CLAIM_TTL_MS);
|
|
11
11
|
}
|
|
12
12
|
export async function getIndexingStatus(store, artifactId) {
|
|
13
|
-
|
|
14
|
-
try {
|
|
15
|
-
const status = await snap.get(INDEXING_STATUS, { id: artifactId });
|
|
16
|
-
return status;
|
|
17
|
-
}
|
|
18
|
-
finally {
|
|
19
|
-
await snap.close();
|
|
20
|
-
}
|
|
13
|
+
return store.view.get(INDEXING_STATUS, { id: artifactId });
|
|
21
14
|
}
|
|
22
15
|
async function mustGetIndexingStatus(store, artifactId) {
|
|
23
16
|
const row = await getIndexingStatus(store, artifactId);
|
|
@@ -27,34 +20,20 @@ async function mustGetIndexingStatus(store, artifactId) {
|
|
|
27
20
|
return row;
|
|
28
21
|
}
|
|
29
22
|
export async function listIndexingByKnowledge(store, { knowledgeId }) {
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
})
|
|
37
|
-
.toArray();
|
|
38
|
-
return rows;
|
|
39
|
-
}
|
|
40
|
-
finally {
|
|
41
|
-
await snap.close();
|
|
42
|
-
}
|
|
23
|
+
return store.view
|
|
24
|
+
.find(INDEXING_STATUS_BY_KNOWLEDGE, {
|
|
25
|
+
gte: { knowledgeId },
|
|
26
|
+
lte: { knowledgeId }
|
|
27
|
+
})
|
|
28
|
+
.toArray();
|
|
43
29
|
}
|
|
44
30
|
export async function listIndexingByChat(store, { chatId }) {
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
})
|
|
52
|
-
.toArray();
|
|
53
|
-
return rows;
|
|
54
|
-
}
|
|
55
|
-
finally {
|
|
56
|
-
await snap.close();
|
|
57
|
-
}
|
|
31
|
+
return store.view
|
|
32
|
+
.find(INDEXING_STATUS_BY_CHAT, {
|
|
33
|
+
gte: { chatId },
|
|
34
|
+
lte: { chatId }
|
|
35
|
+
})
|
|
36
|
+
.toArray();
|
|
58
37
|
}
|
|
59
38
|
export async function claimIndexing(store, { artifactId, knowledgeId, at }) {
|
|
60
39
|
if (!at || at < 0) {
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { AssistantFileProcessorStatus } from '../../../../spec/hyperschema/types.d.ts';
|
|
2
2
|
import type { IndexingProgress } from '../rag/index.ts';
|
|
3
|
+
import type { CachedText } from '../rag/text-cache.ts';
|
|
3
4
|
export type LocalWork = Pick<AssistantFileProcessorStatus, 'state' | 'reason' | 'retryAt'>;
|
|
4
5
|
export type IndexingProgressEntry = Omit<Parameters<IndexingProgress>[0], 'error'> & {
|
|
5
6
|
error: string | null;
|
|
@@ -19,6 +20,7 @@ interface DocumentState {
|
|
|
19
20
|
loaded?: LoadedArtifact;
|
|
20
21
|
sourceMisses?: number;
|
|
21
22
|
artifactIdMemo?: string;
|
|
23
|
+
prepared?: CachedText;
|
|
22
24
|
embedFailures?: number;
|
|
23
25
|
preferredUntil?: number;
|
|
24
26
|
}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { AbortSignal } from 'bare-abort-controller';
|
|
2
2
|
import type { QvacKnowledge } from '../../../../spec/mesh/hyperschema/types.d.ts';
|
|
3
|
-
import
|
|
3
|
+
import type { PdfPage } from '@qvac/harness/pdf-pages';
|
|
4
|
+
import { type RagArtifactPayload, type RagExtractors, type RagPreparedChunk } from 'rag-extract';
|
|
4
5
|
export interface BuildEmbedder {
|
|
5
6
|
model: string;
|
|
6
7
|
embed(input: {
|
|
@@ -13,6 +14,19 @@ export type BuildProgress = (processedChunks: number, totalChunks: number) => vo
|
|
|
13
14
|
export declare class EmbedderFailure extends Error {
|
|
14
15
|
constructor(cause: unknown);
|
|
15
16
|
}
|
|
17
|
+
export interface ReadPdfPagesOptions {
|
|
18
|
+
readonly signal: AbortSignal;
|
|
19
|
+
readonly beforeOcr?: () => Promise<void>;
|
|
20
|
+
}
|
|
21
|
+
export type ReadPdfPages = (bytes: Uint8Array, options: ReadPdfPagesOptions) => Promise<readonly PdfPage[]>;
|
|
22
|
+
export interface KnowledgeExtractors extends RagExtractors {
|
|
23
|
+
readonly pdfPages?: ReadPdfPages;
|
|
24
|
+
}
|
|
25
|
+
export interface PreparedText {
|
|
26
|
+
readonly content: string;
|
|
27
|
+
readonly chunks: readonly RagPreparedChunk[];
|
|
28
|
+
readonly artifactId?: string | undefined;
|
|
29
|
+
}
|
|
16
30
|
export interface ArtifactBuildInput {
|
|
17
31
|
embedder: BuildEmbedder;
|
|
18
32
|
knowledge: QvacKnowledge;
|
|
@@ -22,7 +36,10 @@ export interface ArtifactBuildInput {
|
|
|
22
36
|
onProgress?: BuildProgress;
|
|
23
37
|
onExtracting?: (pages: number, totalPages: number) => Promise<void>;
|
|
24
38
|
onExtracted?: () => void | Promise<void>;
|
|
25
|
-
|
|
39
|
+
onChunked?: (prepared: PreparedText) => void | Promise<void>;
|
|
40
|
+
kept?: PreparedText | null;
|
|
41
|
+
extractors?: KnowledgeExtractors;
|
|
26
42
|
}
|
|
27
43
|
export declare function yieldTurn(signal: AbortSignal): Promise<void>;
|
|
44
|
+
export declare function prepareText(input: ArtifactBuildInput): Promise<PreparedText>;
|
|
28
45
|
export declare function buildArtifactPayload(input: ArtifactBuildInput): Promise<RagArtifactPayload>;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { chunkRagDocuments, extractRagDocumentFromBytes } from 'rag-extract';
|
|
1
|
+
import { chunkRagDocuments, classifyRagDocument, extractRagDocumentFromBytes, RAG_DOCUMENT_KIND } from 'rag-extract';
|
|
2
2
|
import { AbortedError } from '@qvac/harness/race-abort';
|
|
3
|
-
import { errorMessage } from '../../../guards.js';
|
|
3
|
+
import { errorMessage, isRecord } from '../../../guards.js';
|
|
4
4
|
import { IndexingFailure } from './failure.js';
|
|
5
5
|
const EMBED_BATCH = 8;
|
|
6
6
|
export class EmbedderFailure extends Error {
|
|
@@ -25,8 +25,17 @@ function pageProgress(onExtracting) {
|
|
|
25
25
|
settled: () => chain
|
|
26
26
|
};
|
|
27
27
|
}
|
|
28
|
-
export async function
|
|
29
|
-
|
|
28
|
+
export async function prepareText(input) {
|
|
29
|
+
if (input.kept) {
|
|
30
|
+
await input.onExtracted?.();
|
|
31
|
+
return input.kept;
|
|
32
|
+
}
|
|
33
|
+
const { knowledge, artifactId, bytes, signal, extractors } = input;
|
|
34
|
+
const pdfPages = extractors?.pdfPages;
|
|
35
|
+
const { fileName, mimeType } = knowledge;
|
|
36
|
+
if (pdfPages && classifyRagDocument({ fileName, mimeType, bytes }) === RAG_DOCUMENT_KIND.pdf) {
|
|
37
|
+
return pageChunks(input, pdfPages);
|
|
38
|
+
}
|
|
30
39
|
const pages = pageProgress(input.onExtracting);
|
|
31
40
|
const document = await extractRagDocumentFromBytes({
|
|
32
41
|
id: artifactId,
|
|
@@ -34,37 +43,132 @@ export async function buildArtifactPayload(input) {
|
|
|
34
43
|
attachmentId: knowledge.id,
|
|
35
44
|
chatId: knowledge.chatId ?? undefined,
|
|
36
45
|
blobId: knowledge.blobId,
|
|
37
|
-
fileName
|
|
38
|
-
mimeType
|
|
46
|
+
fileName,
|
|
47
|
+
mimeType,
|
|
39
48
|
bytes
|
|
40
49
|
}, { extractors, signal, onProgress: pages.report });
|
|
41
50
|
await pages.settled();
|
|
42
51
|
await yieldTurn(signal);
|
|
43
|
-
await onExtracted?.();
|
|
44
|
-
const chunks = chunkRagDocuments([document], {});
|
|
45
|
-
|
|
52
|
+
await input.onExtracted?.();
|
|
53
|
+
const chunks = document.content.trim() ? chunkRagDocuments([document], {}) : [];
|
|
54
|
+
return { content: document.content, chunks };
|
|
55
|
+
}
|
|
56
|
+
async function pageChunks(input, pdfPages) {
|
|
57
|
+
const { knowledge, artifactId, signal, embedder } = input;
|
|
58
|
+
let read = 0;
|
|
59
|
+
const pages = await pdfPages(input.bytes, {
|
|
60
|
+
signal,
|
|
61
|
+
beforeOcr: async () => {
|
|
62
|
+
if (!knowledge.chatId) {
|
|
63
|
+
await embedder.idle?.(signal);
|
|
64
|
+
}
|
|
65
|
+
await input.onExtracting?.(++read, 0);
|
|
66
|
+
}
|
|
67
|
+
});
|
|
68
|
+
await yieldTurn(signal);
|
|
69
|
+
await input.onExtracted?.();
|
|
70
|
+
const unread = pages.filter((page) => page.unread).length;
|
|
71
|
+
if (unread > 0) {
|
|
72
|
+
throw new IndexingFailure('failed', `${knowledge.fileName}: ${unread} of its pages need OCR`);
|
|
73
|
+
}
|
|
74
|
+
const document = {
|
|
75
|
+
id: artifactId,
|
|
76
|
+
artifactId,
|
|
77
|
+
attachmentId: knowledge.id,
|
|
78
|
+
chatId: knowledge.chatId ?? undefined,
|
|
79
|
+
blobId: knowledge.blobId,
|
|
80
|
+
title: knowledge.fileName,
|
|
81
|
+
mimeType: knowledge.mimeType,
|
|
82
|
+
content: ''
|
|
83
|
+
};
|
|
84
|
+
const chunks = withoutShortPages(pages).flatMap(({ page, text }) => chunkRagDocuments([{ ...document, content: text }], {}).map((chunk) => ({ chunk, page })));
|
|
85
|
+
return {
|
|
86
|
+
content: pages.map(({ text }) => text).join('\n\n'),
|
|
87
|
+
chunks: chunks.map(({ chunk, page }, index) => numbered(chunk, { page, index }))
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
const SHORT_PAGE_WORDS = 12;
|
|
91
|
+
const UNSPACED = /[\p{sc=Han}\p{sc=Hiragana}\p{sc=Katakana}\p{sc=Thai}\p{sc=Lao}\p{sc=Khmer}\p{sc=Myanmar}]/gu;
|
|
92
|
+
const wordsIn = (text) => text.split(/\s+/).filter(Boolean).length + (text.match(UNSPACED)?.length ?? 0);
|
|
93
|
+
const oneParagraph = (text) => text.trim().replace(/\n\s*\n/g, '\n');
|
|
94
|
+
function withoutShortPages(pages) {
|
|
95
|
+
const out = [];
|
|
96
|
+
let held = '';
|
|
97
|
+
for (const { page, text } of pages) {
|
|
98
|
+
const short = wordsIn(text) < SHORT_PAGE_WORDS;
|
|
99
|
+
held = [held, short ? oneParagraph(text) : text].filter(Boolean).join('\n');
|
|
100
|
+
if (short && wordsIn(held) < SHORT_PAGE_WORDS) {
|
|
101
|
+
continue;
|
|
102
|
+
}
|
|
103
|
+
out.push({ page, text: held });
|
|
104
|
+
held = '';
|
|
105
|
+
}
|
|
106
|
+
const last = out.pop();
|
|
107
|
+
if (!last) {
|
|
108
|
+
return pages;
|
|
109
|
+
}
|
|
110
|
+
return [...out, { page: last.page, text: [last.text, held].filter(Boolean).join('\n') }];
|
|
111
|
+
}
|
|
112
|
+
function numbered(chunk, { page, index }) {
|
|
113
|
+
const parsed = JSON.parse(chunk.metadataJson);
|
|
114
|
+
const metadata = isRecord(parsed) ? parsed : {};
|
|
115
|
+
return {
|
|
116
|
+
...chunk,
|
|
117
|
+
id: `${chunk.id.slice(0, chunk.id.lastIndexOf(':') + 1)}${index}`,
|
|
118
|
+
metadataJson: JSON.stringify({ ...metadata, chunkIndex: index, page })
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
export async function buildArtifactPayload(input) {
|
|
122
|
+
const { embedder, knowledge, signal, onProgress } = input;
|
|
123
|
+
const prepared = await prepareText(input);
|
|
124
|
+
const { chunks } = prepared;
|
|
125
|
+
if (chunks.length === 0) {
|
|
46
126
|
throw new IndexingFailure('no-text', `no extractable text: ${knowledge.fileName}`);
|
|
47
127
|
}
|
|
128
|
+
await input.onChunked?.(prepared);
|
|
48
129
|
await onProgress?.(0, chunks.length);
|
|
130
|
+
const positionsByText = new Map();
|
|
131
|
+
for (let i = 0; i < chunks.length; i++) {
|
|
132
|
+
const text = chunks[i].content;
|
|
133
|
+
const positions = positionsByText.get(text);
|
|
134
|
+
if (positions) {
|
|
135
|
+
positions.push(i);
|
|
136
|
+
}
|
|
137
|
+
else {
|
|
138
|
+
positionsByText.set(text, [i]);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
const unique = [...positionsByText];
|
|
49
142
|
const vectors = [];
|
|
50
|
-
|
|
143
|
+
let processed = 0;
|
|
144
|
+
for (let start = 0; start < unique.length; start += EMBED_BATCH) {
|
|
51
145
|
await yieldTurn(signal);
|
|
52
|
-
|
|
53
|
-
|
|
146
|
+
await embedder.idle?.(signal);
|
|
147
|
+
if (signal.aborted) {
|
|
148
|
+
throw new AbortedError();
|
|
54
149
|
}
|
|
55
|
-
const batch =
|
|
150
|
+
const batch = unique.slice(start, start + EMBED_BATCH);
|
|
56
151
|
let rows;
|
|
57
152
|
try {
|
|
58
|
-
rows = await embedder.embed({ texts: batch.map((
|
|
153
|
+
rows = await embedder.embed({ texts: batch.map(([text]) => text), signal });
|
|
59
154
|
}
|
|
60
155
|
catch (err) {
|
|
61
156
|
throw new EmbedderFailure(err);
|
|
62
157
|
}
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
158
|
+
if (signal.aborted) {
|
|
159
|
+
throw new AbortedError();
|
|
160
|
+
}
|
|
161
|
+
if (rows.length !== batch.length) {
|
|
162
|
+
throw new Error(`embedder returned ${rows.length} vectors for ${batch.length} chunks`);
|
|
163
|
+
}
|
|
164
|
+
for (let i = 0; i < batch.length; i++) {
|
|
165
|
+
const [, positions] = batch[i];
|
|
166
|
+
for (const position of positions) {
|
|
167
|
+
vectors[position] = rows[i];
|
|
168
|
+
}
|
|
169
|
+
processed += positions.length;
|
|
170
|
+
}
|
|
171
|
+
await onProgress?.(processed, chunks.length);
|
|
68
172
|
}
|
|
69
173
|
const dimensions = vectors[0]?.length ?? 0;
|
|
70
174
|
if (dimensions <= 0 || vectors.some((v) => v.length !== dimensions)) {
|
|
@@ -33,7 +33,8 @@ export function failureReason(err) {
|
|
|
33
33
|
if ((isRecord(err) && err.code === 'ENOSPC') || OUT_OF_SPACE.test(message)) {
|
|
34
34
|
return 'out-of-space';
|
|
35
35
|
}
|
|
36
|
-
|
|
36
|
+
const locked = (isRecord(err) && err.code === 'PASSWORD') || PASSWORD.test(message);
|
|
37
|
+
return locked ? 'password-protected' : 'failed';
|
|
37
38
|
}
|
|
38
39
|
export function failureDetail(err) {
|
|
39
40
|
const detail = errorMessage(err);
|