@qvac/core 1.0.0-rc.2 → 1.0.0-rc.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +238 -12
- package/dist-lib/index.d.ts +3 -1
- package/dist-lib/index.js +1 -0
- package/dist-lib/lib/api/approval-rules.d.ts +1 -1
- package/dist-lib/lib/api/approval-rules.js +5 -1
- package/dist-lib/lib/api/chunk-delta.d.ts +7 -2
- package/dist-lib/lib/api/chunk-delta.js +89 -7
- package/dist-lib/lib/api/chunks.js +14 -8
- package/dist-lib/lib/api/config.d.ts +10 -3
- package/dist-lib/lib/api/config.js +51 -6
- package/dist-lib/lib/api/custom-models.js +3 -5
- package/dist-lib/lib/api/devices.js +3 -1
- package/dist-lib/lib/api/files.d.ts +1 -1
- package/dist-lib/lib/api/files.js +65 -8
- package/dist-lib/lib/api/index.d.ts +12 -2
- package/dist-lib/lib/api/index.js +6 -1
- package/dist-lib/lib/api/instrument.js +12 -4
- package/dist-lib/lib/api/members.js +7 -13
- package/dist-lib/lib/api/mesh.d.ts +1 -3
- package/dist-lib/lib/api/mesh.js +2 -2
- package/dist-lib/lib/api/notifications.js +8 -4
- package/dist-lib/lib/api/projects.js +5 -1
- package/dist-lib/lib/api/rag.js +32 -19
- package/dist-lib/lib/api/served.js +2 -1
- package/dist-lib/lib/api/workspace.js +0 -1
- package/dist-lib/lib/assistant/harness.d.ts +8 -2
- package/dist-lib/lib/assistant/harness.js +17 -4
- package/dist-lib/lib/assistant/history.d.ts +1 -1
- package/dist-lib/lib/assistant/history.js +19 -13
- package/dist-lib/lib/assistant/message.d.ts +6 -1
- package/dist-lib/lib/assistant/message.js +22 -2
- package/dist-lib/lib/assistant/model-round.d.ts +5 -0
- package/dist-lib/lib/assistant/model-round.js +47 -19
- package/dist-lib/lib/assistant/pdf-budget.d.ts +10 -2
- package/dist-lib/lib/assistant/pdf-budget.js +10 -11
- package/dist-lib/lib/assistant/pdf-pages.d.ts +25 -0
- package/dist-lib/lib/assistant/pdf-pages.js +75 -0
- package/dist-lib/lib/assistant/round-host.js +5 -3
- package/dist-lib/lib/assistant-client.d.ts +2 -1
- package/dist-lib/lib/assistant-client.js +24 -6
- package/dist-lib/lib/assistant-tree.d.ts +2 -0
- package/dist-lib/lib/assistant-tree.js +8 -5
- package/dist-lib/lib/assistant.d.ts +21 -7
- package/dist-lib/lib/assistant.js +481 -244
- package/dist-lib/lib/client.js +10 -1
- package/dist-lib/lib/compaction.d.ts +3 -1
- package/dist-lib/lib/compaction.js +18 -4
- package/dist-lib/lib/constants.d.ts +3 -0
- package/dist-lib/lib/constants.js +3 -0
- package/dist-lib/lib/core/agent-copies.js +7 -1
- package/dist-lib/lib/core/agent.js +1 -0
- package/dist-lib/lib/core/approval-rules.d.ts +5 -1
- package/dist-lib/lib/core/approval-rules.js +16 -1
- package/dist-lib/lib/core/chat-activity.js +13 -16
- package/dist-lib/lib/core/chat-copy.js +1 -1
- package/dist-lib/lib/core/chat-fork.js +2 -1
- package/dist-lib/lib/core/chat-move.d.ts +4 -0
- package/dist-lib/lib/core/chat-move.js +42 -11
- package/dist-lib/lib/core/chat.d.ts +0 -1
- package/dist-lib/lib/core/chat.js +1 -1
- package/dist-lib/lib/core/chunk.d.ts +14 -3
- package/dist-lib/lib/core/chunk.js +116 -25
- package/dist-lib/lib/core/collections.d.ts +12 -1
- package/dist-lib/lib/core/collections.js +12 -1
- package/dist-lib/lib/core/config-value.d.ts +7 -0
- package/dist-lib/lib/core/config-value.js +15 -0
- package/dist-lib/lib/core/contract/activity.d.ts +3 -2
- package/dist-lib/lib/core/contract/activity.js +39 -21
- package/dist-lib/lib/core/contract/agent-join.js +15 -19
- package/dist-lib/lib/core/contract/agents.js +8 -7
- package/dist-lib/lib/core/contract/chat.d.ts +3 -0
- package/dist-lib/lib/core/contract/chat.js +12 -1
- package/dist-lib/lib/core/contract/chunks.js +16 -6
- package/dist-lib/lib/core/contract/devices.js +6 -4
- package/dist-lib/lib/core/contract/index.js +2 -1
- package/dist-lib/lib/core/contract/invites.js +12 -31
- package/dist-lib/lib/core/contract/joined-meshes.js +2 -1
- package/dist-lib/lib/core/contract/mesh-rules.js +4 -0
- package/dist-lib/lib/core/contract/model-requests.js +7 -6
- package/dist-lib/lib/core/contract/project-device-keys.js +15 -6
- package/dist-lib/lib/core/contract/project-host-keys.js +5 -4
- package/dist-lib/lib/core/contract/projects.js +12 -13
- package/dist-lib/lib/core/contract/public-seeds.js +5 -5
- package/dist-lib/lib/core/contract/roles.js +2 -1
- package/dist-lib/lib/core/contract/run-requests.d.ts +1 -1
- package/dist-lib/lib/core/contract/run-requests.js +19 -13
- package/dist-lib/lib/core/contract/shipped-rows.d.ts +2 -0
- package/dist-lib/lib/core/contract/shipped-rows.js +22 -0
- package/dist-lib/lib/core/contract/workspace.js +6 -12
- package/dist-lib/lib/core/contract/writers.js +20 -6
- package/dist-lib/lib/core/deleted-agent-runs.d.ts +7 -0
- package/dist-lib/lib/core/deleted-agent-runs.js +46 -0
- package/dist-lib/lib/core/devices.d.ts +2 -1
- package/dist-lib/lib/core/devices.js +4 -2
- package/dist-lib/lib/core/document-attachment.d.ts +6 -3
- package/dist-lib/lib/core/document-attachment.js +10 -5
- package/dist-lib/lib/core/files/attachments.js +3 -3
- package/dist-lib/lib/core/files/index.d.ts +8 -5
- package/dist-lib/lib/core/files/index.js +111 -18
- package/dist-lib/lib/core/files/processing/index.d.ts +1 -1
- package/dist-lib/lib/core/files/processing/index.js +8 -2
- package/dist-lib/lib/core/files/processing/source.d.ts +1 -0
- package/dist-lib/lib/core/files/processing/source.js +3 -0
- package/dist-lib/lib/core/files/processing/state.d.ts +1 -0
- package/dist-lib/lib/core/files/processing/state.js +7 -0
- package/dist-lib/lib/core/files/rag/build.d.ts +1 -0
- package/dist-lib/lib/core/files/rag/build.js +14 -1
- package/dist-lib/lib/core/files/rag/index.d.ts +1 -1
- package/dist-lib/lib/core/files/rag/index.js +1 -1
- package/dist-lib/lib/core/files/rag/produce.js +1 -0
- package/dist-lib/lib/core/files/rag/retrieval/full-text.d.ts +2 -0
- package/dist-lib/lib/core/files/rag/retrieval/full-text.js +7 -1
- package/dist-lib/lib/core/first-event.d.ts +6 -0
- package/dist-lib/lib/core/first-event.js +13 -0
- package/dist-lib/lib/core/invite.d.ts +2 -2
- package/dist-lib/lib/core/invite.js +5 -4
- package/dist-lib/lib/core/local/index.d.ts +16 -1
- package/dist-lib/lib/core/local/index.js +119 -10
- package/dist-lib/lib/core/mesh.d.ts +14 -3
- package/dist-lib/lib/core/mesh.js +127 -54
- package/dist-lib/lib/core/migrations/chat-chunks.d.ts +4 -0
- package/dist-lib/lib/core/migrations/chat-chunks.js +47 -0
- package/dist-lib/lib/core/migrations/file-index.d.ts +1 -1
- package/dist-lib/lib/core/migrations/file-index.js +6 -2
- package/dist-lib/lib/core/migrations/index.d.ts +2 -1
- package/dist-lib/lib/core/migrations/index.js +66 -9
- package/dist-lib/lib/core/notification.js +96 -23
- package/dist-lib/lib/core/pdf/figure.d.ts +22 -0
- package/dist-lib/lib/core/pdf/figure.js +64 -0
- package/dist-lib/lib/core/pdf/rasterize.d.ts +10 -1
- package/dist-lib/lib/core/pdf/rasterize.js +72 -22
- package/dist-lib/lib/core/peer-duplex.js +57 -5
- package/dist-lib/lib/core/public-mesh.js +8 -2
- package/dist-lib/lib/core/run-requests.js +2 -3
- package/dist-lib/lib/core/senders.d.ts +7 -6
- package/dist-lib/lib/core/senders.js +23 -30
- package/dist-lib/lib/core/shared-agents.d.ts +1 -0
- package/dist-lib/lib/core/shared-agents.js +47 -19
- package/dist-lib/lib/core/skill-secrets.d.ts +3 -2
- package/dist-lib/lib/core/skill-secrets.js +4 -1
- package/dist-lib/lib/core/sync/background.d.ts +3 -15
- package/dist-lib/lib/core/sync/background.js +25 -102
- package/dist-lib/lib/core/sync/devices.js +6 -3
- package/dist-lib/lib/core/sync/index.js +25 -3
- package/dist-lib/lib/core/sync-status.js +3 -3
- package/dist-lib/lib/core/vault-managed-skills.d.ts +18 -0
- package/dist-lib/lib/core/vault-managed-skills.js +105 -0
- package/dist-lib/lib/core.d.ts +66 -10
- package/dist-lib/lib/core.js +768 -179
- package/dist-lib/lib/credential-pusher.d.ts +11 -7
- package/dist-lib/lib/credential-pusher.js +40 -8
- package/dist-lib/lib/device-tree.d.ts +18 -0
- package/dist-lib/lib/device-tree.js +32 -0
- package/dist-lib/lib/doctor/redact.js +1 -1
- package/dist-lib/lib/errors.d.ts +12 -0
- package/dist-lib/lib/errors.js +12 -0
- package/dist-lib/lib/fs-drive-bridge.d.ts +7 -0
- package/dist-lib/lib/fs-drive-bridge.js +308 -30
- package/dist-lib/lib/integrations/setup-routes.js +7 -5
- package/dist-lib/lib/knowledge-search.d.ts +1 -1
- package/dist-lib/lib/knowledge-search.js +5 -46
- package/dist-lib/lib/kv-cache-retention.d.ts +35 -0
- package/dist-lib/lib/kv-cache-retention.js +98 -0
- package/dist-lib/lib/latest-frame.d.ts +10 -0
- package/dist-lib/lib/latest-frame.js +48 -0
- package/dist-lib/lib/mesh-scoped-engine.js +1 -0
- package/dist-lib/lib/model-download.d.ts +6 -1
- package/dist-lib/lib/model-download.js +32 -17
- package/dist-lib/lib/oauth/resolve.js +13 -5
- package/dist-lib/lib/reconnecting-watch.js +19 -11
- package/dist-lib/lib/run-notices.d.ts +2 -0
- package/dist-lib/lib/run-notices.js +5 -0
- package/dist-lib/lib/run-replay.d.ts +13 -1
- package/dist-lib/lib/run-replay.js +58 -21
- package/dist-lib/schema/build.js +66 -57
- package/dist-lib/schema/config.d.ts +1 -1
- package/dist-lib/schema/config.js +1 -1
- package/dist-lib/schema/content.js +2 -1
- package/dist-lib/schema/engine.js +70 -6
- package/dist-lib/schema/field-order.js +7 -1
- package/dist-lib/schema/indexes.d.ts +4 -0
- package/dist-lib/schema/indexes.js +18 -0
- package/dist-lib/schema/local.js +9 -2
- package/dist-lib/schema/qvac.js +116 -6
- package/dist-lib/spec/bind.js +15 -0
- package/dist-lib/spec/calls.js +5 -0
- package/dist-lib/spec/capabilities.d.ts +10 -0
- package/dist-lib/spec/config/hyperconf/index.js +1 -1
- package/dist-lib/spec/hrpc/hrpc.json +65 -0
- package/dist-lib/spec/hrpc/index.js +63 -3
- package/dist-lib/spec/hrpc/messages.js +1537 -1395
- package/dist-lib/spec/hyperschema/index.js +1537 -1395
- package/dist-lib/spec/hyperschema/schema.json +126 -1
- package/dist-lib/spec/hyperschema/types.d.ts +33 -1
- package/dist-lib/spec/local/hyperdb/messages.js +110 -54
- package/dist-lib/spec/local/hyperschema/index.js +56 -28
- package/dist-lib/spec/local/hyperschema/schema.json +55 -20
- package/dist-lib/spec/local/hyperschema/types.d.ts +8 -1
- package/dist-lib/spec/mesh/hyperdb/db.json +255 -51
- package/dist-lib/spec/mesh/hyperdb/index.js +1063 -404
- package/dist-lib/spec/mesh/hyperdb/messages.js +1673 -1202
- package/dist-lib/spec/mesh/hyperdispatch/dispatch.json +62 -56
- package/dist-lib/spec/mesh/hyperdispatch/index.js +156 -139
- package/dist-lib/spec/mesh/hyperdispatch/messages.js +809 -632
- package/dist-lib/spec/mesh/hyperschema/index.js +809 -632
- package/dist-lib/spec/mesh/hyperschema/schema.json +293 -174
- package/dist-lib/spec/mesh/hyperschema/types.d.ts +31 -1
- package/package.json +32 -26
- package/worker/assistant-sidecar.mjs +3 -0
- package/dist-lib/lib/core/files/processing/settle.d.ts +0 -13
- package/dist-lib/lib/core/files/processing/settle.js +0 -86
- package/dist-lib/lib/core/migrations/device-indexes.d.ts +0 -2
- package/dist-lib/lib/core/migrations/device-indexes.js +0 -7
|
@@ -10,6 +10,64 @@ const MAX_DRIVE_BYTES = 8 * 1024 * 1024;
|
|
|
10
10
|
const STAGING_DIR = userTmpPath('qvac-fs-drive-host');
|
|
11
11
|
const STAGING_MAX_AGE_MS = 60 * 60 * 1000;
|
|
12
12
|
const noop = () => { };
|
|
13
|
+
function waitFor(promise, signal) {
|
|
14
|
+
if (signal.aborted) {
|
|
15
|
+
return Promise.reject(signal.reason ?? new Error('drive operation cancelled'));
|
|
16
|
+
}
|
|
17
|
+
return new Promise((resolve, reject) => {
|
|
18
|
+
let settled = false;
|
|
19
|
+
const finish = (result) => {
|
|
20
|
+
if (settled)
|
|
21
|
+
return;
|
|
22
|
+
settled = true;
|
|
23
|
+
signal.removeEventListener('abort', onAbort);
|
|
24
|
+
if (result.ok) {
|
|
25
|
+
resolve(result.value);
|
|
26
|
+
}
|
|
27
|
+
else {
|
|
28
|
+
reject(result.error);
|
|
29
|
+
}
|
|
30
|
+
};
|
|
31
|
+
const onAbort = () => finish({ ok: false, error: signal.reason ?? new Error('drive operation cancelled') });
|
|
32
|
+
signal.addEventListener('abort', onAbort, { once: true });
|
|
33
|
+
promise.then((value) => finish({ ok: true, value }), (error) => finish({ ok: false, error }));
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
async function* abortableIterable(source, signal) {
|
|
37
|
+
const iterator = source[Symbol.asyncIterator]();
|
|
38
|
+
let returning = null;
|
|
39
|
+
const returnIterator = () => {
|
|
40
|
+
if (!returning) {
|
|
41
|
+
returning = Promise.resolve(iterator.return?.());
|
|
42
|
+
}
|
|
43
|
+
return returning;
|
|
44
|
+
};
|
|
45
|
+
const cancel = () => {
|
|
46
|
+
if ('destroy' in source && typeof source.destroy === 'function') {
|
|
47
|
+
source.destroy();
|
|
48
|
+
}
|
|
49
|
+
void returnIterator().catch(noop);
|
|
50
|
+
};
|
|
51
|
+
signal.addEventListener('abort', cancel, { once: true });
|
|
52
|
+
try {
|
|
53
|
+
while (true) {
|
|
54
|
+
const next = await waitFor(iterator.next(), signal);
|
|
55
|
+
if (next.done) {
|
|
56
|
+
return;
|
|
57
|
+
}
|
|
58
|
+
yield next.value;
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
finally {
|
|
62
|
+
signal.removeEventListener('abort', cancel);
|
|
63
|
+
if (signal.aborted) {
|
|
64
|
+
cancel();
|
|
65
|
+
}
|
|
66
|
+
else {
|
|
67
|
+
await returnIterator().catch(noop);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
13
71
|
export async function sweepDriveStaging(now = Date.now()) {
|
|
14
72
|
const names = await fs.promises.readdir(STAGING_DIR).catch(() => []);
|
|
15
73
|
await Promise.all(names.map(async (name) => {
|
|
@@ -45,38 +103,141 @@ async function stagedPath(prefix) {
|
|
|
45
103
|
function stripMetadata(entry) {
|
|
46
104
|
return { ...entry, value: { ...entry.value, metadata: null } };
|
|
47
105
|
}
|
|
48
|
-
async function pipeCapped(source, target, { cap, signal }) {
|
|
106
|
+
export async function pipeCapped(source, target, { cap, signal }) {
|
|
107
|
+
const iterator = source[Symbol.asyncIterator]();
|
|
49
108
|
const sink = fs.createWriteStream(target, { mode: 0o600 });
|
|
109
|
+
const sinkClosed = new Promise((resolve) => sink.once('close', resolve));
|
|
50
110
|
let sinkError = null;
|
|
111
|
+
let rejectSink = null;
|
|
112
|
+
const sinkFailure = new Promise((_, reject) => {
|
|
113
|
+
rejectSink = reject;
|
|
114
|
+
});
|
|
115
|
+
void sinkFailure.catch(noop);
|
|
51
116
|
sink.on('error', (err) => {
|
|
52
117
|
sinkError = err;
|
|
118
|
+
rejectSink?.(err);
|
|
53
119
|
});
|
|
54
120
|
let written = 0;
|
|
55
121
|
let truncated = false;
|
|
122
|
+
let completed = false;
|
|
123
|
+
let returning = null;
|
|
124
|
+
const returnIterator = () => {
|
|
125
|
+
if (!returning) {
|
|
126
|
+
returning = Promise.resolve(iterator.return?.());
|
|
127
|
+
}
|
|
128
|
+
return returning;
|
|
129
|
+
};
|
|
130
|
+
const abortError = () => signal.reason instanceof Error ? signal.reason : new Error('drive read aborted');
|
|
131
|
+
const abort = () => {
|
|
132
|
+
if ('destroy' in source && typeof source.destroy === 'function') {
|
|
133
|
+
source.destroy(abortError());
|
|
134
|
+
}
|
|
135
|
+
void returnIterator().catch(noop);
|
|
136
|
+
sink.destroy(abortError());
|
|
137
|
+
};
|
|
138
|
+
const next = async () => {
|
|
139
|
+
if (signal.aborted) {
|
|
140
|
+
throw abortError();
|
|
141
|
+
}
|
|
142
|
+
let onAbort = null;
|
|
143
|
+
try {
|
|
144
|
+
return await Promise.race([
|
|
145
|
+
iterator.next(),
|
|
146
|
+
sinkFailure,
|
|
147
|
+
new Promise((_, reject) => {
|
|
148
|
+
onAbort = () => reject(abortError());
|
|
149
|
+
signal.addEventListener('abort', onAbort, { once: true });
|
|
150
|
+
})
|
|
151
|
+
]);
|
|
152
|
+
}
|
|
153
|
+
finally {
|
|
154
|
+
if (onAbort) {
|
|
155
|
+
signal.removeEventListener('abort', onAbort);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
};
|
|
159
|
+
signal.addEventListener('abort', abort, { once: true });
|
|
160
|
+
let failure = null;
|
|
56
161
|
try {
|
|
57
|
-
|
|
58
|
-
if (signal.aborted)
|
|
162
|
+
while (true) {
|
|
163
|
+
if (signal.aborted) {
|
|
164
|
+
throw abortError();
|
|
165
|
+
}
|
|
166
|
+
const item = await next();
|
|
167
|
+
if (signal.aborted) {
|
|
168
|
+
throw abortError();
|
|
169
|
+
}
|
|
170
|
+
if (item.done) {
|
|
59
171
|
break;
|
|
60
|
-
|
|
172
|
+
}
|
|
173
|
+
const chunk = item.value;
|
|
174
|
+
if (sinkError) {
|
|
61
175
|
throw sinkError;
|
|
176
|
+
}
|
|
62
177
|
if (written >= cap) {
|
|
63
178
|
truncated = true;
|
|
64
179
|
break;
|
|
65
180
|
}
|
|
66
181
|
const room = cap - written;
|
|
67
182
|
const slice = chunk.byteLength > room ? chunk.subarray(0, room) : chunk;
|
|
68
|
-
sink.write(slice)
|
|
183
|
+
if (!sink.write(slice)) {
|
|
184
|
+
await new Promise((resolve, reject) => {
|
|
185
|
+
const onDrain = () => {
|
|
186
|
+
sink.off('error', onError);
|
|
187
|
+
signal.removeEventListener('abort', onAbort);
|
|
188
|
+
resolve();
|
|
189
|
+
};
|
|
190
|
+
const onError = (err) => {
|
|
191
|
+
sink.off('drain', onDrain);
|
|
192
|
+
signal.removeEventListener('abort', onAbort);
|
|
193
|
+
reject(err);
|
|
194
|
+
};
|
|
195
|
+
const onAbort = () => {
|
|
196
|
+
sink.off('drain', onDrain);
|
|
197
|
+
sink.off('error', onError);
|
|
198
|
+
reject(abortError());
|
|
199
|
+
};
|
|
200
|
+
sink.once('drain', onDrain);
|
|
201
|
+
sink.once('error', onError);
|
|
202
|
+
signal.addEventListener('abort', onAbort, { once: true });
|
|
203
|
+
});
|
|
204
|
+
}
|
|
69
205
|
written += slice.byteLength;
|
|
70
206
|
if (slice.byteLength < chunk.byteLength) {
|
|
71
207
|
truncated = true;
|
|
208
|
+
break;
|
|
72
209
|
}
|
|
73
210
|
}
|
|
211
|
+
completed = true;
|
|
212
|
+
}
|
|
213
|
+
catch (err) {
|
|
214
|
+
failure = err;
|
|
215
|
+
if ('destroy' in source && typeof source.destroy === 'function') {
|
|
216
|
+
source.destroy(err instanceof Error ? err : new Error(String(err)));
|
|
217
|
+
}
|
|
74
218
|
}
|
|
75
219
|
finally {
|
|
76
|
-
|
|
220
|
+
signal.removeEventListener('abort', abort);
|
|
221
|
+
if (!completed || truncated) {
|
|
222
|
+
await returnIterator().catch((err) => (failure ??= err));
|
|
223
|
+
}
|
|
224
|
+
if (signal.aborted || sinkError || sink.destroyed) {
|
|
225
|
+
if (!sink.destroyed) {
|
|
226
|
+
sink.destroy();
|
|
227
|
+
}
|
|
228
|
+
await sinkClosed;
|
|
229
|
+
}
|
|
230
|
+
else {
|
|
231
|
+
sink.end();
|
|
232
|
+
await sinkClosed;
|
|
233
|
+
}
|
|
77
234
|
}
|
|
78
235
|
if (sinkError) {
|
|
79
|
-
|
|
236
|
+
failure ??= sinkError;
|
|
237
|
+
}
|
|
238
|
+
if (failure) {
|
|
239
|
+
await fs.promises.unlink(target).catch(noop);
|
|
240
|
+
throw failure;
|
|
80
241
|
}
|
|
81
242
|
return truncated;
|
|
82
243
|
}
|
|
@@ -84,42 +245,138 @@ function folderRange(folder) {
|
|
|
84
245
|
const trimmed = folder === '/' ? '' : folder.replace(/\/+$/, '');
|
|
85
246
|
return { gt: `${trimmed}/`, lt: `${trimmed}0` };
|
|
86
247
|
}
|
|
248
|
+
function childDirKey(folder, key) {
|
|
249
|
+
const prefix = folder === '/' ? '/' : `${folder.replace(/\/+$/, '')}/`;
|
|
250
|
+
const child = key.slice(prefix.length);
|
|
251
|
+
const slash = child.indexOf('/');
|
|
252
|
+
return slash === -1 ? null : `${prefix}${child.slice(0, slash)}`;
|
|
253
|
+
}
|
|
254
|
+
function directChild(folder, entry) {
|
|
255
|
+
const dir = childDirKey(folder, entry.key);
|
|
256
|
+
return dir ? { key: dir, dir: true } : stripMetadata(entry);
|
|
257
|
+
}
|
|
258
|
+
async function* childKeys(drive, options) {
|
|
259
|
+
const { folder } = options;
|
|
260
|
+
let range = options.start;
|
|
261
|
+
while (true) {
|
|
262
|
+
let past = null;
|
|
263
|
+
for await (const raw of abortableIterable(drive.entries(range, { ignore: options.ignore }), options.signal)) {
|
|
264
|
+
yield raw;
|
|
265
|
+
const dir = childDirKey(folder, raw.key);
|
|
266
|
+
if (dir) {
|
|
267
|
+
past = `${dir}0`;
|
|
268
|
+
break;
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
if (!past) {
|
|
272
|
+
return;
|
|
273
|
+
}
|
|
274
|
+
range = { gte: past, lt: options.range.lt };
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
async function shallowEntries(drive, request, options) {
|
|
278
|
+
const { signal, range } = options;
|
|
279
|
+
const entries = [];
|
|
280
|
+
let current = null;
|
|
281
|
+
let currentLastKey = null;
|
|
282
|
+
let lastEmittedKey = null;
|
|
283
|
+
let cursor = null;
|
|
284
|
+
const cursorDir = request.cursor ? childDirKey(request.folder, request.cursor) : null;
|
|
285
|
+
const start = cursorDir
|
|
286
|
+
? { gte: `${cursorDir}0`, lt: range.lt }
|
|
287
|
+
: { gt: request.cursor ?? range.gt, lt: range.lt };
|
|
288
|
+
for await (const raw of childKeys(drive, { ...options, folder: request.folder, start })) {
|
|
289
|
+
if (signal.aborted) {
|
|
290
|
+
throw new Error('drive list cancelled');
|
|
291
|
+
}
|
|
292
|
+
const child = directChild(request.folder, raw);
|
|
293
|
+
if (current && current.key !== child.key) {
|
|
294
|
+
if (entries.length >= request.limit) {
|
|
295
|
+
cursor = lastEmittedKey;
|
|
296
|
+
break;
|
|
297
|
+
}
|
|
298
|
+
entries.push(current);
|
|
299
|
+
lastEmittedKey = currentLastKey;
|
|
300
|
+
current = child;
|
|
301
|
+
}
|
|
302
|
+
current ??= child;
|
|
303
|
+
currentLastKey = raw.key;
|
|
304
|
+
}
|
|
305
|
+
if (!current || cursor) {
|
|
306
|
+
return { entries, cursor };
|
|
307
|
+
}
|
|
308
|
+
if (entries.length >= request.limit) {
|
|
309
|
+
return { entries, cursor: lastEmittedKey };
|
|
310
|
+
}
|
|
311
|
+
entries.push(current);
|
|
312
|
+
return { entries, cursor };
|
|
313
|
+
}
|
|
87
314
|
async function handleRequest(acquired, request, signal) {
|
|
88
315
|
const drive = acquired.drive;
|
|
89
316
|
switch (request.op) {
|
|
90
317
|
case 'entry': {
|
|
91
|
-
|
|
318
|
+
if (signal.aborted) {
|
|
319
|
+
throw new Error('drive entry cancelled');
|
|
320
|
+
}
|
|
321
|
+
const entry = await waitFor(drive.entry(request.key, { timeout: DRIVE_OP_TIMEOUT_MS, wait: true }), signal);
|
|
322
|
+
if (signal.aborted) {
|
|
323
|
+
throw new Error('drive entry cancelled');
|
|
324
|
+
}
|
|
92
325
|
return { entry: entry ? stripMetadata(entry) : null };
|
|
93
326
|
}
|
|
94
327
|
case 'get': {
|
|
95
|
-
const entry = await drive.entry(request.key, { timeout: DRIVE_OP_TIMEOUT_MS, wait: true });
|
|
328
|
+
const entry = await waitFor(drive.entry(request.key, { timeout: DRIVE_OP_TIMEOUT_MS, wait: true }), signal);
|
|
329
|
+
if (signal.aborted) {
|
|
330
|
+
throw new Error('drive get cancelled');
|
|
331
|
+
}
|
|
96
332
|
if (!entry) {
|
|
97
333
|
return { path: null };
|
|
98
334
|
}
|
|
99
335
|
if ((entry.value.blob?.byteLength ?? 0) > MAX_DRIVE_BYTES) {
|
|
100
336
|
return { error: `get exceeds ${MAX_DRIVE_BYTES} bytes: ${request.key}` };
|
|
101
337
|
}
|
|
102
|
-
const buffer = await drive.get(request.key, { timeout: DRIVE_OP_TIMEOUT_MS, wait: true });
|
|
338
|
+
const buffer = await waitFor(drive.get(request.key, { timeout: DRIVE_OP_TIMEOUT_MS, wait: true }), signal);
|
|
339
|
+
if (signal.aborted) {
|
|
340
|
+
throw new Error('drive get cancelled');
|
|
341
|
+
}
|
|
103
342
|
if (!buffer) {
|
|
104
343
|
return { path: null };
|
|
105
344
|
}
|
|
106
345
|
const target = await stagedPath('drive-get');
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
346
|
+
try {
|
|
347
|
+
if (signal.aborted) {
|
|
348
|
+
throw new Error('drive get cancelled');
|
|
349
|
+
}
|
|
350
|
+
await fs.promises.writeFile(target, buffer, { mode: 0o600 });
|
|
351
|
+
if (signal.aborted) {
|
|
352
|
+
throw new Error('drive get cancelled');
|
|
353
|
+
}
|
|
354
|
+
return { path: target };
|
|
355
|
+
}
|
|
356
|
+
catch (err) {
|
|
357
|
+
await fs.promises.unlink(target).catch(noop);
|
|
358
|
+
throw err;
|
|
359
|
+
}
|
|
112
360
|
}
|
|
113
361
|
case 'read': {
|
|
114
362
|
const target = await stagedPath('drive-read');
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
363
|
+
try {
|
|
364
|
+
if (signal.aborted) {
|
|
365
|
+
throw new Error('drive read cancelled');
|
|
366
|
+
}
|
|
367
|
+
const truncated = await pipeCapped(drive.createReadStream(request.key), target, {
|
|
368
|
+
cap: MAX_DRIVE_BYTES,
|
|
369
|
+
signal
|
|
370
|
+
});
|
|
371
|
+
if (signal.aborted) {
|
|
372
|
+
throw new Error('drive read cancelled');
|
|
373
|
+
}
|
|
374
|
+
return { path: target, truncated };
|
|
375
|
+
}
|
|
376
|
+
catch (err) {
|
|
377
|
+
await fs.promises.unlink(target).catch(noop);
|
|
378
|
+
throw err;
|
|
379
|
+
}
|
|
123
380
|
}
|
|
124
381
|
case 'list': {
|
|
125
382
|
const skip = new Set(request.skip);
|
|
@@ -128,7 +385,12 @@ async function handleRequest(acquired, request, signal) {
|
|
|
128
385
|
const ignore = (key) => key.split('/').some((seg) => skip.has(seg));
|
|
129
386
|
const range = folderRange(request.folder);
|
|
130
387
|
const gt = request.cursor ?? range.gt;
|
|
131
|
-
|
|
388
|
+
if (request.recursive === false) {
|
|
389
|
+
return shallowEntries(drive, request, { signal, ignore, range });
|
|
390
|
+
}
|
|
391
|
+
for await (const raw of abortableIterable(drive.entries({ gt, lt: range.lt }, { ignore }), signal)) {
|
|
392
|
+
if (signal.aborted)
|
|
393
|
+
throw new Error('drive list cancelled');
|
|
132
394
|
entries.push(stripMetadata(raw));
|
|
133
395
|
if (entries.length >= request.limit) {
|
|
134
396
|
cursor = raw.key;
|
|
@@ -141,18 +403,34 @@ async function handleRequest(acquired, request, signal) {
|
|
|
141
403
|
if (!acquired.writable) {
|
|
142
404
|
return { error: `drive is read-only: cannot write ${request.key}` };
|
|
143
405
|
}
|
|
406
|
+
const stagingRoot = await fs.promises
|
|
407
|
+
.realpath(HARNESS_STAGING_DIR)
|
|
408
|
+
.catch(() => path.resolve(HARNESS_STAGING_DIR));
|
|
144
409
|
const resolvedPath = path.resolve(request.path);
|
|
145
|
-
|
|
410
|
+
const realPath = await fs.promises.realpath(resolvedPath);
|
|
411
|
+
if (path.relative(stagingRoot, realPath).startsWith('..') || realPath === stagingRoot) {
|
|
146
412
|
return { error: `write path is outside the staging directory: ${request.path}` };
|
|
147
413
|
}
|
|
148
|
-
|
|
414
|
+
if (signal.aborted) {
|
|
415
|
+
throw new Error('drive write cancelled');
|
|
416
|
+
}
|
|
417
|
+
const source = Readable.from(fs.createReadStream(realPath));
|
|
149
418
|
const sink = drive.createWriteStream(request.key, request.executable ? { executable: true } : undefined);
|
|
150
|
-
|
|
419
|
+
const abort = () => {
|
|
151
420
|
source.destroy();
|
|
152
421
|
sink.destroy();
|
|
153
|
-
}
|
|
154
|
-
|
|
155
|
-
|
|
422
|
+
};
|
|
423
|
+
signal.addEventListener('abort', abort, { once: true });
|
|
424
|
+
try {
|
|
425
|
+
await pipelinePromise(source, sink);
|
|
426
|
+
if (signal.aborted) {
|
|
427
|
+
throw new Error('drive write cancelled');
|
|
428
|
+
}
|
|
429
|
+
await fs.promises.unlink(realPath).catch(noop);
|
|
430
|
+
}
|
|
431
|
+
finally {
|
|
432
|
+
signal.removeEventListener('abort', abort);
|
|
433
|
+
}
|
|
156
434
|
return {};
|
|
157
435
|
}
|
|
158
436
|
case 'search': {
|
|
@@ -47,10 +47,7 @@ function prioritizeVaultPicker(setup) {
|
|
|
47
47
|
function derivedRoutes(entry) {
|
|
48
48
|
const routes = [];
|
|
49
49
|
for (const key of credentialKeysFor(entry)) {
|
|
50
|
-
|
|
51
|
-
if (route) {
|
|
52
|
-
routes.push(route);
|
|
53
|
-
}
|
|
50
|
+
routes.push(credentialRoute(key));
|
|
54
51
|
}
|
|
55
52
|
const install = installRoute(entry);
|
|
56
53
|
if (install) {
|
|
@@ -79,7 +76,12 @@ function credentialRoute(credentialKey) {
|
|
|
79
76
|
fields: [{ key: credentialKey, label: token.label, secret: true, required: true }]
|
|
80
77
|
};
|
|
81
78
|
}
|
|
82
|
-
return
|
|
79
|
+
return {
|
|
80
|
+
kind: SETUP_ROUTE_KIND.token,
|
|
81
|
+
label: `Enter ${credentialKey}`,
|
|
82
|
+
credentialKey,
|
|
83
|
+
fields: [{ key: credentialKey, label: credentialKey, secret: true, required: true }]
|
|
84
|
+
};
|
|
83
85
|
}
|
|
84
86
|
function installRoute(entry) {
|
|
85
87
|
const steps = entry.metadata.install ?? [];
|
|
@@ -5,4 +5,4 @@ export interface SearchKnowledgeOptions {
|
|
|
5
5
|
readonly signal: AbortSignal;
|
|
6
6
|
readonly preparing: (active: boolean) => void;
|
|
7
7
|
}
|
|
8
|
-
export declare function searchKnowledge(engine: Pick<ScopedCapabilities, 'ragSearch'
|
|
8
|
+
export declare function searchKnowledge(engine: Pick<ScopedCapabilities, 'ragSearch'>, request: Omit<AssistantRagSearchRequest, 'meshId'>, options: SearchKnowledgeOptions): Promise<AssistantRagSearchResponse>;
|
|
@@ -1,56 +1,15 @@
|
|
|
1
1
|
import { raceAbort } from '@qvac/harness/race-abort';
|
|
2
2
|
export async function searchKnowledge(engine, request, options) {
|
|
3
3
|
const { signal, preparing } = options;
|
|
4
|
-
const search = await raceAbort(engine.ragSearch(request), signal);
|
|
5
|
-
|
|
6
|
-
if (!search.enabled ||
|
|
7
|
-
!search.hasKnowledge ||
|
|
8
|
-
!attachmentIds?.length ||
|
|
9
|
-
!search.pendingFiles?.length) {
|
|
4
|
+
const search = await raceAbort(engine.ragSearch({ ...request, immediate: true }), signal);
|
|
5
|
+
if (!search.enabled || !search.hasKnowledge || !search.chatFilesIndexingHere) {
|
|
10
6
|
return search;
|
|
11
7
|
}
|
|
12
|
-
|
|
13
|
-
const iterator = watch[Symbol.asyncIterator]();
|
|
14
|
-
let wanted = null;
|
|
15
|
-
let waiting = false;
|
|
8
|
+
preparing(true);
|
|
16
9
|
try {
|
|
17
|
-
|
|
18
|
-
const next = await raceAbort(iterator.next(), signal);
|
|
19
|
-
if (next.done) {
|
|
20
|
-
throw new Error('file status closed while preparing documents');
|
|
21
|
-
}
|
|
22
|
-
const files = next.value.files;
|
|
23
|
-
const ids = (wanted ??= attachedPending(attachmentIds, {
|
|
24
|
-
files,
|
|
25
|
-
pendingFiles: search.pendingFiles
|
|
26
|
-
}));
|
|
27
|
-
if (ids.size === 0) {
|
|
28
|
-
return search;
|
|
29
|
-
}
|
|
30
|
-
if (!files.some((file) => ids.has(file.fileId) && pending(file))) {
|
|
31
|
-
return await raceAbort(engine.ragSearch(request), signal);
|
|
32
|
-
}
|
|
33
|
-
if (!waiting) {
|
|
34
|
-
waiting = true;
|
|
35
|
-
preparing(true);
|
|
36
|
-
}
|
|
37
|
-
}
|
|
10
|
+
return await raceAbort(engine.ragSearch(request), signal);
|
|
38
11
|
}
|
|
39
12
|
finally {
|
|
40
|
-
|
|
41
|
-
if (waiting) {
|
|
42
|
-
preparing(false);
|
|
43
|
-
}
|
|
13
|
+
preparing(false);
|
|
44
14
|
}
|
|
45
15
|
}
|
|
46
|
-
function pending(file) {
|
|
47
|
-
const rag = file.processors.find((processor) => processor.processorId === 'rag');
|
|
48
|
-
return (!file.readyForChat &&
|
|
49
|
-
['pending', 'running', 'uploading'].includes(file.state) &&
|
|
50
|
-
rag?.state !== 'not-applicable');
|
|
51
|
-
}
|
|
52
|
-
function attachedPending(attachmentIds, { files, pendingFiles }) {
|
|
53
|
-
return new Set(files
|
|
54
|
-
.filter((file) => attachmentIds.includes(file.fileId) && pendingFiles.includes(file.fileName))
|
|
55
|
-
.map((file) => file.fileId));
|
|
56
|
-
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import type { LoggerInterface } from '@qvac/logging';
|
|
2
|
+
import type { Capabilities } from '../spec/capabilities.d.ts';
|
|
3
|
+
import type { AssistantChat } from '../spec/hyperschema/types.d.ts';
|
|
4
|
+
type RetentionEngine = Pick<Capabilities, 'configGet' | 'configSet' | 'listChats'>;
|
|
5
|
+
export interface KvCacheRetentionOptions {
|
|
6
|
+
readonly active: () => boolean;
|
|
7
|
+
readonly running: (chatId: string) => boolean;
|
|
8
|
+
readonly deleteChatKvCache: (chatId: string) => Promise<boolean>;
|
|
9
|
+
readonly defaultMs?: number | undefined;
|
|
10
|
+
readonly logger?: LoggerInterface | null;
|
|
11
|
+
readonly now?: () => number;
|
|
12
|
+
readonly tickMs?: number;
|
|
13
|
+
}
|
|
14
|
+
interface IdleWindow {
|
|
15
|
+
readonly from: number;
|
|
16
|
+
readonly cutoff: number;
|
|
17
|
+
}
|
|
18
|
+
export declare function kvCacheRetentionMs(value: unknown, defaultMs?: number): number | null;
|
|
19
|
+
export declare function kvCacheSweptKey(meshId: string): string;
|
|
20
|
+
export declare function idleChatIds(chats: ReadonlyArray<AssistantChat>, { from, cutoff }: IdleWindow): string[];
|
|
21
|
+
export declare class KvCacheRetention {
|
|
22
|
+
private readonly _engine;
|
|
23
|
+
private readonly _opts;
|
|
24
|
+
private readonly _now;
|
|
25
|
+
private readonly _queued;
|
|
26
|
+
private _queue;
|
|
27
|
+
private _timer;
|
|
28
|
+
constructor(engine: RetentionEngine, opts: KvCacheRetentionOptions);
|
|
29
|
+
sweep(meshId: string): Promise<void>;
|
|
30
|
+
start(meshIds: () => Iterable<string>): void;
|
|
31
|
+
stop(): void;
|
|
32
|
+
private _sweep;
|
|
33
|
+
private _deleteIdle;
|
|
34
|
+
}
|
|
35
|
+
export {};
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import { KV_CACHE_RETENTION_DAYS_DEFAULT, KV_CACHE_RETENTION_KEY } from './constants.js';
|
|
2
|
+
import { errorMessage } from './guards.js';
|
|
3
|
+
import { unref } from './unref.js';
|
|
4
|
+
const DAY_MS = 24 * 60 * 60_000;
|
|
5
|
+
const DEFAULT_MS = KV_CACHE_RETENTION_DAYS_DEFAULT * DAY_MS;
|
|
6
|
+
const SWEEP_TICK_MS = 12 * 60 * 60_000;
|
|
7
|
+
export function kvCacheRetentionMs(value, defaultMs = DEFAULT_MS) {
|
|
8
|
+
if (typeof value !== 'number' || !Number.isFinite(value)) {
|
|
9
|
+
return defaultMs > 0 ? defaultMs : null;
|
|
10
|
+
}
|
|
11
|
+
return value > 0 ? value * DAY_MS : null;
|
|
12
|
+
}
|
|
13
|
+
export function kvCacheSweptKey(meshId) {
|
|
14
|
+
return `kvCache.sweptThrough.${meshId}`;
|
|
15
|
+
}
|
|
16
|
+
export function idleChatIds(chats, { from, cutoff }) {
|
|
17
|
+
return chats
|
|
18
|
+
.filter((chat) => {
|
|
19
|
+
const at = chat.lastActivityAt || chat.createdAt;
|
|
20
|
+
return at >= from && at < cutoff;
|
|
21
|
+
})
|
|
22
|
+
.map((chat) => chat.id);
|
|
23
|
+
}
|
|
24
|
+
export class KvCacheRetention {
|
|
25
|
+
_engine;
|
|
26
|
+
_opts;
|
|
27
|
+
_now;
|
|
28
|
+
_queued;
|
|
29
|
+
_queue;
|
|
30
|
+
_timer;
|
|
31
|
+
constructor(engine, opts) {
|
|
32
|
+
this._engine = engine;
|
|
33
|
+
this._opts = opts;
|
|
34
|
+
this._now = opts.now ?? Date.now;
|
|
35
|
+
this._queued = new Set();
|
|
36
|
+
this._queue = Promise.resolve();
|
|
37
|
+
this._timer = null;
|
|
38
|
+
}
|
|
39
|
+
sweep(meshId) {
|
|
40
|
+
if (this._queued.has(meshId)) {
|
|
41
|
+
return this._queue;
|
|
42
|
+
}
|
|
43
|
+
this._queued.add(meshId);
|
|
44
|
+
this._queue = this._queue.then(() => {
|
|
45
|
+
this._queued.delete(meshId);
|
|
46
|
+
return this._sweep(meshId).catch((err) => this._opts.logger?.warn('kv-cache-retention', 'sweep failed', {
|
|
47
|
+
meshId,
|
|
48
|
+
error: errorMessage(err)
|
|
49
|
+
}));
|
|
50
|
+
});
|
|
51
|
+
return this._queue;
|
|
52
|
+
}
|
|
53
|
+
start(meshIds) {
|
|
54
|
+
this._timer = setInterval(() => {
|
|
55
|
+
for (const meshId of meshIds()) {
|
|
56
|
+
void this.sweep(meshId);
|
|
57
|
+
}
|
|
58
|
+
}, this._opts.tickMs ?? SWEEP_TICK_MS);
|
|
59
|
+
unref(this._timer);
|
|
60
|
+
}
|
|
61
|
+
stop() {
|
|
62
|
+
if (this._timer) {
|
|
63
|
+
clearInterval(this._timer);
|
|
64
|
+
}
|
|
65
|
+
this._timer = null;
|
|
66
|
+
}
|
|
67
|
+
async _sweep(meshId) {
|
|
68
|
+
const retention = await this._engine.configGet({ key: KV_CACHE_RETENTION_KEY });
|
|
69
|
+
const retentionMs = kvCacheRetentionMs(retention.value, this._opts.defaultMs);
|
|
70
|
+
if (retentionMs === null || !this._opts.active()) {
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
73
|
+
const markKey = kvCacheSweptKey(meshId);
|
|
74
|
+
const mark = await this._engine.configGet({ key: markKey });
|
|
75
|
+
const from = typeof mark.value === 'number' ? mark.value : 0;
|
|
76
|
+
const cutoff = this._now() - retentionMs;
|
|
77
|
+
if (cutoff <= from) {
|
|
78
|
+
return;
|
|
79
|
+
}
|
|
80
|
+
const { chats } = await this._engine.listChats({ meshId });
|
|
81
|
+
if (await this._deleteIdle(idleChatIds(chats ?? [], { from, cutoff }))) {
|
|
82
|
+
await this._engine.configSet({ key: markKey, value: cutoff });
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
async _deleteIdle(chatIds) {
|
|
86
|
+
let complete = true;
|
|
87
|
+
for (const chatId of chatIds) {
|
|
88
|
+
if (!this._opts.active())
|
|
89
|
+
return false;
|
|
90
|
+
if (this._opts.running(chatId))
|
|
91
|
+
continue;
|
|
92
|
+
if (!(await this._opts.deleteChatKvCache(chatId))) {
|
|
93
|
+
complete = false;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
return complete;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
type ReadCallback = (error: Error | null) => void;
|
|
2
|
+
export interface LatestFrame<Frame> {
|
|
3
|
+
offer(frame: Frame): void;
|
|
4
|
+
end(): void;
|
|
5
|
+
read(callback: ReadCallback): void;
|
|
6
|
+
clear(): void;
|
|
7
|
+
release(): void;
|
|
8
|
+
}
|
|
9
|
+
export declare function latestFrame<Frame>(push: (frame: Frame | null) => void): LatestFrame<Frame>;
|
|
10
|
+
export {};
|