@qvac/core 0.10.3 → 0.10.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/CHANGELOG.md +37 -1
  2. package/dist-lib/lib/assistant/history.js +18 -2
  3. package/dist-lib/lib/assistant/message.d.ts +17 -0
  4. package/dist-lib/lib/assistant/message.js +28 -0
  5. package/dist-lib/lib/assistant/video-budget.d.ts +13 -0
  6. package/dist-lib/lib/assistant/video-budget.js +29 -0
  7. package/dist-lib/lib/assistant.d.ts +22 -2
  8. package/dist-lib/lib/assistant.js +374 -15
  9. package/dist-lib/lib/core/chat-fork.js +1 -0
  10. package/dist-lib/lib/core/chat-search.js +1 -1
  11. package/dist-lib/lib/core/chunk.d.ts +2 -2
  12. package/dist-lib/lib/core/chunk.js +5 -1
  13. package/dist-lib/lib/core/document-attachment.js +3 -0
  14. package/dist-lib/lib/core/files/index.js +4 -4
  15. package/dist-lib/lib/core/files/records/chunks.js +21 -0
  16. package/dist-lib/lib/core/files/records/purge.js +4 -3
  17. package/dist-lib/lib/core/files/records/source.js +3 -5
  18. package/dist-lib/lib/core/files/upload.js +3 -2
  19. package/dist-lib/lib/core/video/attachment.d.ts +1 -0
  20. package/dist-lib/lib/core/video/attachment.js +7 -0
  21. package/dist-lib/lib/core/video/digest.d.ts +16 -0
  22. package/dist-lib/lib/core/video/digest.js +57 -0
  23. package/dist-lib/lib/core/video/frames.d.ts +32 -0
  24. package/dist-lib/lib/core/video/frames.js +224 -0
  25. package/dist-lib/lib/core.js +19 -1
  26. package/dist-lib/lib/credential-pusher.d.ts +5 -0
  27. package/dist-lib/lib/credential-pusher.js +42 -11
  28. package/dist-lib/lib/run-notices.d.ts +1 -0
  29. package/dist-lib/lib/run-notices.js +2 -1
  30. package/dist-lib/lib/task.js +1 -1
  31. package/dist-lib/schema/engine.js +4 -2
  32. package/dist-lib/schema/indexes.js +2 -2
  33. package/dist-lib/schema/qvac.js +4 -2
  34. package/dist-lib/spec/hrpc/messages.js +16 -6
  35. package/dist-lib/spec/hyperschema/index.js +16 -6
  36. package/dist-lib/spec/hyperschema/schema.json +10 -0
  37. package/dist-lib/spec/hyperschema/types.d.ts +2 -0
  38. package/dist-lib/spec/mesh/hyperdb/messages.js +26 -11
  39. package/dist-lib/spec/mesh/hyperdispatch/messages.js +18 -8
  40. package/dist-lib/spec/mesh/hyperschema/index.js +18 -8
  41. package/dist-lib/spec/mesh/hyperschema/schema.json +11 -1
  42. package/dist-lib/spec/mesh/hyperschema/types.d.ts +3 -1
  43. package/package.json +3 -1
  44. package/tui/app.mjs +34 -1
  45. package/tui/lib/render.mjs +5 -0
  46. package/tui/main.mjs +8 -0
@@ -1,3 +1,5 @@
1
+ import b4a from 'b4a';
2
+ import { listChunks, updateOp } from '../../chunk.js';
1
3
  import { hasLiveSource } from './routing.js';
2
4
  export async function syncAttachmentChunk(mesh, chunk) {
3
5
  if (chunk.type !== 'attachment')
@@ -6,6 +8,7 @@ export async function syncAttachmentChunk(mesh, chunk) {
6
8
  if (!attachment?.deletedAt || typeof attachment.blobId !== 'string') {
7
9
  return;
8
10
  }
11
+ await tombstoneDerived(mesh, chunk, attachment.deletedAt);
9
12
  if (await hasLiveSource(mesh, chunk.chatId, attachment.blobId)) {
10
13
  return;
11
14
  }
@@ -13,3 +16,21 @@ export async function syncAttachmentChunk(mesh, chunk) {
13
16
  await mesh.files.clear(attachment.blobId, chunk.id);
14
17
  await mesh.fileIndex?.markCleared(attachment.blobId);
15
18
  }
19
+ async function tombstoneDerived(mesh, source, at) {
20
+ const ours = mesh.ourDeviceId;
21
+ if (!ours || !source.attachment) {
22
+ return;
23
+ }
24
+ const batch = mesh.batch();
25
+ for (const chunk of await listChunks(mesh, { chatId: source.chatId, type: 'attachment' })) {
26
+ const { attachment } = chunk;
27
+ if (chunk.sourceId !== source.attachment.id || !attachment || attachment.deletedAt) {
28
+ continue;
29
+ }
30
+ if (b4a.equals(chunk.originDeviceId, ours)) {
31
+ const tombstone = { ...attachment, deletedAt: at };
32
+ batch.dispatch('update-chunk', updateOp(chunk, { id: chunk.id, attachment: tombstone }));
33
+ }
34
+ }
35
+ await batch.flush();
36
+ }
@@ -1,7 +1,8 @@
1
1
  import b4a from 'b4a';
2
2
  import { fileKeyOf } from '../local-index.js';
3
3
  import { referencedBlobs } from './sweep.js';
4
- import { CHUNKS_BY_CHAT, KNOWLEDGE_ARTIFACTS_BY_KNOWLEDGE, KNOWLEDGE_BY_CHAT, PURGED_CHATS } from '../../collections.js';
4
+ import { findChunksOfType } from '../../chunk.js';
5
+ import { KNOWLEDGE_ARTIFACTS_BY_KNOWLEDGE, KNOWLEDGE_BY_CHAT, PURGED_CHATS } from '../../collections.js';
5
6
  export async function reclaimPurgedFiles(mesh) {
6
7
  const index = mesh.fileIndex;
7
8
  if (!index) {
@@ -42,8 +43,8 @@ export async function chatReferences(mesh, chatId) {
42
43
  const inChat = { gte: { chatId }, lte: { chatId } };
43
44
  const snap = mesh.view.snapshot();
44
45
  try {
45
- for await (const chunk of snap.find(CHUNKS_BY_CHAT, inChat)) {
46
- if (chunk.type === 'attachment' && chunk.attachment) {
46
+ for (const chunk of await findChunksOfType(snap, { chatId, type: 'attachment' })) {
47
+ if (chunk.attachment) {
47
48
  references.push(chunk.attachment.blobId);
48
49
  }
49
50
  }
@@ -1,5 +1,6 @@
1
1
  import { mediaKindOrdinal } from '../../../../schema/media-kind.js';
2
- import { ATTACHMENTS_BY_MEDIA_KIND, CHATS, CHUNKS, CHUNKS_BY_CHAT, KNOWLEDGE } from '../../collections.js';
2
+ import { ATTACHMENTS_BY_MEDIA_KIND, CHATS, CHUNKS, KNOWLEDGE } from '../../collections.js';
3
+ import { findChunksOfType } from '../../chunk.js';
3
4
  export function attachmentRecord(chunk) {
4
5
  const file = chunk.attachment;
5
6
  if (chunk.type !== 'attachment' || !file) {
@@ -64,10 +65,7 @@ export async function scopedFile(view, { chatId, fileId }) {
64
65
  if (row) {
65
66
  return row.chatId === chatId && !row.deletedAt ? row : null;
66
67
  }
67
- for await (const chunk of view.find(CHUNKS_BY_CHAT, {
68
- gte: { chatId },
69
- lte: { chatId }
70
- })) {
68
+ for (const chunk of await findChunksOfType(view, { chatId, type: 'attachment' })) {
71
69
  if (chunk.attachment?.id === fileId) {
72
70
  return chunk.attachment.deletedAt ? null : attachmentRecord(chunk);
73
71
  }
@@ -23,7 +23,7 @@ export async function* uploadFile(mesh, frames, { kind, maxBytes, deviceId, inac
23
23
  if (first.done) {
24
24
  return;
25
25
  }
26
- const { chatId, runId, fileName, mimeType, raw } = first.value;
26
+ const { chatId, runId, fileName, mimeType, raw, sourceId } = first.value;
27
27
  const originDeviceId = kind === 'knowledge' ? deviceId : first.value.originDeviceId;
28
28
  if (typeof chatId !== 'string' ||
29
29
  typeof fileName !== 'string' ||
@@ -89,7 +89,8 @@ export async function* uploadFile(mesh, frames, { kind, maxBytes, deviceId, inac
89
89
  runId,
90
90
  type: 'attachment',
91
91
  attachment,
92
- originDeviceId
92
+ originDeviceId,
93
+ sourceId
93
94
  });
94
95
  chunkId = appended.id;
95
96
  }
@@ -0,0 +1 @@
1
+ export declare function isVideoAttachment(fileName: string, mimeType: string): boolean;
@@ -0,0 +1,7 @@
1
+ import { mediaKindOf } from '../../../schema/media-kind.js';
2
+ const VIDEO_CONTAINERS = ['.mp4', '.m4v', '.mov'];
3
+ export function isVideoAttachment(fileName, mimeType) {
4
+ const name = fileName.toLowerCase();
5
+ return (mediaKindOf(mimeType, fileName) === 'video' ||
6
+ VIDEO_CONTAINERS.some((extension) => name.endsWith(extension)));
7
+ }
@@ -0,0 +1,16 @@
1
+ export declare const VIDEO_DIGEST_MIME = "application/vnd.qvac.video-frames";
2
+ export declare const VIDEO_AUDIO: readonly ['none', 'silent', 'speech', 'unheard'];
3
+ export type VideoAudio = (typeof VIDEO_AUDIO)[number];
4
+ export interface VideoDigestHeader {
5
+ readonly duration: number;
6
+ readonly times: ReadonlyArray<number>;
7
+ readonly audio: VideoAudio;
8
+ readonly transcript?: string;
9
+ }
10
+ export interface VideoDigest extends VideoDigestHeader {
11
+ readonly frames: ReadonlyArray<Uint8Array>;
12
+ }
13
+ export declare function parseVideoDigestHeader(json: string): VideoDigestHeader;
14
+ export declare function videoDigestFileName(videoFileName: string): string;
15
+ export declare function packVideoDigest(digest: VideoDigest): Uint8Array;
16
+ export declare function unpackVideoDigest(bytes: Uint8Array): VideoDigest;
@@ -0,0 +1,57 @@
1
+ import b4a from 'b4a';
2
+ import * as z from 'zod';
3
+ export const VIDEO_DIGEST_MIME = 'application/vnd.qvac.video-frames';
4
+ export const VIDEO_AUDIO = ['none', 'silent', 'speech', 'unheard'];
5
+ const HEADER_BYTES = 4;
6
+ const VideoDigestHeaderSchema = z.object({
7
+ duration: z.number().nonnegative(),
8
+ times: z.array(z.number().nonnegative()),
9
+ audio: z.enum(VIDEO_AUDIO),
10
+ transcript: z.string().optional()
11
+ });
12
+ const HeaderSchema = VideoDigestHeaderSchema.extend({
13
+ version: z.literal(1),
14
+ sizes: z.array(z.number().int().positive())
15
+ });
16
+ export function parseVideoDigestHeader(json) {
17
+ return VideoDigestHeaderSchema.parse(JSON.parse(json));
18
+ }
19
+ export function videoDigestFileName(videoFileName) {
20
+ return `${videoFileName}.frames`;
21
+ }
22
+ export function packVideoDigest(digest) {
23
+ const header = b4a.from(JSON.stringify({
24
+ version: 1,
25
+ duration: digest.duration,
26
+ times: digest.times,
27
+ sizes: digest.frames.map((frame) => frame.byteLength),
28
+ audio: digest.audio,
29
+ transcript: digest.transcript
30
+ }));
31
+ const length = b4a.alloc(HEADER_BYTES);
32
+ b4a.writeUInt32LE(length, header.byteLength, 0);
33
+ return b4a.concat([length, header, ...digest.frames]);
34
+ }
35
+ export function unpackVideoDigest(bytes) {
36
+ const length = b4a.readUInt32LE(bytes, 0);
37
+ const header = HeaderSchema.parse(JSON.parse(b4a.toString(bytes.subarray(HEADER_BYTES, HEADER_BYTES + length))));
38
+ if (header.sizes.length !== header.times.length) {
39
+ throw new Error('video digest: a frame time without its frame');
40
+ }
41
+ let offset = HEADER_BYTES + length;
42
+ const frames = header.sizes.map((size) => {
43
+ const frame = bytes.subarray(offset, offset + size);
44
+ offset += size;
45
+ return frame;
46
+ });
47
+ if (offset !== bytes.byteLength) {
48
+ throw new Error('video digest: frames do not fill the blob');
49
+ }
50
+ return {
51
+ duration: header.duration,
52
+ times: header.times,
53
+ audio: header.audio,
54
+ transcript: header.transcript,
55
+ frames
56
+ };
57
+ }
@@ -0,0 +1,32 @@
1
+ export interface Orientation {
2
+ readonly rotate: 0 | 90 | 180 | 270;
3
+ readonly flipH: boolean;
4
+ readonly flipV: boolean;
5
+ }
6
+ export interface VideoProbe extends Orientation {
7
+ readonly hasVideo: boolean;
8
+ readonly hasAudio: boolean;
9
+ readonly duration: number;
10
+ readonly width: number;
11
+ readonly height: number;
12
+ readonly aspect: number;
13
+ }
14
+ export interface SampledFrame {
15
+ readonly time: number;
16
+ readonly width: number;
17
+ readonly height: number;
18
+ readonly jpeg: Uint8Array;
19
+ }
20
+ export interface SampleOptions {
21
+ readonly times: ReadonlyArray<number>;
22
+ readonly maxEdge: number;
23
+ readonly probe: VideoProbe;
24
+ }
25
+ export interface Size {
26
+ readonly width: number;
27
+ readonly height: number;
28
+ }
29
+ export declare function probeVideo(file: string): Promise<VideoProbe>;
30
+ export declare function frameTimes(duration: number, count: number): number[];
31
+ export declare function frameSize(probe: VideoProbe, maxEdge: number): Size;
32
+ export declare function sampleFrames(file: string, opts: SampleOptions): Promise<SampledFrame[]>;
@@ -0,0 +1,224 @@
1
+ import b4a from 'b4a';
2
+ import fs from 'fs';
3
+ const AV_TIME_BASE = 1_000_000;
4
+ const YIELD_EVERY_PACKETS = 64;
5
+ const NO_PTS = -1;
6
+ const UPRIGHT = { rotate: 0, flipH: false, flipV: false };
7
+ const CORRECTIONS = new Map([
8
+ ['0,-1,1,0', { rotate: 270, flipH: false, flipV: false }],
9
+ ['-1,0,0,-1', { rotate: 180, flipH: false, flipV: false }],
10
+ ['0,1,-1,0', { rotate: 90, flipH: false, flipV: false }],
11
+ ['-1,0,0,1', { rotate: 0, flipH: true, flipV: false }],
12
+ ['1,0,0,-1', { rotate: 0, flipH: false, flipV: true }],
13
+ ['0,1,1,0', { rotate: 270, flipH: true, flipV: false }],
14
+ ['0,-1,-1,0', { rotate: 270, flipH: false, flipV: true }]
15
+ ]);
16
+ let loaded = null;
17
+ function loadFfmpeg() {
18
+ loaded ??= import('bare-ffmpeg').then((mod) => mod.default);
19
+ return loaded;
20
+ }
21
+ function fileIO(ff, fd) {
22
+ const size = fs.fstatSync(fd).size;
23
+ const { SET, END, SIZE } = ff.constants.seek;
24
+ let offset = 0;
25
+ return new ff.IOContext(4096, {
26
+ onread: (buffer, requested) => {
27
+ const read = fs.readSync(fd, buffer, 0, requested, offset);
28
+ offset += read;
29
+ return read;
30
+ },
31
+ onseek: (to, whence) => {
32
+ if (whence === SIZE) {
33
+ return size;
34
+ }
35
+ if (whence === SET) {
36
+ offset = to;
37
+ }
38
+ else if (whence === END) {
39
+ offset = size + to;
40
+ }
41
+ else {
42
+ offset += to;
43
+ }
44
+ return offset;
45
+ }
46
+ });
47
+ }
48
+ function openInput(ff, file) {
49
+ const fd = fs.openSync(file, 'r');
50
+ try {
51
+ const input = new ff.InputFormatContext(fileIO(ff, fd));
52
+ return {
53
+ input,
54
+ close: () => {
55
+ input.destroy();
56
+ fs.closeSync(fd);
57
+ }
58
+ };
59
+ }
60
+ catch (err) {
61
+ fs.closeSync(fd);
62
+ throw err;
63
+ }
64
+ }
65
+ export async function probeVideo(file) {
66
+ const ff = await loadFfmpeg();
67
+ const { input, close } = openInput(ff, file);
68
+ try {
69
+ const hasAudio = Boolean(input.getBestStream(ff.constants.mediaTypes.AUDIO));
70
+ const video = input.getBestStream(ff.constants.mediaTypes.VIDEO);
71
+ const formatDuration = input.duration > 0 ? input.duration / AV_TIME_BASE : 0;
72
+ if (!video) {
73
+ return {
74
+ hasVideo: false,
75
+ hasAudio,
76
+ duration: formatDuration,
77
+ width: 0,
78
+ height: 0,
79
+ aspect: 1,
80
+ ...UPRIGHT
81
+ };
82
+ }
83
+ const { numerator, denominator } = video.timeBase;
84
+ const streamDuration = video.duration > 0 && denominator ? (video.duration * numerator) / denominator : 0;
85
+ return {
86
+ hasVideo: true,
87
+ hasAudio,
88
+ duration: streamDuration || formatDuration,
89
+ width: video.codecParameters.width,
90
+ height: video.codecParameters.height,
91
+ aspect: pixelAspect(video),
92
+ ...orientation(ff, video)
93
+ };
94
+ }
95
+ finally {
96
+ close();
97
+ }
98
+ }
99
+ function orientation(ff, stream) {
100
+ const matrix = stream.sideData.find((entry) => entry.type === ff.constants.packetSideDataType.DISPLAYMATRIX);
101
+ if (!matrix) {
102
+ return UPRIGHT;
103
+ }
104
+ const key = [0, 4, 12, 16]
105
+ .map((offset) => snap(b4a.readInt32LE(matrix.data, offset) / 65536))
106
+ .join(',');
107
+ return CORRECTIONS.get(key) ?? UPRIGHT;
108
+ }
109
+ function snap(value) {
110
+ return Math.abs(value - Math.round(value)) < 1e-3 ? Math.round(value) + 0 : value;
111
+ }
112
+ export function frameTimes(duration, count) {
113
+ const slice = duration / count;
114
+ return Array.from({ length: count }, (_, i) => (i + 0.5) * slice);
115
+ }
116
+ export function frameSize(probe, maxEdge) {
117
+ const size = fit(probe.width * probe.aspect, probe.height, maxEdge);
118
+ return probe.rotate % 180 ? { width: size.height, height: size.width } : size;
119
+ }
120
+ function fit(width, height, maxEdge) {
121
+ const scale = Math.min(1, maxEdge / Math.max(width, height));
122
+ const even = (n) => Math.max(2, Math.round((n * scale) / 2) * 2);
123
+ return { width: even(width), height: even(height) };
124
+ }
125
+ function toRaster(ff, frame, size) {
126
+ const { RGBA } = ff.constants.pixelFormats;
127
+ const scaler = new ff.Scaler(frame.format, frame.width, frame.height, RGBA, size.width, size.height);
128
+ const rgba = new ff.Frame();
129
+ try {
130
+ rgba.width = size.width;
131
+ rgba.height = size.height;
132
+ rgba.format = RGBA;
133
+ rgba.alloc();
134
+ scaler.scale(frame, rgba);
135
+ const image = new ff.Image(RGBA, size.width, size.height);
136
+ image.read(rgba);
137
+ return { width: size.width, height: size.height, data: image.data };
138
+ }
139
+ finally {
140
+ rgba.destroy();
141
+ scaler.destroy();
142
+ }
143
+ }
144
+ function* received(decoder, frame) {
145
+ while (decoder.receiveFrame(frame)) {
146
+ yield frame;
147
+ }
148
+ }
149
+ async function* decodedFrames(ff, input, stream) {
150
+ const decoder = stream.decoder();
151
+ const packet = new ff.Packet();
152
+ const frame = new ff.Frame();
153
+ try {
154
+ decoder.open();
155
+ let packets = 0;
156
+ while (input.readFrame(packet)) {
157
+ if (packet.streamIndex === stream.index && decoder.sendPacket(packet)) {
158
+ yield* received(decoder, frame);
159
+ }
160
+ packet.unref();
161
+ if (++packets % YIELD_EVERY_PACKETS === 0) {
162
+ await new Promise((resolve) => setTimeout(resolve, 0));
163
+ }
164
+ }
165
+ decoder.sendPacket(new ff.Packet());
166
+ yield* received(decoder, frame);
167
+ }
168
+ finally {
169
+ frame.destroy();
170
+ packet.destroy();
171
+ decoder.destroy();
172
+ }
173
+ }
174
+ function pixelAspect(stream) {
175
+ const sar = stream.codecParameters.sampleAspectRatio;
176
+ return sar && sar.numerator > 0 && sar.denominator > 0 ? sar.numerator / sar.denominator : 1;
177
+ }
178
+ async function pick(ff, input, opts) {
179
+ const stream = input.getBestStream(ff.constants.mediaTypes.VIDEO);
180
+ if (!stream) {
181
+ return [];
182
+ }
183
+ const targets = [...opts.times].sort((x, y) => x - y);
184
+ const tb = stream.timeBase.numerator / stream.timeBase.denominator;
185
+ const aspect = pixelAspect(stream);
186
+ const picked = [];
187
+ let origin = null;
188
+ for await (const frame of decodedFrames(ff, input, stream)) {
189
+ const pts = frame.pts === NO_PTS ? frame.packetDTS : frame.pts;
190
+ origin ??= pts;
191
+ const time = (pts - origin) * tb;
192
+ while (picked.length < targets.length && time >= targets[picked.length]) {
193
+ const size = fit(frame.width * aspect, frame.height, opts.maxEdge);
194
+ picked.push({ time, raster: toRaster(ff, frame, size) });
195
+ }
196
+ if (picked.length === targets.length) {
197
+ break;
198
+ }
199
+ }
200
+ return picked;
201
+ }
202
+ export async function sampleFrames(file, opts) {
203
+ const ff = await loadFfmpeg();
204
+ const media = await import('bare-media');
205
+ const { input, close } = openInput(ff, file);
206
+ let picked;
207
+ try {
208
+ picked = await pick(ff, input, opts);
209
+ }
210
+ finally {
211
+ close();
212
+ }
213
+ const { rotate, flipH, flipV } = opts.probe;
214
+ const orientate = media.image.orientate;
215
+ const frames = [];
216
+ for (const { time, raster } of picked) {
217
+ const upright = rotate || flipH || flipV
218
+ ? await orientate(raster, { transform: { rotate, flipH, flipV } })
219
+ : raster;
220
+ const jpeg = await media.image.encode(upright, { mimetype: 'image/jpeg' });
221
+ frames.push({ time, width: upright.width, height: upright.height, jpeg });
222
+ }
223
+ return frames;
224
+ }
@@ -3411,15 +3411,23 @@ export class Core extends ReadyResource {
3411
3411
  if (!mesh.writable)
3412
3412
  return;
3413
3413
  const removed = [];
3414
+ const live = [];
3414
3415
  for await (const row of mesh.view.find(DEVICES)) {
3415
3416
  const ours = mesh === account || ownedBy(row, mesh.ourMemberId);
3416
3417
  if (row.revoked && ours && !b4a.equals(row.id, self)) {
3417
3418
  removed.push(row.id);
3418
3419
  }
3420
+ else if (!row.revoked && ours) {
3421
+ live.push(row);
3422
+ }
3419
3423
  }
3424
+ const adopter = await earliestPaired(account, live);
3425
+ const adopts = adopter !== undefined && b4a.equals(adopter.id, self);
3420
3426
  for (const id of removed) {
3421
3427
  const device = await account.view.get(DEVICES, { id });
3422
- if (device?.revokedBy && b4a.equals(device.revokedBy, account.localWriterKey)) {
3428
+ const by = device?.revokedBy;
3429
+ const left = device?.revoked && device.writerKey && by && b4a.equals(by, device.writerKey);
3430
+ if ((by && b4a.equals(by, account.localWriterKey)) || (left && adopts)) {
3423
3431
  await rehomeAgents(mesh, id, self).catch((err) => this._log?.warn(['p2p', 'mesh'], "handing over a removed device's agents failed", err));
3424
3432
  }
3425
3433
  }
@@ -4741,6 +4749,16 @@ function unmarkedNotice(losers) {
4741
4749
  return `another device of yours opened a shared mesh at the same moment, so ${ids} is an ordinary mesh now`;
4742
4750
  }
4743
4751
  function noop() { }
4752
+ async function earliestPaired(account, rows) {
4753
+ const paired = [];
4754
+ for (const row of rows) {
4755
+ const device = await account.view.get(DEVICES, { id: row.id });
4756
+ if (device && !device.revoked) {
4757
+ paired.push(device);
4758
+ }
4759
+ }
4760
+ return paired.sort((a, b) => a.joinedAt - b.joinedAt || b4a.compare(a.id, b.id))[0];
4761
+ }
4744
4762
  function ownedBy(device, member) {
4745
4763
  const owner = device?.memberId ?? device?.claimedMemberId;
4746
4764
  return !!owner && !!member && b4a.equals(owner, member);
@@ -35,6 +35,11 @@ export interface CredentialEngine extends ConfigEngine {
35
35
  }): AsyncIterable<unknown> & {
36
36
  destroy(): void;
37
37
  };
38
+ agentsWatch(req: {
39
+ meshId: string;
40
+ }): AsyncIterable<unknown> & {
41
+ destroy(): void;
42
+ };
38
43
  account(req: {}): Promise<{
39
44
  meshId?: string | null;
40
45
  }>;
@@ -174,14 +174,22 @@ async function resolveSettings(engine, scopes, { catalog, logger }) {
174
174
  }
175
175
  return entries;
176
176
  }
177
+ async function accountScopes(engine) {
178
+ const { meshId } = await engine.account({});
179
+ const ids = meshId
180
+ ? ((await engine.listAgents({ meshId })).agents ?? []).map((agent) => agent.id)
181
+ : [];
182
+ return { meshId, scopes: [undefined, ...ids] };
183
+ }
177
184
  export function startCredentialPusher(harness, engine, logger) {
178
185
  const last = new Map();
179
186
  const watches = new Map();
187
+ const agentLists = new Map();
180
188
  let disposed = false;
181
189
  let timer = null;
182
190
  const pushLock = new ScopeLock({ debounce: true });
183
191
  const watch = (key) => {
184
- const stream = engine.configWatch({ key: scopedKey(undefined, key) });
192
+ const stream = engine.configWatch({ key });
185
193
  watches.set(key, stream);
186
194
  (async () => {
187
195
  for await (const _ of stream) {
@@ -197,10 +205,11 @@ export function startCredentialPusher(harness, engine, logger) {
197
205
  logger?.warn('credential-pusher', `credential watch ended for ${key}`, errorMessage(err));
198
206
  });
199
207
  };
200
- const syncWatches = (oauthKeys) => {
208
+ const syncWatches = (oauthKeys, scopes) => {
201
209
  if (disposed)
202
210
  return;
203
- const wanted = new Set([...oauthKeys, ...PLAIN_KEYS]);
211
+ const keys = new Set([...oauthKeys, ...PLAIN_KEYS]);
212
+ const wanted = new Set(scopes.flatMap((agentId) => [...keys].map((key) => scopedKey(agentId, key))));
204
213
  for (const [key, stream] of watches) {
205
214
  if (wanted.has(key))
206
215
  continue;
@@ -217,11 +226,34 @@ export function startCredentialPusher(harness, engine, logger) {
217
226
  }
218
227
  for (const mk of last.keys()) {
219
228
  const key = mk.includes('\0') ? mk.slice(mk.indexOf('\0') + 1) : mk;
220
- if (!wanted.has(key)) {
229
+ if (!keys.has(key)) {
221
230
  last.delete(mk);
222
231
  }
223
232
  }
224
233
  };
234
+ const syncAgentsWatch = (meshId) => {
235
+ if (disposed)
236
+ return;
237
+ for (const [id, stream] of agentLists) {
238
+ if (id === meshId)
239
+ continue;
240
+ try {
241
+ stream.destroy();
242
+ }
243
+ catch { }
244
+ agentLists.delete(id);
245
+ }
246
+ if (!meshId || agentLists.has(meshId)) {
247
+ return;
248
+ }
249
+ const stream = engine.agentsWatch({ meshId });
250
+ agentLists.set(meshId, stream);
251
+ (async () => {
252
+ for await (const _ of stream) {
253
+ schedule();
254
+ }
255
+ })().catch(noop);
256
+ };
225
257
  const armTimer = (delay) => {
226
258
  if (disposed)
227
259
  return;
@@ -253,12 +285,9 @@ export function startCredentialPusher(harness, engine, logger) {
253
285
  }
254
286
  const owners = credentialOwners(skills);
255
287
  const specs = oauthSpecs(skills);
256
- syncWatches(specs.map((spec) => spec.credentialKey));
257
- const { meshId } = await engine.account({});
258
- const ids = meshId
259
- ? ((await engine.listAgents({ meshId })).agents ?? []).map((agent) => agent.id)
260
- : [];
261
- const scopes = [undefined, ...ids];
288
+ const { meshId, scopes } = await accountScopes(engine);
289
+ syncWatches(specs.map((spec) => spec.credentialKey), scopes);
290
+ syncAgentsWatch(meshId);
262
291
  const resolution = { engine, last, owners, specs, logger };
263
292
  for (const agentId of scopes) {
264
293
  entries.push(...(await resolveScope(agentId, resolution)));
@@ -326,15 +355,17 @@ export function startCredentialPusher(harness, engine, logger) {
326
355
  clearTimeout(timer);
327
356
  }
328
357
  timer = null;
329
- for (const stream of watches.values()) {
358
+ for (const stream of [...watches.values(), ...agentLists.values()]) {
330
359
  try {
331
360
  stream.destroy();
332
361
  }
333
362
  catch { }
334
363
  }
335
364
  watches.clear();
365
+ agentLists.clear();
336
366
  pushLock.destroy();
337
367
  await pushLock.flush();
338
368
  };
339
369
  return Object.assign(stop, { sync });
340
370
  }
371
+ function noop() { }
@@ -13,6 +13,7 @@ export declare const RUN_FAILURE_REASONS: {
13
13
  readonly runtimeEndedTurn: 'the model runtime ended this turn';
14
14
  readonly capabilityUnavailable: 'this device cannot run a capability without a model';
15
15
  readonly stalled: 'the agent stopped responding';
16
+ readonly videoNeedsVision: 'this model cannot see video: switch to a model that reads images to ask about one';
16
17
  };
17
18
  export type AbortOutcome = {
18
19
  readonly kind: 'replay';
@@ -11,7 +11,8 @@ export const RUN_FAILURE_REASONS = {
11
11
  harnessDied: 'the model runtime died',
12
12
  runtimeEndedTurn: 'the model runtime ended this turn',
13
13
  capabilityUnavailable: 'this device cannot run a capability without a model',
14
- stalled: 'the agent stopped responding'
14
+ stalled: 'the agent stopped responding',
15
+ videoNeedsVision: 'this model cannot see video: switch to a model that reads images to ask about one'
15
16
  };
16
17
  export function knownAbortOutcome(reason) {
17
18
  if (reason === RUN_ABORT_REASONS.suspended ||
@@ -55,7 +55,7 @@ function collect(all, { chatId, runId, state }) {
55
55
  if (call) {
56
56
  toolCalls.push({ name: call.name, arguments: call.arguments });
57
57
  }
58
- if (chunk.attachment && !chunk.attachment.deletedAt) {
58
+ if (chunk.attachment && !chunk.attachment.deletedAt && !chunk.sourceId) {
59
59
  attachments.push(chunk.attachment);
60
60
  }
61
61
  if (chunk.sources) {
@@ -579,7 +579,8 @@ export function registerEngineTypes(ns) {
579
579
  { name: 'senderMemberId', type: 'fixed32' },
580
580
  { name: 'capability', type: 'string' },
581
581
  { name: 'capabilityArgs', type: 'json' },
582
- { name: 'enhance', type: '@assistant/enhance-mode' }
582
+ { name: 'enhance', type: '@assistant/enhance-mode' },
583
+ { name: 'sourceId', type: 'string' }
583
584
  ]);
584
585
  ns.register({
585
586
  name: 'chunk-list',
@@ -695,7 +696,8 @@ export function registerEngineTypes(ns) {
695
696
  { name: 'raw', type: 'bool' },
696
697
  { name: 'fileId', type: 'string' },
697
698
  { name: 'totalBytes', type: 'uint' },
698
- { name: 'meshId', type: 'string' }
699
+ { name: 'meshId', type: 'string' },
700
+ { name: 'sourceId', type: 'string' }
699
701
  ]
700
702
  });
701
703
  ns.register({
@@ -91,7 +91,7 @@ export function runRequestsMap(record) {
91
91
 
92
92
  // not chatId-prefixed, so one range lists every audio attachment in the mesh without filtering
93
93
  export function attachmentsByMediaKindMap(record) {
94
- if (record.type !== 'attachment') return []
94
+ if (record.type !== 'attachment' || record.sourceId) return []
95
95
  const attachment = record.attachment
96
96
  if (!attachment || attachment.deletedAt) {
97
97
  return []
@@ -112,7 +112,7 @@ function attachmentGenerated(record) {
112
112
 
113
113
  // origin in the key makes "only generated audio" a range query, not a post-filter
114
114
  export function attachmentsByMediaKindOriginMap(record) {
115
- if (record.type !== 'attachment') return []
115
+ if (record.type !== 'attachment' || record.sourceId) return []
116
116
  const attachment = record.attachment
117
117
  if (!attachment || attachment.deletedAt) {
118
118
  return []