@qvac/core 0.10.3 → 0.10.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +37 -1
- package/dist-lib/lib/assistant/history.js +18 -2
- package/dist-lib/lib/assistant/message.d.ts +17 -0
- package/dist-lib/lib/assistant/message.js +28 -0
- package/dist-lib/lib/assistant/video-budget.d.ts +13 -0
- package/dist-lib/lib/assistant/video-budget.js +29 -0
- package/dist-lib/lib/assistant.d.ts +22 -2
- package/dist-lib/lib/assistant.js +374 -15
- package/dist-lib/lib/core/chat-fork.js +1 -0
- package/dist-lib/lib/core/chat-search.js +1 -1
- package/dist-lib/lib/core/chunk.d.ts +2 -2
- package/dist-lib/lib/core/chunk.js +5 -1
- package/dist-lib/lib/core/document-attachment.js +3 -0
- package/dist-lib/lib/core/files/index.js +4 -4
- package/dist-lib/lib/core/files/records/chunks.js +21 -0
- package/dist-lib/lib/core/files/records/purge.js +4 -3
- package/dist-lib/lib/core/files/records/source.js +3 -5
- package/dist-lib/lib/core/files/upload.js +3 -2
- package/dist-lib/lib/core/video/attachment.d.ts +1 -0
- package/dist-lib/lib/core/video/attachment.js +7 -0
- package/dist-lib/lib/core/video/digest.d.ts +16 -0
- package/dist-lib/lib/core/video/digest.js +57 -0
- package/dist-lib/lib/core/video/frames.d.ts +32 -0
- package/dist-lib/lib/core/video/frames.js +224 -0
- package/dist-lib/lib/core.js +19 -1
- package/dist-lib/lib/credential-pusher.d.ts +5 -0
- package/dist-lib/lib/credential-pusher.js +42 -11
- package/dist-lib/lib/run-notices.d.ts +1 -0
- package/dist-lib/lib/run-notices.js +2 -1
- package/dist-lib/lib/task.js +1 -1
- package/dist-lib/schema/engine.js +4 -2
- package/dist-lib/schema/indexes.js +2 -2
- package/dist-lib/schema/qvac.js +4 -2
- package/dist-lib/spec/hrpc/messages.js +16 -6
- package/dist-lib/spec/hyperschema/index.js +16 -6
- package/dist-lib/spec/hyperschema/schema.json +10 -0
- package/dist-lib/spec/hyperschema/types.d.ts +2 -0
- package/dist-lib/spec/mesh/hyperdb/messages.js +26 -11
- package/dist-lib/spec/mesh/hyperdispatch/messages.js +18 -8
- package/dist-lib/spec/mesh/hyperschema/index.js +18 -8
- package/dist-lib/spec/mesh/hyperschema/schema.json +11 -1
- package/dist-lib/spec/mesh/hyperschema/types.d.ts +3 -1
- package/package.json +3 -1
- package/tui/app.mjs +34 -1
- package/tui/lib/render.mjs +5 -0
- package/tui/main.mjs +8 -0
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import b4a from 'b4a';
|
|
2
|
+
import { listChunks, updateOp } from '../../chunk.js';
|
|
1
3
|
import { hasLiveSource } from './routing.js';
|
|
2
4
|
export async function syncAttachmentChunk(mesh, chunk) {
|
|
3
5
|
if (chunk.type !== 'attachment')
|
|
@@ -6,6 +8,7 @@ export async function syncAttachmentChunk(mesh, chunk) {
|
|
|
6
8
|
if (!attachment?.deletedAt || typeof attachment.blobId !== 'string') {
|
|
7
9
|
return;
|
|
8
10
|
}
|
|
11
|
+
await tombstoneDerived(mesh, chunk, attachment.deletedAt);
|
|
9
12
|
if (await hasLiveSource(mesh, chunk.chatId, attachment.blobId)) {
|
|
10
13
|
return;
|
|
11
14
|
}
|
|
@@ -13,3 +16,21 @@ export async function syncAttachmentChunk(mesh, chunk) {
|
|
|
13
16
|
await mesh.files.clear(attachment.blobId, chunk.id);
|
|
14
17
|
await mesh.fileIndex?.markCleared(attachment.blobId);
|
|
15
18
|
}
|
|
19
|
+
async function tombstoneDerived(mesh, source, at) {
|
|
20
|
+
const ours = mesh.ourDeviceId;
|
|
21
|
+
if (!ours || !source.attachment) {
|
|
22
|
+
return;
|
|
23
|
+
}
|
|
24
|
+
const batch = mesh.batch();
|
|
25
|
+
for (const chunk of await listChunks(mesh, { chatId: source.chatId, type: 'attachment' })) {
|
|
26
|
+
const { attachment } = chunk;
|
|
27
|
+
if (chunk.sourceId !== source.attachment.id || !attachment || attachment.deletedAt) {
|
|
28
|
+
continue;
|
|
29
|
+
}
|
|
30
|
+
if (b4a.equals(chunk.originDeviceId, ours)) {
|
|
31
|
+
const tombstone = { ...attachment, deletedAt: at };
|
|
32
|
+
batch.dispatch('update-chunk', updateOp(chunk, { id: chunk.id, attachment: tombstone }));
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
await batch.flush();
|
|
36
|
+
}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import b4a from 'b4a';
|
|
2
2
|
import { fileKeyOf } from '../local-index.js';
|
|
3
3
|
import { referencedBlobs } from './sweep.js';
|
|
4
|
-
import {
|
|
4
|
+
import { findChunksOfType } from '../../chunk.js';
|
|
5
|
+
import { KNOWLEDGE_ARTIFACTS_BY_KNOWLEDGE, KNOWLEDGE_BY_CHAT, PURGED_CHATS } from '../../collections.js';
|
|
5
6
|
export async function reclaimPurgedFiles(mesh) {
|
|
6
7
|
const index = mesh.fileIndex;
|
|
7
8
|
if (!index) {
|
|
@@ -42,8 +43,8 @@ export async function chatReferences(mesh, chatId) {
|
|
|
42
43
|
const inChat = { gte: { chatId }, lte: { chatId } };
|
|
43
44
|
const snap = mesh.view.snapshot();
|
|
44
45
|
try {
|
|
45
|
-
for
|
|
46
|
-
if (chunk.
|
|
46
|
+
for (const chunk of await findChunksOfType(snap, { chatId, type: 'attachment' })) {
|
|
47
|
+
if (chunk.attachment) {
|
|
47
48
|
references.push(chunk.attachment.blobId);
|
|
48
49
|
}
|
|
49
50
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { mediaKindOrdinal } from '../../../../schema/media-kind.js';
|
|
2
|
-
import { ATTACHMENTS_BY_MEDIA_KIND, CHATS, CHUNKS,
|
|
2
|
+
import { ATTACHMENTS_BY_MEDIA_KIND, CHATS, CHUNKS, KNOWLEDGE } from '../../collections.js';
|
|
3
|
+
import { findChunksOfType } from '../../chunk.js';
|
|
3
4
|
export function attachmentRecord(chunk) {
|
|
4
5
|
const file = chunk.attachment;
|
|
5
6
|
if (chunk.type !== 'attachment' || !file) {
|
|
@@ -64,10 +65,7 @@ export async function scopedFile(view, { chatId, fileId }) {
|
|
|
64
65
|
if (row) {
|
|
65
66
|
return row.chatId === chatId && !row.deletedAt ? row : null;
|
|
66
67
|
}
|
|
67
|
-
for
|
|
68
|
-
gte: { chatId },
|
|
69
|
-
lte: { chatId }
|
|
70
|
-
})) {
|
|
68
|
+
for (const chunk of await findChunksOfType(view, { chatId, type: 'attachment' })) {
|
|
71
69
|
if (chunk.attachment?.id === fileId) {
|
|
72
70
|
return chunk.attachment.deletedAt ? null : attachmentRecord(chunk);
|
|
73
71
|
}
|
|
@@ -23,7 +23,7 @@ export async function* uploadFile(mesh, frames, { kind, maxBytes, deviceId, inac
|
|
|
23
23
|
if (first.done) {
|
|
24
24
|
return;
|
|
25
25
|
}
|
|
26
|
-
const { chatId, runId, fileName, mimeType, raw } = first.value;
|
|
26
|
+
const { chatId, runId, fileName, mimeType, raw, sourceId } = first.value;
|
|
27
27
|
const originDeviceId = kind === 'knowledge' ? deviceId : first.value.originDeviceId;
|
|
28
28
|
if (typeof chatId !== 'string' ||
|
|
29
29
|
typeof fileName !== 'string' ||
|
|
@@ -89,7 +89,8 @@ export async function* uploadFile(mesh, frames, { kind, maxBytes, deviceId, inac
|
|
|
89
89
|
runId,
|
|
90
90
|
type: 'attachment',
|
|
91
91
|
attachment,
|
|
92
|
-
originDeviceId
|
|
92
|
+
originDeviceId,
|
|
93
|
+
sourceId
|
|
93
94
|
});
|
|
94
95
|
chunkId = appended.id;
|
|
95
96
|
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function isVideoAttachment(fileName: string, mimeType: string): boolean;
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { mediaKindOf } from '../../../schema/media-kind.js';
|
|
2
|
+
const VIDEO_CONTAINERS = ['.mp4', '.m4v', '.mov'];
|
|
3
|
+
export function isVideoAttachment(fileName, mimeType) {
|
|
4
|
+
const name = fileName.toLowerCase();
|
|
5
|
+
return (mediaKindOf(mimeType, fileName) === 'video' ||
|
|
6
|
+
VIDEO_CONTAINERS.some((extension) => name.endsWith(extension)));
|
|
7
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
export declare const VIDEO_DIGEST_MIME = "application/vnd.qvac.video-frames";
|
|
2
|
+
export declare const VIDEO_AUDIO: readonly ['none', 'silent', 'speech', 'unheard'];
|
|
3
|
+
export type VideoAudio = (typeof VIDEO_AUDIO)[number];
|
|
4
|
+
export interface VideoDigestHeader {
|
|
5
|
+
readonly duration: number;
|
|
6
|
+
readonly times: ReadonlyArray<number>;
|
|
7
|
+
readonly audio: VideoAudio;
|
|
8
|
+
readonly transcript?: string;
|
|
9
|
+
}
|
|
10
|
+
export interface VideoDigest extends VideoDigestHeader {
|
|
11
|
+
readonly frames: ReadonlyArray<Uint8Array>;
|
|
12
|
+
}
|
|
13
|
+
export declare function parseVideoDigestHeader(json: string): VideoDigestHeader;
|
|
14
|
+
export declare function videoDigestFileName(videoFileName: string): string;
|
|
15
|
+
export declare function packVideoDigest(digest: VideoDigest): Uint8Array;
|
|
16
|
+
export declare function unpackVideoDigest(bytes: Uint8Array): VideoDigest;
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import b4a from 'b4a';
|
|
2
|
+
import * as z from 'zod';
|
|
3
|
+
export const VIDEO_DIGEST_MIME = 'application/vnd.qvac.video-frames';
|
|
4
|
+
export const VIDEO_AUDIO = ['none', 'silent', 'speech', 'unheard'];
|
|
5
|
+
const HEADER_BYTES = 4;
|
|
6
|
+
const VideoDigestHeaderSchema = z.object({
|
|
7
|
+
duration: z.number().nonnegative(),
|
|
8
|
+
times: z.array(z.number().nonnegative()),
|
|
9
|
+
audio: z.enum(VIDEO_AUDIO),
|
|
10
|
+
transcript: z.string().optional()
|
|
11
|
+
});
|
|
12
|
+
const HeaderSchema = VideoDigestHeaderSchema.extend({
|
|
13
|
+
version: z.literal(1),
|
|
14
|
+
sizes: z.array(z.number().int().positive())
|
|
15
|
+
});
|
|
16
|
+
export function parseVideoDigestHeader(json) {
|
|
17
|
+
return VideoDigestHeaderSchema.parse(JSON.parse(json));
|
|
18
|
+
}
|
|
19
|
+
export function videoDigestFileName(videoFileName) {
|
|
20
|
+
return `${videoFileName}.frames`;
|
|
21
|
+
}
|
|
22
|
+
export function packVideoDigest(digest) {
|
|
23
|
+
const header = b4a.from(JSON.stringify({
|
|
24
|
+
version: 1,
|
|
25
|
+
duration: digest.duration,
|
|
26
|
+
times: digest.times,
|
|
27
|
+
sizes: digest.frames.map((frame) => frame.byteLength),
|
|
28
|
+
audio: digest.audio,
|
|
29
|
+
transcript: digest.transcript
|
|
30
|
+
}));
|
|
31
|
+
const length = b4a.alloc(HEADER_BYTES);
|
|
32
|
+
b4a.writeUInt32LE(length, header.byteLength, 0);
|
|
33
|
+
return b4a.concat([length, header, ...digest.frames]);
|
|
34
|
+
}
|
|
35
|
+
export function unpackVideoDigest(bytes) {
|
|
36
|
+
const length = b4a.readUInt32LE(bytes, 0);
|
|
37
|
+
const header = HeaderSchema.parse(JSON.parse(b4a.toString(bytes.subarray(HEADER_BYTES, HEADER_BYTES + length))));
|
|
38
|
+
if (header.sizes.length !== header.times.length) {
|
|
39
|
+
throw new Error('video digest: a frame time without its frame');
|
|
40
|
+
}
|
|
41
|
+
let offset = HEADER_BYTES + length;
|
|
42
|
+
const frames = header.sizes.map((size) => {
|
|
43
|
+
const frame = bytes.subarray(offset, offset + size);
|
|
44
|
+
offset += size;
|
|
45
|
+
return frame;
|
|
46
|
+
});
|
|
47
|
+
if (offset !== bytes.byteLength) {
|
|
48
|
+
throw new Error('video digest: frames do not fill the blob');
|
|
49
|
+
}
|
|
50
|
+
return {
|
|
51
|
+
duration: header.duration,
|
|
52
|
+
times: header.times,
|
|
53
|
+
audio: header.audio,
|
|
54
|
+
transcript: header.transcript,
|
|
55
|
+
frames
|
|
56
|
+
};
|
|
57
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
export interface Orientation {
|
|
2
|
+
readonly rotate: 0 | 90 | 180 | 270;
|
|
3
|
+
readonly flipH: boolean;
|
|
4
|
+
readonly flipV: boolean;
|
|
5
|
+
}
|
|
6
|
+
export interface VideoProbe extends Orientation {
|
|
7
|
+
readonly hasVideo: boolean;
|
|
8
|
+
readonly hasAudio: boolean;
|
|
9
|
+
readonly duration: number;
|
|
10
|
+
readonly width: number;
|
|
11
|
+
readonly height: number;
|
|
12
|
+
readonly aspect: number;
|
|
13
|
+
}
|
|
14
|
+
export interface SampledFrame {
|
|
15
|
+
readonly time: number;
|
|
16
|
+
readonly width: number;
|
|
17
|
+
readonly height: number;
|
|
18
|
+
readonly jpeg: Uint8Array;
|
|
19
|
+
}
|
|
20
|
+
export interface SampleOptions {
|
|
21
|
+
readonly times: ReadonlyArray<number>;
|
|
22
|
+
readonly maxEdge: number;
|
|
23
|
+
readonly probe: VideoProbe;
|
|
24
|
+
}
|
|
25
|
+
export interface Size {
|
|
26
|
+
readonly width: number;
|
|
27
|
+
readonly height: number;
|
|
28
|
+
}
|
|
29
|
+
export declare function probeVideo(file: string): Promise<VideoProbe>;
|
|
30
|
+
export declare function frameTimes(duration: number, count: number): number[];
|
|
31
|
+
export declare function frameSize(probe: VideoProbe, maxEdge: number): Size;
|
|
32
|
+
export declare function sampleFrames(file: string, opts: SampleOptions): Promise<SampledFrame[]>;
|
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
import b4a from 'b4a';
|
|
2
|
+
import fs from 'fs';
|
|
3
|
+
const AV_TIME_BASE = 1_000_000;
|
|
4
|
+
const YIELD_EVERY_PACKETS = 64;
|
|
5
|
+
const NO_PTS = -1;
|
|
6
|
+
const UPRIGHT = { rotate: 0, flipH: false, flipV: false };
|
|
7
|
+
const CORRECTIONS = new Map([
|
|
8
|
+
['0,-1,1,0', { rotate: 270, flipH: false, flipV: false }],
|
|
9
|
+
['-1,0,0,-1', { rotate: 180, flipH: false, flipV: false }],
|
|
10
|
+
['0,1,-1,0', { rotate: 90, flipH: false, flipV: false }],
|
|
11
|
+
['-1,0,0,1', { rotate: 0, flipH: true, flipV: false }],
|
|
12
|
+
['1,0,0,-1', { rotate: 0, flipH: false, flipV: true }],
|
|
13
|
+
['0,1,1,0', { rotate: 270, flipH: true, flipV: false }],
|
|
14
|
+
['0,-1,-1,0', { rotate: 270, flipH: false, flipV: true }]
|
|
15
|
+
]);
|
|
16
|
+
let loaded = null;
|
|
17
|
+
function loadFfmpeg() {
|
|
18
|
+
loaded ??= import('bare-ffmpeg').then((mod) => mod.default);
|
|
19
|
+
return loaded;
|
|
20
|
+
}
|
|
21
|
+
function fileIO(ff, fd) {
|
|
22
|
+
const size = fs.fstatSync(fd).size;
|
|
23
|
+
const { SET, END, SIZE } = ff.constants.seek;
|
|
24
|
+
let offset = 0;
|
|
25
|
+
return new ff.IOContext(4096, {
|
|
26
|
+
onread: (buffer, requested) => {
|
|
27
|
+
const read = fs.readSync(fd, buffer, 0, requested, offset);
|
|
28
|
+
offset += read;
|
|
29
|
+
return read;
|
|
30
|
+
},
|
|
31
|
+
onseek: (to, whence) => {
|
|
32
|
+
if (whence === SIZE) {
|
|
33
|
+
return size;
|
|
34
|
+
}
|
|
35
|
+
if (whence === SET) {
|
|
36
|
+
offset = to;
|
|
37
|
+
}
|
|
38
|
+
else if (whence === END) {
|
|
39
|
+
offset = size + to;
|
|
40
|
+
}
|
|
41
|
+
else {
|
|
42
|
+
offset += to;
|
|
43
|
+
}
|
|
44
|
+
return offset;
|
|
45
|
+
}
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
function openInput(ff, file) {
|
|
49
|
+
const fd = fs.openSync(file, 'r');
|
|
50
|
+
try {
|
|
51
|
+
const input = new ff.InputFormatContext(fileIO(ff, fd));
|
|
52
|
+
return {
|
|
53
|
+
input,
|
|
54
|
+
close: () => {
|
|
55
|
+
input.destroy();
|
|
56
|
+
fs.closeSync(fd);
|
|
57
|
+
}
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
catch (err) {
|
|
61
|
+
fs.closeSync(fd);
|
|
62
|
+
throw err;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
export async function probeVideo(file) {
|
|
66
|
+
const ff = await loadFfmpeg();
|
|
67
|
+
const { input, close } = openInput(ff, file);
|
|
68
|
+
try {
|
|
69
|
+
const hasAudio = Boolean(input.getBestStream(ff.constants.mediaTypes.AUDIO));
|
|
70
|
+
const video = input.getBestStream(ff.constants.mediaTypes.VIDEO);
|
|
71
|
+
const formatDuration = input.duration > 0 ? input.duration / AV_TIME_BASE : 0;
|
|
72
|
+
if (!video) {
|
|
73
|
+
return {
|
|
74
|
+
hasVideo: false,
|
|
75
|
+
hasAudio,
|
|
76
|
+
duration: formatDuration,
|
|
77
|
+
width: 0,
|
|
78
|
+
height: 0,
|
|
79
|
+
aspect: 1,
|
|
80
|
+
...UPRIGHT
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
const { numerator, denominator } = video.timeBase;
|
|
84
|
+
const streamDuration = video.duration > 0 && denominator ? (video.duration * numerator) / denominator : 0;
|
|
85
|
+
return {
|
|
86
|
+
hasVideo: true,
|
|
87
|
+
hasAudio,
|
|
88
|
+
duration: streamDuration || formatDuration,
|
|
89
|
+
width: video.codecParameters.width,
|
|
90
|
+
height: video.codecParameters.height,
|
|
91
|
+
aspect: pixelAspect(video),
|
|
92
|
+
...orientation(ff, video)
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
finally {
|
|
96
|
+
close();
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
function orientation(ff, stream) {
|
|
100
|
+
const matrix = stream.sideData.find((entry) => entry.type === ff.constants.packetSideDataType.DISPLAYMATRIX);
|
|
101
|
+
if (!matrix) {
|
|
102
|
+
return UPRIGHT;
|
|
103
|
+
}
|
|
104
|
+
const key = [0, 4, 12, 16]
|
|
105
|
+
.map((offset) => snap(b4a.readInt32LE(matrix.data, offset) / 65536))
|
|
106
|
+
.join(',');
|
|
107
|
+
return CORRECTIONS.get(key) ?? UPRIGHT;
|
|
108
|
+
}
|
|
109
|
+
function snap(value) {
|
|
110
|
+
return Math.abs(value - Math.round(value)) < 1e-3 ? Math.round(value) + 0 : value;
|
|
111
|
+
}
|
|
112
|
+
export function frameTimes(duration, count) {
|
|
113
|
+
const slice = duration / count;
|
|
114
|
+
return Array.from({ length: count }, (_, i) => (i + 0.5) * slice);
|
|
115
|
+
}
|
|
116
|
+
export function frameSize(probe, maxEdge) {
|
|
117
|
+
const size = fit(probe.width * probe.aspect, probe.height, maxEdge);
|
|
118
|
+
return probe.rotate % 180 ? { width: size.height, height: size.width } : size;
|
|
119
|
+
}
|
|
120
|
+
function fit(width, height, maxEdge) {
|
|
121
|
+
const scale = Math.min(1, maxEdge / Math.max(width, height));
|
|
122
|
+
const even = (n) => Math.max(2, Math.round((n * scale) / 2) * 2);
|
|
123
|
+
return { width: even(width), height: even(height) };
|
|
124
|
+
}
|
|
125
|
+
function toRaster(ff, frame, size) {
|
|
126
|
+
const { RGBA } = ff.constants.pixelFormats;
|
|
127
|
+
const scaler = new ff.Scaler(frame.format, frame.width, frame.height, RGBA, size.width, size.height);
|
|
128
|
+
const rgba = new ff.Frame();
|
|
129
|
+
try {
|
|
130
|
+
rgba.width = size.width;
|
|
131
|
+
rgba.height = size.height;
|
|
132
|
+
rgba.format = RGBA;
|
|
133
|
+
rgba.alloc();
|
|
134
|
+
scaler.scale(frame, rgba);
|
|
135
|
+
const image = new ff.Image(RGBA, size.width, size.height);
|
|
136
|
+
image.read(rgba);
|
|
137
|
+
return { width: size.width, height: size.height, data: image.data };
|
|
138
|
+
}
|
|
139
|
+
finally {
|
|
140
|
+
rgba.destroy();
|
|
141
|
+
scaler.destroy();
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
function* received(decoder, frame) {
|
|
145
|
+
while (decoder.receiveFrame(frame)) {
|
|
146
|
+
yield frame;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
async function* decodedFrames(ff, input, stream) {
|
|
150
|
+
const decoder = stream.decoder();
|
|
151
|
+
const packet = new ff.Packet();
|
|
152
|
+
const frame = new ff.Frame();
|
|
153
|
+
try {
|
|
154
|
+
decoder.open();
|
|
155
|
+
let packets = 0;
|
|
156
|
+
while (input.readFrame(packet)) {
|
|
157
|
+
if (packet.streamIndex === stream.index && decoder.sendPacket(packet)) {
|
|
158
|
+
yield* received(decoder, frame);
|
|
159
|
+
}
|
|
160
|
+
packet.unref();
|
|
161
|
+
if (++packets % YIELD_EVERY_PACKETS === 0) {
|
|
162
|
+
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
decoder.sendPacket(new ff.Packet());
|
|
166
|
+
yield* received(decoder, frame);
|
|
167
|
+
}
|
|
168
|
+
finally {
|
|
169
|
+
frame.destroy();
|
|
170
|
+
packet.destroy();
|
|
171
|
+
decoder.destroy();
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
function pixelAspect(stream) {
|
|
175
|
+
const sar = stream.codecParameters.sampleAspectRatio;
|
|
176
|
+
return sar && sar.numerator > 0 && sar.denominator > 0 ? sar.numerator / sar.denominator : 1;
|
|
177
|
+
}
|
|
178
|
+
async function pick(ff, input, opts) {
|
|
179
|
+
const stream = input.getBestStream(ff.constants.mediaTypes.VIDEO);
|
|
180
|
+
if (!stream) {
|
|
181
|
+
return [];
|
|
182
|
+
}
|
|
183
|
+
const targets = [...opts.times].sort((x, y) => x - y);
|
|
184
|
+
const tb = stream.timeBase.numerator / stream.timeBase.denominator;
|
|
185
|
+
const aspect = pixelAspect(stream);
|
|
186
|
+
const picked = [];
|
|
187
|
+
let origin = null;
|
|
188
|
+
for await (const frame of decodedFrames(ff, input, stream)) {
|
|
189
|
+
const pts = frame.pts === NO_PTS ? frame.packetDTS : frame.pts;
|
|
190
|
+
origin ??= pts;
|
|
191
|
+
const time = (pts - origin) * tb;
|
|
192
|
+
while (picked.length < targets.length && time >= targets[picked.length]) {
|
|
193
|
+
const size = fit(frame.width * aspect, frame.height, opts.maxEdge);
|
|
194
|
+
picked.push({ time, raster: toRaster(ff, frame, size) });
|
|
195
|
+
}
|
|
196
|
+
if (picked.length === targets.length) {
|
|
197
|
+
break;
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
return picked;
|
|
201
|
+
}
|
|
202
|
+
export async function sampleFrames(file, opts) {
|
|
203
|
+
const ff = await loadFfmpeg();
|
|
204
|
+
const media = await import('bare-media');
|
|
205
|
+
const { input, close } = openInput(ff, file);
|
|
206
|
+
let picked;
|
|
207
|
+
try {
|
|
208
|
+
picked = await pick(ff, input, opts);
|
|
209
|
+
}
|
|
210
|
+
finally {
|
|
211
|
+
close();
|
|
212
|
+
}
|
|
213
|
+
const { rotate, flipH, flipV } = opts.probe;
|
|
214
|
+
const orientate = media.image.orientate;
|
|
215
|
+
const frames = [];
|
|
216
|
+
for (const { time, raster } of picked) {
|
|
217
|
+
const upright = rotate || flipH || flipV
|
|
218
|
+
? await orientate(raster, { transform: { rotate, flipH, flipV } })
|
|
219
|
+
: raster;
|
|
220
|
+
const jpeg = await media.image.encode(upright, { mimetype: 'image/jpeg' });
|
|
221
|
+
frames.push({ time, width: upright.width, height: upright.height, jpeg });
|
|
222
|
+
}
|
|
223
|
+
return frames;
|
|
224
|
+
}
|
package/dist-lib/lib/core.js
CHANGED
|
@@ -3411,15 +3411,23 @@ export class Core extends ReadyResource {
|
|
|
3411
3411
|
if (!mesh.writable)
|
|
3412
3412
|
return;
|
|
3413
3413
|
const removed = [];
|
|
3414
|
+
const live = [];
|
|
3414
3415
|
for await (const row of mesh.view.find(DEVICES)) {
|
|
3415
3416
|
const ours = mesh === account || ownedBy(row, mesh.ourMemberId);
|
|
3416
3417
|
if (row.revoked && ours && !b4a.equals(row.id, self)) {
|
|
3417
3418
|
removed.push(row.id);
|
|
3418
3419
|
}
|
|
3420
|
+
else if (!row.revoked && ours) {
|
|
3421
|
+
live.push(row);
|
|
3422
|
+
}
|
|
3419
3423
|
}
|
|
3424
|
+
const adopter = await earliestPaired(account, live);
|
|
3425
|
+
const adopts = adopter !== undefined && b4a.equals(adopter.id, self);
|
|
3420
3426
|
for (const id of removed) {
|
|
3421
3427
|
const device = await account.view.get(DEVICES, { id });
|
|
3422
|
-
|
|
3428
|
+
const by = device?.revokedBy;
|
|
3429
|
+
const left = device?.revoked && device.writerKey && by && b4a.equals(by, device.writerKey);
|
|
3430
|
+
if ((by && b4a.equals(by, account.localWriterKey)) || (left && adopts)) {
|
|
3423
3431
|
await rehomeAgents(mesh, id, self).catch((err) => this._log?.warn(['p2p', 'mesh'], "handing over a removed device's agents failed", err));
|
|
3424
3432
|
}
|
|
3425
3433
|
}
|
|
@@ -4741,6 +4749,16 @@ function unmarkedNotice(losers) {
|
|
|
4741
4749
|
return `another device of yours opened a shared mesh at the same moment, so ${ids} is an ordinary mesh now`;
|
|
4742
4750
|
}
|
|
4743
4751
|
function noop() { }
|
|
4752
|
+
async function earliestPaired(account, rows) {
|
|
4753
|
+
const paired = [];
|
|
4754
|
+
for (const row of rows) {
|
|
4755
|
+
const device = await account.view.get(DEVICES, { id: row.id });
|
|
4756
|
+
if (device && !device.revoked) {
|
|
4757
|
+
paired.push(device);
|
|
4758
|
+
}
|
|
4759
|
+
}
|
|
4760
|
+
return paired.sort((a, b) => a.joinedAt - b.joinedAt || b4a.compare(a.id, b.id))[0];
|
|
4761
|
+
}
|
|
4744
4762
|
function ownedBy(device, member) {
|
|
4745
4763
|
const owner = device?.memberId ?? device?.claimedMemberId;
|
|
4746
4764
|
return !!owner && !!member && b4a.equals(owner, member);
|
|
@@ -35,6 +35,11 @@ export interface CredentialEngine extends ConfigEngine {
|
|
|
35
35
|
}): AsyncIterable<unknown> & {
|
|
36
36
|
destroy(): void;
|
|
37
37
|
};
|
|
38
|
+
agentsWatch(req: {
|
|
39
|
+
meshId: string;
|
|
40
|
+
}): AsyncIterable<unknown> & {
|
|
41
|
+
destroy(): void;
|
|
42
|
+
};
|
|
38
43
|
account(req: {}): Promise<{
|
|
39
44
|
meshId?: string | null;
|
|
40
45
|
}>;
|
|
@@ -174,14 +174,22 @@ async function resolveSettings(engine, scopes, { catalog, logger }) {
|
|
|
174
174
|
}
|
|
175
175
|
return entries;
|
|
176
176
|
}
|
|
177
|
+
async function accountScopes(engine) {
|
|
178
|
+
const { meshId } = await engine.account({});
|
|
179
|
+
const ids = meshId
|
|
180
|
+
? ((await engine.listAgents({ meshId })).agents ?? []).map((agent) => agent.id)
|
|
181
|
+
: [];
|
|
182
|
+
return { meshId, scopes: [undefined, ...ids] };
|
|
183
|
+
}
|
|
177
184
|
export function startCredentialPusher(harness, engine, logger) {
|
|
178
185
|
const last = new Map();
|
|
179
186
|
const watches = new Map();
|
|
187
|
+
const agentLists = new Map();
|
|
180
188
|
let disposed = false;
|
|
181
189
|
let timer = null;
|
|
182
190
|
const pushLock = new ScopeLock({ debounce: true });
|
|
183
191
|
const watch = (key) => {
|
|
184
|
-
const stream = engine.configWatch({ key
|
|
192
|
+
const stream = engine.configWatch({ key });
|
|
185
193
|
watches.set(key, stream);
|
|
186
194
|
(async () => {
|
|
187
195
|
for await (const _ of stream) {
|
|
@@ -197,10 +205,11 @@ export function startCredentialPusher(harness, engine, logger) {
|
|
|
197
205
|
logger?.warn('credential-pusher', `credential watch ended for ${key}`, errorMessage(err));
|
|
198
206
|
});
|
|
199
207
|
};
|
|
200
|
-
const syncWatches = (oauthKeys) => {
|
|
208
|
+
const syncWatches = (oauthKeys, scopes) => {
|
|
201
209
|
if (disposed)
|
|
202
210
|
return;
|
|
203
|
-
const
|
|
211
|
+
const keys = new Set([...oauthKeys, ...PLAIN_KEYS]);
|
|
212
|
+
const wanted = new Set(scopes.flatMap((agentId) => [...keys].map((key) => scopedKey(agentId, key))));
|
|
204
213
|
for (const [key, stream] of watches) {
|
|
205
214
|
if (wanted.has(key))
|
|
206
215
|
continue;
|
|
@@ -217,11 +226,34 @@ export function startCredentialPusher(harness, engine, logger) {
|
|
|
217
226
|
}
|
|
218
227
|
for (const mk of last.keys()) {
|
|
219
228
|
const key = mk.includes('\0') ? mk.slice(mk.indexOf('\0') + 1) : mk;
|
|
220
|
-
if (!
|
|
229
|
+
if (!keys.has(key)) {
|
|
221
230
|
last.delete(mk);
|
|
222
231
|
}
|
|
223
232
|
}
|
|
224
233
|
};
|
|
234
|
+
const syncAgentsWatch = (meshId) => {
|
|
235
|
+
if (disposed)
|
|
236
|
+
return;
|
|
237
|
+
for (const [id, stream] of agentLists) {
|
|
238
|
+
if (id === meshId)
|
|
239
|
+
continue;
|
|
240
|
+
try {
|
|
241
|
+
stream.destroy();
|
|
242
|
+
}
|
|
243
|
+
catch { }
|
|
244
|
+
agentLists.delete(id);
|
|
245
|
+
}
|
|
246
|
+
if (!meshId || agentLists.has(meshId)) {
|
|
247
|
+
return;
|
|
248
|
+
}
|
|
249
|
+
const stream = engine.agentsWatch({ meshId });
|
|
250
|
+
agentLists.set(meshId, stream);
|
|
251
|
+
(async () => {
|
|
252
|
+
for await (const _ of stream) {
|
|
253
|
+
schedule();
|
|
254
|
+
}
|
|
255
|
+
})().catch(noop);
|
|
256
|
+
};
|
|
225
257
|
const armTimer = (delay) => {
|
|
226
258
|
if (disposed)
|
|
227
259
|
return;
|
|
@@ -253,12 +285,9 @@ export function startCredentialPusher(harness, engine, logger) {
|
|
|
253
285
|
}
|
|
254
286
|
const owners = credentialOwners(skills);
|
|
255
287
|
const specs = oauthSpecs(skills);
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
? ((await engine.listAgents({ meshId })).agents ?? []).map((agent) => agent.id)
|
|
260
|
-
: [];
|
|
261
|
-
const scopes = [undefined, ...ids];
|
|
288
|
+
const { meshId, scopes } = await accountScopes(engine);
|
|
289
|
+
syncWatches(specs.map((spec) => spec.credentialKey), scopes);
|
|
290
|
+
syncAgentsWatch(meshId);
|
|
262
291
|
const resolution = { engine, last, owners, specs, logger };
|
|
263
292
|
for (const agentId of scopes) {
|
|
264
293
|
entries.push(...(await resolveScope(agentId, resolution)));
|
|
@@ -326,15 +355,17 @@ export function startCredentialPusher(harness, engine, logger) {
|
|
|
326
355
|
clearTimeout(timer);
|
|
327
356
|
}
|
|
328
357
|
timer = null;
|
|
329
|
-
for (const stream of watches.values()) {
|
|
358
|
+
for (const stream of [...watches.values(), ...agentLists.values()]) {
|
|
330
359
|
try {
|
|
331
360
|
stream.destroy();
|
|
332
361
|
}
|
|
333
362
|
catch { }
|
|
334
363
|
}
|
|
335
364
|
watches.clear();
|
|
365
|
+
agentLists.clear();
|
|
336
366
|
pushLock.destroy();
|
|
337
367
|
await pushLock.flush();
|
|
338
368
|
};
|
|
339
369
|
return Object.assign(stop, { sync });
|
|
340
370
|
}
|
|
371
|
+
function noop() { }
|
|
@@ -13,6 +13,7 @@ export declare const RUN_FAILURE_REASONS: {
|
|
|
13
13
|
readonly runtimeEndedTurn: 'the model runtime ended this turn';
|
|
14
14
|
readonly capabilityUnavailable: 'this device cannot run a capability without a model';
|
|
15
15
|
readonly stalled: 'the agent stopped responding';
|
|
16
|
+
readonly videoNeedsVision: 'this model cannot see video: switch to a model that reads images to ask about one';
|
|
16
17
|
};
|
|
17
18
|
export type AbortOutcome = {
|
|
18
19
|
readonly kind: 'replay';
|
|
@@ -11,7 +11,8 @@ export const RUN_FAILURE_REASONS = {
|
|
|
11
11
|
harnessDied: 'the model runtime died',
|
|
12
12
|
runtimeEndedTurn: 'the model runtime ended this turn',
|
|
13
13
|
capabilityUnavailable: 'this device cannot run a capability without a model',
|
|
14
|
-
stalled: 'the agent stopped responding'
|
|
14
|
+
stalled: 'the agent stopped responding',
|
|
15
|
+
videoNeedsVision: 'this model cannot see video: switch to a model that reads images to ask about one'
|
|
15
16
|
};
|
|
16
17
|
export function knownAbortOutcome(reason) {
|
|
17
18
|
if (reason === RUN_ABORT_REASONS.suspended ||
|
package/dist-lib/lib/task.js
CHANGED
|
@@ -55,7 +55,7 @@ function collect(all, { chatId, runId, state }) {
|
|
|
55
55
|
if (call) {
|
|
56
56
|
toolCalls.push({ name: call.name, arguments: call.arguments });
|
|
57
57
|
}
|
|
58
|
-
if (chunk.attachment && !chunk.attachment.deletedAt) {
|
|
58
|
+
if (chunk.attachment && !chunk.attachment.deletedAt && !chunk.sourceId) {
|
|
59
59
|
attachments.push(chunk.attachment);
|
|
60
60
|
}
|
|
61
61
|
if (chunk.sources) {
|
|
@@ -579,7 +579,8 @@ export function registerEngineTypes(ns) {
|
|
|
579
579
|
{ name: 'senderMemberId', type: 'fixed32' },
|
|
580
580
|
{ name: 'capability', type: 'string' },
|
|
581
581
|
{ name: 'capabilityArgs', type: 'json' },
|
|
582
|
-
{ name: 'enhance', type: '@assistant/enhance-mode' }
|
|
582
|
+
{ name: 'enhance', type: '@assistant/enhance-mode' },
|
|
583
|
+
{ name: 'sourceId', type: 'string' }
|
|
583
584
|
]);
|
|
584
585
|
ns.register({
|
|
585
586
|
name: 'chunk-list',
|
|
@@ -695,7 +696,8 @@ export function registerEngineTypes(ns) {
|
|
|
695
696
|
{ name: 'raw', type: 'bool' },
|
|
696
697
|
{ name: 'fileId', type: 'string' },
|
|
697
698
|
{ name: 'totalBytes', type: 'uint' },
|
|
698
|
-
{ name: 'meshId', type: 'string' }
|
|
699
|
+
{ name: 'meshId', type: 'string' },
|
|
700
|
+
{ name: 'sourceId', type: 'string' }
|
|
699
701
|
]
|
|
700
702
|
});
|
|
701
703
|
ns.register({
|
|
@@ -91,7 +91,7 @@ export function runRequestsMap(record) {
|
|
|
91
91
|
|
|
92
92
|
// not chatId-prefixed, so one range lists every audio attachment in the mesh without filtering
|
|
93
93
|
export function attachmentsByMediaKindMap(record) {
|
|
94
|
-
if (record.type !== 'attachment') return []
|
|
94
|
+
if (record.type !== 'attachment' || record.sourceId) return []
|
|
95
95
|
const attachment = record.attachment
|
|
96
96
|
if (!attachment || attachment.deletedAt) {
|
|
97
97
|
return []
|
|
@@ -112,7 +112,7 @@ function attachmentGenerated(record) {
|
|
|
112
112
|
|
|
113
113
|
// origin in the key makes "only generated audio" a range query, not a post-filter
|
|
114
114
|
export function attachmentsByMediaKindOriginMap(record) {
|
|
115
|
-
if (record.type !== 'attachment') return []
|
|
115
|
+
if (record.type !== 'attachment' || record.sourceId) return []
|
|
116
116
|
const attachment = record.attachment
|
|
117
117
|
if (!attachment || attachment.deletedAt) {
|
|
118
118
|
return []
|