@qvac/core 0.10.2 → 0.10.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +66 -1
- package/dist-lib/lib/api/agent-folders.d.ts +22 -0
- package/dist-lib/lib/api/agent-folders.js +115 -0
- package/dist-lib/lib/api/agents.d.ts +3 -1
- package/dist-lib/lib/api/agents.js +4 -2
- package/dist-lib/lib/api/index.d.ts +1 -0
- package/dist-lib/lib/api/index.js +4 -0
- package/dist-lib/lib/assistant/history.js +18 -2
- package/dist-lib/lib/assistant/message.d.ts +17 -0
- package/dist-lib/lib/assistant/message.js +28 -0
- package/dist-lib/lib/assistant/video-budget.d.ts +13 -0
- package/dist-lib/lib/assistant/video-budget.js +29 -0
- package/dist-lib/lib/assistant-tree.js +2 -1
- package/dist-lib/lib/assistant.d.ts +22 -2
- package/dist-lib/lib/assistant.js +377 -15
- package/dist-lib/lib/core/chat-fork.js +1 -0
- package/dist-lib/lib/core/chat-search.js +1 -1
- package/dist-lib/lib/core/chat.d.ts +1 -0
- package/dist-lib/lib/core/chat.js +2 -1
- package/dist-lib/lib/core/chunk.d.ts +2 -2
- package/dist-lib/lib/core/chunk.js +5 -1
- package/dist-lib/lib/core/document-attachment.js +3 -0
- package/dist-lib/lib/core/files/index.js +4 -4
- package/dist-lib/lib/core/files/records/chunks.js +21 -0
- package/dist-lib/lib/core/files/records/purge.js +4 -3
- package/dist-lib/lib/core/files/records/source.js +3 -5
- package/dist-lib/lib/core/files/upload.js +3 -2
- package/dist-lib/lib/core/video/attachment.d.ts +1 -0
- package/dist-lib/lib/core/video/attachment.js +7 -0
- package/dist-lib/lib/core/video/digest.d.ts +16 -0
- package/dist-lib/lib/core/video/digest.js +57 -0
- package/dist-lib/lib/core/video/frames.d.ts +32 -0
- package/dist-lib/lib/core/video/frames.js +224 -0
- package/dist-lib/lib/core.js +25 -5
- package/dist-lib/lib/credential-pusher.d.ts +5 -0
- package/dist-lib/lib/credential-pusher.js +42 -11
- package/dist-lib/lib/errors.d.ts +20 -0
- package/dist-lib/lib/errors.js +20 -0
- package/dist-lib/lib/lent-harness.d.ts +2 -0
- package/dist-lib/lib/lent-harness.js +51 -0
- package/dist-lib/lib/mesh-scoped-engine.js +3 -0
- package/dist-lib/lib/run-notices.d.ts +1 -0
- package/dist-lib/lib/run-notices.js +2 -1
- package/dist-lib/lib/task.js +1 -1
- package/dist-lib/schema/engine.js +53 -2
- package/dist-lib/schema/indexes.js +2 -2
- package/dist-lib/schema/qvac.js +4 -2
- package/dist-lib/spec/bind.js +9 -0
- package/dist-lib/spec/calls.js +3 -0
- package/dist-lib/spec/capabilities.d.ts +6 -0
- package/dist-lib/spec/hrpc/hrpc.json +39 -0
- package/dist-lib/spec/hrpc/index.js +39 -3
- package/dist-lib/spec/hrpc/messages.js +177 -6
- package/dist-lib/spec/hyperschema/index.js +177 -6
- package/dist-lib/spec/hyperschema/schema.json +126 -0
- package/dist-lib/spec/hyperschema/types.d.ts +35 -0
- package/dist-lib/spec/mesh/hyperdb/messages.js +26 -11
- package/dist-lib/spec/mesh/hyperdispatch/messages.js +18 -8
- package/dist-lib/spec/mesh/hyperschema/index.js +18 -8
- package/dist-lib/spec/mesh/hyperschema/schema.json +11 -1
- package/dist-lib/spec/mesh/hyperschema/types.d.ts +3 -1
- package/package.json +4 -1
- package/tui/README.md +10 -0
- package/tui/app.mjs +63 -2
- package/tui/lib/folders-command.mjs +92 -0
- package/tui/lib/knowledge-import.mjs +3 -21
- package/tui/lib/local-path.mjs +21 -0
- package/tui/lib/render.mjs +5 -0
- package/tui/main.mjs +8 -0
|
@@ -4,7 +4,7 @@ import { attachmentFiles } from './attachments.js';
|
|
|
4
4
|
import { isKnowledgeRoutedAttachment, getKnowledge, listKnowledge, getKnowledgeArtifact, listKnowledgeArtifactsByKnowledge, addKnowledgeArtifact, hasLiveSource, syncAttachmentChunk, syncKnowledgeTombstone, syncArtifactArrival, sweepTombstonedBlobs, reclaimPurgedFiles, referencedBlobs, scopedFile, attachmentRecord, isProcessorOutput, isProcessorArtifactId } from './records/index.js';
|
|
5
5
|
import { knowledgeFiles } from './knowledge.js';
|
|
6
6
|
import { randomId } from '../id.js';
|
|
7
|
-
import { getChunk, listChunks } from '../chunk.js';
|
|
7
|
+
import { findChunksOfType, getChunk, listChunks } from '../chunk.js';
|
|
8
8
|
import { settleFiles, isClaimLive, listIndexingByChat, FileState, FileProcessing } from './processing/index.js';
|
|
9
9
|
import { DeadlineScheduler } from '../../deadline.js';
|
|
10
10
|
import { liveDeviceForWriter } from '../contract/activity.js';
|
|
@@ -14,7 +14,7 @@ import { RagProcessor } from './rag/index.js';
|
|
|
14
14
|
import { PreviewProcessor, PREVIEW_VERSION, previewId } from './preview/index.js';
|
|
15
15
|
import { CoreError } from '../../errors.js';
|
|
16
16
|
import { errorMessage } from '../../guards.js';
|
|
17
|
-
import { ACTIVE_KNOWLEDGE_BY_CHAT, AGENTS, CHATS,
|
|
17
|
+
import { ACTIVE_KNOWLEDGE_BY_CHAT, AGENTS, CHATS, INDEXING_STATUS_BY_CHAT, KNOWLEDGE, KNOWLEDGE_ARTIFACTS_BY_KNOWLEDGE } from '../collections.js';
|
|
18
18
|
export { fullTextBudgetChars, fullTextHits, retrieveRagKnowledge, searchRagChunks, referencesPendingFile } from './rag/index.js';
|
|
19
19
|
export { isKnowledgeRoutedAttachment } from './records/index.js';
|
|
20
20
|
export { assertRemovableAt } from './records/index.js';
|
|
@@ -457,13 +457,13 @@ export class Files extends EventEmitter {
|
|
|
457
457
|
}
|
|
458
458
|
const agent = await snap.get(AGENTS, { id: chat.agentId });
|
|
459
459
|
const knowledge = await snap.find(ACTIVE_KNOWLEDGE_BY_CHAT, range).toArray();
|
|
460
|
-
const chunks = await snap
|
|
460
|
+
const chunks = await findChunksOfType(snap, { chatId, type: 'attachment' });
|
|
461
461
|
const statuses = await snap.find(INDEXING_STATUS_BY_CHAT, range).toArray();
|
|
462
462
|
const sources = new Map(knowledge.map((row) => [row.id, row]));
|
|
463
463
|
const ragIds = new Set(sources.keys());
|
|
464
464
|
for (const chunk of chunks) {
|
|
465
465
|
const file = chunk.attachment;
|
|
466
|
-
if (!file || file.deletedAt || sources.has(file.id)) {
|
|
466
|
+
if (!file || file.deletedAt || chunk.sourceId || sources.has(file.id)) {
|
|
467
467
|
continue;
|
|
468
468
|
}
|
|
469
469
|
sources.set(file.id, { ...file, chatId, updatedAt: file.createdAt });
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import b4a from 'b4a';
|
|
2
|
+
import { listChunks, updateOp } from '../../chunk.js';
|
|
1
3
|
import { hasLiveSource } from './routing.js';
|
|
2
4
|
export async function syncAttachmentChunk(mesh, chunk) {
|
|
3
5
|
if (chunk.type !== 'attachment')
|
|
@@ -6,6 +8,7 @@ export async function syncAttachmentChunk(mesh, chunk) {
|
|
|
6
8
|
if (!attachment?.deletedAt || typeof attachment.blobId !== 'string') {
|
|
7
9
|
return;
|
|
8
10
|
}
|
|
11
|
+
await tombstoneDerived(mesh, chunk, attachment.deletedAt);
|
|
9
12
|
if (await hasLiveSource(mesh, chunk.chatId, attachment.blobId)) {
|
|
10
13
|
return;
|
|
11
14
|
}
|
|
@@ -13,3 +16,21 @@ export async function syncAttachmentChunk(mesh, chunk) {
|
|
|
13
16
|
await mesh.files.clear(attachment.blobId, chunk.id);
|
|
14
17
|
await mesh.fileIndex?.markCleared(attachment.blobId);
|
|
15
18
|
}
|
|
19
|
+
async function tombstoneDerived(mesh, source, at) {
|
|
20
|
+
const ours = mesh.ourDeviceId;
|
|
21
|
+
if (!ours || !source.attachment) {
|
|
22
|
+
return;
|
|
23
|
+
}
|
|
24
|
+
const batch = mesh.batch();
|
|
25
|
+
for (const chunk of await listChunks(mesh, { chatId: source.chatId, type: 'attachment' })) {
|
|
26
|
+
const { attachment } = chunk;
|
|
27
|
+
if (chunk.sourceId !== source.attachment.id || !attachment || attachment.deletedAt) {
|
|
28
|
+
continue;
|
|
29
|
+
}
|
|
30
|
+
if (b4a.equals(chunk.originDeviceId, ours)) {
|
|
31
|
+
const tombstone = { ...attachment, deletedAt: at };
|
|
32
|
+
batch.dispatch('update-chunk', updateOp(chunk, { id: chunk.id, attachment: tombstone }));
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
await batch.flush();
|
|
36
|
+
}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import b4a from 'b4a';
|
|
2
2
|
import { fileKeyOf } from '../local-index.js';
|
|
3
3
|
import { referencedBlobs } from './sweep.js';
|
|
4
|
-
import {
|
|
4
|
+
import { findChunksOfType } from '../../chunk.js';
|
|
5
|
+
import { KNOWLEDGE_ARTIFACTS_BY_KNOWLEDGE, KNOWLEDGE_BY_CHAT, PURGED_CHATS } from '../../collections.js';
|
|
5
6
|
export async function reclaimPurgedFiles(mesh) {
|
|
6
7
|
const index = mesh.fileIndex;
|
|
7
8
|
if (!index) {
|
|
@@ -42,8 +43,8 @@ export async function chatReferences(mesh, chatId) {
|
|
|
42
43
|
const inChat = { gte: { chatId }, lte: { chatId } };
|
|
43
44
|
const snap = mesh.view.snapshot();
|
|
44
45
|
try {
|
|
45
|
-
for
|
|
46
|
-
if (chunk.
|
|
46
|
+
for (const chunk of await findChunksOfType(snap, { chatId, type: 'attachment' })) {
|
|
47
|
+
if (chunk.attachment) {
|
|
47
48
|
references.push(chunk.attachment.blobId);
|
|
48
49
|
}
|
|
49
50
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { mediaKindOrdinal } from '../../../../schema/media-kind.js';
|
|
2
|
-
import { ATTACHMENTS_BY_MEDIA_KIND, CHATS, CHUNKS,
|
|
2
|
+
import { ATTACHMENTS_BY_MEDIA_KIND, CHATS, CHUNKS, KNOWLEDGE } from '../../collections.js';
|
|
3
|
+
import { findChunksOfType } from '../../chunk.js';
|
|
3
4
|
export function attachmentRecord(chunk) {
|
|
4
5
|
const file = chunk.attachment;
|
|
5
6
|
if (chunk.type !== 'attachment' || !file) {
|
|
@@ -64,10 +65,7 @@ export async function scopedFile(view, { chatId, fileId }) {
|
|
|
64
65
|
if (row) {
|
|
65
66
|
return row.chatId === chatId && !row.deletedAt ? row : null;
|
|
66
67
|
}
|
|
67
|
-
for
|
|
68
|
-
gte: { chatId },
|
|
69
|
-
lte: { chatId }
|
|
70
|
-
})) {
|
|
68
|
+
for (const chunk of await findChunksOfType(view, { chatId, type: 'attachment' })) {
|
|
71
69
|
if (chunk.attachment?.id === fileId) {
|
|
72
70
|
return chunk.attachment.deletedAt ? null : attachmentRecord(chunk);
|
|
73
71
|
}
|
|
@@ -23,7 +23,7 @@ export async function* uploadFile(mesh, frames, { kind, maxBytes, deviceId, inac
|
|
|
23
23
|
if (first.done) {
|
|
24
24
|
return;
|
|
25
25
|
}
|
|
26
|
-
const { chatId, runId, fileName, mimeType, raw } = first.value;
|
|
26
|
+
const { chatId, runId, fileName, mimeType, raw, sourceId } = first.value;
|
|
27
27
|
const originDeviceId = kind === 'knowledge' ? deviceId : first.value.originDeviceId;
|
|
28
28
|
if (typeof chatId !== 'string' ||
|
|
29
29
|
typeof fileName !== 'string' ||
|
|
@@ -89,7 +89,8 @@ export async function* uploadFile(mesh, frames, { kind, maxBytes, deviceId, inac
|
|
|
89
89
|
runId,
|
|
90
90
|
type: 'attachment',
|
|
91
91
|
attachment,
|
|
92
|
-
originDeviceId
|
|
92
|
+
originDeviceId,
|
|
93
|
+
sourceId
|
|
93
94
|
});
|
|
94
95
|
chunkId = appended.id;
|
|
95
96
|
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function isVideoAttachment(fileName: string, mimeType: string): boolean;
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { mediaKindOf } from '../../../schema/media-kind.js';
|
|
2
|
+
const VIDEO_CONTAINERS = ['.mp4', '.m4v', '.mov'];
|
|
3
|
+
export function isVideoAttachment(fileName, mimeType) {
|
|
4
|
+
const name = fileName.toLowerCase();
|
|
5
|
+
return (mediaKindOf(mimeType, fileName) === 'video' ||
|
|
6
|
+
VIDEO_CONTAINERS.some((extension) => name.endsWith(extension)));
|
|
7
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
export declare const VIDEO_DIGEST_MIME = "application/vnd.qvac.video-frames";
|
|
2
|
+
export declare const VIDEO_AUDIO: readonly ['none', 'silent', 'speech', 'unheard'];
|
|
3
|
+
export type VideoAudio = (typeof VIDEO_AUDIO)[number];
|
|
4
|
+
export interface VideoDigestHeader {
|
|
5
|
+
readonly duration: number;
|
|
6
|
+
readonly times: ReadonlyArray<number>;
|
|
7
|
+
readonly audio: VideoAudio;
|
|
8
|
+
readonly transcript?: string;
|
|
9
|
+
}
|
|
10
|
+
export interface VideoDigest extends VideoDigestHeader {
|
|
11
|
+
readonly frames: ReadonlyArray<Uint8Array>;
|
|
12
|
+
}
|
|
13
|
+
export declare function parseVideoDigestHeader(json: string): VideoDigestHeader;
|
|
14
|
+
export declare function videoDigestFileName(videoFileName: string): string;
|
|
15
|
+
export declare function packVideoDigest(digest: VideoDigest): Uint8Array;
|
|
16
|
+
export declare function unpackVideoDigest(bytes: Uint8Array): VideoDigest;
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import b4a from 'b4a';
|
|
2
|
+
import * as z from 'zod';
|
|
3
|
+
export const VIDEO_DIGEST_MIME = 'application/vnd.qvac.video-frames';
|
|
4
|
+
export const VIDEO_AUDIO = ['none', 'silent', 'speech', 'unheard'];
|
|
5
|
+
const HEADER_BYTES = 4;
|
|
6
|
+
const VideoDigestHeaderSchema = z.object({
|
|
7
|
+
duration: z.number().nonnegative(),
|
|
8
|
+
times: z.array(z.number().nonnegative()),
|
|
9
|
+
audio: z.enum(VIDEO_AUDIO),
|
|
10
|
+
transcript: z.string().optional()
|
|
11
|
+
});
|
|
12
|
+
const HeaderSchema = VideoDigestHeaderSchema.extend({
|
|
13
|
+
version: z.literal(1),
|
|
14
|
+
sizes: z.array(z.number().int().positive())
|
|
15
|
+
});
|
|
16
|
+
export function parseVideoDigestHeader(json) {
|
|
17
|
+
return VideoDigestHeaderSchema.parse(JSON.parse(json));
|
|
18
|
+
}
|
|
19
|
+
export function videoDigestFileName(videoFileName) {
|
|
20
|
+
return `${videoFileName}.frames`;
|
|
21
|
+
}
|
|
22
|
+
export function packVideoDigest(digest) {
|
|
23
|
+
const header = b4a.from(JSON.stringify({
|
|
24
|
+
version: 1,
|
|
25
|
+
duration: digest.duration,
|
|
26
|
+
times: digest.times,
|
|
27
|
+
sizes: digest.frames.map((frame) => frame.byteLength),
|
|
28
|
+
audio: digest.audio,
|
|
29
|
+
transcript: digest.transcript
|
|
30
|
+
}));
|
|
31
|
+
const length = b4a.alloc(HEADER_BYTES);
|
|
32
|
+
b4a.writeUInt32LE(length, header.byteLength, 0);
|
|
33
|
+
return b4a.concat([length, header, ...digest.frames]);
|
|
34
|
+
}
|
|
35
|
+
export function unpackVideoDigest(bytes) {
|
|
36
|
+
const length = b4a.readUInt32LE(bytes, 0);
|
|
37
|
+
const header = HeaderSchema.parse(JSON.parse(b4a.toString(bytes.subarray(HEADER_BYTES, HEADER_BYTES + length))));
|
|
38
|
+
if (header.sizes.length !== header.times.length) {
|
|
39
|
+
throw new Error('video digest: a frame time without its frame');
|
|
40
|
+
}
|
|
41
|
+
let offset = HEADER_BYTES + length;
|
|
42
|
+
const frames = header.sizes.map((size) => {
|
|
43
|
+
const frame = bytes.subarray(offset, offset + size);
|
|
44
|
+
offset += size;
|
|
45
|
+
return frame;
|
|
46
|
+
});
|
|
47
|
+
if (offset !== bytes.byteLength) {
|
|
48
|
+
throw new Error('video digest: frames do not fill the blob');
|
|
49
|
+
}
|
|
50
|
+
return {
|
|
51
|
+
duration: header.duration,
|
|
52
|
+
times: header.times,
|
|
53
|
+
audio: header.audio,
|
|
54
|
+
transcript: header.transcript,
|
|
55
|
+
frames
|
|
56
|
+
};
|
|
57
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
export interface Orientation {
|
|
2
|
+
readonly rotate: 0 | 90 | 180 | 270;
|
|
3
|
+
readonly flipH: boolean;
|
|
4
|
+
readonly flipV: boolean;
|
|
5
|
+
}
|
|
6
|
+
export interface VideoProbe extends Orientation {
|
|
7
|
+
readonly hasVideo: boolean;
|
|
8
|
+
readonly hasAudio: boolean;
|
|
9
|
+
readonly duration: number;
|
|
10
|
+
readonly width: number;
|
|
11
|
+
readonly height: number;
|
|
12
|
+
readonly aspect: number;
|
|
13
|
+
}
|
|
14
|
+
export interface SampledFrame {
|
|
15
|
+
readonly time: number;
|
|
16
|
+
readonly width: number;
|
|
17
|
+
readonly height: number;
|
|
18
|
+
readonly jpeg: Uint8Array;
|
|
19
|
+
}
|
|
20
|
+
export interface SampleOptions {
|
|
21
|
+
readonly times: ReadonlyArray<number>;
|
|
22
|
+
readonly maxEdge: number;
|
|
23
|
+
readonly probe: VideoProbe;
|
|
24
|
+
}
|
|
25
|
+
export interface Size {
|
|
26
|
+
readonly width: number;
|
|
27
|
+
readonly height: number;
|
|
28
|
+
}
|
|
29
|
+
export declare function probeVideo(file: string): Promise<VideoProbe>;
|
|
30
|
+
export declare function frameTimes(duration: number, count: number): number[];
|
|
31
|
+
export declare function frameSize(probe: VideoProbe, maxEdge: number): Size;
|
|
32
|
+
export declare function sampleFrames(file: string, opts: SampleOptions): Promise<SampledFrame[]>;
|
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
import b4a from 'b4a';
|
|
2
|
+
import fs from 'fs';
|
|
3
|
+
const AV_TIME_BASE = 1_000_000;
|
|
4
|
+
const YIELD_EVERY_PACKETS = 64;
|
|
5
|
+
const NO_PTS = -1;
|
|
6
|
+
const UPRIGHT = { rotate: 0, flipH: false, flipV: false };
|
|
7
|
+
const CORRECTIONS = new Map([
|
|
8
|
+
['0,-1,1,0', { rotate: 270, flipH: false, flipV: false }],
|
|
9
|
+
['-1,0,0,-1', { rotate: 180, flipH: false, flipV: false }],
|
|
10
|
+
['0,1,-1,0', { rotate: 90, flipH: false, flipV: false }],
|
|
11
|
+
['-1,0,0,1', { rotate: 0, flipH: true, flipV: false }],
|
|
12
|
+
['1,0,0,-1', { rotate: 0, flipH: false, flipV: true }],
|
|
13
|
+
['0,1,1,0', { rotate: 270, flipH: true, flipV: false }],
|
|
14
|
+
['0,-1,-1,0', { rotate: 270, flipH: false, flipV: true }]
|
|
15
|
+
]);
|
|
16
|
+
let loaded = null;
|
|
17
|
+
function loadFfmpeg() {
|
|
18
|
+
loaded ??= import('bare-ffmpeg').then((mod) => mod.default);
|
|
19
|
+
return loaded;
|
|
20
|
+
}
|
|
21
|
+
function fileIO(ff, fd) {
|
|
22
|
+
const size = fs.fstatSync(fd).size;
|
|
23
|
+
const { SET, END, SIZE } = ff.constants.seek;
|
|
24
|
+
let offset = 0;
|
|
25
|
+
return new ff.IOContext(4096, {
|
|
26
|
+
onread: (buffer, requested) => {
|
|
27
|
+
const read = fs.readSync(fd, buffer, 0, requested, offset);
|
|
28
|
+
offset += read;
|
|
29
|
+
return read;
|
|
30
|
+
},
|
|
31
|
+
onseek: (to, whence) => {
|
|
32
|
+
if (whence === SIZE) {
|
|
33
|
+
return size;
|
|
34
|
+
}
|
|
35
|
+
if (whence === SET) {
|
|
36
|
+
offset = to;
|
|
37
|
+
}
|
|
38
|
+
else if (whence === END) {
|
|
39
|
+
offset = size + to;
|
|
40
|
+
}
|
|
41
|
+
else {
|
|
42
|
+
offset += to;
|
|
43
|
+
}
|
|
44
|
+
return offset;
|
|
45
|
+
}
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
function openInput(ff, file) {
|
|
49
|
+
const fd = fs.openSync(file, 'r');
|
|
50
|
+
try {
|
|
51
|
+
const input = new ff.InputFormatContext(fileIO(ff, fd));
|
|
52
|
+
return {
|
|
53
|
+
input,
|
|
54
|
+
close: () => {
|
|
55
|
+
input.destroy();
|
|
56
|
+
fs.closeSync(fd);
|
|
57
|
+
}
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
catch (err) {
|
|
61
|
+
fs.closeSync(fd);
|
|
62
|
+
throw err;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
export async function probeVideo(file) {
|
|
66
|
+
const ff = await loadFfmpeg();
|
|
67
|
+
const { input, close } = openInput(ff, file);
|
|
68
|
+
try {
|
|
69
|
+
const hasAudio = Boolean(input.getBestStream(ff.constants.mediaTypes.AUDIO));
|
|
70
|
+
const video = input.getBestStream(ff.constants.mediaTypes.VIDEO);
|
|
71
|
+
const formatDuration = input.duration > 0 ? input.duration / AV_TIME_BASE : 0;
|
|
72
|
+
if (!video) {
|
|
73
|
+
return {
|
|
74
|
+
hasVideo: false,
|
|
75
|
+
hasAudio,
|
|
76
|
+
duration: formatDuration,
|
|
77
|
+
width: 0,
|
|
78
|
+
height: 0,
|
|
79
|
+
aspect: 1,
|
|
80
|
+
...UPRIGHT
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
const { numerator, denominator } = video.timeBase;
|
|
84
|
+
const streamDuration = video.duration > 0 && denominator ? (video.duration * numerator) / denominator : 0;
|
|
85
|
+
return {
|
|
86
|
+
hasVideo: true,
|
|
87
|
+
hasAudio,
|
|
88
|
+
duration: streamDuration || formatDuration,
|
|
89
|
+
width: video.codecParameters.width,
|
|
90
|
+
height: video.codecParameters.height,
|
|
91
|
+
aspect: pixelAspect(video),
|
|
92
|
+
...orientation(ff, video)
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
finally {
|
|
96
|
+
close();
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
function orientation(ff, stream) {
|
|
100
|
+
const matrix = stream.sideData.find((entry) => entry.type === ff.constants.packetSideDataType.DISPLAYMATRIX);
|
|
101
|
+
if (!matrix) {
|
|
102
|
+
return UPRIGHT;
|
|
103
|
+
}
|
|
104
|
+
const key = [0, 4, 12, 16]
|
|
105
|
+
.map((offset) => snap(b4a.readInt32LE(matrix.data, offset) / 65536))
|
|
106
|
+
.join(',');
|
|
107
|
+
return CORRECTIONS.get(key) ?? UPRIGHT;
|
|
108
|
+
}
|
|
109
|
+
function snap(value) {
|
|
110
|
+
return Math.abs(value - Math.round(value)) < 1e-3 ? Math.round(value) + 0 : value;
|
|
111
|
+
}
|
|
112
|
+
export function frameTimes(duration, count) {
|
|
113
|
+
const slice = duration / count;
|
|
114
|
+
return Array.from({ length: count }, (_, i) => (i + 0.5) * slice);
|
|
115
|
+
}
|
|
116
|
+
export function frameSize(probe, maxEdge) {
|
|
117
|
+
const size = fit(probe.width * probe.aspect, probe.height, maxEdge);
|
|
118
|
+
return probe.rotate % 180 ? { width: size.height, height: size.width } : size;
|
|
119
|
+
}
|
|
120
|
+
function fit(width, height, maxEdge) {
|
|
121
|
+
const scale = Math.min(1, maxEdge / Math.max(width, height));
|
|
122
|
+
const even = (n) => Math.max(2, Math.round((n * scale) / 2) * 2);
|
|
123
|
+
return { width: even(width), height: even(height) };
|
|
124
|
+
}
|
|
125
|
+
function toRaster(ff, frame, size) {
|
|
126
|
+
const { RGBA } = ff.constants.pixelFormats;
|
|
127
|
+
const scaler = new ff.Scaler(frame.format, frame.width, frame.height, RGBA, size.width, size.height);
|
|
128
|
+
const rgba = new ff.Frame();
|
|
129
|
+
try {
|
|
130
|
+
rgba.width = size.width;
|
|
131
|
+
rgba.height = size.height;
|
|
132
|
+
rgba.format = RGBA;
|
|
133
|
+
rgba.alloc();
|
|
134
|
+
scaler.scale(frame, rgba);
|
|
135
|
+
const image = new ff.Image(RGBA, size.width, size.height);
|
|
136
|
+
image.read(rgba);
|
|
137
|
+
return { width: size.width, height: size.height, data: image.data };
|
|
138
|
+
}
|
|
139
|
+
finally {
|
|
140
|
+
rgba.destroy();
|
|
141
|
+
scaler.destroy();
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
function* received(decoder, frame) {
|
|
145
|
+
while (decoder.receiveFrame(frame)) {
|
|
146
|
+
yield frame;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
async function* decodedFrames(ff, input, stream) {
|
|
150
|
+
const decoder = stream.decoder();
|
|
151
|
+
const packet = new ff.Packet();
|
|
152
|
+
const frame = new ff.Frame();
|
|
153
|
+
try {
|
|
154
|
+
decoder.open();
|
|
155
|
+
let packets = 0;
|
|
156
|
+
while (input.readFrame(packet)) {
|
|
157
|
+
if (packet.streamIndex === stream.index && decoder.sendPacket(packet)) {
|
|
158
|
+
yield* received(decoder, frame);
|
|
159
|
+
}
|
|
160
|
+
packet.unref();
|
|
161
|
+
if (++packets % YIELD_EVERY_PACKETS === 0) {
|
|
162
|
+
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
decoder.sendPacket(new ff.Packet());
|
|
166
|
+
yield* received(decoder, frame);
|
|
167
|
+
}
|
|
168
|
+
finally {
|
|
169
|
+
frame.destroy();
|
|
170
|
+
packet.destroy();
|
|
171
|
+
decoder.destroy();
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
function pixelAspect(stream) {
|
|
175
|
+
const sar = stream.codecParameters.sampleAspectRatio;
|
|
176
|
+
return sar && sar.numerator > 0 && sar.denominator > 0 ? sar.numerator / sar.denominator : 1;
|
|
177
|
+
}
|
|
178
|
+
async function pick(ff, input, opts) {
|
|
179
|
+
const stream = input.getBestStream(ff.constants.mediaTypes.VIDEO);
|
|
180
|
+
if (!stream) {
|
|
181
|
+
return [];
|
|
182
|
+
}
|
|
183
|
+
const targets = [...opts.times].sort((x, y) => x - y);
|
|
184
|
+
const tb = stream.timeBase.numerator / stream.timeBase.denominator;
|
|
185
|
+
const aspect = pixelAspect(stream);
|
|
186
|
+
const picked = [];
|
|
187
|
+
let origin = null;
|
|
188
|
+
for await (const frame of decodedFrames(ff, input, stream)) {
|
|
189
|
+
const pts = frame.pts === NO_PTS ? frame.packetDTS : frame.pts;
|
|
190
|
+
origin ??= pts;
|
|
191
|
+
const time = (pts - origin) * tb;
|
|
192
|
+
while (picked.length < targets.length && time >= targets[picked.length]) {
|
|
193
|
+
const size = fit(frame.width * aspect, frame.height, opts.maxEdge);
|
|
194
|
+
picked.push({ time, raster: toRaster(ff, frame, size) });
|
|
195
|
+
}
|
|
196
|
+
if (picked.length === targets.length) {
|
|
197
|
+
break;
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
return picked;
|
|
201
|
+
}
|
|
202
|
+
export async function sampleFrames(file, opts) {
|
|
203
|
+
const ff = await loadFfmpeg();
|
|
204
|
+
const media = await import('bare-media');
|
|
205
|
+
const { input, close } = openInput(ff, file);
|
|
206
|
+
let picked;
|
|
207
|
+
try {
|
|
208
|
+
picked = await pick(ff, input, opts);
|
|
209
|
+
}
|
|
210
|
+
finally {
|
|
211
|
+
close();
|
|
212
|
+
}
|
|
213
|
+
const { rotate, flipH, flipV } = opts.probe;
|
|
214
|
+
const orientate = media.image.orientate;
|
|
215
|
+
const frames = [];
|
|
216
|
+
for (const { time, raster } of picked) {
|
|
217
|
+
const upright = rotate || flipH || flipV
|
|
218
|
+
? await orientate(raster, { transform: { rotate, flipH, flipV } })
|
|
219
|
+
: raster;
|
|
220
|
+
const jpeg = await media.image.encode(upright, { mimetype: 'image/jpeg' });
|
|
221
|
+
frames.push({ time, width: upright.width, height: upright.height, jpeg });
|
|
222
|
+
}
|
|
223
|
+
return frames;
|
|
224
|
+
}
|
package/dist-lib/lib/core.js
CHANGED
|
@@ -4,6 +4,7 @@ import BlindPeering from 'blind-peering';
|
|
|
4
4
|
import Corestore from 'corestore';
|
|
5
5
|
import CorestoreStorage from 'hypercore-storage';
|
|
6
6
|
import { constants as RocksConstants } from 'rocksdb-native';
|
|
7
|
+
import path from 'path';
|
|
7
8
|
import Rache from 'rache';
|
|
8
9
|
import Hyperswarm from 'hyperswarm';
|
|
9
10
|
import Wakeup from 'protomux-wakeup';
|
|
@@ -409,6 +410,7 @@ export class Core extends ReadyResource {
|
|
|
409
410
|
platform: () => this._localDevice?.platform ?? null,
|
|
410
411
|
computeLender: () => this._computeLender(),
|
|
411
412
|
pushSkillSecret: (request) => this.pushSkillSecret(request),
|
|
413
|
+
storageRoot: path.dirname(this._storagePath),
|
|
412
414
|
listModels: this._hostsModelSource || this._listModels
|
|
413
415
|
? async (request) => (await this._readModelList(request)) ?? { models: [] }
|
|
414
416
|
: null,
|
|
@@ -2148,13 +2150,13 @@ export class Core extends ReadyResource {
|
|
|
2148
2150
|
}
|
|
2149
2151
|
const rows = (await this.local.listMeshes()).filter((row) => !b4a.equals(row.key, joined));
|
|
2150
2152
|
const meshes = rows.filter((row) => !(account && b4a.equals(account, row.key)));
|
|
2151
|
-
for (const mesh of meshes) {
|
|
2152
|
-
await this._departIfPooled(mesh.key, 'mesh');
|
|
2153
|
-
}
|
|
2154
2153
|
await this.local.adoptAccountMesh(this._requireDeviceKeyPair().publicKey, joined, {
|
|
2155
2154
|
account,
|
|
2156
2155
|
keepName
|
|
2157
2156
|
});
|
|
2157
|
+
for (const mesh of meshes) {
|
|
2158
|
+
await this._departIfPooled(mesh.key, 'mesh');
|
|
2159
|
+
}
|
|
2158
2160
|
if (account) {
|
|
2159
2161
|
await this._departIfPooled(account, 'account');
|
|
2160
2162
|
}
|
|
@@ -2278,7 +2280,7 @@ export class Core extends ReadyResource {
|
|
|
2278
2280
|
if (this.closing)
|
|
2279
2281
|
return null;
|
|
2280
2282
|
const key = await this._storedAccountKey();
|
|
2281
|
-
const held = key ? this._pooled(b4a.toHex(key)) : null;
|
|
2283
|
+
const held = key ? (this._pooled(b4a.toHex(key)) ?? this._opening.get(b4a.toHex(key))) : null;
|
|
2282
2284
|
const mesh = key ? await this._openPooledMesh(b4a.toHex(key)).catch(noop) : null;
|
|
2283
2285
|
if (mesh && !mesh.ourMemberId) {
|
|
2284
2286
|
await mesh.refreshOurMember();
|
|
@@ -3409,15 +3411,23 @@ export class Core extends ReadyResource {
|
|
|
3409
3411
|
if (!mesh.writable)
|
|
3410
3412
|
return;
|
|
3411
3413
|
const removed = [];
|
|
3414
|
+
const live = [];
|
|
3412
3415
|
for await (const row of mesh.view.find(DEVICES)) {
|
|
3413
3416
|
const ours = mesh === account || ownedBy(row, mesh.ourMemberId);
|
|
3414
3417
|
if (row.revoked && ours && !b4a.equals(row.id, self)) {
|
|
3415
3418
|
removed.push(row.id);
|
|
3416
3419
|
}
|
|
3420
|
+
else if (!row.revoked && ours) {
|
|
3421
|
+
live.push(row);
|
|
3422
|
+
}
|
|
3417
3423
|
}
|
|
3424
|
+
const adopter = await earliestPaired(account, live);
|
|
3425
|
+
const adopts = adopter !== undefined && b4a.equals(adopter.id, self);
|
|
3418
3426
|
for (const id of removed) {
|
|
3419
3427
|
const device = await account.view.get(DEVICES, { id });
|
|
3420
|
-
|
|
3428
|
+
const by = device?.revokedBy;
|
|
3429
|
+
const left = device?.revoked && device.writerKey && by && b4a.equals(by, device.writerKey);
|
|
3430
|
+
if ((by && b4a.equals(by, account.localWriterKey)) || (left && adopts)) {
|
|
3421
3431
|
await rehomeAgents(mesh, id, self).catch((err) => this._log?.warn(['p2p', 'mesh'], "handing over a removed device's agents failed", err));
|
|
3422
3432
|
}
|
|
3423
3433
|
}
|
|
@@ -4739,6 +4749,16 @@ function unmarkedNotice(losers) {
|
|
|
4739
4749
|
return `another device of yours opened a shared mesh at the same moment, so ${ids} is an ordinary mesh now`;
|
|
4740
4750
|
}
|
|
4741
4751
|
function noop() { }
|
|
4752
|
+
async function earliestPaired(account, rows) {
|
|
4753
|
+
const paired = [];
|
|
4754
|
+
for (const row of rows) {
|
|
4755
|
+
const device = await account.view.get(DEVICES, { id: row.id });
|
|
4756
|
+
if (device && !device.revoked) {
|
|
4757
|
+
paired.push(device);
|
|
4758
|
+
}
|
|
4759
|
+
}
|
|
4760
|
+
return paired.sort((a, b) => a.joinedAt - b.joinedAt || b4a.compare(a.id, b.id))[0];
|
|
4761
|
+
}
|
|
4742
4762
|
function ownedBy(device, member) {
|
|
4743
4763
|
const owner = device?.memberId ?? device?.claimedMemberId;
|
|
4744
4764
|
return !!owner && !!member && b4a.equals(owner, member);
|
|
@@ -35,6 +35,11 @@ export interface CredentialEngine extends ConfigEngine {
|
|
|
35
35
|
}): AsyncIterable<unknown> & {
|
|
36
36
|
destroy(): void;
|
|
37
37
|
};
|
|
38
|
+
agentsWatch(req: {
|
|
39
|
+
meshId: string;
|
|
40
|
+
}): AsyncIterable<unknown> & {
|
|
41
|
+
destroy(): void;
|
|
42
|
+
};
|
|
38
43
|
account(req: {}): Promise<{
|
|
39
44
|
meshId?: string | null;
|
|
40
45
|
}>;
|