@neta-art/cohub-cli 10.18.0 → 10.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/board-export.d.ts +8 -2
- package/dist/board-export.js +75 -42
- package/dist/board-gpu/page.js +339 -133
- package/dist/board-gpu/protocol.d.ts +1 -0
- package/dist/board-kernel.d.ts +1 -1
- package/dist/board-kernel.js +1113 -188
- package/dist/board-media-export.d.ts +16 -0
- package/dist/board-media-export.js +147 -0
- package/dist/board-media.d.ts +2 -1
- package/dist/board-media.js +23 -0
- package/dist/commands/boards/authoring.js +45 -3
- package/dist/commands/space-displays.d.ts +1 -1
- package/dist/commands/space-displays.js +70 -18
- package/dist/output.d.ts +1 -0
- package/dist/output.js +3 -0
- package/dist/safe-remote-image.d.ts +4 -0
- package/dist/safe-remote-image.js +8 -0
- package/package.json +2 -2
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { type BoardMediaSpan, type BoardMediaState } from "@neta-art/cohub/board";
|
|
2
|
+
export declare function createBoardMediaFiles(spaceId: string, dir: string, signal?: AbortSignal): {
|
|
3
|
+
path(src: string, kind: "video" | "audio"): Promise<string>;
|
|
4
|
+
audible(src: string, kind: "video" | "audio"): Promise<boolean>;
|
|
5
|
+
};
|
|
6
|
+
export type BoardMediaFiles = ReturnType<typeof createBoardMediaFiles>;
|
|
7
|
+
export declare function createBoardVideoFrames(files: BoardMediaFiles, dir: string, options: {
|
|
8
|
+
fps: number;
|
|
9
|
+
signal?: AbortSignal;
|
|
10
|
+
}): {
|
|
11
|
+
frame(state: BoardMediaState, rate: number, edge: number): Promise<Uint8Array | null>;
|
|
12
|
+
};
|
|
13
|
+
export declare function boardAudioMix(spans: readonly BoardMediaSpan[], files: BoardMediaFiles, firstInput: number): Promise<{
|
|
14
|
+
inputs: string[];
|
|
15
|
+
filter: string;
|
|
16
|
+
} | null>;
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
import { execFile as execFileCallback } from "node:child_process";
|
|
2
|
+
import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { promisify } from "node:util";
|
|
5
|
+
import { parseBoardMediaSource } from "@neta-art/cohub/board";
|
|
6
|
+
import { createClient } from "./client.js";
|
|
7
|
+
import { downloadPublicAudio, downloadPublicVideo } from "./safe-remote-image.js";
|
|
8
|
+
const execFile = promisify(execFileCallback);
|
|
9
|
+
const CHUNK_SECONDS = 4;
|
|
10
|
+
const MAX_DECODE_FPS = 60;
|
|
11
|
+
async function ffmpeg(args, signal) {
|
|
12
|
+
try {
|
|
13
|
+
return await execFile("ffmpeg", ["-v", "error", ...args], { encoding: "utf8", maxBuffer: 2 * 1024 * 1024, signal });
|
|
14
|
+
}
|
|
15
|
+
catch (error) {
|
|
16
|
+
const cause = error;
|
|
17
|
+
if (cause.code === "ENOENT")
|
|
18
|
+
throw new Error("Exporting video and audio needs ffmpeg. Install ffmpeg and try again.");
|
|
19
|
+
throw new Error(`ffmpeg failed: ${(cause.stderr ?? cause.message ?? "unknown error").trim().split("\n").slice(-4).join("\n")}`);
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
export function createBoardMediaFiles(spaceId, dir, signal) {
|
|
23
|
+
const files = new Map();
|
|
24
|
+
const audible = new Map();
|
|
25
|
+
const client = createClient();
|
|
26
|
+
return {
|
|
27
|
+
path(src, kind) {
|
|
28
|
+
let pending = files.get(src);
|
|
29
|
+
if (!pending) {
|
|
30
|
+
const path = join(dir, `media-${files.size}`);
|
|
31
|
+
pending = (async () => {
|
|
32
|
+
const source = parseBoardMediaSource(src);
|
|
33
|
+
if (!source)
|
|
34
|
+
throw new Error(`Invalid media source ${src}`);
|
|
35
|
+
const { bytes } = source.kind === "file"
|
|
36
|
+
? { bytes: new Uint8Array(await (await client.space(spaceId).files.download(source.value)).blob.arrayBuffer()) }
|
|
37
|
+
: await (kind === "video" ? downloadPublicVideo : downloadPublicAudio)(source.value);
|
|
38
|
+
await writeFile(path, bytes);
|
|
39
|
+
return path;
|
|
40
|
+
})();
|
|
41
|
+
files.set(src, pending);
|
|
42
|
+
}
|
|
43
|
+
return pending;
|
|
44
|
+
},
|
|
45
|
+
async audible(src, kind) {
|
|
46
|
+
let pending = audible.get(src);
|
|
47
|
+
if (!pending) {
|
|
48
|
+
pending = this.path(src, kind).then(async (path) => {
|
|
49
|
+
const { stdout } = await execFile("ffprobe", ["-v", "error", "-select_streams", "a", "-show_entries", "stream=index", "-of", "csv=p=0", path], { signal });
|
|
50
|
+
return stdout.trim().length > 0;
|
|
51
|
+
});
|
|
52
|
+
audible.set(src, pending);
|
|
53
|
+
}
|
|
54
|
+
return pending;
|
|
55
|
+
},
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
export function createBoardVideoFrames(files, dir, options) {
|
|
59
|
+
const chunks = new Map();
|
|
60
|
+
let decoded = 0;
|
|
61
|
+
function lookup(chunk, time, edge) {
|
|
62
|
+
if (!chunk || edge > chunk.edge)
|
|
63
|
+
return null;
|
|
64
|
+
const index = chunk.count === 1 ? (Math.abs(time - chunk.start) < 1 ? 0 : -1) : Math.round(((time - chunk.start) * chunk.fps) / 1000);
|
|
65
|
+
return index >= 0 && index < chunk.count ? join(chunk.dir, `${String(index + 1).padStart(5, "0")}.jpg`) : null;
|
|
66
|
+
}
|
|
67
|
+
async function decode(state, rate, edge) {
|
|
68
|
+
const path = await files.path(state.src, "video");
|
|
69
|
+
const previous = chunks.get(state.id);
|
|
70
|
+
if (previous)
|
|
71
|
+
await rm(previous.dir, { recursive: true, force: true });
|
|
72
|
+
const chunkDir = join(dir, `frames-${decoded++}`);
|
|
73
|
+
await mkdir(chunkDir);
|
|
74
|
+
const fps = rate > 0 ? Math.min(MAX_DECODE_FPS, Math.max(1, Math.ceil(options.fps * rate))) : 0;
|
|
75
|
+
const scale = `scale=w=${edge}:h=${edge}:force_original_aspect_ratio=decrease`;
|
|
76
|
+
await ffmpeg([
|
|
77
|
+
"-ss", (state.time / 1000).toFixed(3), "-i", path,
|
|
78
|
+
...(fps ? ["-t", String(CHUNK_SECONDS), "-vf", `fps=${fps},${scale}`] : ["-frames:v", "1", "-vf", scale]),
|
|
79
|
+
"-q:v", "3", join(chunkDir, "%05d.jpg"),
|
|
80
|
+
], options.signal);
|
|
81
|
+
const count = fps ? Math.max(1, Math.round(CHUNK_SECONDS * fps)) : 1;
|
|
82
|
+
const chunk = { src: state.src, start: state.time, fps: fps || 1, count, edge, dir: chunkDir };
|
|
83
|
+
chunks.set(state.id, chunk);
|
|
84
|
+
return chunk;
|
|
85
|
+
}
|
|
86
|
+
return {
|
|
87
|
+
async frame(state, rate, edge) {
|
|
88
|
+
const current = chunks.get(state.id);
|
|
89
|
+
let file = current?.src === state.src ? lookup(current, state.time, edge) : null;
|
|
90
|
+
if (!file)
|
|
91
|
+
file = lookup(await decode(state, rate, edge), state.time, edge);
|
|
92
|
+
return file ? new Uint8Array(await readFile(file).catch(() => Buffer.alloc(0))) : null;
|
|
93
|
+
},
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
function volumeExpression(span) {
|
|
97
|
+
const points = span.volume.map((point) => ({ t: (point.at - span.start) / 1000, v: point.value }));
|
|
98
|
+
if (points.every((point) => point.v === points[0]?.v))
|
|
99
|
+
return String(points[0]?.v ?? 1);
|
|
100
|
+
let expression = String(points.at(-1)?.v ?? 1);
|
|
101
|
+
for (let index = points.length - 2; index >= 0; index -= 1) {
|
|
102
|
+
const a = points[index];
|
|
103
|
+
const b = points[index + 1];
|
|
104
|
+
const slope = b.t > a.t ? (b.v - a.v) / (b.t - a.t) : 0;
|
|
105
|
+
expression = `if(lt(t,${b.t.toFixed(4)}),${a.v.toFixed(4)}+${slope.toFixed(6)}*(t-${a.t.toFixed(4)}),${expression})`;
|
|
106
|
+
}
|
|
107
|
+
return `'${expression}'`;
|
|
108
|
+
}
|
|
109
|
+
function tempo(rate) {
|
|
110
|
+
const stages = [];
|
|
111
|
+
let rest = rate;
|
|
112
|
+
while (rest > 2) {
|
|
113
|
+
stages.push("atempo=2");
|
|
114
|
+
rest /= 2;
|
|
115
|
+
}
|
|
116
|
+
while (rest < 0.5) {
|
|
117
|
+
stages.push("atempo=0.5");
|
|
118
|
+
rest /= 0.5;
|
|
119
|
+
}
|
|
120
|
+
if (Math.abs(rest - 1) > 1e-4)
|
|
121
|
+
stages.push(`atempo=${rest.toFixed(6)}`);
|
|
122
|
+
return stages;
|
|
123
|
+
}
|
|
124
|
+
export async function boardAudioMix(spans, files, firstInput) {
|
|
125
|
+
const inputs = [];
|
|
126
|
+
const chains = [];
|
|
127
|
+
for (const span of spans) {
|
|
128
|
+
if (!(await files.audible(span.src, span.kind)))
|
|
129
|
+
continue;
|
|
130
|
+
const index = firstInput + chains.length;
|
|
131
|
+
inputs.push("-i", await files.path(span.src, span.kind));
|
|
132
|
+
const length = ((span.end - span.start) * span.rate) / 1000;
|
|
133
|
+
const delay = Math.max(0, Math.round(span.start));
|
|
134
|
+
const filters = [
|
|
135
|
+
`atrim=start=${(span.from / 1000).toFixed(4)}:duration=${length.toFixed(4)}`,
|
|
136
|
+
"asetpts=PTS-STARTPTS",
|
|
137
|
+
...tempo(span.rate),
|
|
138
|
+
`volume=${volumeExpression(span)}:eval=frame`,
|
|
139
|
+
`adelay=${delay}:all=1`,
|
|
140
|
+
];
|
|
141
|
+
chains.push(`[${index}:a]${filters.join(",")}[a${chains.length}]`);
|
|
142
|
+
}
|
|
143
|
+
if (!chains.length)
|
|
144
|
+
return null;
|
|
145
|
+
const mixed = chains.map((_, index) => `[a${index}]`).join("");
|
|
146
|
+
return { inputs, filter: `${chains.join(";")};${mixed}amix=inputs=${chains.length}:duration=longest:dropout_transition=0:normalize=0[aout]` };
|
|
147
|
+
}
|
package/dist/board-media.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { type BoardDocument, type BoardMediaFileResolver } from "@neta-art/cohub/board";
|
|
1
|
+
import { type BoardDocument, type BoardItem, type BoardMediaFileResolver } from "@neta-art/cohub/board";
|
|
2
2
|
export declare function boardMediaFileResolver(spaceId: string): BoardMediaFileResolver;
|
|
3
3
|
export declare function settleBoardDocument(spaceId: string, document: BoardDocument): Promise<BoardDocument>;
|
|
4
4
|
export declare function settleBoardPatch<T extends Record<string, unknown>>(spaceId: string, patch: T, current: (ids: string[]) => Promise<Record<string, unknown> | null>): Promise<T>;
|
|
5
|
+
export declare function settleBoardMediaDurations(spaceId: string, document: BoardDocument, ids: readonly string[]): Promise<Record<string, BoardItem>>;
|
package/dist/board-media.js
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
|
+
import { execFile } from "node:child_process";
|
|
1
2
|
import { resolve } from "node:path";
|
|
3
|
+
import { promisify } from "node:util";
|
|
2
4
|
import { diffMergePatch, mergePatch, parseBoardItem, settleBoardMediaSizes, } from "@neta-art/cohub/board";
|
|
3
5
|
import { createClient } from "./client.js";
|
|
4
6
|
import { localMediaFacts } from "./local-media.js";
|
|
@@ -41,3 +43,24 @@ export async function settleBoardPatch(spaceId, patch, current) {
|
|
|
41
43
|
const adopted = Object.entries(settled).map(([id, item]) => [id, mergePatch(items[id], diffMergePatch(created[id], item))]);
|
|
42
44
|
return { ...patch, items: { ...items, ...Object.fromEntries(adopted) } };
|
|
43
45
|
}
|
|
46
|
+
const run = promisify(execFile);
|
|
47
|
+
export async function settleBoardMediaDurations(spaceId, document, ids) {
|
|
48
|
+
const local = process.env.COHUB_SPACE_ID === spaceId;
|
|
49
|
+
const settled = {};
|
|
50
|
+
for (const id of ids) {
|
|
51
|
+
const item = document.items[id];
|
|
52
|
+
if ((item?.type !== "video" && item?.type !== "audio") || item.props.snapshot?.durationMs)
|
|
53
|
+
continue;
|
|
54
|
+
const file = item.props.src.startsWith("http") ? null : resolve(WORKSPACE_ROOT, item.props.src);
|
|
55
|
+
const input = file && local && file.startsWith(`${WORKSPACE_ROOT}/`)
|
|
56
|
+
? file
|
|
57
|
+
: file ? (await createClient().space(spaceId).files.resolve(item.props.src, { purpose: "playback" }).catch(() => null))?.url : null;
|
|
58
|
+
const seconds = input
|
|
59
|
+
? Number((await run("ffprobe", ["-v", "error", "-show_entries", "format=duration", "-of", "csv=p=0", input], { timeout: 15_000 }).catch(() => ({ stdout: "" }))).stdout)
|
|
60
|
+
: Number.NaN;
|
|
61
|
+
if (!(seconds > 0))
|
|
62
|
+
throw new Error(`The length of ${id} is unknown; pass --duration.`);
|
|
63
|
+
settled[id] = { ...item, props: { ...item.props, snapshot: { ...item.props.snapshot, durationMs: Math.round(seconds * 1000) } } };
|
|
64
|
+
}
|
|
65
|
+
return settled;
|
|
66
|
+
}
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import { readBoardJsonObject } from "../../board-command-support.js";
|
|
2
2
|
import { loadBoardDocument, resolveBundledFonts } from "../../board-export.js";
|
|
3
|
-
import { settleBoardDocument } from "../../board-media.js";
|
|
4
|
-
import { boardFrameRegions, boardPathTrack, boardRegionStep, boardSequencePatch, buildBoardScene, createBoardAnimationSampler, createNodeBoardRenderer, inspectBoardItems, } from "../../board-kernel.js";
|
|
5
|
-
import { parseBoardTimes } from "../../board-time.js";
|
|
3
|
+
import { settleBoardDocument, settleBoardMediaDurations } from "../../board-media.js";
|
|
4
|
+
import { boardFrameRegions, boardPathTrack, boardPlacePatch, boardRegionStep, boardSequencePatch, buildBoardScene, createBoardAnimationSampler, createNodeBoardRenderer, inspectBoardItems, } from "../../board-kernel.js";
|
|
5
|
+
import { parseBoardTime, parseBoardTimes } from "../../board-time.js";
|
|
6
6
|
import { handleHttp, json as outJson, jsonRequested, table } from "../../output.js";
|
|
7
7
|
import { resolveSpace } from "../../space.js";
|
|
8
8
|
const DIRECTIONS = ["left", "right", "up", "down"];
|
|
@@ -97,6 +97,48 @@ Viewers step with N, the arrow keys and swipes; #focus=<frame>&present presents.
|
|
|
97
97
|
handleHttp(cause);
|
|
98
98
|
}
|
|
99
99
|
});
|
|
100
|
+
boards
|
|
101
|
+
.command("place <board>")
|
|
102
|
+
.description("Put items or animations on a timeline as clips; prints a patch for apply, does not write")
|
|
103
|
+
.requiredOption("--animation <id>", "Animation to place them in; created when missing")
|
|
104
|
+
.requiredOption("--targets <ids>", "Comma-separated item or animation ids, laid end to end")
|
|
105
|
+
.option("--at <time>", "Where the first one starts", "0")
|
|
106
|
+
.option("--duration <time>", "Length of each; defaults to what is left of the media or animation, else 3s")
|
|
107
|
+
.option("--from <time>", "Where media or an animation starts playing in its own time", "0")
|
|
108
|
+
.option("--rate <number>", "Playback speed of media and animations", "1")
|
|
109
|
+
.option("--together", "Start them all at --at instead of one after another")
|
|
110
|
+
.addHelpText("after", `
|
|
111
|
+
Each target is shown only during its clip (a visible track); video, audio and
|
|
112
|
+
animations also play (a props.time or time track), and sound in exports while
|
|
113
|
+
their time moves forward. Placing a target that is already on the timeline moves it.
|
|
114
|
+
cohub boards place cut.board --animation cut --targets intro,talk,outro | cohub boards apply cut.board -i -
|
|
115
|
+
cohub boards place cut.board --animation cut --targets music --at 0 --duration 30s --from 12s`)
|
|
116
|
+
.action(async (target, options) => {
|
|
117
|
+
try {
|
|
118
|
+
const targets = options.targets.split(",").map((id) => id.trim()).filter(Boolean);
|
|
119
|
+
if (!targets.length)
|
|
120
|
+
throw new Error("--targets needs at least one id.");
|
|
121
|
+
const rate = Number(options.rate);
|
|
122
|
+
if (!Number.isFinite(rate) || rate <= 0)
|
|
123
|
+
throw new Error("--rate must be a positive number.");
|
|
124
|
+
const spaceId = await resolveSpace(boards);
|
|
125
|
+
const { document: loaded } = await loadBoardDocument(spaceId, target);
|
|
126
|
+
const durations = options.duration ? {} : await settleBoardMediaDurations(spaceId, loaded, targets);
|
|
127
|
+
const document = { ...loaded, items: { ...loaded.items, ...durations } };
|
|
128
|
+
const patch = boardPlacePatch(document, options.animation, targets, {
|
|
129
|
+
at: parseBoardTime(options.at, "--at"),
|
|
130
|
+
from: parseBoardTime(options.from, "--from"),
|
|
131
|
+
rate,
|
|
132
|
+
...(options.duration ? { duration: parseBoardTime(options.duration, "--duration") } : {}),
|
|
133
|
+
together: options.together,
|
|
134
|
+
});
|
|
135
|
+
const items = Object.fromEntries(Object.entries(durations).map(([id, item]) => [id, { props: { snapshot: { durationMs: item.props.snapshot?.durationMs } } }]));
|
|
136
|
+
outJson(Object.keys(items).length ? { ...patch, items } : patch);
|
|
137
|
+
}
|
|
138
|
+
catch (cause) {
|
|
139
|
+
handleHttp(cause);
|
|
140
|
+
}
|
|
141
|
+
});
|
|
100
142
|
boards
|
|
101
143
|
.command("path <target>")
|
|
102
144
|
.description("Sample a polyline into a position track; does not write")
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
import type
|
|
1
|
+
import { type Command } from "commander";
|
|
2
2
|
export declare function registerSpaceDisplays(spacesCmd: Command): void;
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
|
-
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
2
|
+
import { mkdir, readdir, readFile, rm, writeFile } from "node:fs/promises";
|
|
3
3
|
import { homedir } from "node:os";
|
|
4
4
|
import { dirname, join, resolve } from "node:path";
|
|
5
5
|
import { compileDisplayPlan, DISPLAY_ELEMENT_REF_PATTERN, DISPLAY_ELEMENT_ROLES, DISPLAY_SYSTEM_ACTIONS, DISPLAY_HANDOVER_DEFAULT_MS, DISPLAY_HANDOVER_MAX_MS, DISPLAY_HANDOVER_WAIT_MS, DisplayActionError, } from "@neta-art/cohub";
|
|
6
|
+
import { Option } from "commander";
|
|
6
7
|
import { createClient } from "../client.js";
|
|
7
|
-
import { error, handleHttp, json as outJson, jsonRequested, ok, table } from "../output.js";
|
|
8
|
+
import { error, handleHttp, json as outJson, jsonRequested, ok, table, warn } from "../output.js";
|
|
8
9
|
import { resolveSpace } from "../space.js";
|
|
9
10
|
import { webUrl } from "../web.js";
|
|
10
11
|
const DIRECTIONS = ["up", "down", "left", "right"];
|
|
@@ -15,6 +16,9 @@ const HANDOVER_RETRY_MS = 60_000;
|
|
|
15
16
|
const HANDOVER_RETRY_GAP_MS = 2_000;
|
|
16
17
|
const CACHE_DIR = join(homedir(), ".cache", "cohub-cli");
|
|
17
18
|
const FRAMES_PATH = join(CACHE_DIR, "display-frames.json");
|
|
19
|
+
// In the workspace, where a Runtime's fenced file tools reach them.
|
|
20
|
+
const IMAGE_DIR = join(".cache", "cohub", "displays");
|
|
21
|
+
const KEPT_IMAGES = 20;
|
|
18
22
|
const NO_DISPLAY_HINT = "Share one first: set up phone control in the Cohub Android app, run `cohub runtime up --display` on a computer, or `cohub spaces displays start` for a virtual screen.";
|
|
19
23
|
function parseSize(value) {
|
|
20
24
|
const match = /^(\d{1,5})x(\d{1,5})$/i.exec(value.trim());
|
|
@@ -94,18 +98,30 @@ function needsHint(display) {
|
|
|
94
98
|
const names = needs.map((need) => (need === "screenRecording" ? "Screen Recording" : "Accessibility")).join(" and ");
|
|
95
99
|
return `${display.name || display.id} needs ${names}: allow it for the Runtime's terminal in System Settings → Privacy & Security, then restart the Runtime.`;
|
|
96
100
|
}
|
|
97
|
-
function
|
|
98
|
-
|
|
101
|
+
async function writeOnce(path, content) {
|
|
102
|
+
await writeFile(path, content, { flag: "wx" }).catch((e) => {
|
|
103
|
+
if (e.code !== "EEXIST")
|
|
104
|
+
throw e;
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
async function imageDir() {
|
|
108
|
+
const dir = resolve(IMAGE_DIR);
|
|
109
|
+
await mkdir(dir, { recursive: true });
|
|
110
|
+
await Promise.all([writeOnce(join(dir, "..", ".gitignore"), "*\n"), writeOnce(join(dir, ".nomedia"), "")]);
|
|
111
|
+
const images = (await readdir(dir)).filter((name) => name.endsWith(".jpg")).sort();
|
|
112
|
+
await Promise.all(images.slice(0, -(KEPT_IMAGES - 1)).map((name) => rm(join(dir, name), { force: true })));
|
|
113
|
+
return dir;
|
|
99
114
|
}
|
|
100
115
|
async function saveImage(spaceId, display, image, output) {
|
|
101
|
-
const
|
|
116
|
+
const name = `${new Date().toISOString().replace(/[:.]/g, "-")}-${display.id}.jpg`;
|
|
117
|
+
const path = output ? resolve(output) : join(await imageDir().catch(() => join(CACHE_DIR, "displays")), name);
|
|
102
118
|
await mkdir(dirname(path), { recursive: true });
|
|
103
119
|
await writeFile(path, Buffer.from(image.data, "base64"));
|
|
104
120
|
await rememberFrame(spaceId, display, image);
|
|
105
121
|
return { path, width: image.width, height: image.height };
|
|
106
122
|
}
|
|
107
123
|
async function screenshot(api, spaceId, display, maxSize, output) {
|
|
108
|
-
return saveImage(spaceId, display, await api.capture(display.id, { maxSize }), output
|
|
124
|
+
return saveImage(spaceId, display, await api.capture(display.id, { maxSize }), output);
|
|
109
125
|
}
|
|
110
126
|
function formatElement(element, frame, indent = true, box = true) {
|
|
111
127
|
const scaleX = frame.width - 1 || 1;
|
|
@@ -144,7 +160,7 @@ async function act(spacesCmd, opts, actions) {
|
|
|
144
160
|
try {
|
|
145
161
|
const launching = actions.some((action) => action.type === "launch" || action.type === "open");
|
|
146
162
|
const { spaceId, api, display } = await target(spacesCmd, opts, launching);
|
|
147
|
-
if (!display.input && actions.
|
|
163
|
+
if (!display.input && actions.some((action) => action.type !== "wait"))
|
|
148
164
|
return error("This display is view only", needsHint(display) ?? "Allow control on the device first.");
|
|
149
165
|
const frame = await frameOf(spaceId, display, opts.size);
|
|
150
166
|
const tree = opts.tree ?? "compact";
|
|
@@ -175,7 +191,7 @@ async function act(spacesCmd, opts, actions) {
|
|
|
175
191
|
const { image: _, ...screen } = result.screen;
|
|
176
192
|
return outJson({ display: display.id, ...result, screen: { ...screen, ...(image ? { image } : {}) }, frame: { width: shown.width, height: shown.height } });
|
|
177
193
|
}
|
|
178
|
-
printResult(display, result, shown, tree === "compact", image?.path);
|
|
194
|
+
printResult(display, actions, result, shown, tree === "compact", image?.path);
|
|
179
195
|
}
|
|
180
196
|
catch (e) {
|
|
181
197
|
if (e instanceof DisplayActionError)
|
|
@@ -189,14 +205,49 @@ function isLostAnswer(e) {
|
|
|
189
205
|
const status = e.status;
|
|
190
206
|
return status === undefined ? e instanceof TypeError : status >= 500 && status !== 501;
|
|
191
207
|
}
|
|
192
|
-
function
|
|
208
|
+
function targetText(target) {
|
|
209
|
+
return target.name ? `${target.role} ${JSON.stringify(target.name)}` : target.role;
|
|
210
|
+
}
|
|
211
|
+
function atPoint(action) {
|
|
212
|
+
return action.x !== undefined && action.ref === undefined && action.name === undefined && action.role === undefined;
|
|
213
|
+
}
|
|
214
|
+
function actionText(action) {
|
|
215
|
+
const point = atPoint(action) ? ` (${action.x},${action.y})` : "";
|
|
216
|
+
switch (action.type) {
|
|
217
|
+
case "tap": return `${action.count === 2 ? "double-tap" : action.count === 3 ? "triple-tap" : "tap"}${point}`;
|
|
218
|
+
case "long_press": return `long press${point}`;
|
|
219
|
+
case "swipe": return `swipe${point} to (${action.to_x},${action.to_y})`;
|
|
220
|
+
case "scroll": return `scroll ${action.direction ?? "down"}`;
|
|
221
|
+
case "type": return `type ${[...(action.text ?? "")].length} characters`; // never echo it: it may be a secret
|
|
222
|
+
case "key": return `key ${action.key}`;
|
|
223
|
+
case "system": return `press ${action.action}`;
|
|
224
|
+
case "launch": return `launch ${action.app}`;
|
|
225
|
+
case "open": return `open ${action.url}`;
|
|
226
|
+
case "wait": return "wait";
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
function stepText(action, step) {
|
|
230
|
+
const parts = [action ? actionText(action) : "step"];
|
|
231
|
+
if (step.target)
|
|
232
|
+
parts.push(`${action && atPoint(action) ? "on " : action?.type === "type" ? "into " : ""}${targetText(step.target)}`);
|
|
233
|
+
if (action?.type !== "wait")
|
|
234
|
+
parts.push(`· ${step.effect}`);
|
|
235
|
+
parts.push(`· ${step.findMs + step.actMs + step.waitedMs} ms`);
|
|
236
|
+
return parts.join(" ");
|
|
237
|
+
}
|
|
238
|
+
function printResult(display, actions, result, frame, compact, imageFile) {
|
|
193
239
|
const name = display.name || display.id;
|
|
194
240
|
if (result.steps.length > 0 || result.stopped) {
|
|
195
241
|
const done = result.steps.length;
|
|
196
|
-
|
|
197
|
-
|
|
242
|
+
const still = result.steps.findIndex((step, index) => step.effect === "none" && actions[index]?.type !== "wait");
|
|
243
|
+
if (!result.stopped) {
|
|
244
|
+
if (still >= 0)
|
|
245
|
+
warn(`Step ${still + 1} changed nothing on ${name}: it may not have landed. Look before repeating it.`);
|
|
246
|
+
else
|
|
247
|
+
ok(`${done} step${done === 1 ? "" : "s"} on ${name}`);
|
|
248
|
+
}
|
|
198
249
|
for (const [index, step] of result.steps.entries())
|
|
199
|
-
console.log(` ${index + 1}. ${
|
|
250
|
+
console.log(` ${index + 1}. ${stepText(actions[index], step)}`);
|
|
200
251
|
if (result.stopped) {
|
|
201
252
|
console.log(`✗ Stopped (${result.stopped.reason}): ${result.stopped.message}`);
|
|
202
253
|
for (const candidate of result.stopped.candidates ?? [])
|
|
@@ -397,7 +448,7 @@ export function registerSpaceDisplays(spacesCmd) {
|
|
|
397
448
|
.command("capture")
|
|
398
449
|
.description("Save a screenshot; later coordinates are its pixels")
|
|
399
450
|
.option("-d, --display <id>", "Display id; defaults to the only one, or the agent screen")
|
|
400
|
-
.option("-o, --output <path>",
|
|
451
|
+
.option("-o, --output <path>", `Where to write the image (default: ${IMAGE_DIR}/, the latest ${KEPT_IMAGES} kept)`)
|
|
401
452
|
.option("--max-size <px>", `Longest edge in pixels (default ${DEFAULT_MAX_SIZE})`)
|
|
402
453
|
.option("--json", "Output as JSON")
|
|
403
454
|
.action(async (opts) => {
|
|
@@ -418,7 +469,8 @@ export function registerSpaceDisplays(spacesCmd) {
|
|
|
418
469
|
.description("Read the interface as elements: refs to act on, roles, names and boxes")
|
|
419
470
|
.option("-d, --display <id>", "Display id; defaults to the only one, or the agent screen")
|
|
420
471
|
.option("--max <n>", "Most elements to return (default 300)")
|
|
421
|
-
.option("--
|
|
472
|
+
.option("--full", "Every element, nested, instead of what can be acted on or read")
|
|
473
|
+
.addOption(new Option("--compact").hideHelp())
|
|
422
474
|
.option("--json", "Output as JSON")
|
|
423
475
|
.action(async (opts) => {
|
|
424
476
|
try {
|
|
@@ -427,14 +479,14 @@ export function registerSpaceDisplays(spacesCmd) {
|
|
|
427
479
|
return error("This display has no element tree", "Use capture and coordinates instead.");
|
|
428
480
|
const tree = await api.tree(display.id, {
|
|
429
481
|
...(opts.max ? { maxElements: parseNumber("max", opts.max) } : {}),
|
|
430
|
-
...(opts.
|
|
482
|
+
...(opts.full ? {} : { compact: true }),
|
|
431
483
|
});
|
|
432
484
|
if (jsonRequested(opts))
|
|
433
485
|
return outJson(tree);
|
|
434
486
|
const frame = await frameOf(spaceId, display);
|
|
435
487
|
if (tree.app)
|
|
436
488
|
console.log(tree.app);
|
|
437
|
-
console.log(formatTree(tree, frame, opts.
|
|
489
|
+
console.log(formatTree(tree, frame, !opts.full));
|
|
438
490
|
console.log(`\n Boxes are x,y and size in ${frame.width}x${frame.height} pixels. Act with: tap --name <text> · type <text> --into <ref> · act < plan.json`);
|
|
439
491
|
if (tree.truncated)
|
|
440
492
|
console.log(" More elements exist; pass --max to read more.");
|
|
@@ -462,8 +514,8 @@ export function registerSpaceDisplays(spacesCmd) {
|
|
|
462
514
|
handleHttp(e);
|
|
463
515
|
}
|
|
464
516
|
});
|
|
465
|
-
withAct(displays.command("look").description("Read the screen as `act` reports it, without acting"))
|
|
466
|
-
.action((opts) => act(spacesCmd, opts, []));
|
|
517
|
+
withAct(displays.command("look").description("Read the screen as `act` reports it, without acting; --until-* waits first"))
|
|
518
|
+
.action((opts) => act(spacesCmd, opts, untilOf(opts) ? [{ type: "wait" }] : []));
|
|
467
519
|
withAct(displays.command("launch <app>").description("Launch an app by id (see `apps`) and wait until it is in front"))
|
|
468
520
|
.action((app, opts) => act(spacesCmd, { untilApp: app, ...opts }, [{ type: "launch", app }]));
|
|
469
521
|
withAct(displays.command("open <url>").description("Open a link, or a page of the system's settings"))
|
package/dist/output.d.ts
CHANGED
|
@@ -36,6 +36,7 @@ export declare function jsonRequested(opts?: {
|
|
|
36
36
|
json?: boolean;
|
|
37
37
|
}): boolean;
|
|
38
38
|
export declare function ok(msg: string): void;
|
|
39
|
+
export declare function warn(msg: string): void;
|
|
39
40
|
export declare function error(msg: string, detail?: string): never;
|
|
40
41
|
export declare function handleHttp(e: unknown): never;
|
|
41
42
|
export declare function spinner(): {
|
package/dist/output.js
CHANGED
|
@@ -95,6 +95,9 @@ export function jsonRequested(opts) {
|
|
|
95
95
|
export function ok(msg) {
|
|
96
96
|
console.log(`\n ✓ ${msg}\n`);
|
|
97
97
|
}
|
|
98
|
+
export function warn(msg) {
|
|
99
|
+
console.log(`\n ⚠ ${msg}\n`);
|
|
100
|
+
}
|
|
98
101
|
export function error(msg, detail) {
|
|
99
102
|
process.stderr.write(`\n ✗ ${msg}\n`);
|
|
100
103
|
if (detail)
|
|
@@ -29,4 +29,8 @@ export declare function downloadPublicAudio(input: string, options?: RemoteImage
|
|
|
29
29
|
bytes: Uint8Array;
|
|
30
30
|
mimeType: string;
|
|
31
31
|
}>;
|
|
32
|
+
export declare function downloadPublicVideo(input: string, options?: RemoteImageDownloadOptions): Promise<{
|
|
33
|
+
bytes: Uint8Array;
|
|
34
|
+
mimeType: string;
|
|
35
|
+
}>;
|
|
32
36
|
export {};
|
|
@@ -18,6 +18,7 @@ const AUDIO_MIME_TYPES = new Set([
|
|
|
18
18
|
"audio/wav", "audio/x-wav", "audio/wave", "audio/flac", "audio/x-flac",
|
|
19
19
|
"audio/webm", "audio/opus", "application/ogg",
|
|
20
20
|
]);
|
|
21
|
+
const VIDEO_MIME_TYPES = new Set(["video/mp4", "video/webm", "video/quicktime", "video/ogg", "video/x-matroska"]);
|
|
21
22
|
const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
|
|
22
23
|
async function defaultLookup(hostname) {
|
|
23
24
|
const records = await dns.lookup(hostname, { all: true, verbatim: true });
|
|
@@ -145,6 +146,13 @@ export function downloadPublicAudio(input, options = {}) {
|
|
|
145
146
|
...options,
|
|
146
147
|
});
|
|
147
148
|
}
|
|
149
|
+
export function downloadPublicVideo(input, options = {}) {
|
|
150
|
+
return downloadPublicMedia(input, VIDEO_MIME_TYPES, {
|
|
151
|
+
maxBytes: 512 * 1024 * 1024,
|
|
152
|
+
timeoutMs: 120_000,
|
|
153
|
+
...options,
|
|
154
|
+
});
|
|
155
|
+
}
|
|
148
156
|
async function downloadPublicMedia(input, mimeTypes, options) {
|
|
149
157
|
const lookup = options.lookup ?? defaultLookup;
|
|
150
158
|
const requester = options.requester ?? requestPinned;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@neta-art/cohub-cli",
|
|
3
|
-
"version": "10.
|
|
3
|
+
"version": "10.19.0",
|
|
4
4
|
"description": "CLI for Cohub — spaces, sessions, and agent collaboration.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"engines": {
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
"NOTICE"
|
|
19
19
|
],
|
|
20
20
|
"dependencies": {
|
|
21
|
-
"@neta-art/cohub": "13.
|
|
21
|
+
"@neta-art/cohub": "13.10.0",
|
|
22
22
|
"@neta-art/generation": "0.2.15",
|
|
23
23
|
"commander": "^15.0.0",
|
|
24
24
|
"sharp": "^0.35.4",
|