@agent-video/pi 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/extensions/video.ts +83 -0
- package/lib/mcp.ts +403 -0
- package/lib/schema.ts +34 -0
- package/package.json +21 -4
- package/README.md +0 -3
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pi extension: the `video` tool over the agent-video binary.
|
|
3
|
+
*
|
|
4
|
+
* A thin MCP stdio client, not an MCP framework: the extension lazily spawns
|
|
5
|
+
* the platform binary's `mcp` subcommand once per session (consent tokens and
|
|
6
|
+
* background downloads are in-process state there), speaks newline-delimited
|
|
7
|
+
* JSON-RPC to it, and maps content blocks straight through — pi's
|
|
8
|
+
* ToolResultMessage uses the same `{ type: "image", data, mimeType }` shape.
|
|
9
|
+
* The client itself lives in ../lib/mcp.ts, dependency-free so its boundary
|
|
10
|
+
* behavior is testable; this file is binary resolution plus pi glue.
|
|
11
|
+
*
|
|
12
|
+
* Binary resolution: `AGENT_VIDEO_BIN` env override, then the exact-pinned
|
|
13
|
+
* `@agent-video/<platform>` package installed via this package's dependency
|
|
14
|
+
* on `@agent-video/cli`. No PATH fallback by design.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { spawn } from "node:child_process";
|
|
18
|
+
import { createRequire } from "node:module";
|
|
19
|
+
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
|
20
|
+
import { runVideoCall, VideoServer } from "../lib/mcp.ts";
|
|
21
|
+
import {
|
|
22
|
+
CONFIRM_PARAMETERS,
|
|
23
|
+
CONFIRM_TOOL_DESCRIPTION,
|
|
24
|
+
PARAMETERS,
|
|
25
|
+
PROMPT_SNIPPET,
|
|
26
|
+
TOOL_DESCRIPTION,
|
|
27
|
+
} from "../lib/schema.ts";
|
|
28
|
+
|
|
29
|
+
const require = createRequire(import.meta.url);
|
|
30
|
+
|
|
31
|
+
const RELEASES_URL = "https://github.com/doggo-devs/agent-video/releases";
|
|
32
|
+
|
|
33
|
+
function resolveBinary(): string {
|
|
34
|
+
const override = process.env["AGENT_VIDEO_BIN"];
|
|
35
|
+
if (override) return override;
|
|
36
|
+
const pkg = `@agent-video/${process.platform}-${process.arch}`;
|
|
37
|
+
const bin = process.platform === "win32" ? "agent-video.exe" : "agent-video";
|
|
38
|
+
try {
|
|
39
|
+
return require.resolve(`${pkg}/bin/${bin}`);
|
|
40
|
+
} catch {
|
|
41
|
+
throw new Error(
|
|
42
|
+
`agent-video core binary not found: ${pkg} is not installed. ` +
|
|
43
|
+
`Your platform may be unsupported by the npm distribution, or optional ` +
|
|
44
|
+
`dependencies were skipped at install time. Set AGENT_VIDEO_BIN to a ` +
|
|
45
|
+
`binary from ${RELEASES_URL} as a manual override.`,
|
|
46
|
+
);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export default function (pi: ExtensionAPI) {
|
|
51
|
+
const server = new VideoServer(() =>
|
|
52
|
+
spawn(resolveBinary(), ["mcp"], { stdio: ["pipe", "pipe", "pipe"] }),
|
|
53
|
+
);
|
|
54
|
+
// Both tools ride one server: consent tokens are in-process state there,
|
|
55
|
+
// so a confirm must reach the same spawn that issued the token.
|
|
56
|
+
const register = (
|
|
57
|
+
name: string,
|
|
58
|
+
label: string,
|
|
59
|
+
description: string,
|
|
60
|
+
parameters: typeof PARAMETERS | typeof CONFIRM_PARAMETERS,
|
|
61
|
+
promptSnippet?: string,
|
|
62
|
+
) =>
|
|
63
|
+
pi.registerTool({
|
|
64
|
+
name,
|
|
65
|
+
label,
|
|
66
|
+
description,
|
|
67
|
+
promptSnippet,
|
|
68
|
+
parameters,
|
|
69
|
+
async execute(_toolCallId, params, signal, onUpdate) {
|
|
70
|
+
const content = await runVideoCall(
|
|
71
|
+
server,
|
|
72
|
+
name,
|
|
73
|
+
params,
|
|
74
|
+
(message) => onUpdate?.({ content: [{ type: "text", text: message }] }),
|
|
75
|
+
signal,
|
|
76
|
+
);
|
|
77
|
+
return { content, details: {} };
|
|
78
|
+
},
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
register("video", "Video", TOOL_DESCRIPTION, PARAMETERS, PROMPT_SNIPPET);
|
|
82
|
+
register("video_confirm", "Video consent", CONFIRM_TOOL_DESCRIPTION, CONFIRM_PARAMETERS);
|
|
83
|
+
}
|
package/lib/mcp.ts
ADDED
|
@@ -0,0 +1,403 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The MCP stdio client behind the pi `video` tool.
|
|
3
|
+
*
|
|
4
|
+
* Everything that crosses the process boundary lives here, free of pi and
|
|
5
|
+
* schema imports so tests can drive it with an in-memory fake child. The
|
|
6
|
+
* boundary rule: bytes from the server are validated, never asserted — a
|
|
7
|
+
* malformed response becomes a diagnosable Error naming what was wrong,
|
|
8
|
+
* not a TypeError three frames later. The server's Failed body carries a
|
|
9
|
+
* machine-readable fault `kind` (crates/harness/src/schema.rs); this module
|
|
10
|
+
* retains it on VideoCallError and renders it in the message for the host.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { createInterface } from "node:readline";
|
|
14
|
+
|
|
15
|
+
/** Bound on the initialize handshake. Generous — startup is local process
|
|
16
|
+
* spawn plus one JSON round-trip — so it only fires on a binary that wedges
|
|
17
|
+
* before speaking MCP, which would otherwise block every video call forever
|
|
18
|
+
* (the abort signal covers tool-call waiters, not the shared handshake). */
|
|
19
|
+
const INIT_TIMEOUT_MS = 30_000;
|
|
20
|
+
|
|
21
|
+
// ── wire shapes ──
|
|
22
|
+
|
|
23
|
+
export type Text = { type: "text"; text: string };
|
|
24
|
+
export type Image = { type: "image"; data: string; mimeType: string };
|
|
25
|
+
export type Block = Text | Image;
|
|
26
|
+
|
|
27
|
+
export interface McpCallResult {
|
|
28
|
+
content: Block[];
|
|
29
|
+
isError?: boolean;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** The slice of a spawned MCP server process this client touches —
|
|
33
|
+
* structurally satisfied by `node:child_process.ChildProcess`, small
|
|
34
|
+
* enough for tests to fake in memory. */
|
|
35
|
+
export interface McpChild {
|
|
36
|
+
stdin: NodeJS.WritableStream | null;
|
|
37
|
+
stdout: NodeJS.ReadableStream | null;
|
|
38
|
+
stderr: NodeJS.ReadableStream | null;
|
|
39
|
+
kill(): void;
|
|
40
|
+
once(event: "exit", listener: (code: number | null, signal: string | null) => void): void;
|
|
41
|
+
once(event: "error", listener: (error: Error) => void): void;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
// ── boundary validation ──
|
|
45
|
+
|
|
46
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
47
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** A short, safe rendering of a foreign value for error messages. */
|
|
51
|
+
function render(value: unknown): string {
|
|
52
|
+
const json = JSON.stringify(value);
|
|
53
|
+
if (json === undefined) return typeof value;
|
|
54
|
+
return json.length > 200 ? `${json.slice(0, 200)}…` : json;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** Narrow a raw `tools/call` result from the wire. The server is a separate
|
|
58
|
+
* process: its output is a claim, not a type. Rejections name the offending
|
|
59
|
+
* field so a protocol bug is diagnosable from the error alone. */
|
|
60
|
+
export function parseCallResult(value: unknown): McpCallResult {
|
|
61
|
+
if (!isRecord(value)) {
|
|
62
|
+
throw new Error(`malformed MCP result: expected an object, got ${render(value)}`);
|
|
63
|
+
}
|
|
64
|
+
const content = value["content"];
|
|
65
|
+
if (!Array.isArray(content)) {
|
|
66
|
+
throw new Error(`malformed MCP result: "content" is not an array in ${render(value)}`);
|
|
67
|
+
}
|
|
68
|
+
const blocks: Block[] = [];
|
|
69
|
+
for (const [i, item] of content.entries()) {
|
|
70
|
+
if (!isRecord(item) || typeof item["type"] !== "string") {
|
|
71
|
+
throw new Error(`malformed MCP result: content[${i}] lacks a string "type": ${render(item)}`);
|
|
72
|
+
}
|
|
73
|
+
const string = (key: string): string => {
|
|
74
|
+
const field = item[key];
|
|
75
|
+
if (typeof field !== "string") {
|
|
76
|
+
throw new Error(`malformed MCP result: content[${i}].${key} is not a string`);
|
|
77
|
+
}
|
|
78
|
+
return field;
|
|
79
|
+
};
|
|
80
|
+
switch (item["type"]) {
|
|
81
|
+
case "text":
|
|
82
|
+
blocks.push({ type: "text", text: string("text") });
|
|
83
|
+
break;
|
|
84
|
+
case "image":
|
|
85
|
+
blocks.push({
|
|
86
|
+
type: "image",
|
|
87
|
+
data: string("data"),
|
|
88
|
+
mimeType: item["mimeType"] === undefined ? "image/jpeg" : string("mimeType"),
|
|
89
|
+
});
|
|
90
|
+
break;
|
|
91
|
+
default:
|
|
92
|
+
// Future MCP content types are ignored; known types must be complete.
|
|
93
|
+
break;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
const isError = value["isError"];
|
|
97
|
+
if (isError !== undefined && typeof isError !== "boolean") {
|
|
98
|
+
throw new Error(`malformed MCP result: "isError" is not a boolean: ${render(isError)}`);
|
|
99
|
+
}
|
|
100
|
+
return isError === undefined ? { content: blocks } : { content: blocks, isError };
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** Narrow tool parameters at the host boundary. pi hands them over shaped
|
|
104
|
+
* by the schema, but this module never assumes the host validated; deeper
|
|
105
|
+
* argument checking stays the server's job (single source of truth). */
|
|
106
|
+
export function parseArgs(tool: string, params: unknown): Record<string, unknown> {
|
|
107
|
+
if (!isRecord(params)) {
|
|
108
|
+
throw new Error(`${tool}: tool parameters must be an object, got ${render(params)}`);
|
|
109
|
+
}
|
|
110
|
+
if (typeof params["source"] !== "string") {
|
|
111
|
+
throw new Error(`${tool}: "source" must be a string, got ${render(params["source"])}`);
|
|
112
|
+
}
|
|
113
|
+
return params;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
// ── failure unwrapping ──
|
|
117
|
+
|
|
118
|
+
/** Retry guidance per fault kind, mirroring the contract of
|
|
119
|
+
* `VideoBody::Failed` (crates/harness/src/schema.rs): each kind names who
|
|
120
|
+
* can fix the failure and whether retrying can possibly succeed. */
|
|
121
|
+
const KIND_GUIDANCE = {
|
|
122
|
+
bad_input: "the source or arguments are wrong; retrying the identical call cannot succeed",
|
|
123
|
+
environment: "this installation lacks something — tell the user; retrying cannot help",
|
|
124
|
+
transient: "plausibly a passing condition; retrying the same call may succeed",
|
|
125
|
+
internal: "an agent-video bug; retrying cannot help",
|
|
126
|
+
} as const;
|
|
127
|
+
|
|
128
|
+
/** Failure categories understood by this client. */
|
|
129
|
+
export type FaultKind = keyof typeof KIND_GUIDANCE;
|
|
130
|
+
|
|
131
|
+
function isFaultKind(kind: string): kind is FaultKind {
|
|
132
|
+
return Object.hasOwn(KIND_GUIDANCE, kind);
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/** A tool failure with a machine-readable category and model-readable guidance.
|
|
136
|
+
* Unknown remote categories survive in remoteKind without acquiring retry semantics. */
|
|
137
|
+
export class VideoCallError extends Error {
|
|
138
|
+
readonly kind: FaultKind | undefined;
|
|
139
|
+
|
|
140
|
+
private constructor(
|
|
141
|
+
message: string,
|
|
142
|
+
readonly remoteKind: string | undefined,
|
|
143
|
+
) {
|
|
144
|
+
const kind = remoteKind !== undefined && isFaultKind(remoteKind) ? remoteKind : undefined;
|
|
145
|
+
const guidance = kind === undefined ? undefined : KIND_GUIDANCE[kind];
|
|
146
|
+
super(
|
|
147
|
+
remoteKind === undefined
|
|
148
|
+
? message
|
|
149
|
+
: guidance === undefined
|
|
150
|
+
? `${message} (kind: ${remoteKind})`
|
|
151
|
+
: `${message} (kind: ${remoteKind} — ${guidance})`,
|
|
152
|
+
);
|
|
153
|
+
this.name = "VideoCallError";
|
|
154
|
+
this.kind = kind;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/** Decode a Failed body, retaining plain prose and unknown categories from other servers. */
|
|
158
|
+
static fromText(text: string): VideoCallError {
|
|
159
|
+
let body: unknown;
|
|
160
|
+
try {
|
|
161
|
+
body = JSON.parse(text);
|
|
162
|
+
} catch {
|
|
163
|
+
return new VideoCallError(text, undefined);
|
|
164
|
+
}
|
|
165
|
+
if (!isRecord(body) || typeof body["error"] !== "string") {
|
|
166
|
+
return new VideoCallError(text, undefined);
|
|
167
|
+
}
|
|
168
|
+
return new VideoCallError(
|
|
169
|
+
body["error"],
|
|
170
|
+
typeof body["kind"] === "string" ? body["kind"] : undefined,
|
|
171
|
+
);
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
// ── the client ──
|
|
176
|
+
|
|
177
|
+
interface Pending {
|
|
178
|
+
resolve: (result: unknown) => void;
|
|
179
|
+
reject: (error: Error) => void;
|
|
180
|
+
onProgress?: (message: string) => void;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
/** The one result an aborted call resolves to. The contract: the server
|
|
184
|
+
* keeps working (downloads stay pollable; a late response finds no waiter
|
|
185
|
+
* and is dropped) — cancellation only detaches this call's waiter, never
|
|
186
|
+
* the shared child or other pending calls. Constructed per call: no
|
|
187
|
+
* mutable object is ever shared across calls, so passing `result.content`
|
|
188
|
+
* through to the host stays safe. */
|
|
189
|
+
function cancelled(): McpCallResult {
|
|
190
|
+
return {
|
|
191
|
+
content: [
|
|
192
|
+
{
|
|
193
|
+
type: "text",
|
|
194
|
+
text: "call cancelled — the server keeps working; repeat the same source to pick up the result",
|
|
195
|
+
},
|
|
196
|
+
],
|
|
197
|
+
};
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
/** One MCP server child per pi session, restarted on death at the next
|
|
201
|
+
* call. The child is long-lived on purpose: consent tokens and background
|
|
202
|
+
* downloads are in-process state over there, so every call must ride the
|
|
203
|
+
* same process for tokens to stay valid. */
|
|
204
|
+
export class VideoServer {
|
|
205
|
+
private child: McpChild | undefined;
|
|
206
|
+
private ready: Promise<void> | undefined;
|
|
207
|
+
private pending = new Map<number, Pending>();
|
|
208
|
+
private nextId = 2; // 1 is the initialize request
|
|
209
|
+
|
|
210
|
+
constructor(private readonly spawnServer: () => McpChild) {}
|
|
211
|
+
|
|
212
|
+
/** Call one of the server's tools by name; progress notifications stream
|
|
213
|
+
* to `onProgress`. */
|
|
214
|
+
async call(
|
|
215
|
+
tool: string,
|
|
216
|
+
args: Record<string, unknown>,
|
|
217
|
+
onProgress: ((message: string) => void) | undefined,
|
|
218
|
+
signal: AbortSignal | undefined,
|
|
219
|
+
): Promise<McpCallResult> {
|
|
220
|
+
// A listener added to an already-aborted signal never fires, so aborts
|
|
221
|
+
// that land before the listener below exists are answered by checks:
|
|
222
|
+
// once on entry, once after the shared handshake (which an abort must
|
|
223
|
+
// ride out, not kill — other calls share the spawn). Either way the
|
|
224
|
+
// request is never sent and no pending entry is left behind.
|
|
225
|
+
if (signal?.aborted) return cancelled();
|
|
226
|
+
await this.start();
|
|
227
|
+
if (signal?.aborted) return cancelled();
|
|
228
|
+
const id = this.nextId++;
|
|
229
|
+
const result = new Promise<unknown>((resolve, reject) => {
|
|
230
|
+
this.pending.set(id, { resolve, reject, onProgress });
|
|
231
|
+
this.send({
|
|
232
|
+
jsonrpc: "2.0",
|
|
233
|
+
id,
|
|
234
|
+
method: "tools/call",
|
|
235
|
+
params: { name: tool, arguments: args, _meta: { progressToken: id } },
|
|
236
|
+
});
|
|
237
|
+
});
|
|
238
|
+
// No await separates the post-handshake check from this attachment, so
|
|
239
|
+
// an abort cannot slip between: every abort either returned above or
|
|
240
|
+
// fires here, detaching the waiter and resolving it with the cancel
|
|
241
|
+
// notice — no promise is left dangling.
|
|
242
|
+
signal?.addEventListener(
|
|
243
|
+
"abort",
|
|
244
|
+
() => {
|
|
245
|
+
const waiter = this.pending.get(id);
|
|
246
|
+
if (waiter === undefined) return; // already settled
|
|
247
|
+
this.pending.delete(id);
|
|
248
|
+
waiter.resolve(cancelled());
|
|
249
|
+
},
|
|
250
|
+
{ once: true },
|
|
251
|
+
);
|
|
252
|
+
return parseCallResult(await result);
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
/** Spawn + initialize once; concurrent callers share the handshake. */
|
|
256
|
+
private start(): Promise<void> {
|
|
257
|
+
if (this.child !== undefined && this.ready !== undefined) return this.ready;
|
|
258
|
+
const child = this.spawnServer();
|
|
259
|
+
this.child = child;
|
|
260
|
+
this.wireStdout(child);
|
|
261
|
+
// The server never logs to stderr, but crashes do: linker errors,
|
|
262
|
+
// panic backtraces, signal messages. Keep a bounded tail so the exit
|
|
263
|
+
// error can say why instead of just that.
|
|
264
|
+
let stderrTail = "";
|
|
265
|
+
child.stderr?.on("data", (chunk: Buffer) => {
|
|
266
|
+
stderrTail = (stderrTail + chunk.toString("utf8")).slice(-4096);
|
|
267
|
+
});
|
|
268
|
+
child.stderr?.on("error", () => {});
|
|
269
|
+
child.once("exit", (code, signal) => {
|
|
270
|
+
if (this.child !== child) return; // superseded; a respawn owns the state now
|
|
271
|
+
const tail = stderrTail.trim();
|
|
272
|
+
const why = tail === "" ? "" : `\nstderr tail:\n${tail}`;
|
|
273
|
+
this.fail(new Error(`agent-video mcp exited (code ${code}, signal ${signal})${why}`));
|
|
274
|
+
});
|
|
275
|
+
child.once("error", (e) => {
|
|
276
|
+
if (this.child !== child) return;
|
|
277
|
+
this.fail(new Error(`agent-video mcp failed: ${e.message}`));
|
|
278
|
+
});
|
|
279
|
+
// A write into the window between server death and the 'exit' event
|
|
280
|
+
// surfaces as a stream error; without a listener Node throws it as an
|
|
281
|
+
// uncaught exception into the host. The 'exit' handler owns recovery.
|
|
282
|
+
child.stdin?.on("error", () => {});
|
|
283
|
+
child.stdout?.on("error", () => {});
|
|
284
|
+
|
|
285
|
+
this.ready = new Promise<void>((resolve, reject) => {
|
|
286
|
+
// A binary that hangs at startup must not pin the first call (and
|
|
287
|
+
// every call behind it) forever: kill it and fail the handshake;
|
|
288
|
+
// `fail` clears the state, so the next call respawns cleanly.
|
|
289
|
+
const deadline = setTimeout(() => {
|
|
290
|
+
child.kill();
|
|
291
|
+
this.fail(
|
|
292
|
+
new Error(
|
|
293
|
+
`agent-video mcp did not answer initialize within ${INIT_TIMEOUT_MS / 1000}s — killed; the next call restarts it`,
|
|
294
|
+
),
|
|
295
|
+
);
|
|
296
|
+
}, INIT_TIMEOUT_MS);
|
|
297
|
+
this.pending.set(1, {
|
|
298
|
+
resolve: () => {
|
|
299
|
+
clearTimeout(deadline);
|
|
300
|
+
this.send({ jsonrpc: "2.0", method: "notifications/initialized" });
|
|
301
|
+
resolve();
|
|
302
|
+
},
|
|
303
|
+
reject: (error) => {
|
|
304
|
+
clearTimeout(deadline);
|
|
305
|
+
reject(error);
|
|
306
|
+
},
|
|
307
|
+
});
|
|
308
|
+
this.send({
|
|
309
|
+
jsonrpc: "2.0",
|
|
310
|
+
id: 1,
|
|
311
|
+
method: "initialize",
|
|
312
|
+
params: {
|
|
313
|
+
protocolVersion: "2024-11-05",
|
|
314
|
+
capabilities: {},
|
|
315
|
+
clientInfo: { name: "pi-agent-video", version: "0" },
|
|
316
|
+
},
|
|
317
|
+
});
|
|
318
|
+
});
|
|
319
|
+
return this.ready;
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
/** Reject all in-flight calls and forget the child; next call respawns. */
|
|
323
|
+
private fail(error: Error): void {
|
|
324
|
+
for (const p of this.pending.values()) p.reject(error);
|
|
325
|
+
this.pending.clear();
|
|
326
|
+
this.child = undefined;
|
|
327
|
+
this.ready = undefined;
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
private send(message: unknown): void {
|
|
331
|
+
this.child?.stdin?.write(`${JSON.stringify(message)}\n`);
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
private wireStdout(child: McpChild): void {
|
|
335
|
+
if (child.stdout === null) return;
|
|
336
|
+
const lines = createInterface({ input: child.stdout, terminal: false, crlfDelay: Infinity });
|
|
337
|
+
lines.on("line", (line) => {
|
|
338
|
+
if (this.child === child && line.trim() !== "") this.dispatch(line);
|
|
339
|
+
});
|
|
340
|
+
const close = () => lines.close();
|
|
341
|
+
lines.on("error", close);
|
|
342
|
+
child.once("exit", close);
|
|
343
|
+
child.once("error", close);
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
private dispatch(line: string): void {
|
|
347
|
+
let message: unknown;
|
|
348
|
+
try {
|
|
349
|
+
message = JSON.parse(line);
|
|
350
|
+
} catch {
|
|
351
|
+
return; // not ours to understand; skip
|
|
352
|
+
}
|
|
353
|
+
if (!isRecord(message)) return;
|
|
354
|
+
if (message["method"] === "notifications/progress") {
|
|
355
|
+
const params = message["params"];
|
|
356
|
+
if (!isRecord(params)) return;
|
|
357
|
+
const token = params["progressToken"];
|
|
358
|
+
const text = params["message"];
|
|
359
|
+
if (typeof token === "number" && typeof text === "string") {
|
|
360
|
+
this.pending.get(token)?.onProgress?.(text);
|
|
361
|
+
}
|
|
362
|
+
return;
|
|
363
|
+
}
|
|
364
|
+
const id = message["id"];
|
|
365
|
+
if (typeof id !== "number") return;
|
|
366
|
+
const waiter = this.pending.get(id);
|
|
367
|
+
if (waiter === undefined) return; // cancelled or unknown; drop
|
|
368
|
+
this.pending.delete(id);
|
|
369
|
+
const error = message["error"];
|
|
370
|
+
if (error !== undefined) {
|
|
371
|
+
const detail =
|
|
372
|
+
isRecord(error) && typeof error["message"] === "string"
|
|
373
|
+
? error["message"]
|
|
374
|
+
: `MCP request failed: ${render(error)}`;
|
|
375
|
+
waiter.reject(new Error(detail));
|
|
376
|
+
} else {
|
|
377
|
+
// Raw and unvalidated on purpose: the initialize waiter ignores
|
|
378
|
+
// its value, and `call` narrows through parseCallResult.
|
|
379
|
+
waiter.resolve(message["result"]);
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
/** One tool call, unwrapped the way pi's tool contract expects: content
|
|
385
|
+
* blocks on success, a thrown Error on failure — with the Failed body's
|
|
386
|
+
* fault kind retained on the error and rendered in its message for the model. */
|
|
387
|
+
export async function runVideoCall(
|
|
388
|
+
server: VideoServer,
|
|
389
|
+
tool: string,
|
|
390
|
+
params: unknown,
|
|
391
|
+
onProgress: ((message: string) => void) | undefined,
|
|
392
|
+
signal: AbortSignal | undefined,
|
|
393
|
+
): Promise<Block[]> {
|
|
394
|
+
const result = await server.call(tool, parseArgs(tool, params), onProgress, signal);
|
|
395
|
+
const content = result.content;
|
|
396
|
+
if (result.isError === true) {
|
|
397
|
+
// pi marks failure by throw; unwrap the Failed body so the model
|
|
398
|
+
// sees the message and the fault kind, not an envelope.
|
|
399
|
+
const text = content.find((b): b is Text => b.type === "text")?.text ?? `${tool} call failed`;
|
|
400
|
+
throw VideoCallError.fromText(text);
|
|
401
|
+
}
|
|
402
|
+
return content;
|
|
403
|
+
}
|
package/lib/schema.ts
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Schema data for the pi `video` tool, derived from the Rust tool schema.
|
|
3
|
+
* Generated by scripts/gen-surfaces.ts — do not edit.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { StringEnum } from '@earendil-works/pi-ai'
|
|
7
|
+
import { Type } from 'typebox'
|
|
8
|
+
|
|
9
|
+
export const TOOL_DESCRIPTION = "Watch a video (local path or URL): `want` picks the view — grids of timestamped frames (default), the spoken transcript, or probe metadata alone. `from`/`until` window any of them. Local files render on the spot; a URL answers with download options to confirm first."
|
|
10
|
+
|
|
11
|
+
export const PROMPT_SNIPPET = 'Watch a video (local path or URL) as timestamped frame grids'
|
|
12
|
+
|
|
13
|
+
export const PARAMETERS = Type.Object(
|
|
14
|
+
{
|
|
15
|
+
from: Type.Optional(Type.Union([Type.String(), Type.Number()], { description: "Window start: seconds (83) or timestamp (\"1:23\", \"1:02:03\"). Omit to start from the beginning." })),
|
|
16
|
+
look: Type.Optional(StringEnum(["overview","detailed","read_text","track_motion"] as const, { description: "How to look: overview (fast triage), detailed (miss nothing), read_text (few large frames to read on-screen text/UI), track_motion (dense small frames for fast action). Subtitle text always rides along in metadata regardless; this only affects frame size/density. Omit for a balanced default." })),
|
|
17
|
+
raw: Type.Optional(Type.String({ description: "Expert escape hatch: verbatim agent-video core CLI arguments starting with the subcommand, e.g. \"process --strategy keyframes --base-fps 0.2\". Mutually exclusive with every other argument. Input/output plumbing is injected by the server." })),
|
|
18
|
+
source: Type.String({ description: "Local video file path or URL. Call with only `source` first; the response tells you what to do next." }),
|
|
19
|
+
strategy: Type.Optional(StringEnum(["adaptive","keyframes","uniform"] as const, { description: "Which moments become frames: adaptive (default: keyframes plus fill in sparse stretches, right for most footage), keyframes (cheapest scan, frames land on cuts — triage for edited footage, blind to changes inside static shots), uniform (evenly spaced in time — use when keyframes mislead: screen recordings, slides, long static takes). Orthogonal to look, which sets frame size/density." })),
|
|
20
|
+
until: Type.Optional(Type.Union([Type.String(), Type.Number()], { description: "Window end, same formats. Omit to go to the end of the video." })),
|
|
21
|
+
want: Type.Optional(StringEnum(["grids","transcript","plan"] as const, { description: "Which view to return over the window: grids (default — frame grids you read directly), transcript (the spoken text as \"MM:SS text\" lines; whisper runs once and the result then rides along with every grid), plan (probe metadata and follow-up calls, decoding nothing). For a long video, transcript first is the cheapest way to find what is worth looking at." })),
|
|
22
|
+
},
|
|
23
|
+
{ additionalProperties: false }
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
export const CONFIRM_TOOL_DESCRIPTION = "Authorize one pending action a `video` response asked for. Pass the source it concerned and one token copied from that response's `tokens`."
|
|
27
|
+
|
|
28
|
+
export const CONFIRM_PARAMETERS = Type.Object(
|
|
29
|
+
{
|
|
30
|
+
source: Type.String({ description: "The exact source the pending action concerned." }),
|
|
31
|
+
token: Type.String({ description: "One token copied verbatim from a `confirm_required` response. Tokens stay valid until a newer plan for the same source replaces them." }),
|
|
32
|
+
},
|
|
33
|
+
{ additionalProperties: false }
|
|
34
|
+
)
|
package/package.json
CHANGED
|
@@ -1,6 +1,23 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@agent-video/pi",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.1.1",
|
|
4
|
+
"description": "Watch a video (local path or URL): `want` picks the view — grids of timestamped frames (default), the spoken transcript, or probe metadata alone. `from`/`until` window any of them. Local files render on the spot; a URL answers with download options to confirm first.",
|
|
5
|
+
"license": "Apache-2.0",
|
|
6
|
+
"repository": {
|
|
7
|
+
"type": "git",
|
|
8
|
+
"url": "git+https://github.com/doggo-devs/agent-video.git",
|
|
9
|
+
"directory": "surfaces/pi"
|
|
10
|
+
},
|
|
11
|
+
"dependencies": {
|
|
12
|
+
"@agent-video/cli": "0.1.1"
|
|
13
|
+
},
|
|
14
|
+
"peerDependencies": {
|
|
15
|
+
"@earendil-works/pi-ai": "*",
|
|
16
|
+
"@earendil-works/pi-coding-agent": "*",
|
|
17
|
+
"typebox": "*"
|
|
18
|
+
},
|
|
19
|
+
"files": [
|
|
20
|
+
"extensions",
|
|
21
|
+
"lib"
|
|
22
|
+
]
|
|
23
|
+
}
|
package/README.md
DELETED