@tandryukha/aidemo 0.8.0 → 0.14.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/record-demo/SKILL.md +5 -2
- package/README.md +16 -4
- package/docs/AUTHORING.md +609 -45
- package/package.json +3 -2
- package/src/anchors.ts +128 -0
- package/src/attention.ts +365 -0
- package/src/caption-render.ts +17 -8
- package/src/captions.ts +70 -6
- package/src/capture.ts +113 -0
- package/src/cards.ts +15 -6
- package/src/cli.ts +580 -22
- package/src/compose.ts +1006 -93
- package/src/cursor-overlay.ts +7 -113
- package/src/cursor.ts +9 -3
- package/src/distribute.ts +37 -4
- package/src/expr.ts +130 -0
- package/src/ffmpeg.ts +156 -0
- package/src/frame.ts +205 -0
- package/src/frames.ts +159 -0
- package/src/golden.ts +27 -0
- package/src/guide.ts +102 -0
- package/src/i18n.ts +7 -4
- package/src/import-trace.ts +629 -0
- package/src/inspect.ts +307 -0
- package/src/lint.ts +601 -0
- package/src/login.ts +143 -0
- package/src/mcp/jobs.ts +4 -1
- package/src/mcp/server.ts +403 -26
- package/src/player.ts +975 -32
- package/src/profile.ts +128 -0
- package/src/project.ts +12 -2
- package/src/recorder.ts +252 -15
- package/src/setup.ts +248 -0
- package/src/starter.ts +244 -0
- package/src/stills.ts +15 -4
- package/src/types.ts +643 -5
- package/src/variants.ts +1 -1
- package/src/voice-local.ts +7 -1
- package/src/voice.ts +60 -1
- package/src/walkthrough.ts +250 -0
- package/src/zoom.ts +85 -39
package/src/mcp/server.ts
CHANGED
|
@@ -6,12 +6,13 @@ import { readFile } from "node:fs/promises";
|
|
|
6
6
|
import { resolve } from "node:path";
|
|
7
7
|
import { ENGINE_ROOT, engineVersion, captionsAutoOffline } from "../config.js";
|
|
8
8
|
import { Project, parseStoryboard } from "../project.js";
|
|
9
|
-
import { StoryboardSchema } from "../types.js";
|
|
9
|
+
import { StoryboardSchema, SeedCookieSchema, type SeedCookie, type Storyboard } from "../types.js";
|
|
10
10
|
import { generateVoice } from "../voice.js";
|
|
11
11
|
import { record, type RecordOptions } from "../recorder.js";
|
|
12
12
|
import {
|
|
13
13
|
buildProbeGolden,
|
|
14
14
|
diffGolden,
|
|
15
|
+
driftFilesForDiff,
|
|
15
16
|
readProbeGolden,
|
|
16
17
|
writeProbeGolden,
|
|
17
18
|
} from "../golden.js";
|
|
@@ -21,9 +22,15 @@ import { compose } from "../compose.js";
|
|
|
21
22
|
import { exportGif } from "../gif.js";
|
|
22
23
|
import { buildEmbed } from "../embed.js";
|
|
23
24
|
import { extractStills, storyboardHasStills } from "../stills.js";
|
|
25
|
+
import { extractFrames } from "../frames.js";
|
|
26
|
+
import { inspectPage } from "../inspect.js";
|
|
27
|
+
import { importTrace, scaffoldImported } from "../import-trace.js";
|
|
28
|
+
import { exportWalkthrough } from "../walkthrough.js";
|
|
29
|
+
import { GUIDE_TOPIC_NAMES, guideHeadings, guidePath as guideFilePath, sliceGuide } from "../guide.js";
|
|
30
|
+
import { lintStoryboard, logLint } from "../lint.js";
|
|
24
31
|
import { localizeStoryboard } from "../i18n.js";
|
|
25
32
|
import { scaffoldDemo, doctorReport, buildFeedback, fileFeedback } from "../distribute.js";
|
|
26
|
-
import { readJson, log, CanceledError, type SceneProgress } from "../util.js";
|
|
33
|
+
import { readJson, writeJson, log, CanceledError, type SceneProgress } from "../util.js";
|
|
27
34
|
import { JobManager, JobBusyError, type Job, type JobKind } from "./jobs.js";
|
|
28
35
|
|
|
29
36
|
/**
|
|
@@ -116,6 +123,16 @@ const DIR_INPUT = z
|
|
|
116
123
|
* placeholders resolve to these values (else the declared default) across all
|
|
117
124
|
* stages of the run.
|
|
118
125
|
*/
|
|
126
|
+
/** Output shape of one lint finding (src/lint.ts LintIssue). */
|
|
127
|
+
const LINT_ISSUE_SHAPE = z.object({
|
|
128
|
+
severity: z.enum(["error", "warn", "info"]),
|
|
129
|
+
code: z.string(),
|
|
130
|
+
scene: z.string().optional(),
|
|
131
|
+
action: z.number().optional(),
|
|
132
|
+
message: z.string(),
|
|
133
|
+
fix: z.string().optional(),
|
|
134
|
+
});
|
|
135
|
+
|
|
119
136
|
const PARAMS_INPUT = z
|
|
120
137
|
.record(z.string(), z.string())
|
|
121
138
|
.optional()
|
|
@@ -148,7 +165,46 @@ const RECORD_INPUT_SHAPE = {
|
|
|
148
165
|
.string()
|
|
149
166
|
.optional()
|
|
150
167
|
.describe("Chrome user-data dir (logged-in profile)"),
|
|
168
|
+
fresh: z
|
|
169
|
+
.boolean()
|
|
170
|
+
.optional()
|
|
171
|
+
.describe(
|
|
172
|
+
"run against a WIPED throwaway profile — use for any demo whose story " +
|
|
173
|
+
"starts at a first-run gate, onboarding, an empty state or a one-shot " +
|
|
174
|
+
"flow, since carried-over cookies/localStorage silently record the " +
|
|
175
|
+
"wrong story. Not for logged-in demos (a fresh profile has no login)."
|
|
176
|
+
),
|
|
151
177
|
capture: z.enum(["playwright", "native", "obs"]).optional(),
|
|
178
|
+
fromScene: z
|
|
179
|
+
.string()
|
|
180
|
+
.optional()
|
|
181
|
+
.describe(
|
|
182
|
+
"resume: keep the previous take's scenes before this scene id (footage + " +
|
|
183
|
+
"timeline, verified unchanged by hash) and record from it — after a " +
|
|
184
|
+
"late-scene failure or a change to the tail of the storyboard"
|
|
185
|
+
),
|
|
186
|
+
storageState: z
|
|
187
|
+
.string()
|
|
188
|
+
.optional()
|
|
189
|
+
.describe(
|
|
190
|
+
"absolute path to a Playwright storageState JSON (cookies + per-origin " +
|
|
191
|
+
"localStorage) to seed into the profile before the first action — for " +
|
|
192
|
+
"cookie-gated sites. Also available as storyboard setup.storageState."
|
|
193
|
+
),
|
|
194
|
+
cookies: z
|
|
195
|
+
.array(SeedCookieSchema)
|
|
196
|
+
.optional()
|
|
197
|
+
.describe(
|
|
198
|
+
"cookies to seed before the first action (name, value, domain[, path, " +
|
|
199
|
+
"secure, httpOnly, sameSite, expires]). Also storyboard setup.cookies."
|
|
200
|
+
),
|
|
201
|
+
profileSeeded: z
|
|
202
|
+
.boolean()
|
|
203
|
+
.optional()
|
|
204
|
+
.describe(
|
|
205
|
+
"the profile is seeded on purpose (login / cookie gate): silence the " +
|
|
206
|
+
"carried-over-state warning. Implied by storageState/cookies."
|
|
207
|
+
),
|
|
152
208
|
params: PARAMS_INPUT,
|
|
153
209
|
};
|
|
154
210
|
|
|
@@ -179,7 +235,7 @@ export function buildMcpServer(): { server: McpServer; jobs: JobManager } {
|
|
|
179
235
|
.catch(() => {});
|
|
180
236
|
};
|
|
181
237
|
|
|
182
|
-
const guidePath =
|
|
238
|
+
const guidePath = guideFilePath();
|
|
183
239
|
|
|
184
240
|
server.registerTool(
|
|
185
241
|
"get_authoring_guide",
|
|
@@ -188,16 +244,51 @@ export function buildMcpServer(): { server: McpServer; jobs: JobManager } {
|
|
|
188
244
|
description:
|
|
189
245
|
"The canonical guide to authoring demos: storyboard schema, action " +
|
|
190
246
|
"vocabulary, demo-director principles, ChatGPT-app recording facts. " +
|
|
191
|
-
"Call this FIRST before authoring or editing a storyboard."
|
|
192
|
-
|
|
193
|
-
|
|
247
|
+
"Call this FIRST before authoring or editing a storyboard. Pass " +
|
|
248
|
+
`\`topic\` (${GUIDE_TOPIC_NAMES.join(" | ")}, or any H2 heading prefix) ` +
|
|
249
|
+
"for one slice instead of the whole ~1000-line guide; start with " +
|
|
250
|
+
"`core`, then fetch `attention`/`polish`/`chatgpt-apps` as needed.",
|
|
251
|
+
inputSchema: {
|
|
252
|
+
topic: z
|
|
253
|
+
.string()
|
|
254
|
+
.optional()
|
|
255
|
+
.describe(
|
|
256
|
+
`Slice: ${GUIDE_TOPIC_NAMES.join(", ")} — or an H2 heading prefix. Omit for the full guide.`
|
|
257
|
+
),
|
|
258
|
+
},
|
|
259
|
+
outputSchema: {
|
|
260
|
+
guide: z.string(),
|
|
261
|
+
engineVersion: z.string(),
|
|
262
|
+
topic: z.string().optional(),
|
|
263
|
+
topics: z.array(z.string()),
|
|
264
|
+
sections: z.array(z.string()),
|
|
265
|
+
},
|
|
194
266
|
annotations: { readOnlyHint: true },
|
|
195
267
|
},
|
|
196
|
-
async () =>
|
|
197
|
-
|
|
198
|
-
|
|
268
|
+
async ({ topic }) => {
|
|
269
|
+
const md = await readFile(guidePath, "utf8");
|
|
270
|
+
const sections = guideHeadings(md);
|
|
271
|
+
if (!topic) {
|
|
272
|
+
return jsonResult({ guide: md, engineVersion: engineVersion(), topics: GUIDE_TOPIC_NAMES, sections });
|
|
273
|
+
}
|
|
274
|
+
const slice = sliceGuide(md, topic);
|
|
275
|
+
if (slice == null) {
|
|
276
|
+
return errorResult({
|
|
277
|
+
message:
|
|
278
|
+
`Unknown guide topic "${topic}". Topics: ${GUIDE_TOPIC_NAMES.join(", ")}; ` +
|
|
279
|
+
`or an H2 heading prefix: ${sections.join(" · ")}`,
|
|
280
|
+
topics: GUIDE_TOPIC_NAMES,
|
|
281
|
+
sections,
|
|
282
|
+
});
|
|
283
|
+
}
|
|
284
|
+
return jsonResult({
|
|
285
|
+
guide: slice,
|
|
199
286
|
engineVersion: engineVersion(),
|
|
200
|
-
|
|
287
|
+
topic,
|
|
288
|
+
topics: GUIDE_TOPIC_NAMES,
|
|
289
|
+
sections,
|
|
290
|
+
});
|
|
291
|
+
}
|
|
201
292
|
);
|
|
202
293
|
|
|
203
294
|
server.registerTool(
|
|
@@ -242,6 +333,7 @@ export function buildMcpServer(): { server: McpServer; jobs: JobManager } {
|
|
|
242
333
|
z.object({ path: z.string(), message: z.string(), code: z.string() })
|
|
243
334
|
),
|
|
244
335
|
warnings: z.array(z.string()),
|
|
336
|
+
lint: z.array(LINT_ISSUE_SHAPE).optional(),
|
|
245
337
|
},
|
|
246
338
|
annotations: { readOnlyHint: true },
|
|
247
339
|
},
|
|
@@ -300,7 +392,78 @@ export function buildMcpServer(): { server: McpServer; jobs: JobManager } {
|
|
|
300
392
|
sceneCount: parsed.storyboard.scenes.length,
|
|
301
393
|
issues: [],
|
|
302
394
|
warnings: parsed.warnings,
|
|
395
|
+
lint: lintStoryboard(parsed.storyboard).issues,
|
|
396
|
+
});
|
|
397
|
+
}
|
|
398
|
+
);
|
|
399
|
+
|
|
400
|
+
server.registerTool(
|
|
401
|
+
"lint_storyboard",
|
|
402
|
+
{
|
|
403
|
+
title: "Lint a storyboard (preflight, no browser)",
|
|
404
|
+
description:
|
|
405
|
+
"Browser-free preflight over a storyboard: a per-scene pacing forecast " +
|
|
406
|
+
"(which scenes compose will mostly freeze-hold or cut because the " +
|
|
407
|
+
"narration and the recorded action don't match), selector/wait " +
|
|
408
|
+
"pitfalls (type+Enter with no wait, :text-is on nested labels, focus " +
|
|
409
|
+
"without a zoom block, anchored waitForChange regexes), and no-op keys. " +
|
|
410
|
+
"Run it after every storyboard edit, before probe/render. Pass exactly " +
|
|
411
|
+
"one of dir / path / json. lang lints a narrations[lang] translation at " +
|
|
412
|
+
"that language's speaking rate.",
|
|
413
|
+
inputSchema: {
|
|
414
|
+
dir: z.string().optional().describe("demo dir → generated/storyboard.json"),
|
|
415
|
+
path: z.string().optional().describe("path to a storyboard .json file"),
|
|
416
|
+
json: z.string().optional().describe("storyboard JSON as a string"),
|
|
417
|
+
lang: z.string().optional(),
|
|
418
|
+
params: PARAMS_INPUT,
|
|
419
|
+
},
|
|
420
|
+
outputSchema: {
|
|
421
|
+
issues: z.array(LINT_ISSUE_SHAPE),
|
|
422
|
+
estimate: z.array(
|
|
423
|
+
z.object({
|
|
424
|
+
id: z.string(),
|
|
425
|
+
words: z.number(),
|
|
426
|
+
narrationMs: z.number(),
|
|
427
|
+
actionMs: z.number(),
|
|
428
|
+
holdPct: z.number(),
|
|
429
|
+
overrunMs: z.number(),
|
|
430
|
+
})
|
|
431
|
+
),
|
|
432
|
+
narrationTotalMs: z.number(),
|
|
433
|
+
wordsPerSec: z.number(),
|
|
434
|
+
},
|
|
435
|
+
annotations: { readOnlyHint: true },
|
|
436
|
+
},
|
|
437
|
+
async (args) => {
|
|
438
|
+
const sources = [args.dir, args.path, args.json].filter((s) => s != null).length;
|
|
439
|
+
if (sources !== 1) {
|
|
440
|
+
return errorResult({ message: "pass exactly one of: dir, path, json" });
|
|
441
|
+
}
|
|
442
|
+
let raw: unknown;
|
|
443
|
+
try {
|
|
444
|
+
raw =
|
|
445
|
+
args.json != null
|
|
446
|
+
? JSON.parse(args.json)
|
|
447
|
+
: await readJson<unknown>(
|
|
448
|
+
args.path != null
|
|
449
|
+
? resolve(args.path)
|
|
450
|
+
: new Project(args.dir as string).storyboardPath
|
|
451
|
+
);
|
|
452
|
+
} catch (err) {
|
|
453
|
+
return errorResult({ message: (err as Error).message });
|
|
454
|
+
}
|
|
455
|
+
const parsed = parseStoryboard(raw, {
|
|
456
|
+
relaxed: true,
|
|
457
|
+
params: args.params,
|
|
458
|
+
strict: args.params != null,
|
|
303
459
|
});
|
|
460
|
+
if (!parsed.ok) {
|
|
461
|
+
return errorResult({
|
|
462
|
+
message: "storyboard fails schema validation — run validate_storyboard",
|
|
463
|
+
issues: parsed.issues,
|
|
464
|
+
});
|
|
465
|
+
}
|
|
466
|
+
return jsonResult({ ...lintStoryboard(parsed.storyboard, { lang: args.lang }) });
|
|
304
467
|
}
|
|
305
468
|
);
|
|
306
469
|
|
|
@@ -310,11 +473,21 @@ export function buildMcpServer(): { server: McpServer; jobs: JobManager } {
|
|
|
310
473
|
title: "Scaffold a new demo",
|
|
311
474
|
description:
|
|
312
475
|
"Create demos/<name>/ with a starter brief + storyboard. dir is the " +
|
|
313
|
-
"repo to scaffold into (absolute path recommended; default: server cwd)."
|
|
476
|
+
"repo to scaffold into (absolute path recommended; default: server cwd). " +
|
|
477
|
+
"With fromUrl the page is inspected first (no LLM): its headings become " +
|
|
478
|
+
"scenes and its unique selectors become the beats + a `_candidates` list — " +
|
|
479
|
+
"you then write the narration and turn the candidate hover into the real click.",
|
|
314
480
|
inputSchema: {
|
|
315
481
|
name: z.string().describe("demo name (creates demos/<name>/)"),
|
|
316
482
|
dir: z.string().optional(),
|
|
317
483
|
force: z.boolean().optional(),
|
|
484
|
+
fromUrl: z.string().optional().describe("draft the storyboard from this live page"),
|
|
485
|
+
headless: z.boolean().optional().describe("fromUrl: run Chrome headless (default true)"),
|
|
486
|
+
profile: z.string().optional().describe("fromUrl: Chrome user-data dir (logged-in pages)"),
|
|
487
|
+
viewport: z
|
|
488
|
+
.object({ width: z.number(), height: z.number() })
|
|
489
|
+
.optional()
|
|
490
|
+
.describe("fromUrl: viewport (default 1280x720)"),
|
|
318
491
|
},
|
|
319
492
|
outputSchema: {
|
|
320
493
|
demoDir: z.string(),
|
|
@@ -326,7 +499,13 @@ export function buildMcpServer(): { server: McpServer; jobs: JobManager } {
|
|
|
326
499
|
const demoDir = await scaffoldDemo(
|
|
327
500
|
args.dir ? resolve(args.dir) : process.cwd(),
|
|
328
501
|
args.name,
|
|
329
|
-
{
|
|
502
|
+
{
|
|
503
|
+
force: args.force,
|
|
504
|
+
fromUrl: args.fromUrl,
|
|
505
|
+
headed: args.headless === false,
|
|
506
|
+
profileDir: args.profile,
|
|
507
|
+
viewport: args.viewport,
|
|
508
|
+
}
|
|
330
509
|
);
|
|
331
510
|
const project = new Project(demoDir);
|
|
332
511
|
return jsonResult({
|
|
@@ -631,13 +810,29 @@ export function buildMcpServer(): { server: McpServer; jobs: JobManager } {
|
|
|
631
810
|
|
|
632
811
|
/** Record options wired to a job's progress fields + abort signal. */
|
|
633
812
|
function recordOpts(
|
|
634
|
-
args: {
|
|
635
|
-
|
|
813
|
+
args: {
|
|
814
|
+
headless?: boolean;
|
|
815
|
+
profile?: string;
|
|
816
|
+
fresh?: boolean;
|
|
817
|
+
capture?: "playwright" | "native" | "obs";
|
|
818
|
+
storageState?: string;
|
|
819
|
+
cookies?: SeedCookie[];
|
|
820
|
+
profileSeeded?: boolean;
|
|
821
|
+
fromScene?: string;
|
|
822
|
+
},
|
|
823
|
+
job: Job,
|
|
824
|
+
reloadStoryboard?: () => Promise<Storyboard>
|
|
636
825
|
): RecordOptions {
|
|
637
826
|
return {
|
|
638
827
|
profileDir: args.profile,
|
|
828
|
+
fresh: args.fresh,
|
|
829
|
+
storageState: args.storageState,
|
|
830
|
+
cookies: args.cookies,
|
|
831
|
+
profileSeeded: args.profileSeeded,
|
|
832
|
+
reloadStoryboard,
|
|
639
833
|
headed: !args.headless,
|
|
640
834
|
capture: args.capture,
|
|
835
|
+
fromScene: args.fromScene,
|
|
641
836
|
signal: job.controller.signal,
|
|
642
837
|
onSceneStart: (sceneId, index, total) => {
|
|
643
838
|
job.currentScene = sceneId;
|
|
@@ -688,14 +883,17 @@ export function buildMcpServer(): { server: McpServer; jobs: JobManager } {
|
|
|
688
883
|
},
|
|
689
884
|
(project, args) => async (job) =>
|
|
690
885
|
jobs.runStage(job, "probe", async () => {
|
|
691
|
-
const
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
886
|
+
const load = () =>
|
|
887
|
+
project.loadStoryboard({
|
|
888
|
+
relaxed: true,
|
|
889
|
+
params: args.params,
|
|
890
|
+
});
|
|
891
|
+
const storyboard = await load();
|
|
892
|
+
logLint(lintStoryboard(storyboard), log);
|
|
695
893
|
const goldenMode = !!(args.golden || args.updateGolden);
|
|
696
894
|
const probeScenes: ProbeGoldenScene[] = [];
|
|
697
895
|
const timeline = await record(project, storyboard, {
|
|
698
|
-
...recordOpts(args, job),
|
|
896
|
+
...recordOpts(args, job, load),
|
|
699
897
|
...(goldenMode ? { probe: probeScenes } : {}),
|
|
700
898
|
});
|
|
701
899
|
const base = {
|
|
@@ -729,13 +927,18 @@ export function buildMcpServer(): { server: McpServer; jobs: JobManager } {
|
|
|
729
927
|
},
|
|
730
928
|
};
|
|
731
929
|
}
|
|
732
|
-
const
|
|
930
|
+
const actual = buildProbeGolden(storyboard, probeScenes);
|
|
931
|
+
const diffs = diffGolden(expected, actual);
|
|
932
|
+
const drift = diffs.length ? await driftFilesForDiff(project, actual, diffs) : [];
|
|
733
933
|
return {
|
|
734
934
|
...base,
|
|
735
935
|
golden: {
|
|
736
936
|
path: project.goldenProbePath,
|
|
737
937
|
match: diffs.length === 0,
|
|
738
938
|
diffs,
|
|
939
|
+
// Nearest-selector suggestions for the actions that flipped
|
|
940
|
+
// (logs/drift-<scene>-<n>.json — read them, fix, re-probe).
|
|
941
|
+
...(drift.length ? { drift } : {}),
|
|
739
942
|
},
|
|
740
943
|
};
|
|
741
944
|
}
|
|
@@ -743,14 +946,60 @@ export function buildMcpServer(): { server: McpServer; jobs: JobManager } {
|
|
|
743
946
|
})
|
|
744
947
|
);
|
|
745
948
|
|
|
949
|
+
server.registerTool(
|
|
950
|
+
"import_trace",
|
|
951
|
+
{
|
|
952
|
+
title: "Draft a storyboard from a Playwright trace or test",
|
|
953
|
+
description:
|
|
954
|
+
"Turn a Playwright trace.zip (context.tracing / `--trace on`) or a *.spec.ts " +
|
|
955
|
+
"test file into demos/<name>/ with a draft storyboard: the run's own actions " +
|
|
956
|
+
"and selectors become scenes (cut at navigations and async payoffs), narration " +
|
|
957
|
+
"is left as placeholders, and `_notes` lists what was approximated. No LLM, " +
|
|
958
|
+
"no browser. Then write the narration and probe.",
|
|
959
|
+
inputSchema: {
|
|
960
|
+
file: z.string().describe("absolute path to trace.zip or the test file"),
|
|
961
|
+
name: z.string().describe("demo name (creates demos/<name>/)"),
|
|
962
|
+
dir: z.string().optional().describe("repo to scaffold into (default: server cwd)"),
|
|
963
|
+
force: z.boolean().optional(),
|
|
964
|
+
},
|
|
965
|
+
outputSchema: {
|
|
966
|
+
demoDir: z.string(),
|
|
967
|
+
storyboardPath: z.string(),
|
|
968
|
+
steps: z.number(),
|
|
969
|
+
scenes: z.number(),
|
|
970
|
+
notes: z.array(z.string()),
|
|
971
|
+
},
|
|
972
|
+
},
|
|
973
|
+
async (args) => {
|
|
974
|
+
try {
|
|
975
|
+
const result = await importTrace(resolve(args.file), args.name);
|
|
976
|
+
const demoDir = await scaffoldImported(args.dir ? resolve(args.dir) : process.cwd(), args.name, result, {
|
|
977
|
+
force: args.force,
|
|
978
|
+
});
|
|
979
|
+
const project = new Project(demoDir);
|
|
980
|
+
return jsonResult({
|
|
981
|
+
demoDir,
|
|
982
|
+
storyboardPath: project.storyboardPath,
|
|
983
|
+
steps: result.steps,
|
|
984
|
+
scenes: (result.storyboard.scenes as unknown[]).length,
|
|
985
|
+
notes: result.notes,
|
|
986
|
+
});
|
|
987
|
+
} catch (err) {
|
|
988
|
+
return errorResult({ message: (err as Error).message });
|
|
989
|
+
}
|
|
990
|
+
}
|
|
991
|
+
);
|
|
992
|
+
|
|
746
993
|
registerJob(
|
|
747
994
|
"record",
|
|
748
995
|
"Drive the storyboard in Chrome and record raw video + timeline.json.",
|
|
749
996
|
RECORD_INPUT_SHAPE,
|
|
750
997
|
(project, args) => async (job) =>
|
|
751
998
|
jobs.runStage(job, "record", async () => {
|
|
752
|
-
const
|
|
753
|
-
const
|
|
999
|
+
const load = () => project.loadStoryboard({ params: args.params });
|
|
1000
|
+
const storyboard = await load();
|
|
1001
|
+
logLint(lintStoryboard(storyboard), log);
|
|
1002
|
+
const timeline = await record(project, storyboard, recordOpts(args, job, load));
|
|
754
1003
|
return {
|
|
755
1004
|
rawVideo: await project.resolveRawVideo(),
|
|
756
1005
|
timeline: project.timelinePath,
|
|
@@ -772,11 +1021,13 @@ export function buildMcpServer(): { server: McpServer; jobs: JobManager } {
|
|
|
772
1021
|
(project, args) => async (job) =>
|
|
773
1022
|
jobs.runStage(job, "render", async () => {
|
|
774
1023
|
const signal = job.controller.signal;
|
|
775
|
-
const
|
|
1024
|
+
const load = () => project.loadStoryboard({ params: args.params });
|
|
1025
|
+
const storyboard = await load();
|
|
776
1026
|
// Language variant (if any): voice/captions/compose run on a lang-scoped
|
|
777
1027
|
// project + localized storyboard; the take is recorded ONCE on the base.
|
|
778
1028
|
const lp = langProject(project, args.lang);
|
|
779
1029
|
const sb = args.lang ? localizeStoryboard(storyboard, args.lang) : storyboard;
|
|
1030
|
+
logLint(lintStoryboard(storyboard, { lang: args.lang }), log);
|
|
780
1031
|
// Each sub-stage below is wrapped in runSubStage: it refreshes its OWN
|
|
781
1032
|
// stable logs/<stage>.log (not just logs/render.log) and updates
|
|
782
1033
|
// job.stage/currentScene/scenesTotal/scenesDone — same fields a
|
|
@@ -788,7 +1039,7 @@ export function buildMcpServer(): { server: McpServer; jobs: JobManager } {
|
|
|
788
1039
|
);
|
|
789
1040
|
throwIfAborted(signal);
|
|
790
1041
|
await jobs.runSubStage(job, "record", () =>
|
|
791
|
-
record(project, storyboard, recordOpts(args, job))
|
|
1042
|
+
record(project, storyboard, recordOpts(args, job, load))
|
|
792
1043
|
);
|
|
793
1044
|
throwIfAborted(signal);
|
|
794
1045
|
await jobs.runSubStage(job, "captions", async () => {
|
|
@@ -800,7 +1051,9 @@ export function buildMcpServer(): { server: McpServer; jobs: JobManager } {
|
|
|
800
1051
|
}
|
|
801
1052
|
});
|
|
802
1053
|
throwIfAborted(signal);
|
|
803
|
-
await jobs.runSubStage(job, "compose", () =>
|
|
1054
|
+
const report = await jobs.runSubStage(job, "compose", () =>
|
|
1055
|
+
compose(lp, sb, sceneProgress(job))
|
|
1056
|
+
);
|
|
804
1057
|
let gifPath: string | undefined;
|
|
805
1058
|
if (args.gif) {
|
|
806
1059
|
gifPath = await jobs.runSubStage(job, "gif", () => exportGif(lp));
|
|
@@ -811,12 +1064,22 @@ export function buildMcpServer(): { server: McpServer; jobs: JobManager } {
|
|
|
811
1064
|
if (storyboardHasStills(storyboard)) {
|
|
812
1065
|
stills = await jobs.runSubStage(job, "stills", () => extractStills(project));
|
|
813
1066
|
}
|
|
1067
|
+
let walkthrough: string | undefined;
|
|
1068
|
+
if (storyboard.output?.walkthrough) {
|
|
1069
|
+
walkthrough = (
|
|
1070
|
+
await jobs.runSubStage(job, "walkthrough", () => exportWalkthrough(lp, sb))
|
|
1071
|
+
).index;
|
|
1072
|
+
}
|
|
814
1073
|
return {
|
|
815
1074
|
output: lp.outputPath,
|
|
816
1075
|
...(gifPath ? { gif: gifPath } : {}),
|
|
817
1076
|
...(stills && stills.length ? { stills } : {}),
|
|
1077
|
+
...(walkthrough ? { walkthrough } : {}),
|
|
818
1078
|
timeline: lp.timelinePath,
|
|
819
1079
|
captionsSrt: lp.captionsSrtPath,
|
|
1080
|
+
report: lp.reportPath,
|
|
1081
|
+
durationMs: report.durationMs,
|
|
1082
|
+
warnings: report.warnings,
|
|
820
1083
|
};
|
|
821
1084
|
})
|
|
822
1085
|
);
|
|
@@ -897,12 +1160,15 @@ export function buildMcpServer(): { server: McpServer; jobs: JobManager } {
|
|
|
897
1160
|
const lp = langProject(project, args.lang);
|
|
898
1161
|
const storyboard = await project.loadStoryboard({ params: args.params });
|
|
899
1162
|
const sb = args.lang ? localizeStoryboard(storyboard, args.lang) : storyboard;
|
|
900
|
-
await compose(lp, sb, sceneProgress(job));
|
|
1163
|
+
const report = await compose(lp, sb, sceneProgress(job));
|
|
901
1164
|
let gifPath: string | undefined;
|
|
902
1165
|
if (args.gif) gifPath = await exportGif(lp);
|
|
903
1166
|
return {
|
|
904
1167
|
output: lp.outputPath,
|
|
905
1168
|
...(gifPath ? { gif: gifPath } : {}),
|
|
1169
|
+
report: lp.reportPath,
|
|
1170
|
+
durationMs: report.durationMs,
|
|
1171
|
+
warnings: report.warnings,
|
|
906
1172
|
};
|
|
907
1173
|
})
|
|
908
1174
|
);
|
|
@@ -944,6 +1210,112 @@ export function buildMcpServer(): { server: McpServer; jobs: JobManager } {
|
|
|
944
1210
|
}))
|
|
945
1211
|
);
|
|
946
1212
|
|
|
1213
|
+
registerJob(
|
|
1214
|
+
"frames",
|
|
1215
|
+
"Dump evenly spaced PNG frames from the final video (or the raw take) into " +
|
|
1216
|
+
"output/frames/ for review — look at them instead of hand-running " +
|
|
1217
|
+
"`ffmpeg -ss`. Needs only the video (no key).",
|
|
1218
|
+
{
|
|
1219
|
+
dir: DIR_INPUT,
|
|
1220
|
+
everySec: z.number().optional().describe("seconds between frames (default 3)"),
|
|
1221
|
+
source: z
|
|
1222
|
+
.enum(["final", "raw", "take"])
|
|
1223
|
+
.optional()
|
|
1224
|
+
.describe(
|
|
1225
|
+
"final = output/final-demo.mp4 (default); raw = the latest raw file; " +
|
|
1226
|
+
"take = the whole recorded take, across the raw files a resumed take is spliced from"
|
|
1227
|
+
),
|
|
1228
|
+
width: z.number().optional().describe("frame width in px, aspect kept (default 640)"),
|
|
1229
|
+
out: z.string().optional().describe("output directory (default <dir>/output/frames)"),
|
|
1230
|
+
},
|
|
1231
|
+
(project, args) => async (job) =>
|
|
1232
|
+
jobs.runStage(job, "frames", async () => {
|
|
1233
|
+
const res = await extractFrames(project, {
|
|
1234
|
+
everySec: args.everySec,
|
|
1235
|
+
source: args.source,
|
|
1236
|
+
width: args.width,
|
|
1237
|
+
outDir: args.out,
|
|
1238
|
+
});
|
|
1239
|
+
return {
|
|
1240
|
+
source: res.source,
|
|
1241
|
+
durationMs: res.durationMs,
|
|
1242
|
+
everySec: res.everySec,
|
|
1243
|
+
frames: res.files,
|
|
1244
|
+
};
|
|
1245
|
+
})
|
|
1246
|
+
);
|
|
1247
|
+
|
|
1248
|
+
registerJob(
|
|
1249
|
+
"walkthrough",
|
|
1250
|
+
"Export output/walkthrough/ from the final video: index.html (one card per " +
|
|
1251
|
+
"scene — payoff frame, title, narration, jump-to-time; ← → keyboard nav), " +
|
|
1252
|
+
"guide.md (README/SOP-ready), per-scene PNGs, SRT/VTT and a JSON manifest. " +
|
|
1253
|
+
"Needs only the rendered video + report.json (no key, no browser).",
|
|
1254
|
+
{
|
|
1255
|
+
dir: DIR_INPUT,
|
|
1256
|
+
lang: z.string().optional().describe("language variant (final-demo.<lang>.mp4)"),
|
|
1257
|
+
width: z.number().optional().describe("frame width in px (default 960)"),
|
|
1258
|
+
out: z.string().optional().describe("output directory (default <dir>/output/walkthrough)"),
|
|
1259
|
+
},
|
|
1260
|
+
(project, args) => async (job) =>
|
|
1261
|
+
jobs.runStage(job, "walkthrough", async () => {
|
|
1262
|
+
const lp = args.lang ? new Project(project.dir, args.lang) : project;
|
|
1263
|
+
const storyboard = await project.loadStoryboard({ relaxed: true });
|
|
1264
|
+
const sb = args.lang ? localizeStoryboard(storyboard, args.lang) : storyboard;
|
|
1265
|
+
const res = await exportWalkthrough(lp, sb, { width: args.width, outDir: args.out });
|
|
1266
|
+
return {
|
|
1267
|
+
dir: res.dir,
|
|
1268
|
+
index: res.index,
|
|
1269
|
+
guide: res.guide,
|
|
1270
|
+
manifest: res.manifest,
|
|
1271
|
+
scenes: res.scenes.length,
|
|
1272
|
+
};
|
|
1273
|
+
})
|
|
1274
|
+
);
|
|
1275
|
+
|
|
1276
|
+
registerJob(
|
|
1277
|
+
"inspect",
|
|
1278
|
+
"Open a URL in the recording profile (logged-in state included) and list " +
|
|
1279
|
+
"every visible interactive element with UNIQUE selectors ranked " +
|
|
1280
|
+
"data-testid → id → aria-label → role/text → name/placeholder → class → " +
|
|
1281
|
+
"path, plus headings and iframes. Use it BEFORE writing targets — no " +
|
|
1282
|
+
"selector guessing, no wasted probe. Writes logs/inspect-<n>.json and a " +
|
|
1283
|
+
"screenshot in the demo dir.",
|
|
1284
|
+
{
|
|
1285
|
+
dir: DIR_INPUT,
|
|
1286
|
+
url: z.string().describe("page to inspect (absolute URL)"),
|
|
1287
|
+
headless: z.boolean().optional().describe("default true"),
|
|
1288
|
+
profile: z.string().optional().describe("Chrome user-data dir (default: the recording profile)"),
|
|
1289
|
+
limit: z.number().int().min(5).max(400).optional().describe("max elements (default 80)"),
|
|
1290
|
+
viewport: z
|
|
1291
|
+
.object({ width: z.number().int(), height: z.number().int() })
|
|
1292
|
+
.optional()
|
|
1293
|
+
.describe("default 1280x720 (use the storyboard's video size)"),
|
|
1294
|
+
frames: z
|
|
1295
|
+
.record(z.string(), z.string())
|
|
1296
|
+
.optional()
|
|
1297
|
+
.describe("iframe selectors to scan too, as in the storyboard `frames` block"),
|
|
1298
|
+
},
|
|
1299
|
+
(project, args) => async (job) =>
|
|
1300
|
+
jobs.runStage(job, "inspect", async () => {
|
|
1301
|
+
const stamp = new Date().toISOString().replace(/[:.]/g, "-");
|
|
1302
|
+
const png = project.p("logs", `inspect-${stamp}.png`);
|
|
1303
|
+
const res = await inspectPage({
|
|
1304
|
+
url: args.url,
|
|
1305
|
+
headless: args.headless !== false,
|
|
1306
|
+
profileDir: args.profile,
|
|
1307
|
+
limit: args.limit,
|
|
1308
|
+
viewport: args.viewport,
|
|
1309
|
+
frames: args.frames,
|
|
1310
|
+
screenshotPath: png,
|
|
1311
|
+
});
|
|
1312
|
+
const file = project.p("logs", `inspect-${stamp}.json`);
|
|
1313
|
+
await writeJson(file, res);
|
|
1314
|
+
log(`inspect: ${res.elements.length} element(s), ${res.headings.length} heading(s) → ${file}`);
|
|
1315
|
+
return { ...res, file };
|
|
1316
|
+
})
|
|
1317
|
+
);
|
|
1318
|
+
|
|
947
1319
|
server.registerResource(
|
|
948
1320
|
"authoring-guide",
|
|
949
1321
|
"aidemo://authoring-guide",
|
|
@@ -1004,6 +1376,11 @@ export async function runMcpServer(): Promise<void> {
|
|
|
1004
1376
|
// and break the next record.
|
|
1005
1377
|
void (async () => {
|
|
1006
1378
|
if (jobs.abortActive()) await jobs.waitForIdle(10_000);
|
|
1379
|
+
// Grace for native addons: after a local-TTS job, onnxruntime's worker
|
|
1380
|
+
// threads wind down asynchronously, and process.exit mid-teardown
|
|
1381
|
+
// aborts with a shutdown-time "libc++abi: mutex lock failed" on macOS
|
|
1382
|
+
// (issue #19). Half a second is invisible here — the client is gone.
|
|
1383
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
1007
1384
|
process.exit(0);
|
|
1008
1385
|
})();
|
|
1009
1386
|
};
|