@tandryukha/aidemo 0.8.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/record-demo/SKILL.md +5 -2
- package/README.md +16 -4
- package/docs/AUTHORING.md +734 -48
- package/package.json +3 -2
- package/src/anchors.ts +128 -0
- package/src/attention.ts +365 -0
- package/src/caption-render.ts +17 -8
- package/src/captions.ts +244 -36
- package/src/capture.ts +113 -0
- package/src/cards.ts +15 -6
- package/src/cli.ts +711 -26
- package/src/compose.ts +1318 -101
- package/src/cursor-overlay.ts +7 -113
- package/src/cursor.ts +19 -4
- package/src/distribute.ts +37 -4
- package/src/expr.ts +130 -0
- package/src/ffmpeg.ts +224 -0
- package/src/frame.ts +205 -0
- package/src/frames.ts +208 -0
- package/src/golden.ts +27 -0
- package/src/guide.ts +102 -0
- package/src/i18n.ts +7 -4
- package/src/import-trace.ts +640 -0
- package/src/inspect.ts +316 -0
- package/src/lint.ts +673 -0
- package/src/login.ts +143 -0
- package/src/mcp/jobs.ts +5 -1
- package/src/mcp/server.ts +468 -27
- package/src/player.ts +1286 -41
- package/src/profile.ts +128 -0
- package/src/project.ts +12 -2
- package/src/qa.ts +295 -0
- package/src/recorder.ts +316 -21
- package/src/safe-regex.ts +36 -0
- package/src/setup.ts +248 -0
- package/src/starter.ts +244 -0
- package/src/stills.ts +15 -4
- package/src/types.ts +796 -22
- package/src/variants.ts +1 -1
- package/src/voice-local.ts +7 -1
- package/src/voice.ts +60 -1
- package/src/walkthrough.ts +250 -0
- package/src/zoom.ts +85 -39
package/src/mcp/server.ts
CHANGED
|
@@ -4,14 +4,15 @@ import { z } from "zod";
|
|
|
4
4
|
import { zodToJsonSchema } from "zod-to-json-schema";
|
|
5
5
|
import { readFile } from "node:fs/promises";
|
|
6
6
|
import { resolve } from "node:path";
|
|
7
|
-
import { ENGINE_ROOT, engineVersion, captionsAutoOffline } from "../config.js";
|
|
7
|
+
import { ENGINE_ROOT, engineVersion, captionsAutoOffline, chromeProfileDir } from "../config.js";
|
|
8
8
|
import { Project, parseStoryboard } from "../project.js";
|
|
9
|
-
import { StoryboardSchema } from "../types.js";
|
|
9
|
+
import { StoryboardSchema, SeedCookieSchema, type SeedCookie, type Storyboard } from "../types.js";
|
|
10
10
|
import { generateVoice } from "../voice.js";
|
|
11
11
|
import { record, type RecordOptions } from "../recorder.js";
|
|
12
12
|
import {
|
|
13
13
|
buildProbeGolden,
|
|
14
14
|
diffGolden,
|
|
15
|
+
driftFilesForDiff,
|
|
15
16
|
readProbeGolden,
|
|
16
17
|
writeProbeGolden,
|
|
17
18
|
} from "../golden.js";
|
|
@@ -21,9 +22,17 @@ import { compose } from "../compose.js";
|
|
|
21
22
|
import { exportGif } from "../gif.js";
|
|
22
23
|
import { buildEmbed } from "../embed.js";
|
|
23
24
|
import { extractStills, storyboardHasStills } from "../stills.js";
|
|
25
|
+
import { extractFrames } from "../frames.js";
|
|
26
|
+
import { runQa, logQa } from "../qa.js";
|
|
27
|
+
import { inspectPage } from "../inspect.js";
|
|
28
|
+
import { collectSeeds, runPreflight } from "../setup.js";
|
|
29
|
+
import { importTrace, scaffoldImported } from "../import-trace.js";
|
|
30
|
+
import { exportWalkthrough } from "../walkthrough.js";
|
|
31
|
+
import { GUIDE_TOPIC_NAMES, guideHeadings, guidePath as guideFilePath, sliceGuide } from "../guide.js";
|
|
32
|
+
import { lintStoryboard, logLint } from "../lint.js";
|
|
24
33
|
import { localizeStoryboard } from "../i18n.js";
|
|
25
34
|
import { scaffoldDemo, doctorReport, buildFeedback, fileFeedback } from "../distribute.js";
|
|
26
|
-
import { readJson, log, CanceledError, type SceneProgress } from "../util.js";
|
|
35
|
+
import { readJson, writeJson, log, CanceledError, type SceneProgress } from "../util.js";
|
|
27
36
|
import { JobManager, JobBusyError, type Job, type JobKind } from "./jobs.js";
|
|
28
37
|
|
|
29
38
|
/**
|
|
@@ -116,6 +125,16 @@ const DIR_INPUT = z
|
|
|
116
125
|
* placeholders resolve to these values (else the declared default) across all
|
|
117
126
|
* stages of the run.
|
|
118
127
|
*/
|
|
128
|
+
/** Output shape of one lint finding (src/lint.ts LintIssue). */
|
|
129
|
+
const LINT_ISSUE_SHAPE = z.object({
|
|
130
|
+
severity: z.enum(["error", "warn", "info"]),
|
|
131
|
+
code: z.string(),
|
|
132
|
+
scene: z.string().optional(),
|
|
133
|
+
action: z.number().optional(),
|
|
134
|
+
message: z.string(),
|
|
135
|
+
fix: z.string().optional(),
|
|
136
|
+
});
|
|
137
|
+
|
|
119
138
|
const PARAMS_INPUT = z
|
|
120
139
|
.record(z.string(), z.string())
|
|
121
140
|
.optional()
|
|
@@ -148,7 +167,46 @@ const RECORD_INPUT_SHAPE = {
|
|
|
148
167
|
.string()
|
|
149
168
|
.optional()
|
|
150
169
|
.describe("Chrome user-data dir (logged-in profile)"),
|
|
170
|
+
fresh: z
|
|
171
|
+
.boolean()
|
|
172
|
+
.optional()
|
|
173
|
+
.describe(
|
|
174
|
+
"run against a WIPED throwaway profile — use for any demo whose story " +
|
|
175
|
+
"starts at a first-run gate, onboarding, an empty state or a one-shot " +
|
|
176
|
+
"flow, since carried-over cookies/localStorage silently record the " +
|
|
177
|
+
"wrong story. Not for logged-in demos (a fresh profile has no login)."
|
|
178
|
+
),
|
|
151
179
|
capture: z.enum(["playwright", "native", "obs"]).optional(),
|
|
180
|
+
fromScene: z
|
|
181
|
+
.string()
|
|
182
|
+
.optional()
|
|
183
|
+
.describe(
|
|
184
|
+
"resume: keep the previous take's scenes before this scene id (footage + " +
|
|
185
|
+
"timeline, verified unchanged by hash) and record from it — after a " +
|
|
186
|
+
"late-scene failure or a change to the tail of the storyboard"
|
|
187
|
+
),
|
|
188
|
+
storageState: z
|
|
189
|
+
.string()
|
|
190
|
+
.optional()
|
|
191
|
+
.describe(
|
|
192
|
+
"absolute path to a Playwright storageState JSON (cookies + per-origin " +
|
|
193
|
+
"localStorage) to seed into the profile before the first action — for " +
|
|
194
|
+
"cookie-gated sites. Also available as storyboard setup.storageState."
|
|
195
|
+
),
|
|
196
|
+
cookies: z
|
|
197
|
+
.array(SeedCookieSchema)
|
|
198
|
+
.optional()
|
|
199
|
+
.describe(
|
|
200
|
+
"cookies to seed before the first action (name, value, domain[, path, " +
|
|
201
|
+
"secure, httpOnly, sameSite, expires]). Also storyboard setup.cookies."
|
|
202
|
+
),
|
|
203
|
+
profileSeeded: z
|
|
204
|
+
.boolean()
|
|
205
|
+
.optional()
|
|
206
|
+
.describe(
|
|
207
|
+
"the profile is seeded on purpose (login / cookie gate): silence the " +
|
|
208
|
+
"carried-over-state warning. Implied by storageState/cookies."
|
|
209
|
+
),
|
|
152
210
|
params: PARAMS_INPUT,
|
|
153
211
|
};
|
|
154
212
|
|
|
@@ -179,7 +237,7 @@ export function buildMcpServer(): { server: McpServer; jobs: JobManager } {
|
|
|
179
237
|
.catch(() => {});
|
|
180
238
|
};
|
|
181
239
|
|
|
182
|
-
const guidePath =
|
|
240
|
+
const guidePath = guideFilePath();
|
|
183
241
|
|
|
184
242
|
server.registerTool(
|
|
185
243
|
"get_authoring_guide",
|
|
@@ -188,16 +246,51 @@ export function buildMcpServer(): { server: McpServer; jobs: JobManager } {
|
|
|
188
246
|
description:
|
|
189
247
|
"The canonical guide to authoring demos: storyboard schema, action " +
|
|
190
248
|
"vocabulary, demo-director principles, ChatGPT-app recording facts. " +
|
|
191
|
-
"Call this FIRST before authoring or editing a storyboard."
|
|
192
|
-
|
|
193
|
-
|
|
249
|
+
"Call this FIRST before authoring or editing a storyboard. Pass " +
|
|
250
|
+
`\`topic\` (${GUIDE_TOPIC_NAMES.join(" | ")}, or any H2 heading prefix) ` +
|
|
251
|
+
"for one slice instead of the whole ~1000-line guide; start with " +
|
|
252
|
+
"`core`, then fetch `attention`/`polish`/`chatgpt-apps` as needed.",
|
|
253
|
+
inputSchema: {
|
|
254
|
+
topic: z
|
|
255
|
+
.string()
|
|
256
|
+
.optional()
|
|
257
|
+
.describe(
|
|
258
|
+
`Slice: ${GUIDE_TOPIC_NAMES.join(", ")} — or an H2 heading prefix. Omit for the full guide.`
|
|
259
|
+
),
|
|
260
|
+
},
|
|
261
|
+
outputSchema: {
|
|
262
|
+
guide: z.string(),
|
|
263
|
+
engineVersion: z.string(),
|
|
264
|
+
topic: z.string().optional(),
|
|
265
|
+
topics: z.array(z.string()),
|
|
266
|
+
sections: z.array(z.string()),
|
|
267
|
+
},
|
|
194
268
|
annotations: { readOnlyHint: true },
|
|
195
269
|
},
|
|
196
|
-
async () =>
|
|
197
|
-
|
|
198
|
-
|
|
270
|
+
async ({ topic }) => {
|
|
271
|
+
const md = await readFile(guidePath, "utf8");
|
|
272
|
+
const sections = guideHeadings(md);
|
|
273
|
+
if (!topic) {
|
|
274
|
+
return jsonResult({ guide: md, engineVersion: engineVersion(), topics: GUIDE_TOPIC_NAMES, sections });
|
|
275
|
+
}
|
|
276
|
+
const slice = sliceGuide(md, topic);
|
|
277
|
+
if (slice == null) {
|
|
278
|
+
return errorResult({
|
|
279
|
+
message:
|
|
280
|
+
`Unknown guide topic "${topic}". Topics: ${GUIDE_TOPIC_NAMES.join(", ")}; ` +
|
|
281
|
+
`or an H2 heading prefix: ${sections.join(" · ")}`,
|
|
282
|
+
topics: GUIDE_TOPIC_NAMES,
|
|
283
|
+
sections,
|
|
284
|
+
});
|
|
285
|
+
}
|
|
286
|
+
return jsonResult({
|
|
287
|
+
guide: slice,
|
|
199
288
|
engineVersion: engineVersion(),
|
|
200
|
-
|
|
289
|
+
topic,
|
|
290
|
+
topics: GUIDE_TOPIC_NAMES,
|
|
291
|
+
sections,
|
|
292
|
+
});
|
|
293
|
+
}
|
|
201
294
|
);
|
|
202
295
|
|
|
203
296
|
server.registerTool(
|
|
@@ -242,6 +335,7 @@ export function buildMcpServer(): { server: McpServer; jobs: JobManager } {
|
|
|
242
335
|
z.object({ path: z.string(), message: z.string(), code: z.string() })
|
|
243
336
|
),
|
|
244
337
|
warnings: z.array(z.string()),
|
|
338
|
+
lint: z.array(LINT_ISSUE_SHAPE).optional(),
|
|
245
339
|
},
|
|
246
340
|
annotations: { readOnlyHint: true },
|
|
247
341
|
},
|
|
@@ -300,21 +394,102 @@ export function buildMcpServer(): { server: McpServer; jobs: JobManager } {
|
|
|
300
394
|
sceneCount: parsed.storyboard.scenes.length,
|
|
301
395
|
issues: [],
|
|
302
396
|
warnings: parsed.warnings,
|
|
397
|
+
lint: lintStoryboard(parsed.storyboard).issues,
|
|
303
398
|
});
|
|
304
399
|
}
|
|
305
400
|
);
|
|
306
401
|
|
|
402
|
+
server.registerTool(
|
|
403
|
+
"lint_storyboard",
|
|
404
|
+
{
|
|
405
|
+
title: "Lint a storyboard (preflight, no browser)",
|
|
406
|
+
description:
|
|
407
|
+
"Browser-free preflight over a storyboard: a per-scene pacing forecast " +
|
|
408
|
+
"(which scenes compose will mostly freeze-hold or cut because the " +
|
|
409
|
+
"narration and the recorded action don't match), selector/wait " +
|
|
410
|
+
"pitfalls (type+Enter with no wait, :text-is on nested labels, focus " +
|
|
411
|
+
"without a zoom block, anchored waitForChange regexes), and no-op keys. " +
|
|
412
|
+
"Run it after every storyboard edit, before probe/render. Pass exactly " +
|
|
413
|
+
"one of dir / path / json. lang lints a narrations[lang] translation at " +
|
|
414
|
+
"that language's speaking rate.",
|
|
415
|
+
inputSchema: {
|
|
416
|
+
dir: z.string().optional().describe("demo dir → generated/storyboard.json"),
|
|
417
|
+
path: z.string().optional().describe("path to a storyboard .json file"),
|
|
418
|
+
json: z.string().optional().describe("storyboard JSON as a string"),
|
|
419
|
+
lang: z.string().optional(),
|
|
420
|
+
params: PARAMS_INPUT,
|
|
421
|
+
},
|
|
422
|
+
outputSchema: {
|
|
423
|
+
issues: z.array(LINT_ISSUE_SHAPE),
|
|
424
|
+
estimate: z.array(
|
|
425
|
+
z.object({
|
|
426
|
+
id: z.string(),
|
|
427
|
+
words: z.number(),
|
|
428
|
+
narrationMs: z.number(),
|
|
429
|
+
actionMs: z.number(),
|
|
430
|
+
holdPct: z.number(),
|
|
431
|
+
overrunMs: z.number(),
|
|
432
|
+
})
|
|
433
|
+
),
|
|
434
|
+
narrationTotalMs: z.number(),
|
|
435
|
+
wordsPerSec: z.number(),
|
|
436
|
+
},
|
|
437
|
+
annotations: { readOnlyHint: true },
|
|
438
|
+
},
|
|
439
|
+
async (args) => {
|
|
440
|
+
const sources = [args.dir, args.path, args.json].filter((s) => s != null).length;
|
|
441
|
+
if (sources !== 1) {
|
|
442
|
+
return errorResult({ message: "pass exactly one of: dir, path, json" });
|
|
443
|
+
}
|
|
444
|
+
let raw: unknown;
|
|
445
|
+
try {
|
|
446
|
+
raw =
|
|
447
|
+
args.json != null
|
|
448
|
+
? JSON.parse(args.json)
|
|
449
|
+
: await readJson<unknown>(
|
|
450
|
+
args.path != null
|
|
451
|
+
? resolve(args.path)
|
|
452
|
+
: new Project(args.dir as string).storyboardPath
|
|
453
|
+
);
|
|
454
|
+
} catch (err) {
|
|
455
|
+
return errorResult({ message: (err as Error).message });
|
|
456
|
+
}
|
|
457
|
+
const parsed = parseStoryboard(raw, {
|
|
458
|
+
relaxed: true,
|
|
459
|
+
params: args.params,
|
|
460
|
+
strict: args.params != null,
|
|
461
|
+
});
|
|
462
|
+
if (!parsed.ok) {
|
|
463
|
+
return errorResult({
|
|
464
|
+
message: "storyboard fails schema validation — run validate_storyboard",
|
|
465
|
+
issues: parsed.issues,
|
|
466
|
+
});
|
|
467
|
+
}
|
|
468
|
+
return jsonResult({ ...lintStoryboard(parsed.storyboard, { lang: args.lang }) });
|
|
469
|
+
}
|
|
470
|
+
);
|
|
471
|
+
|
|
307
472
|
server.registerTool(
|
|
308
473
|
"init_demo",
|
|
309
474
|
{
|
|
310
475
|
title: "Scaffold a new demo",
|
|
311
476
|
description:
|
|
312
477
|
"Create demos/<name>/ with a starter brief + storyboard. dir is the " +
|
|
313
|
-
"repo to scaffold into (absolute path recommended; default: server cwd)."
|
|
478
|
+
"repo to scaffold into (absolute path recommended; default: server cwd). " +
|
|
479
|
+
"With fromUrl the page is inspected first (no LLM): its headings become " +
|
|
480
|
+
"scenes and its unique selectors become the beats + a `_candidates` list — " +
|
|
481
|
+
"you then write the narration and turn the candidate hover into the real click.",
|
|
314
482
|
inputSchema: {
|
|
315
483
|
name: z.string().describe("demo name (creates demos/<name>/)"),
|
|
316
484
|
dir: z.string().optional(),
|
|
317
485
|
force: z.boolean().optional(),
|
|
486
|
+
fromUrl: z.string().optional().describe("draft the storyboard from this live page"),
|
|
487
|
+
headless: z.boolean().optional().describe("fromUrl: run Chrome headless (default true)"),
|
|
488
|
+
profile: z.string().optional().describe("fromUrl: Chrome user-data dir (logged-in pages)"),
|
|
489
|
+
viewport: z
|
|
490
|
+
.object({ width: z.number(), height: z.number() })
|
|
491
|
+
.optional()
|
|
492
|
+
.describe("fromUrl: viewport (default 1280x720)"),
|
|
318
493
|
},
|
|
319
494
|
outputSchema: {
|
|
320
495
|
demoDir: z.string(),
|
|
@@ -326,7 +501,13 @@ export function buildMcpServer(): { server: McpServer; jobs: JobManager } {
|
|
|
326
501
|
const demoDir = await scaffoldDemo(
|
|
327
502
|
args.dir ? resolve(args.dir) : process.cwd(),
|
|
328
503
|
args.name,
|
|
329
|
-
{
|
|
504
|
+
{
|
|
505
|
+
force: args.force,
|
|
506
|
+
fromUrl: args.fromUrl,
|
|
507
|
+
headed: args.headless === false,
|
|
508
|
+
profileDir: args.profile,
|
|
509
|
+
viewport: args.viewport,
|
|
510
|
+
}
|
|
330
511
|
);
|
|
331
512
|
const project = new Project(demoDir);
|
|
332
513
|
return jsonResult({
|
|
@@ -631,13 +812,29 @@ export function buildMcpServer(): { server: McpServer; jobs: JobManager } {
|
|
|
631
812
|
|
|
632
813
|
/** Record options wired to a job's progress fields + abort signal. */
|
|
633
814
|
function recordOpts(
|
|
634
|
-
args: {
|
|
635
|
-
|
|
815
|
+
args: {
|
|
816
|
+
headless?: boolean;
|
|
817
|
+
profile?: string;
|
|
818
|
+
fresh?: boolean;
|
|
819
|
+
capture?: "playwright" | "native" | "obs";
|
|
820
|
+
storageState?: string;
|
|
821
|
+
cookies?: SeedCookie[];
|
|
822
|
+
profileSeeded?: boolean;
|
|
823
|
+
fromScene?: string;
|
|
824
|
+
},
|
|
825
|
+
job: Job,
|
|
826
|
+
reloadStoryboard?: () => Promise<Storyboard>
|
|
636
827
|
): RecordOptions {
|
|
637
828
|
return {
|
|
638
829
|
profileDir: args.profile,
|
|
830
|
+
fresh: args.fresh,
|
|
831
|
+
storageState: args.storageState,
|
|
832
|
+
cookies: args.cookies,
|
|
833
|
+
profileSeeded: args.profileSeeded,
|
|
834
|
+
reloadStoryboard,
|
|
639
835
|
headed: !args.headless,
|
|
640
836
|
capture: args.capture,
|
|
837
|
+
fromScene: args.fromScene,
|
|
641
838
|
signal: job.controller.signal,
|
|
642
839
|
onSceneStart: (sceneId, index, total) => {
|
|
643
840
|
job.currentScene = sceneId;
|
|
@@ -688,14 +885,17 @@ export function buildMcpServer(): { server: McpServer; jobs: JobManager } {
|
|
|
688
885
|
},
|
|
689
886
|
(project, args) => async (job) =>
|
|
690
887
|
jobs.runStage(job, "probe", async () => {
|
|
691
|
-
const
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
888
|
+
const load = () =>
|
|
889
|
+
project.loadStoryboard({
|
|
890
|
+
relaxed: true,
|
|
891
|
+
params: args.params,
|
|
892
|
+
});
|
|
893
|
+
const storyboard = await load();
|
|
894
|
+
logLint(lintStoryboard(storyboard), log);
|
|
695
895
|
const goldenMode = !!(args.golden || args.updateGolden);
|
|
696
896
|
const probeScenes: ProbeGoldenScene[] = [];
|
|
697
897
|
const timeline = await record(project, storyboard, {
|
|
698
|
-
...recordOpts(args, job),
|
|
898
|
+
...recordOpts(args, job, load),
|
|
699
899
|
...(goldenMode ? { probe: probeScenes } : {}),
|
|
700
900
|
});
|
|
701
901
|
const base = {
|
|
@@ -729,13 +929,18 @@ export function buildMcpServer(): { server: McpServer; jobs: JobManager } {
|
|
|
729
929
|
},
|
|
730
930
|
};
|
|
731
931
|
}
|
|
732
|
-
const
|
|
932
|
+
const actual = buildProbeGolden(storyboard, probeScenes);
|
|
933
|
+
const diffs = diffGolden(expected, actual);
|
|
934
|
+
const drift = diffs.length ? await driftFilesForDiff(project, actual, diffs) : [];
|
|
733
935
|
return {
|
|
734
936
|
...base,
|
|
735
937
|
golden: {
|
|
736
938
|
path: project.goldenProbePath,
|
|
737
939
|
match: diffs.length === 0,
|
|
738
940
|
diffs,
|
|
941
|
+
// Nearest-selector suggestions for the actions that flipped
|
|
942
|
+
// (logs/drift-<scene>-<n>.json — read them, fix, re-probe).
|
|
943
|
+
...(drift.length ? { drift } : {}),
|
|
739
944
|
},
|
|
740
945
|
};
|
|
741
946
|
}
|
|
@@ -743,14 +948,60 @@ export function buildMcpServer(): { server: McpServer; jobs: JobManager } {
|
|
|
743
948
|
})
|
|
744
949
|
);
|
|
745
950
|
|
|
951
|
+
server.registerTool(
|
|
952
|
+
"import_trace",
|
|
953
|
+
{
|
|
954
|
+
title: "Draft a storyboard from a Playwright trace or test",
|
|
955
|
+
description:
|
|
956
|
+
"Turn a Playwright trace.zip (context.tracing / `--trace on`) or a *.spec.ts " +
|
|
957
|
+
"test file into demos/<name>/ with a draft storyboard: the run's own actions " +
|
|
958
|
+
"and selectors become scenes (cut at navigations and async payoffs), narration " +
|
|
959
|
+
"is left as placeholders, and `_notes` lists what was approximated. No LLM, " +
|
|
960
|
+
"no browser. Then write the narration and probe.",
|
|
961
|
+
inputSchema: {
|
|
962
|
+
file: z.string().describe("absolute path to trace.zip or the test file"),
|
|
963
|
+
name: z.string().describe("demo name (creates demos/<name>/)"),
|
|
964
|
+
dir: z.string().optional().describe("repo to scaffold into (default: server cwd)"),
|
|
965
|
+
force: z.boolean().optional(),
|
|
966
|
+
},
|
|
967
|
+
outputSchema: {
|
|
968
|
+
demoDir: z.string(),
|
|
969
|
+
storyboardPath: z.string(),
|
|
970
|
+
steps: z.number(),
|
|
971
|
+
scenes: z.number(),
|
|
972
|
+
notes: z.array(z.string()),
|
|
973
|
+
},
|
|
974
|
+
},
|
|
975
|
+
async (args) => {
|
|
976
|
+
try {
|
|
977
|
+
const result = await importTrace(resolve(args.file), args.name);
|
|
978
|
+
const demoDir = await scaffoldImported(args.dir ? resolve(args.dir) : process.cwd(), args.name, result, {
|
|
979
|
+
force: args.force,
|
|
980
|
+
});
|
|
981
|
+
const project = new Project(demoDir);
|
|
982
|
+
return jsonResult({
|
|
983
|
+
demoDir,
|
|
984
|
+
storyboardPath: project.storyboardPath,
|
|
985
|
+
steps: result.steps,
|
|
986
|
+
scenes: (result.storyboard.scenes as unknown[]).length,
|
|
987
|
+
notes: result.notes,
|
|
988
|
+
});
|
|
989
|
+
} catch (err) {
|
|
990
|
+
return errorResult({ message: (err as Error).message });
|
|
991
|
+
}
|
|
992
|
+
}
|
|
993
|
+
);
|
|
994
|
+
|
|
746
995
|
registerJob(
|
|
747
996
|
"record",
|
|
748
997
|
"Drive the storyboard in Chrome and record raw video + timeline.json.",
|
|
749
998
|
RECORD_INPUT_SHAPE,
|
|
750
999
|
(project, args) => async (job) =>
|
|
751
1000
|
jobs.runStage(job, "record", async () => {
|
|
752
|
-
const
|
|
753
|
-
const
|
|
1001
|
+
const load = () => project.loadStoryboard({ params: args.params });
|
|
1002
|
+
const storyboard = await load();
|
|
1003
|
+
logLint(lintStoryboard(storyboard), log);
|
|
1004
|
+
const timeline = await record(project, storyboard, recordOpts(args, job, load));
|
|
754
1005
|
return {
|
|
755
1006
|
rawVideo: await project.resolveRawVideo(),
|
|
756
1007
|
timeline: project.timelinePath,
|
|
@@ -772,11 +1023,13 @@ export function buildMcpServer(): { server: McpServer; jobs: JobManager } {
|
|
|
772
1023
|
(project, args) => async (job) =>
|
|
773
1024
|
jobs.runStage(job, "render", async () => {
|
|
774
1025
|
const signal = job.controller.signal;
|
|
775
|
-
const
|
|
1026
|
+
const load = () => project.loadStoryboard({ params: args.params });
|
|
1027
|
+
const storyboard = await load();
|
|
776
1028
|
// Language variant (if any): voice/captions/compose run on a lang-scoped
|
|
777
1029
|
// project + localized storyboard; the take is recorded ONCE on the base.
|
|
778
1030
|
const lp = langProject(project, args.lang);
|
|
779
1031
|
const sb = args.lang ? localizeStoryboard(storyboard, args.lang) : storyboard;
|
|
1032
|
+
logLint(lintStoryboard(storyboard, { lang: args.lang }), log);
|
|
780
1033
|
// Each sub-stage below is wrapped in runSubStage: it refreshes its OWN
|
|
781
1034
|
// stable logs/<stage>.log (not just logs/render.log) and updates
|
|
782
1035
|
// job.stage/currentScene/scenesTotal/scenesDone — same fields a
|
|
@@ -788,7 +1041,7 @@ export function buildMcpServer(): { server: McpServer; jobs: JobManager } {
|
|
|
788
1041
|
);
|
|
789
1042
|
throwIfAborted(signal);
|
|
790
1043
|
await jobs.runSubStage(job, "record", () =>
|
|
791
|
-
record(project, storyboard, recordOpts(args, job))
|
|
1044
|
+
record(project, storyboard, recordOpts(args, job, load))
|
|
792
1045
|
);
|
|
793
1046
|
throwIfAborted(signal);
|
|
794
1047
|
await jobs.runSubStage(job, "captions", async () => {
|
|
@@ -800,7 +1053,9 @@ export function buildMcpServer(): { server: McpServer; jobs: JobManager } {
|
|
|
800
1053
|
}
|
|
801
1054
|
});
|
|
802
1055
|
throwIfAborted(signal);
|
|
803
|
-
await jobs.runSubStage(job, "compose", () =>
|
|
1056
|
+
const report = await jobs.runSubStage(job, "compose", () =>
|
|
1057
|
+
compose(lp, sb, sceneProgress(job))
|
|
1058
|
+
);
|
|
804
1059
|
let gifPath: string | undefined;
|
|
805
1060
|
if (args.gif) {
|
|
806
1061
|
gifPath = await jobs.runSubStage(job, "gif", () => exportGif(lp));
|
|
@@ -811,12 +1066,26 @@ export function buildMcpServer(): { server: McpServer; jobs: JobManager } {
|
|
|
811
1066
|
if (storyboardHasStills(storyboard)) {
|
|
812
1067
|
stills = await jobs.runSubStage(job, "stills", () => extractStills(project));
|
|
813
1068
|
}
|
|
1069
|
+
let walkthrough: string | undefined;
|
|
1070
|
+
if (storyboard.output?.walkthrough) {
|
|
1071
|
+
walkthrough = (
|
|
1072
|
+
await jobs.runSubStage(job, "walkthrough", () => exportWalkthrough(lp, sb))
|
|
1073
|
+
).index;
|
|
1074
|
+
}
|
|
814
1075
|
return {
|
|
815
1076
|
output: lp.outputPath,
|
|
816
1077
|
...(gifPath ? { gif: gifPath } : {}),
|
|
817
1078
|
...(stills && stills.length ? { stills } : {}),
|
|
1079
|
+
...(walkthrough ? { walkthrough } : {}),
|
|
818
1080
|
timeline: lp.timelinePath,
|
|
819
1081
|
captionsSrt: lp.captionsSrtPath,
|
|
1082
|
+
report: lp.reportPath,
|
|
1083
|
+
durationMs: report.durationMs,
|
|
1084
|
+
// warningCount lets an orchestrating agent gate on the render
|
|
1085
|
+
// without parsing the warning list (issue #59).
|
|
1086
|
+
warningCount: report.warnings.length,
|
|
1087
|
+
warnings: report.warnings,
|
|
1088
|
+
...(report.blankOpenMs ? { blankOpenMs: report.blankOpenMs } : {}),
|
|
820
1089
|
};
|
|
821
1090
|
})
|
|
822
1091
|
);
|
|
@@ -897,16 +1166,45 @@ export function buildMcpServer(): { server: McpServer; jobs: JobManager } {
|
|
|
897
1166
|
const lp = langProject(project, args.lang);
|
|
898
1167
|
const storyboard = await project.loadStoryboard({ params: args.params });
|
|
899
1168
|
const sb = args.lang ? localizeStoryboard(storyboard, args.lang) : storyboard;
|
|
900
|
-
await compose(lp, sb, sceneProgress(job));
|
|
1169
|
+
const report = await compose(lp, sb, sceneProgress(job));
|
|
901
1170
|
let gifPath: string | undefined;
|
|
902
1171
|
if (args.gif) gifPath = await exportGif(lp);
|
|
903
1172
|
return {
|
|
904
1173
|
output: lp.outputPath,
|
|
905
1174
|
...(gifPath ? { gif: gifPath } : {}),
|
|
1175
|
+
report: lp.reportPath,
|
|
1176
|
+
durationMs: report.durationMs,
|
|
1177
|
+
// warningCount lets an orchestrating agent gate on the render
|
|
1178
|
+
// without parsing the warning list (issue #59).
|
|
1179
|
+
warningCount: report.warnings.length,
|
|
1180
|
+
warnings: report.warnings,
|
|
1181
|
+
...(report.blankOpenMs ? { blankOpenMs: report.blankOpenMs } : {}),
|
|
906
1182
|
};
|
|
907
1183
|
})
|
|
908
1184
|
);
|
|
909
1185
|
|
|
1186
|
+
registerJob(
|
|
1187
|
+
"qa",
|
|
1188
|
+
"Post-render checks on the FINISHED MP4 — blank opening, blank poster, " +
|
|
1189
|
+
"master loudness, music-bed level, per-scene hold shares + compose " +
|
|
1190
|
+
"warnings, caption readability, frame/chapter hygiene. Run it after " +
|
|
1191
|
+
"compose/render: it sees what a viewer sees, which neither lint (pre-take) " +
|
|
1192
|
+
"nor report.json (structural) can.",
|
|
1193
|
+
{
|
|
1194
|
+
dir: DIR_INPUT,
|
|
1195
|
+
params: PARAMS_INPUT,
|
|
1196
|
+
lang: LANG_INPUT,
|
|
1197
|
+
},
|
|
1198
|
+
(project, args) => async (job) =>
|
|
1199
|
+
jobs.runStage(job, "qa", async () => {
|
|
1200
|
+
const storyboard = await project.loadStoryboard({ params: args.params });
|
|
1201
|
+
const sb = args.lang ? localizeStoryboard(storyboard, args.lang) : storyboard;
|
|
1202
|
+
const res = await runQa(langProject(project, args.lang), sb);
|
|
1203
|
+
logQa(res);
|
|
1204
|
+
return res;
|
|
1205
|
+
})
|
|
1206
|
+
);
|
|
1207
|
+
|
|
910
1208
|
registerJob(
|
|
911
1209
|
"gif",
|
|
912
1210
|
"Convert output/final-demo.mp4 to a README-ready GIF.",
|
|
@@ -944,6 +1242,144 @@ export function buildMcpServer(): { server: McpServer; jobs: JobManager } {
|
|
|
944
1242
|
}))
|
|
945
1243
|
);
|
|
946
1244
|
|
|
1245
|
+
registerJob(
|
|
1246
|
+
"frames",
|
|
1247
|
+
"Dump evenly spaced PNG frames from the final video (or the raw take) into " +
|
|
1248
|
+
"output/frames/ for review — look at them instead of hand-running " +
|
|
1249
|
+
"`ffmpeg -ss`. Needs only the video (no key).",
|
|
1250
|
+
{
|
|
1251
|
+
dir: DIR_INPUT,
|
|
1252
|
+
everySec: z.number().optional().describe("seconds between frames (default 3)"),
|
|
1253
|
+
source: z
|
|
1254
|
+
.enum(["final", "raw", "take"])
|
|
1255
|
+
.optional()
|
|
1256
|
+
.describe(
|
|
1257
|
+
"final = output/final-demo.mp4 (default); raw = the latest raw file; " +
|
|
1258
|
+
"take = the whole recorded take, across the raw files a resumed take is spliced from"
|
|
1259
|
+
),
|
|
1260
|
+
width: z.number().optional().describe("frame width in px, aspect kept (default 640)"),
|
|
1261
|
+
out: z.string().optional().describe("output directory (default <dir>/output/frames)"),
|
|
1262
|
+
},
|
|
1263
|
+
(project, args) => async (job) =>
|
|
1264
|
+
jobs.runStage(job, "frames", async () => {
|
|
1265
|
+
const res = await extractFrames(project, {
|
|
1266
|
+
everySec: args.everySec,
|
|
1267
|
+
source: args.source,
|
|
1268
|
+
width: args.width,
|
|
1269
|
+
outDir: args.out,
|
|
1270
|
+
});
|
|
1271
|
+
return {
|
|
1272
|
+
source: res.source,
|
|
1273
|
+
durationMs: res.durationMs,
|
|
1274
|
+
everySec: res.everySec,
|
|
1275
|
+
frames: res.files,
|
|
1276
|
+
};
|
|
1277
|
+
})
|
|
1278
|
+
);
|
|
1279
|
+
|
|
1280
|
+
registerJob(
|
|
1281
|
+
"walkthrough",
|
|
1282
|
+
"Export output/walkthrough/ from the final video: index.html (one card per " +
|
|
1283
|
+
"scene — payoff frame, title, narration, jump-to-time; ← → keyboard nav), " +
|
|
1284
|
+
"guide.md (README/SOP-ready), per-scene PNGs, SRT/VTT and a JSON manifest. " +
|
|
1285
|
+
"Needs only the rendered video + report.json (no key, no browser).",
|
|
1286
|
+
{
|
|
1287
|
+
dir: DIR_INPUT,
|
|
1288
|
+
lang: z.string().optional().describe("language variant (final-demo.<lang>.mp4)"),
|
|
1289
|
+
width: z.number().optional().describe("frame width in px (default 960)"),
|
|
1290
|
+
out: z.string().optional().describe("output directory (default <dir>/output/walkthrough)"),
|
|
1291
|
+
},
|
|
1292
|
+
(project, args) => async (job) =>
|
|
1293
|
+
jobs.runStage(job, "walkthrough", async () => {
|
|
1294
|
+
const lp = args.lang ? new Project(project.dir, args.lang) : project;
|
|
1295
|
+
const storyboard = await project.loadStoryboard({ relaxed: true });
|
|
1296
|
+
const sb = args.lang ? localizeStoryboard(storyboard, args.lang) : storyboard;
|
|
1297
|
+
const res = await exportWalkthrough(lp, sb, { width: args.width, outDir: args.out });
|
|
1298
|
+
return {
|
|
1299
|
+
dir: res.dir,
|
|
1300
|
+
index: res.index,
|
|
1301
|
+
guide: res.guide,
|
|
1302
|
+
manifest: res.manifest,
|
|
1303
|
+
scenes: res.scenes.length,
|
|
1304
|
+
};
|
|
1305
|
+
})
|
|
1306
|
+
);
|
|
1307
|
+
|
|
1308
|
+
registerJob(
|
|
1309
|
+
"inspect",
|
|
1310
|
+
"Open a URL in the recording profile (logged-in state included) and list " +
|
|
1311
|
+
"every visible interactive element with UNIQUE selectors ranked " +
|
|
1312
|
+
"data-testid → id → aria-label → role/text → name/placeholder → class → " +
|
|
1313
|
+
"path, plus headings and iframes. Use it BEFORE writing targets — no " +
|
|
1314
|
+
"selector guessing, no wasted probe. Writes logs/inspect-<n>.json and a " +
|
|
1315
|
+
"screenshot in the demo dir.",
|
|
1316
|
+
{
|
|
1317
|
+
dir: DIR_INPUT,
|
|
1318
|
+
url: z.string().describe("page to inspect (absolute URL)"),
|
|
1319
|
+
headless: z.boolean().optional().describe("default true"),
|
|
1320
|
+
profile: z.string().optional().describe("Chrome user-data dir (default: the recording profile)"),
|
|
1321
|
+
limit: z.number().int().min(5).max(400).optional().describe("max elements (default 80)"),
|
|
1322
|
+
viewport: z
|
|
1323
|
+
.object({ width: z.number().int(), height: z.number().int() })
|
|
1324
|
+
.optional()
|
|
1325
|
+
.describe("default 1280x720 (use the storyboard's video size)"),
|
|
1326
|
+
frames: z
|
|
1327
|
+
.record(z.string(), z.string())
|
|
1328
|
+
.optional()
|
|
1329
|
+
.describe("iframe selectors to scan too, as in the storyboard `frames` block"),
|
|
1330
|
+
useSetup: z
|
|
1331
|
+
.boolean()
|
|
1332
|
+
.optional()
|
|
1333
|
+
.describe(
|
|
1334
|
+
"honour the storyboard's setup block (storageState/cookies/preflight) so an " +
|
|
1335
|
+
"authenticated page can be scanned. Default true."
|
|
1336
|
+
),
|
|
1337
|
+
storageState: z
|
|
1338
|
+
.string()
|
|
1339
|
+
.optional()
|
|
1340
|
+
.describe("extra Playwright storageState JSON to seed before the scan"),
|
|
1341
|
+
},
|
|
1342
|
+
(project, args) => async (job) =>
|
|
1343
|
+
jobs.runStage(job, "inspect", async () => {
|
|
1344
|
+
const stamp = new Date().toISOString().replace(/[:.]/g, "-");
|
|
1345
|
+
const png = project.p("logs", `inspect-${stamp}.png`);
|
|
1346
|
+
// #53: scan authenticated pages — same seeding surface as probe/record.
|
|
1347
|
+
let storyboard: Storyboard | null = null;
|
|
1348
|
+
if (args.useSetup !== false) {
|
|
1349
|
+
storyboard = await project.loadStoryboard({ relaxed: true }).catch(() => null);
|
|
1350
|
+
}
|
|
1351
|
+
if (storyboard?.setup?.preflight) {
|
|
1352
|
+
await runPreflight(storyboard.setup.preflight, {
|
|
1353
|
+
demoDir: project.dir,
|
|
1354
|
+
storyboardPath: project.storyboardPath,
|
|
1355
|
+
profileDir: args.profile ?? chromeProfileDir(),
|
|
1356
|
+
});
|
|
1357
|
+
storyboard = await project
|
|
1358
|
+
.loadStoryboard({ relaxed: true })
|
|
1359
|
+
.catch(() => storyboard);
|
|
1360
|
+
}
|
|
1361
|
+
const seeds = await collectSeeds(
|
|
1362
|
+
project.dir,
|
|
1363
|
+
storyboard ?? ({ setup: undefined } as never),
|
|
1364
|
+
{ storageState: args.storageState }
|
|
1365
|
+
);
|
|
1366
|
+
const res = await inspectPage({
|
|
1367
|
+
url: args.url,
|
|
1368
|
+
headless: args.headless !== false,
|
|
1369
|
+
profileDir: args.profile,
|
|
1370
|
+
limit: args.limit,
|
|
1371
|
+
viewport: args.viewport ?? storyboard?.video,
|
|
1372
|
+
frames: args.frames ?? storyboard?.frames,
|
|
1373
|
+
seeds,
|
|
1374
|
+
screenshotPath: png,
|
|
1375
|
+
});
|
|
1376
|
+
const file = project.p("logs", `inspect-${stamp}.json`);
|
|
1377
|
+
await writeJson(file, res);
|
|
1378
|
+
log(`inspect: ${res.elements.length} element(s), ${res.headings.length} heading(s) → ${file}`);
|
|
1379
|
+
return { ...res, file };
|
|
1380
|
+
})
|
|
1381
|
+
);
|
|
1382
|
+
|
|
947
1383
|
server.registerResource(
|
|
948
1384
|
"authoring-guide",
|
|
949
1385
|
"aidemo://authoring-guide",
|
|
@@ -1004,6 +1440,11 @@ export async function runMcpServer(): Promise<void> {
|
|
|
1004
1440
|
// and break the next record.
|
|
1005
1441
|
void (async () => {
|
|
1006
1442
|
if (jobs.abortActive()) await jobs.waitForIdle(10_000);
|
|
1443
|
+
// Grace for native addons: after a local-TTS job, onnxruntime's worker
|
|
1444
|
+
// threads wind down asynchronously, and process.exit mid-teardown
|
|
1445
|
+
// aborts with a shutdown-time "libc++abi: mutex lock failed" on macOS
|
|
1446
|
+
// (issue #19). Half a second is invisible here — the client is gone.
|
|
1447
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
1007
1448
|
process.exit(0);
|
|
1008
1449
|
})();
|
|
1009
1450
|
};
|