@foldspace_npm/harness 0.1.30 → 0.1.32

package/src/video/recorder.mjs

@@ -0,0 +1,331 @@
+// Recording one scene of the share video: a fresh conversation, the question
+// typed key by key, the agent thinking and answering, a few seconds of the
+// settled answer - captured with Chrome's own screencast.
+//
+// Page.startScreencast runs inside Chrome, so it needs no macOS Screen
+// Recording permission and captures only the page, never the desktop. Chrome
+// sends a picture when the page repaints; each one is written to disk as it
+// arrives and acknowledged at once, or Chrome stops sending.
+// Never Page.bringToFront: the human may be working in another app.
+
+import fs from "node:fs";
+import path from "node:path";
+
+import { createSettleTracker, looksPending, squash } from "../ask-core.mjs";
+import { sleep } from "../cdp-client.mjs";
+import {
+  clickNewChat,
+  messageInNewChat,
+  openComposer,
+  panelRect,
+  readPanel,
+  typeKeyByKey,
+  waitForReply,
+} from "../panel-driver.mjs";
+
+export const RECORD_DEFAULTS = Object.freeze({
+  capS: 90, // the most one scene records
+  keepS: 3, // of the settled answer
+  pollMs: 500,
+  stableS: 3, // a reply unchanged this long has settled (as ask: 2 polls of 1.5 s)
+  leadMs: 700, // the still panel before the first key
+  minFrames: 3,
+  // Chrome sends up to ~60 pictures a second while the app animates; the video
+  // is 30 fps, so pictures closer together than this are not kept (about half
+  // the disk). The last one of a burst always is.
+  minFrameGapMs: 25,
+  // Every key repaints the composer, so while the question is typed a picture
+  // comes at least this often - unless the window is minimized or covered.
+  typingGapMaxMs: 1000,
+});
+
+/** Pause between keys: a question types in about 1.6 s, never faster than a quick typist. */
+export function keyDelayMs(question) {
+  const length = [...String(question || "")].length || 1;
+  return Math.round(Math.min(75, Math.max(28, 1600 / length)));
+}
+
+/**
+ * Times from the polls: when a reply first showed (text that is not a
+ * progress phrase, or a card), and when what the panel showed last changed.
+ */
+export function createReplyClock() {
+  let firstReply = null;
+  let lastChange = null;
+  let last = null;
+  return {
+    observe({ at, reply = "", cards = [], newCardFrames = 0 }) {
+      const answered = (reply.length >= 2 && !looksPending(reply)) || cards.some(Boolean) || newCardFrames > 0;
+      if (answered && firstReply === null) firstReply = at;
+      const snapshot = `${reply}|${cards.join("|")}|${newCardFrames}`;
+      if (snapshot !== last) lastChange = at;
+      last = snapshot;
+    },
+    get firstReply() {
+      return firstReply;
+    },
+    get lastChange() {
+      return lastChange;
+    },
+  };
+}
+
+/**
+ * Which pictures to keep, by their times in ms. Times only go forward: Chrome's
+ * own stamp can run up to a few tens of ms behind the one before, and the
+ * video looks pictures up by time. A picture closer than `minGapMs` to the
+ * last one kept is held back rather than dropped: if nothing follows it (the
+ * answer's last word, then stillness), it is kept when the recording stops.
+ * offer(t) -> { t, keep, flush }: flush is a held picture to write first.
+ */
+export function createFrameGate({ minGapMs = RECORD_DEFAULTS.minFrameGapMs } = {}) {
+  let lastT = -Infinity;
+  let keptT = -Infinity;
+  let held = null;
+  return {
+    offer(rawT, item) {
+      const t = Math.max(rawT, lastT);
+      lastT = t;
+      if (t - keptT < minGapMs) {
+        held = { t, item };
+        return { t, keep: false };
+      }
+      held = null;
+      keptT = t;
+      return { t, keep: true };
+    },
+    /** The picture still held back, once nothing more will come. */
+    drain() {
+      const last = held;
+      held = null;
+      if (last) keptT = last.t;
+      return last;
+    },
+  };
+}
+
+/** The longest stretch without a picture in [from, to] (ms), counting from each end. */
+export function longestGap(times, from, to) {
+  let longest = 0;
+  let previous = from;
+  for (const t of times) {
+    if (t < from) continue;
+    if (t > to) break;
+    longest = Math.max(longest, t - previous);
+    previous = t;
+  }
+  return Math.max(longest, to - previous);
+}
+
+function screencastSink(page, sceneDir, { minGapMs }) {
+  fs.mkdirSync(path.join(sceneDir, "frames"), { recursive: true });
+  const frames = [];
+  const gate = createFrameGate({ minGapMs });
+  let viewport = null;
+  let hiddenAt = null;
+  const write = (data, at) => {
+    const file = `frames/${String(frames.length + 1).padStart(6, "0")}.jpg`;
+    try {
+      fs.writeFileSync(path.join(sceneDir, file), Buffer.from(data, "base64"));
+      frames.push({ file, at });
+    } catch {
+      // a picture that could not be written is a picture less, not a failed scene
+    }
+  };
+  const onFrame = (params) => {
+    // Acknowledge first: Chrome sends the next picture only after the ack.
+    page.send("Page.screencastFrameAck", { sessionId: params.sessionId }).catch(() => {});
+    const now = Date.now();
+    const stamp = Number(params.metadata?.timestamp) * 1000;
+    // Chrome's own time of the picture when it is sane, else when it arrived.
+    const raw = Number.isFinite(stamp) && Math.abs(stamp - now) < 5000 ? stamp : now;
+    if (!viewport && params.metadata?.deviceWidth) {
+      viewport = { width: params.metadata.deviceWidth, height: params.metadata.deviceHeight };
+    }
+    const { t, keep } = gate.offer(raw, params.data);
+    if (keep) write(params.data, t);
+  };
+  // Chrome stops painting a window that is minimized or covered: the
+  // recording would jump from the question to the answer.
+  const onVisibility = (params) => {
+    if (params?.visible === false && hiddenAt === null) hiddenAt = Date.now();
+  };
+  return {
+    frames,
+    get viewport() {
+      return viewport;
+    },
+    get hiddenAt() {
+      return hiddenAt;
+    },
+    async start() {
+      page.on("Page.screencastFrame", onFrame);
+      page.on("Page.screencastVisibilityChanged", onVisibility);
+      await page.send("Page.startScreencast", {
+        format: "jpeg",
+        quality: 85,
+        maxWidth: 1920,
+        maxHeight: 1200,
+        everyNthFrame: 1,
+      });
+    },
+    async stop() {
+      await page.send("Page.stopScreencast", {}, { timeoutMs: 5000 }).catch(() => {});
+      // A picture already on its way still lands.
+      await sleep(150);
+      page.off("Page.screencastFrame", onFrame);
+      page.off("Page.screencastVisibilityChanged", onVisibility);
+      const held = gate.drain();
+      if (held) write(held.item, held.t);
+    },
+  };
+}
+
+/**
+ * Record one scene into `dir`: frames/*.jpg, frames.json and events.json.
+ * Returns the scene's report; throws only when the page cannot be driven.
+ */
+export async function recordScene(page, { question, dir, agentApiName, options = {} }) {
+  const opts = { ...RECORD_DEFAULTS, ...options };
+  fs.rmSync(dir, { recursive: true, force: true });
+  fs.mkdirSync(dir, { recursive: true });
+
+  const { state } = await openComposer(page, agentApiName);
+  if (!state.composer) {
+    return { ok: false, reason: state.frame ? "no_composer" : "no_panel" };
+  }
+
+  // A fresh conversation for every scene: the panel's New Chat button, as a
+  // person would. Without one, the SDK's message(..., { newChat: true })
+  // starts it and sends the question in one go - then nothing is typed.
+  const newChat = (await clickNewChat(page).catch(() => false)) ? "button" : "sdk";
+  await sleep(900);
+
+  const sink = screencastSink(page, dir, { minGapMs: opts.minFrameGapMs });
+  const panel = [];
+  const sample = async (at) => {
+    const rect = await panelRect(page).catch(() => null);
+    if (rect) panel.push({ at, ...rect });
+  };
+  const events = { newChat };
+  const clock = createReplyClock();
+  let result = { settled: false, reply: "" };
+  const startedAt = Date.now();
+  await sink.start();
+  try {
+    await sample(startedAt);
+    await sleep(opts.leadMs);
+
+    events.typingStart = Date.now();
+    let sent;
+    let typedBy;
+    let enteredAt = null;
+    if (newChat === "button") {
+      ({ sent, typedBy, enteredAt } = await typeKeyByKey(page, question, { delayMs: keyDelayMs(question) }));
+    } else {
+      sent = await messageInNewChat(page, agentApiName, question).catch(() => false);
+      typedBy = "sdk";
+    }
+    // When Enter landed, not when typing returned: it waits after Enter to
+    // check the composer emptied, and the question bubble is already showing.
+    events.sent = enteredAt ?? Date.now();
+    events.typedBy = typedBy;
+    if (!sent) return { ok: false, reason: "not_sent", typedBy, newChat };
+
+    const capAt = startedAt + opts.capS * 1000;
+    const timeoutS = Math.max(1, (capAt - Date.now()) / 1000);
+    result = await waitForReply(page, question, {
+      timeoutS,
+      pollMs: opts.pollMs,
+      tracker: createSettleTracker({ stableNeeded: Math.ceil((opts.stableS * 1000) / opts.pollMs) }),
+      onPoll: ({ at, reply, seen }) => clock.observe({ at, reply, cards: seen.cards, newCardFrames: seen.newCardFrames }),
+    });
+    events.firstReply = clock.firstReply;
+    // The answer settled when the panel last changed, not when the stability
+    // check confirmed it; the poll saw the change up to half a poll late.
+    events.settled = result.settled && clock.lastChange ? clock.lastChange - opts.pollMs / 2 : null;
+    events.settledConfirmed = Date.now();
+    await sample(Date.now());
+    if (result.settled) {
+      const keepUntil = events.settled + opts.keepS * 1000;
+      if (Date.now() < keepUntil) await sleep(Math.min(keepUntil - Date.now(), capAt - Date.now()));
+      await sleep(300);
+    }
+  } finally {
+    await sink.stop();
+  }
+  events.end = Date.now();
+
+  const rel = (at) => (at == null ? null : Math.round(at - startedAt));
+  const viewport = sink.viewport || (panel[0] ? { width: panel[0].viewportWidth, height: panel[0].viewportHeight } : null);
+  const record = {
+    question,
+    newChat,
+    typedBy: events.typedBy,
+    settled: Boolean(result.settled),
+    // Milliseconds since this scene's recording started.
+    typingStart: rel(events.typingStart),
+    sent: rel(events.sent),
+    firstReply: rel(events.firstReply),
+    settledAt: rel(events.settled),
+    settledConfirmed: rel(events.settledConfirmed),
+    end: rel(events.end),
+    viewport,
+    panel: panel.map(({ at, ...rect }) => ({ t: rel(at), ...rect })),
+    frames: sink.frames.length,
+  };
+  fs.writeFileSync(
+    path.join(dir, "frames.json"),
+    `${JSON.stringify(sink.frames.map((frame) => ({ file: frame.file, t: rel(frame.at) })))}\n`,
+  );
+  fs.writeFileSync(path.join(dir, "events.json"), `${JSON.stringify(record, null, 2)}\n`);
+
+  if (sink.frames.length < opts.minFrames) return { ok: false, reason: "no_frames", ...record };
+  // Hidden while it recorded: Chrome said so, or the typing - a repaint every
+  // key - went a second without a picture. Either way part of it is missing.
+  const typingGap = longestGap(
+    sink.frames.map((frame) => frame.at),
+    events.typingStart,
+    events.sent,
+  );
+  if (sink.hiddenAt !== null || typingGap > opts.typingGapMaxMs) {
+    return { ok: false, reason: "hidden", typingGapMs: Math.round(typingGap), ...record };
+  }
+  if (!result.settled) {
+    return { ok: false, reason: squash(result.reply) ? "not_settled" : "no_reply", ...record };
+  }
+  return { ok: true, ...record };
+}
+
+/** Whether a recording folder finished: every scene's answer settled. */
+export function recordingComplete(dir) {
+  try {
+    return JSON.parse(fs.readFileSync(path.join(dir, "record.json"), "utf8")).complete === true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Keep the `keep` newest finished recordings in `root`, and the newest one of
+ * all (the run that just ended, finished or not); delete the rest. A failed
+ * try never pushes out a good recording that `--from` still needs. Each is
+ * one folder named by its time.
+ */
+export function pruneRecordings(root, keep = 3) {
+  let names = [];
+  try {
+    names = fs
+      .readdirSync(root, { withFileTypes: true })
+      .filter((entry) => entry.isDirectory() && /^\d{4}-\d\d-\d\dT/.test(entry.name))
+      .map((entry) => entry.name)
+      .sort();
+  } catch {
+    return [];
+  }
+  const kept = new Set(names.filter((name) => recordingComplete(path.join(root, name))).slice(-keep));
+  if (names.length) kept.add(names[names.length - 1]);
+  const removed = names.filter((name) => !kept.has(name));
+  for (const name of removed) fs.rmSync(path.join(root, name), { recursive: true, force: true });
+  return removed;
+}

package/src/video/scenes.mjs

@@ -0,0 +1,253 @@
+// video/scenes.json: what `foldspace record` asks and what each caption says.
+//
+//   {
+//     "product": "Northwind CRM",
+//     "endCard": true,                                optional, default true
+//     "scenes": [
+//       { "question": "Which deals are at risk this week?",
+//         "capabilities": ["Action", "Agentic UI"],
+//         "line": "The deals at risk, *ranked,*\nstraight from your CRM." }
+//     ]
+//   }
+//
+// Each scene is one of the agent's key use cases. Its caption shows the
+// capabilities it uses as pills and where the video is ("2 of 4"), the
+// question small, and the scene's line large: what the user gets, with the
+// payoff on a blue highlight (*like this*). The capabilities are the names of
+// Foldspace's features; a plain answer has none.
+//
+// Checked up front with errors that name the field, so a typo costs nothing
+// and never a half-made recording.
+
+import fs from "node:fs";
+import path from "node:path";
+
+import { parseMarked, visibleText } from "./timeline.mjs";
+
+export const SCENE_LIMITS = Object.freeze({
+  minScenes: 1,
+  maxScenes: 6,
+  product: 40,
+  question: 120,
+  line: 70, // visible characters, without the asterisks
+  lineBreaks: 1, // a "\n" makes two lines; no more
+  lineMarks: 2,
+  // A line broken by hand longer than this is drawn smaller to fit.
+  lineHalf: 38,
+  maxCapabilities: 3,
+});
+
+// The pills, in this spelling; a file may use any case. File upload is not
+// here on purpose: a recording never uploads. Voice is: the recording types,
+// but the pill says the agent takes speech too, when voice is on for it.
+export const CAPABILITIES = Object.freeze(["Action", "Agentic UI", "Navigation", "Tandem", "Human in the loop", "Voice"]);
+
+// Names an earlier version used, and what took their place.
+const RETIRED = Object.freeze({
+  "generative ui": "Agentic UI",
+  "your backend": "Action",
+  "chat & voice": "Voice",
+});
+
+const TOP_KEYS = ["product", "endCard", "scenes"];
+const SCENE_KEYS = ["question", "capabilities", "line"];
+const ALLOWED = `up to ${SCENE_LIMITS.maxCapabilities} of: ${CAPABILITIES.join(", ")}`;
+
+/** A required line of text, at most `max` characters. */
+function text(value, where, errors, { max }) {
+  if (value === undefined || value === null || (typeof value === "string" && !value.trim())) {
+    errors.push(`${where}: missing`);
+    return null;
+  }
+  if (typeof value !== "string") {
+    errors.push(`${where}: must be text`);
+    return null;
+  }
+  const clean = value.replace(/\r/g, "").trim();
+  if (clean.includes("\n")) errors.push(`${where}: one line only`);
+  if (max && clean.length > max) errors.push(`${where}: ${clean.length} characters, keep it to ${max}`);
+  return clean;
+}
+
+/**
+ * The scene's line: at most two lines (one "\n"), at most 70 characters
+ * without the asterisks, at most two *highlights*, each closed and not empty.
+ */
+function line(value, where, errors, warnings) {
+  if (value === undefined || value === null || (typeof value === "string" && !value.trim())) {
+    errors.push(`${where}: missing - one short line on what the user gets, e.g. "The deals at risk, *ranked,*\\nstraight from your CRM."`);
+    return null;
+  }
+  if (typeof value !== "string") {
+    errors.push(`${where}: must be text`);
+    return null;
+  }
+  const clean = value
+    .replace(/\r/g, "")
+    .split("\n")
+    .map((part) => part.replace(/[ \t]+/g, " ").trim())
+    .join("\n")
+    .trim();
+  const breaks = (clean.match(/\n/g) || []).length;
+  if (breaks > SCENE_LIMITS.lineBreaks) errors.push(`${where}: ${breaks + 1} lines - at most 2 (one "\\n")`);
+  const visible = visibleText(clean).length;
+  if (visible > SCENE_LIMITS.line) {
+    errors.push(`${where}: ${visible} characters without the asterisks, keep it to ${SCENE_LIMITS.line}`);
+  }
+  const marked = parseMarked(clean);
+  if (marked.problem === "unbalanced") errors.push(`${where}: an asterisk without its pair - *highlight* takes one on each side`);
+  else if (marked.problem === "across") errors.push(`${where}: a highlight runs over the "\\n" - close it before the break`);
+  else if (marked.problem === "empty") errors.push(`${where}: an empty highlight - put a word or two between the asterisks`);
+  else if (marked.marks > SCENE_LIMITS.lineMarks) {
+    errors.push(`${where}: ${marked.marks} highlights - at most ${SCENE_LIMITS.lineMarks}; one is best, on the payoff`);
+  }
+  if (breaks === 1) {
+    const long = clean.split("\n").map(visibleText).find((part) => part.length > SCENE_LIMITS.lineHalf);
+    if (long) warnings.push(`${where}: "${long}" is long for one line, so the line is drawn smaller - move the "\\n" or shorten it`);
+  }
+  return clean;
+}
+
+function unknownKeys(object, allowed, where, errors) {
+  for (const key of Object.keys(object)) {
+    if (allowed.includes(key)) continue;
+    const hint = key === "summary" ? ` - the caption's words are now "line"` : "";
+    errors.push(`${where ? `${where}.` : ""}${key}: not a field here (fields: ${allowed.join(", ")})${hint}`);
+  }
+}
+
+const capabilityKey = (name) => String(name).trim().replace(/\s+/g, " ").toLowerCase();
+
+/** None to three of CAPABILITIES, in the order given and spelled as the list spells them. */
+function capabilities(value, where, errors) {
+  if (value === undefined || value === null) {
+    errors.push(`${where}: missing - ${ALLOWED}; [] for a plain answer`);
+    return null;
+  }
+  if (!Array.isArray(value)) {
+    errors.push(`${where}: a list - ${ALLOWED}, e.g. ["Action", "Agentic UI"]`);
+    return null;
+  }
+  if (value.length > SCENE_LIMITS.maxCapabilities) {
+    errors.push(`${where}: ${value.length} - at most ${SCENE_LIMITS.maxCapabilities}, the ones this scene shows best (${CAPABILITIES.join(", ")})`);
+  }
+  const names = [];
+  value.forEach((name, index) => {
+    const key = typeof name === "string" ? capabilityKey(name) : null;
+    const known = key === null ? null : CAPABILITIES.find((one) => capabilityKey(one) === key);
+    if (!known && key !== null && key in RETIRED) {
+      const now = RETIRED[key];
+      errors.push(
+        now
+          ? `${where}[${index}]: ${JSON.stringify(name)} is now "${now}" - ${ALLOWED}`
+          : `${where}[${index}]: ${JSON.stringify(name)} is no longer a pill - leave it out (a plain answer has none); ${ALLOWED}`,
+      );
+    } else if (!known) {
+      errors.push(`${where}[${index}]: ${JSON.stringify(name)} is not a capability - ${ALLOWED}`);
+    } else if (names.includes(known)) {
+      errors.push(`${where}[${index}]: "${known}" twice - name each capability once`);
+    } else {
+      names.push(known);
+    }
+  });
+  return names;
+}
+
+/**
+ * Check a parsed scenes file. Returns { ok, errors, warnings, value } where
+ * value has the defaults filled in and the capabilities spelled as CAPABILITIES.
+ */
+export function validateScenes(input) {
+  const errors = [];
+  const warnings = [];
+  if (!input || typeof input !== "object" || Array.isArray(input)) {
+    return { ok: false, errors: ["the file must hold one JSON object: { product, scenes: [...] }"], warnings, value: null };
+  }
+  unknownKeys(input, TOP_KEYS, "", errors);
+  const product = text(input.product, "product", errors, { max: SCENE_LIMITS.product });
+  if (input.endCard !== undefined && typeof input.endCard !== "boolean") {
+    errors.push("endCard: true or false");
+  }
+
+  const scenes = [];
+  if (!Array.isArray(input.scenes)) {
+    errors.push("scenes: missing - a list of { question, capabilities, line }");
+  } else {
+    if (input.scenes.length < SCENE_LIMITS.minScenes) errors.push("scenes: empty - add 4 or 5");
+    if (input.scenes.length > SCENE_LIMITS.maxScenes) {
+      errors.push(`scenes: ${input.scenes.length} - at most ${SCENE_LIMITS.maxScenes}; 4 or 5 keep the video at about 40 s`);
+    } else if (input.scenes.length && (input.scenes.length < 4 || input.scenes.length > 5)) {
+      warnings.push(`${input.scenes.length} scene${input.scenes.length === 1 ? "" : "s"}: 4 or 5 make a video of about 40 s`);
+    }
+    input.scenes.forEach((scene, index) => {
+      const where = `scenes[${index}]`;
+      if (!scene || typeof scene !== "object" || Array.isArray(scene)) {
+        errors.push(`${where}: must be { question, capabilities, line }`);
+        scenes.push({ question: null, capabilities: null, line: null });
+        return;
+      }
+      unknownKeys(scene, SCENE_KEYS, where, errors);
+      scenes.push({
+        question: text(scene.question, `${where}.question`, errors, { max: SCENE_LIMITS.question }),
+        capabilities: capabilities(scene.capabilities, `${where}.capabilities`, errors),
+        line: line(scene.line, `${where}.line`, errors, warnings),
+      });
+    });
+    const asked = scenes.map((scene) => scene.question);
+    asked.forEach((question, index) => {
+      if (question && asked.indexOf(question) !== index) errors.push(`scenes[${index}].question: the same question twice`);
+    });
+  }
+
+  if (errors.length) return { ok: false, errors, warnings, value: null };
+  return {
+    ok: true,
+    errors,
+    warnings,
+    value: {
+      product,
+      endCard: input.endCard !== false,
+      scenes,
+    },
+  };
+}
+
+/** Where a caption says the video is: "2 of 4" (drawn in capitals). */
+export function scenePlace(n, total) {
+  return `${n} of ${total}`;
+}
+
+/** A file name from a product name: "Northwind CRM" -> "northwind-crm". */
+export function productSlug(product) {
+  const slug = String(product || "")
+    .normalize("NFKD")
+    .replace(/[̀-ͯ]/g, "")
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-+|-+$/g, "")
+    .slice(0, 40);
+  return slug || "agent";
+}
+
+/** video/<product>-<yyyymmdd-hhmm>.mp4, in local time. */
+export function videoBaseName(product, date = new Date()) {
+  const two = (value) => String(value).padStart(2, "0");
+  const stamp =
+    `${date.getFullYear()}${two(date.getMonth() + 1)}${two(date.getDate())}` +
+    `-${two(date.getHours())}${two(date.getMinutes())}`;
+  return `${productSlug(product)}-${stamp}`;
+}
+
+// The videos hold real data from the account: they stay out of git in every
+// project, old ones too (an upgrade never touches .gitignore). scenes.json
+// stays tracked.
+export const VIDEO_GITIGNORE = "# The videos show real data from the account: never saved with the code.\n*.mp4\n*.jpg\n";
+
+/** Write <videoDir>/.gitignore when it is missing; returns whether it wrote one. */
+export function ensureVideoIgnored(videoDir) {
+  const file = path.join(videoDir, ".gitignore");
+  if (fs.existsSync(file)) return false;
+  fs.mkdirSync(videoDir, { recursive: true });
+  fs.writeFileSync(file, VIDEO_GITIGNORE);
+  return true;
+}