@vgai/live 0.5.41 → 0.5.44

package/src/editor-document.ts

@@ -1,7 +1,7 @@
 /**
  * `editor.document` — the session binding for the scoped editor-chrome door.
  *
- * WHY IT IS A SEPARATE OBJECT, and why the verbs are these four, is recorded
+ * WHY IT IS A SEPARATE OBJECT, and why the verbs are these five, is recorded
  * once in the implementation's header
  * (`packages/editor/src/editor-document-probe.ts`); the short version is that
  * `game.page()` is play-mode-gated and rooted at the GAME container, so an
@@ -16,7 +16,7 @@
  * RENDERED rather than what a DOM scrape can find.
  *
  * A field on `LiveEditor` rather than methods on it, so `vgai eval --list`
- * shows the four verbs as one named surface — the same reason `game.input`
+ * shows the six verbs as one named surface — the same reason `game.input`
  * and `game.events` are instance fields.
  */
 
@@ -69,6 +69,41 @@ export class LiveEditorDocument {
     });
   }
 
+  /**
+   * A real pointer DRAG across one matched element — press at `from`, move,
+   * release at `to`, as fractions of the element's box (`[0.5, 0.5]` is its
+   * center). The gesture a direct-manipulation canvas needs; a zero-length
+   * drag is a click at that fraction, which `click` (always the center)
+   * cannot place.
+   */
+  async drag(
+    selector: string,
+    options: DocumentGestureOptions & {
+      from: [number, number];
+      to: [number, number];
+      via?: [number, number][];
+      steps?: number;
+      altKey?: boolean;
+      ctrlKey?: boolean;
+      metaKey?: boolean;
+      shiftKey?: boolean;
+    },
+  ): Promise<DocumentProbeResult> {
+    return this.#probe({
+      action: 'drag',
+      selector,
+      from: options.from,
+      to: options.to,
+      ...(options.via === undefined ? {} : { via: options.via }),
+      ...(options.steps === undefined ? {} : { steps: options.steps }),
+      ...(options.index === undefined ? {} : { index: options.index }),
+      ...(options.altKey === undefined ? {} : { altKey: options.altKey }),
+      ...(options.ctrlKey === undefined ? {} : { ctrlKey: options.ctrlKey }),
+      ...(options.metaKey === undefined ? {} : { metaKey: options.metaKey }),
+      ...(options.shiftKey === undefined ? {} : { shiftKey: options.shiftKey }),
+    });
+  }
+
   /** A real keydown/keyup on the target, or on whatever inside the document has focus. */
   async key(key: string, options?: DocumentKeyOptions): Promise<DocumentProbeResult> {
     return this.#probe({ action: 'key', key, ...(options ?? {}) });
@@ -80,6 +115,41 @@ export class LiveEditorDocument {
     return this.#probe({ action: 'paste', text, ...(options ?? {}) });
   }
 
+  /**
+   * Choose `value` on a `<select>` — a native dropdown's options are drawn by
+   * the OS, so `click` has nothing in the document to resolve, and a plain
+   * `element.value =` is invisible to React. Set through the prototype's own
+   * value setter plus `input`/`change`; `value` is the option's `value`, not
+   * its label. An unknown value is refused with the options it does offer.
+   */
+  async select(
+    selector: string,
+    value: string,
+    options?: DocumentGestureOptions,
+  ): Promise<DocumentProbeResult> {
+    return this.#probe({
+      action: 'select',
+      selector,
+      value,
+      ...(options?.index === undefined ? {} : { index: options.index }),
+    });
+  }
+
+  /**
+   * THE REPL over the open document: run `step` in the editor page against the
+   * object the ACTIVE document published as its context (the mesh document
+   * publishes its `MeshEditSession`, whose `ctx` is the bpy-shaped edit
+   * context — `ctx.ops.mesh.bevel({ offset: 0.1 })`, `ctx.selection`,
+   * `ctx.history`, `session.commit()`). Edit mode, no play. Serialized like
+   * `game.page`: the step's own source travels, so inline every value it
+   * needs and return plain data.
+   */
+  async run<T = unknown>(
+    step: (ctx: unknown, info: { documentId: string }) => T | Promise<T>,
+  ): Promise<T> {
+    return this.#client.documentScript<T>(step.toString());
+  }
+
   #probe(step: DocumentProbeStep): Promise<DocumentProbeResult> {
     return this.#client.documentProbe(step);
   }

package/src/editor.ts

@@ -17,18 +17,27 @@ import type {
   AssetKind,
   AssetPreviewCapture,
   AssetPreviewOptions,
+  AssetPreviewShotSetDefinition,
   AssetPreviewSource,
+  CaptureDimensions,
+  DocumentLookOutcome,
+  EditorChromeCapture,
   EditorClient,
   EditorState,
   EditorView,
+  EditorWorkspaceName,
   HistoryStep,
   InspectedFieldWrite,
   InspectedHierarchy,
   InspectedInspection,
+  LabeledShotSetCapture,
   OpenedDocument,
   PresentedEditorView,
   RagdollGenerationResult,
   ShadingMode,
+  StructureOp,
+  StructureOpOptions,
+  StructureOpResult,
   ViewPreset,
   ViewportCapture,
 } from '@vgai/editor-sdk';
@@ -56,6 +65,23 @@ const EXTENSION_KIND: Record<string, AssetKind> = {
   '.glsl': 'source',
   '.vert': 'source',
   '.frag': 'source',
+  // PROJECT SCRIPTS ARE SOURCE. Without these the guess below falls through to
+  // `'json'`, the asset-document router sends the file to the generic JSON
+  // viewer (`asset-documents.tsx#assetDocumentViewerRoute`: `spec.kind ===
+  // 'json'` is decided before any content routing), and the LIVE MODELING
+  // DOCUMENT never mounts — `editor.openAsset('src/lib/fox/fox.model.ts')`
+  // silently shows a text pane instead of the model. Only `kind: 'source'`
+  // reaches `SourceAssetViewer`, which is what content-routes a project script
+  // to `LiveModuleDocument`. The set matches that viewer's own
+  // `isProjectScriptPath` regex, `/\.(?:[cm]?[jt]sx?)$/`.
+  '.ts': 'source',
+  '.tsx': 'source',
+  '.mts': 'source',
+  '.cts': 'source',
+  '.js': 'source',
+  '.jsx': 'source',
+  '.mjs': 'source',
+  '.cjs': 'source',
 };
 
 /**
@@ -125,9 +151,32 @@ export class LiveEditor {
     return this.#client.currentView();
   }
 
-  /** Capture the same center document the human is currently looking at. */
-  async captureActiveDocument(size?: number): Promise<ActiveDocumentCapture> {
-    return this.#client.captureActiveDocument(size);
+  /**
+   * Capture the same center document the human is currently looking at.
+   *
+   * A number is a SQUARE of that size — the default, and the right shape for
+   * an unstaged look at a model. `{width, height}` asks for a shaped frame, so
+   * a video-aspect look needs no crop afterwards. Both are bounded by the
+   * relay budget (64-1024 per side, total no larger than a 1024 square); see
+   * `@vgai/editor-sdk`'s `CaptureDimensions`.
+   * Supply a view to present and photograph it in one editor request.
+   */
+  async captureActiveDocument(
+    size?: CaptureDimensions,
+    view?: EditorView,
+  ): Promise<ActiveDocumentCapture> {
+    return this.#client.captureActiveDocument(size, view);
+  }
+
+  /**
+   * Photograph the editor PAGE — every panel, tab strip and viewport as the
+   * person sees it. `vgai screenshot editor` is this verb from the shell. The
+   * one door for judging chrome sighted: a skin, a workspace arrangement or a
+   * contributed panel is looked at through this, never guessed at from DOM
+   * probes. Returns the page at its own size.
+   */
+  async captureEditorChrome(): Promise<EditorChromeCapture> {
+    return this.#client.captureEditorChrome();
   }
 
   async play(opts?: { seed?: number }): Promise<void> {
@@ -184,8 +233,57 @@ export class LiveEditor {
    * `assetPreview(..., { stage: 'scene' })`): a framing that silently missed
    * would otherwise be indistinguishable from one that worked.
    */
-  async frame(entityId: string): Promise<void> {
-    await this.#client.frameEntity(entityId);
+  async frame(entityId: string): Promise<void>;
+  /**
+   * Bare `frame()` frames the OPEN Object3D document's subject instead — its
+   * selection if it has one, else the whole model: the toolbar's own Frame
+   * button, reachable from a script. `fit` scales the fitted distance (1 is
+   * that button's tight fit, 1.5 stands back a little for a shot).
+   */
+  async frame(options?: { readonly fit?: number }): Promise<void>;
+  async frame(target?: string | { readonly fit?: number }): Promise<void> {
+    if (typeof target === 'string') {
+      await this.#client.frameEntity(target);
+      return;
+    }
+    await this.#client.frameDocument(target?.fit);
+  }
+
+  /**
+   * WATCH THE AGENT LOOK AROUND THE MODEL.
+   *
+   * Swings the open Object3D document's camera — the camera the human's tab is
+   * showing — around the framed subject by `azimuth`/`elevation` RADIANS,
+   * animated over `duration` seconds (default 0.6), and resolves when the move
+   * ends. This is deliberately not a jump cut: the point of the verb is that a
+   * person watching sees the agent walk around the thing it is working on.
+   *
+   * `await editor.orbit({ azimuth: Math.PI / 2 })` — a quarter turn to the right.
+   *
+   * There is ONE camera, and the human owns it: a drag during the move cancels
+   * it exactly where it is, and the resolved outcome says `cancelledBy:
+   * 'human'` rather than throwing. A second look verb supersedes the first.
+   * The move is drawn by the document's own frame loop, so a document that
+   * isn't being drawn (background tab, inactive panel) doesn't orbit.
+   */
+  async orbit(options: {
+    readonly azimuth?: number;
+    readonly elevation?: number;
+    readonly duration?: number;
+  }): Promise<DocumentLookOutcome> {
+    return this.#client.orbitDocument(options);
+  }
+
+  /**
+   * A slow full revolution of the open document's subject — {@link orbit} with
+   * the turns spelled out and a constant angular rate. Resolves at the end of
+   * the last revolution.
+   */
+  async turntable(options?: {
+    readonly seconds?: number;
+    readonly revolutions?: number;
+  }): Promise<DocumentLookOutcome> {
+    return this.#client.turntableDocument(options);
   }
 
   async view(preset: ViewPreset): Promise<void> {
@@ -206,6 +304,25 @@ export class LiveEditor {
     await this.#client.setInstanceCount(countOrNames);
   }
 
+  /**
+   * Switch the editor's NAMED WORKSPACE — `await editor.workspace('model')`.
+   *
+   * A workspace is a task-named LAYOUT MEMORY over the one dock
+   * (ARCHITECTURE-CORE §Editor chrome): `game` (the default, the editor's
+   * standing arrangement), `model`, `sculpt`, `texture`, `animate`, `look`.
+   * Switching is an EXPLICIT act — nothing in the editor moves chrome on its
+   * own, opening a document included — and this is the session door to it,
+   * beside `Window → Workspace` and the registered actions.
+   *
+   * Resolves once the dock has finished rebuilding, so a capture taken
+   * immediately after photographs the arrangement that was asked for. Each
+   * workspace remembers the user's own hand-tuning per project, so switching
+   * away and back is lossless.
+   */
+  async workspace(id: EditorWorkspaceName): Promise<void> {
+    await this.#client.setWorkspace(id);
+  }
+
   /** `vgai show <viewport <edit|play>|inspector|console|build>`'s four sub-verbs, folded into one action name. */
   async showPanel(name: PanelName): Promise<void> {
     switch (name) {
@@ -232,6 +349,20 @@ export class LiveEditor {
     await this.#client.openAsset(path, kind ?? inferAssetKind(path));
   }
 
+  /**
+   * SELECT a project asset — the browser's single click, which fills the
+   * Inspector without opening a document. `openAsset` is the double click.
+   *
+   * This is how a project's own `asset.inspector` section is reached: select
+   * the file it matches, then `inspect()` lists the verbs that section
+   * declares and `runAction(id)` runs one. Selecting a path nothing matches
+   * is not an error — the Inspector shows what it has, exactly as it does
+   * for a human.
+   */
+  async selectAsset(path: string): Promise<void> {
+    await this.#client.selectAsset(path);
+  }
+
   /**
    * Fit native Rapier bodies and joints to a rigged GLB/glTF, copy the reusable
    * project capability, and open the generated TSX prefab's Setup story.
@@ -261,6 +392,27 @@ export class LiveEditor {
     );
   }
 
+  /**
+   * The same subject photographed as a LABELED SHOT SET instead of the four
+   * views — a caller-supplied definition of turntable yaws and bone-anchored
+   * crops, rendered against the asset's own skeleton, with a contact sheet.
+   * Every source {@link assetPreview} takes works here, GLB bytes included:
+   * a shot set stages its own subject, so it needs no place to stand.
+   *
+   * Sole in-repo caller today: `project.bake.preview`'s `--orbit` lane.
+   */
+  async assetPreviewShots(
+    source: string | AssetPreviewSource,
+    definition: AssetPreviewShotSetDefinition,
+    options?: AssetPreviewOptions,
+  ): Promise<LabeledShotSetCapture> {
+    return this.#client.captureShotSetPreview(
+      typeof source === 'string' ? { assetPath: source } : source,
+      definition,
+      options,
+    );
+  }
+
   async grid(on: boolean): Promise<void> {
     await this.#client.setGrid(on);
   }
@@ -322,6 +474,53 @@ export class LiveEditor {
     return this.#client.runInspectionAction(actionId);
   }
 
+  /**
+   * RESTRUCTURE the authored tree — the hierarchy context menu's own verbs.
+   *
+   * `create`, `delete`, `duplicate`, `reparent`, `reorder`, `wrap`, `unwrap`,
+   * `group`, `ungroup`, `copy`, `cut`, `paste`; `extractComponent` and
+   * `forkComponent` are the two that write whole new files and have their own
+   * doors below. All of them run the SAME `authoring/consumer-actions.ts`
+   * helpers the menu items call, so there is one implementation of each op and
+   * not a second that can disagree with what a human gets.
+   *
+   * It exists because the menu is a POINTER surface: every one of these ops was
+   * reachable only by right-clicking a hierarchy row, which is nothing an agent
+   * can do — so for an ingest root, whose only authoring surface IS the editor,
+   * structure was closed entirely.
+   *
+   * `id`/`ids` default to the current selection. The answer carries the same
+   * per-edit `write` ack `setField` does, so `write.persisted` tells a saved
+   * restructure from a live-only one. An op the active adapter does not provide
+   * REJECTS by name — never a silent no-op.
+   */
+  async structure(op: StructureOp, options?: StructureOpOptions): Promise<StructureOpResult> {
+    return this.#client.structureOp(op, options ?? {});
+  }
+
+  /**
+   * "Extract Component…" — lift the selected native subtree into its own
+   * component file (plus a story) and replace the callsite with it.
+   *
+   * Answers the action's own sentence, which NAMES both new files, because
+   * undo owns the callsite edit and will not remove them.
+   */
+  async extractComponent(options?: { id?: string; name?: string }): Promise<string> {
+    return (await this.#client.extractComponent(options ?? {})).hint;
+  }
+
+  /**
+   * "Fork Component…" — copy the selected instance's component definition to a
+   * new file and retarget THIS CALLSITE at it.
+   *
+   * One callsite is the unit of the edit; when that callsite sits inside a
+   * component rendered many times, every one of those renders now renders the
+   * fork.
+   */
+  async forkComponent(options?: { id?: string }): Promise<string> {
+    return (await this.#client.forkComponent(options ?? {})).hint;
+  }
+
   /**
    * READ the hierarchy panel, as data — the rows a human is looking at right
    * now, nested exactly as the panel nests them.
@@ -353,6 +552,11 @@ export class LiveEditor {
     return this.#client.hierarchy();
   }
 
+  /** Expand every branch through the Hierarchy panel's own action. */
+  async expandHierarchyAll(): Promise<void> {
+    await this.#client.expandHierarchyAll();
+  }
+
   /**
    * Write one editable field from `inspect()` by its stable path, through the
    * same Inspector IO and persistence boundary the human control uses.

package/src/game-client/capture-notes.ts

@@ -2,17 +2,25 @@
  * What a capture knows about ITSELF beyond its pixels — and the one place the
  * caveat sentence is spelled.
  *
- * A PNG is silent about the conditions it was taken under. Two of those
- * conditions change what the frame is worth as evidence, and both are already
- * measured elsewhere in the stack: the editor page reports
+ * A PNG is silent about the conditions it was taken under. Four of those
+ * conditions matter, and each is already measured elsewhere in the stack.
+ * Three change what the frame is worth as EVIDENCE: the editor page reports
  * `loopRecoveryFrame` when the host loop was starved and the runtime had to
  * render one deterministic tick on demand (`command-listener.ts`'s
  * `handleBridgeScreenshot`), and it reports
  * a `flatness.warning` sentence when the frame is nine-tenths one flat surface
- * (`composite-screenshot.ts`'s `measureFlatness`). Until now both stopped at a
+ * (`composite-screenshot.ts`'s `measureFlatness`), and it names the CLIP a
+ * frame of a recorded run belongs to (`command-listener.ts`'s
+ * `screenshotRecordingNotice`). Until now these stopped at a
  * `console.warn` inside the relay transport — visible to a human watching a
  * terminal, invisible to anything that later reads the file.
  *
+ * The fourth is not about the frame at all but about what taking it COST: a
+ * capture written under the served project root goes through the dev server's
+ * file watcher on every shot (measured ~3x slower frame rates), and the
+ * out-path resolver is the only place that knows
+ * (`screenshot-target.ts`'s `underWatchedProjectRoot`).
+ *
  * So the transport seam carries them back as {@link CaptureNotes}, and
  * {@link describeCaptureCaveat} turns them into the ONE sentence every surface
  * says. Its two callers are the transport's own console warning and
@@ -21,7 +29,7 @@
  * record carries beside the frame cannot drift apart.
  */
 
-/** The conditions a capture was taken under, as facts. Both fields are
+/** The conditions a capture was taken under, as facts. Every field is
  *  optional and absent means "not so": an ordinary frame carries no notes. */
 export interface CaptureNotes {
   /** The host loop was starved, so the frame exists only because the runtime
@@ -30,6 +38,16 @@ export interface CaptureNotes {
   /** The page's own near-blank-frame sentence, verbatim (it owns the wording;
    *  see `composite-screenshot.ts`'s `CaptureFlatness.warning`). */
   readonly flatnessWarning?: string;
+  /** The clip this frame is one frame OF — every `vgai play` records
+   *  (`play-recording.ts`). Not a degradation: the still is delivered and is
+   *  the right instrument for a look question. It is here because a still of a
+   *  MOVING game answers a temporal question only by luck, and a reader of the
+   *  persisted record has to be able to find the door that answers it. */
+  readonly recordingPath?: string;
+  /** The SERVED project root this capture was written under, when it was.
+   *  Not a degradation of the frame — a cost of taking it; see
+   *  {@link WATCHED_CAPTURE_PATH_CAVEAT}. */
+  readonly watchedProjectRoot?: string;
 }
 
 /** One capture, as reported to a {@link CaptureListener} after the bytes are
@@ -53,6 +71,20 @@ export type CaptureListener = (capture: CaptureRecord) => void | Promise<void>;
 /** The loop-recovery sentence. Spelled once because it is said in two places
  *  (a live console warning and a persisted record) and a second copy is a
  *  second wording. */
+/**
+ * The cost of writing a capture INSIDE the served project.
+ *
+ * The dev server watches the project root, and its ignore list names build
+ * output only — nothing about capture output — so every PNG written under the
+ * root goes through the watcher. Measured cost class: ~3x slower frame rates
+ * while a capture loop wrote there. Named, not fixed: where a capture goes is
+ * the caller's decision, and this door's job is to stop that decision being
+ * made blind.
+ */
+const WATCHED_CAPTURE_PATH_CAVEAT =
+  'writing captures under the project root triggers the dev server’s file watcher; expect ~3× ' +
+  'slower frame rates — write outside the project or to the session’s own capture dir';
+
 const LOOP_RECOVERY_FRAME_CAVEAT =
   'LOOP-RECOVERY FRAME — the host loop was starved, so the runtime rendered one ' +
   'deterministic tick on demand. It is current, not stale; it was not produced by ordinary presentation.';
@@ -61,9 +93,9 @@ const LOOP_RECOVERY_FRAME_CAVEAT =
  * What a reader must be told about this frame, or `null` when there is nothing
  * to tell.
  *
- * Both notes can be true at once (an on-demand recovery tick that also came
- * out near-blank), and both are said — a capture that is degraded twice over
- * must not report only the first reason.
+ * Any of them can be true at once (an on-demand recovery tick that also came
+ * out near-blank, in a recorded run), and each is said — a capture that is
+ * degraded twice over must not report only the first reason.
  */
 export function describeCaptureCaveat(notes: CaptureNotes): string | null {
   const parts: string[] = [];
@@ -71,5 +103,14 @@ export function describeCaptureCaveat(notes: CaptureNotes): string | null {
   if (typeof notes.flatnessWarning === 'string' && notes.flatnessWarning !== '') {
     parts.push(notes.flatnessWarning);
   }
+  if (typeof notes.recordingPath === 'string' && notes.recordingPath !== '') {
+    parts.push(
+      `ONE FRAME of a recorded run — ${notes.recordingPath} holds the whole of it. A still ` +
+        'answers what it looks like; a temporal question (did the jump land) needs the clip.',
+    );
+  }
+  if (typeof notes.watchedProjectRoot === 'string' && notes.watchedProjectRoot !== '') {
+    parts.push(`${WATCHED_CAPTURE_PATH_CAVEAT} (${notes.watchedProjectRoot}).`);
+  }
   return parts.length === 0 ? null : parts.join(' ');
 }