@kheleshikhi/studio-agent 0.6.1 → 0.8.0

package/README.md

@@ -13,10 +13,140 @@ nerdbro connect
 mkdir my-kheleshikhi-work
 cd my-kheleshikhi-work
 nerdbro agent init --provider codex   # or claude / generic
-nerdbro studio create --title "Ball Lab" --target combined --concept "Projectile range"
+nerdbro studio create --title "Ball Lab" --concept "Projectile range" \
+  --graph-purpose "Range against launch angle"   # target defaults to sceneComposition
 nerdbro studio checkout <asset-id>
 ```
 
+## Mental model: Scene + Experience
+
+Since 0.8.0 the CLI follows ADR-0003, the unified model the server projects:
+
+- **Everything visual is a Scene** (`sceneComposition`). A graph is not a
+  separate asset or target: it is a **Scene math plane** (`mathPlanes`, Scene
+  Source v15, manifest v19) inside the Scene. A graph-only lesson is a Scene
+  holding one plane; a physical + graph lesson is one Scene holding both.
+- **Interaction is an Experience** (`kheleshikhi.experience.v3`): final learner
+  content containing one or more Scenes, with `interaction` (inputs, requests,
+  handlers, readouts, exports, tools), `rounds`, `presentation` and
+  `assessment.outcomes`. `game` carries scoring only (sections and metrics).
+  `requiredCapabilities` are derived by the server, never authored; every
+  canonical capability is available in every host.
+- **Frozen:** the `interactiveScene` part shape and the `mathGraph` and
+  `combined` targets are read so existing assets keep playing, but are never
+  authored for new work. Experience V2 is read through one normalizer and never
+  authored again.
+
+The CLI does not bundle these lists. `nerdbro context` returns the live
+projection — `authoring_targets`, `frozen_authoring_targets`, `deprecations`,
+`scene_schema`, `scene_manifest_schema`, `experience_schema`,
+`experience_capabilities`, `experience_tools` — plus a `summary` with the
+compatibility verdict. `studio create` defaults `--target` to `sceneComposition`
+and refuses a target only when the server's context lists it as frozen
+(`STUDIO_TARGET_FROZEN`, exit `2`, with the server's deprecation text); any
+other target is sent to the server, which remains the authority. Graph intent
+(`--math-plane`, or `--graph-purpose`/`--graph-x`/`--graph-y`/`--graph-sync`)
+sets the brief's `requires_graph`, meaning "this Scene holds a math plane",
+whatever the target.
+
+`nerdbro schema experience` returns the live V3 contract with the universal
+capability list attached (`universal_capabilities`), and
+`nerdbro capabilities` returns that list and the canonical tools beside the
+Studio capability registry. A schema topic the server marks deprecated (for
+example `mathGraph.config`) carries `deprecated` in its result.
+
+A Scene holding a math plane (validated and compiled by the server's Scene
+parser; the plane fields are documented by `nerdbro docs open mathgraph`, and
+`nerdbro schema scene.versions` names the current authoring schema):
+
+```json
+{
+  "schema": "kheleshikhi.scene-source.v15",
+  "id": "parabola-lab",
+  "titleEn": "Parabola",
+  "titleBn": "পরাবৃত্ত",
+  "instructionEn": "Change k and watch the curve.",
+  "instructionBn": "k বদলাও, বক্ররেখা দেখো।",
+  "canvas": { "width": 960, "height": 540 },
+  "variables": [{ "name": "k", "type": "number", "value": 1, "min": -3, "max": 3 }],
+  "bands": [{ "id": "graphs", "kind": "mathPath", "placement": "compiled" }],
+  "instances": [],
+  "bindings": [],
+  "mathPlanes": [
+    {
+      "id": "graph",
+      "band": "graphs",
+      "title": "y equals k x squared",
+      "rect": { "x": 280, "y": 120, "width": 400, "height": 300 },
+      "viewport": { "minX": -5, "maxX": 5, "minY": -4, "maxY": 4 },
+      "axes": { "xLabel": "x", "yLabel": "y" },
+      "series": [{ "id": "curve", "kind": "explicit", "formula": "k*x^2", "label": "y = kx²" }]
+    }
+  ]
+}
+```
+
+An Experience V3 using that kind of Scene, with inputs, requests, readouts,
+exports, a tool, a round, presentation and assessment outcomes (validated by
+the server's Experience parser; read `nerdbro schema experience` for the live
+field lists and `nerdbro capabilities` for tool types):
+
+```json
+{
+  "schema": "kheleshikhi.experience.v3",
+  "id": "ramp",
+  "kind": "lesson",
+  "title": "Ramp",
+  "language": "bn",
+  "scenes": { "main": { "asset": "<approved Scene asset id>" } },
+  "variables": { "placed": false },
+  "interaction": {
+    "inputs": [
+      { "type": "directional2d", "axes": ["moveX"] },
+      { "type": "action", "action": "primary", "labelEn": "Go", "labelBn": "চলো" }
+    ],
+    "requests": {
+      "speed": { "type": "number", "presentation": "dial", "min": 0, "max": 10, "step": 1 },
+      "hint": { "type": "hint", "textEn": "Look at the slope." }
+    },
+    "handlers": [
+      { "topic": "input.action.primary.pressed", "kind": "event", "sceneEvent": "begin" }
+    ],
+    "readouts": [
+      {
+        "topic": "ramp.speed",
+        "kind": "state",
+        "from": { "scene": "main", "variable": "speed" },
+        "display": { "labelEn": "Speed", "labelBn": "বেগ", "unit": "m/s" }
+      }
+    ],
+    "exports": [{ "topic": "session.complete", "kind": "event", "on": "allPlaced" }],
+    "tools": [{ "id": "replay", "type": "replay-scrubber", "config": {} }]
+  },
+  "rounds": {
+    "count": 1,
+    "pool": [{ "id": "one", "promptEn": "Place it", "promptBn": "রাখো", "value": 1 }]
+  },
+  "presentation": { "orientation": "landscape", "aspectRatio": "16:9", "controlsLayout": "standard" },
+  "assessment": {
+    "outcomes": [
+      { "id": "placed", "on": "targetMatched", "status": "correct", "evidence": "placed" },
+      { "id": "missed", "on": "targetMissed", "status": "incorrect", "retry": true },
+      { "id": "done", "on": "allPlaced", "status": "completed" }
+    ]
+  }
+}
+```
+
+### Version negotiation
+
+Every command first reads `/v1/agent/context`. A different `agent_protocol`
+or a CLI older than `minimum_cli_version` exits `5`
+(`AGENT_PROTOCOL_INCOMPATIBLE`, `AGENT_CLI_UPGRADE_REQUIRED`) with the exact
+`npm install -g @kheleshikhi/studio-agent@<version>` to run. A CLI older than
+`recommended_cli_version` keeps working and prints one upgrade notice on
+stderr. The server currently recommends 0.8.0.
+
 The browser pairing page uses the normal NerdBro login. The daemon stores the
 scoped production credential only in memory. It binds to `127.0.0.1` on a
 random port and protects local RPC with a random discovery secret. The
@@ -30,7 +160,7 @@ are CSS layout sizes; they do not simulate a low-density display. Headless mode
 uses the same explicit responsive viewports for automated inspection, but its
 screenshot pixels are not promised to be byte-identical across hardware.
 Chrome's device-metrics API applies the CSS size while leaving the host device
-scale native; NerdBro does not force a DPR value. Version 0.6.1 keeps the
+scale native; NerdBro does not force a DPR value. Since 0.6.2 the CLI keeps the
 Chrome DevTools viewport session alive for the browser tab, so a requested
 desktop, mobile, or custom CSS size remains in effect through inspection and
 capture.
@@ -81,15 +211,61 @@ Use `nerdbro schema experience` for the final learner content contract and
 `nerdbro docs open experiences` for one-Scene, narrated, mixed, and multi-Band
 examples. Use `nerdbro schema scene.variables`, `scene.events`, `scene.gestures`, `scene.diagram`,
 `scene.motions`, `scene.captions`, `scene.audio`, `scene.stateMachines`,
-`expressions`, `stateMachine.actions`, `mathGraph.behaviorProjection`,
-`assessment`, or the `runtime.*` topics to request a narrow projection of
-the parser/runtime contract. `nerdbro schema` returns the topic index and
-summary contracts.
+`scene.versions`, `scene.limits`, `expressions`, `stateMachine.actions`,
+`assessment`, or the other listed topics to request a narrow projection of
+the parser/runtime contract. `nerdbro schema` returns the live topic index and
+summary contracts. The `mathGraph.*` and `runtime.*` topics describe frozen
+shapes kept for existing assets; new graphs are Scene math planes.
 Search Components and Behaviors narrowly. Example search rejects an empty
 query, returns ranked summaries with a small default limit, and leaves full
 source to `example inspect <id>`; normally inspect no more than three relevant
 examples.
 
+`nerdbro experience inspect-browser <id>` opens the canonical Experience
+workspace. `nerdbro experience drive <id> --scenario scenario.json` runs a
+bounded scenario against the mounted Experience Preview of **your local
+`source.json`, saved or not** (0.7.0+): the exact candidate is mounted in the
+daemon's own browser through the non-persisting Draft Preview and bound by its
+canonical hash (`source_origin: "local"`, `source_hash`); nothing is stored.
+`--saved` drives the saved revision instead. It accepts master
+events (`fire`), Scene semantic events (`fireSceneEvent`), learner presses of
+the active Scene's available events (`press`), Timeline play/pause/seek, Cue
+dispatch, `previewState` (a bare master State id with master-variable
+`setup`, or `machine/state` in the active Scene with `setup` naming variables
+that Scene declares — type and min/max checked, reported as
+`probe.sceneSetup`), `select` (`state:repair`), restart,
+waits, assertions, and captures. It never saves or publishes the source.
+Observation paths read the shared `kheleshikhi.studio-observation.v2`
+domains the Maker's Observatory shows — `experience.activeState`,
+`experience.masterVariables.<name>`, `scene.activeStates.<machine>`,
+`timeline.ms`, `probe.active`, `probe.setup.<name>`, `selection.sourcePath` —
+and the older flat keys (`active_state`, `master_variables`, `timeline_ms`,
+`scene_active_states`, `preview_override`, …) still work.
+
+### Source is the authority; probes are not journeys
+
+You author the same canonical Source as the Maker. Every drive returns
+`evidence_mode`: `probe` when it jumped to a State, used synthetic `setup`,
+fired an event a learner could not press, or seeked — useful development
+evidence that never proves a learner path or certifies anything — and
+`journey` when it ran from the initial State with learner-reachable actions
+only. Certification refuses a probed runtime observation.
+
+### Agent Presence
+
+The Maker's open Studio shows what you are doing, through
+`PUT /v1/agent/presence` (scope `STUDIO_OBSERVE`): status, changed source
+paths, the last semantic command and result, where your runtime is, your
+selection, your last capture. The CLI reports it around each
+validate/compile/preview/save/certify/capture/drive and per drive step.
+`nerdbro studio watch <asset> [--once]` and `nerdbro experience watch <id>
+[--once]` report which JSON Pointers your local `source.json` changed
+against the saved head, debounced; they never send the source. Presence is
+overwritten in place, expires after 90 seconds, is cleared on disconnect, and
+carries no reasoning, credentials or local paths. A Maker may **Follow
+Agent**: their Studio mirrors your selection, source path and runtime
+position, but you never control their browser or runtime.
+
 The daemon reuses the authenticated context response for at most 30 seconds,
 then revalidates it; connecting or disconnecting clears the cache. This cuts
 repeated compatibility reads without caching mutable drafts.
@@ -97,9 +273,9 @@ repeated compatibility reads without caching mutable drafts.
 Check out once, edit only `source.json`, and use `studio status` for later
 synchronization. Validate, compile, preview when useful, save, then inspect the
 real Studio. The daemon reuses its short-lived read-only browser credential in
-memory and clears it on disconnect, stop, reconnect, or origin change. A graph
-requirement without a teaching purpose or axis/synchronization meaning is a
-clarification request, not permission to invent filler.
+memory and clears it on disconnect, stop, reconnect, or origin change. A math
+plane requirement without a teaching purpose or axis/synchronization meaning is
+a clarification request, not permission to invent filler.
 
 `studio create` needs the `STUDIO_ASSET_CREATE` pairing scope and returns a
 Maker-owned draft identity and its server starter-source hash. `--scratch`
@@ -107,7 +283,8 @@ only labels the intent to use an ordinary new, unapproved asset as a disposable
 spike. It has no separate lifecycle, automatic expiry, promotion, or garbage
 collection. Save its exact ID for explicit cleanup after success. Use
 `studio brief <asset>` and `studio brief update <asset> --graph-purpose ...`
-to inspect or update authoring intent with a concurrency check.
+(or `--math-plane` / `--no-math-plane`) to inspect or update authoring intent
+with a concurrency check.
 
 `studio save` creates a draft. `studio certify` records technical evidence.
 Neither approves or publishes. `studio export --revision ...` accepts only the
@@ -147,8 +324,28 @@ and has at most 64 steps and a two-minute run budget:
 }
 ```
 
-`set`, `fire`, `press`, `tap`, `drag`, `play`, `pause`, `seek`, and `reset`
-address only declared variables, events, instances, and gestures. `press`
+To start mid-flow without replaying the Scene, jump to a State of one
+machine — always `machine/state`, because two machines may share a State id —
+optionally with synthetic setup for 1–16 declared variables:
+
+```json
+{"version": 1, "steps": [
+  {"op": "previewState", "target": "launch/armed", "setup": {"force": 9, "armed": true}},
+  {"op": "assert", "path": "probe.active", "equals": true},
+  {"op": "select", "target": "instance:cart"},
+  {"op": "assert", "path": "selection.sourcePath", "equals": "/sources/mainScene/document/instances/0"},
+  {"op": "press", "target": "disarmPressed"},
+  {"op": "wait", "path": "scene.activeStates.launch", "equals": "idle"},
+  {"op": "capture", "name": "after-disarm"},
+  {"op": "restart"}
+]}
+```
+
+`set`, `fire`, `press`, `tap`, `drag`, `play`, `pause`, `seek`, `reset`,
+`restart`, `previewState` and `select` address only declared variables,
+events, instances, gestures, machines and States. A `set` on a variable the
+learner has no visible control for is synthetic setup and makes the run a
+probe. `press`
 honours conditional availability; `drag` derives a Scene point from the
 authored gesture range. `wait` and `assert` read allowlisted observation paths,
 never arbitrary code. `capture` records the actual browser pixels. The result
@@ -199,7 +396,12 @@ edits without `--force`. `status` reports `local_changed`, `remote_changed` and
 saving unchanged content creates no draft but still refreshes the lock, and a
 stale base exits `3` (`VOCABULARY_DRAFT_CONFLICT`) with `source.json` untouched.
 `validate` and `certify` default to the lock's head revision. `preview` sends
-the local, unsaved `source.json` and writes `preview.json` beside it.
+the local, unsaved `source.json` and writes `preview.json` beside it; for an
+Experience its summary reports the schema, Scene and audio counts, V3
+interaction block counts, rounds, outcomes and the server-derived
+`required_capabilities`. When a checked-out Experience is not the context's
+`experience_schema` (for example a V2 document), `checkout`, `status` and
+`preview` add a `schema_notice` with the server's deprecation text.
 `inspect-browser` captures the Studio Library workspace to `screenshot.png` and
 `browser.json` in the checkout directory; its bounded `browser.capture` activity
 record is reported locally in the command result only.

package/dist/browser.d.ts

@@ -1,6 +1,15 @@
+import { type RenderRecipe } from "./video.js";
 import { type Page } from "playwright";
 import { AgentApi } from "./protocol.js";
-import { type DriveScenario } from "./scenario.js";
+import { type DriveScenario, type ExperienceScenario } from "./scenario.js";
+/** One semantic step as it happened, for Agent Presence and the activity feed. */
+export type DriveStepReport = {
+    index: number;
+    summary: Record<string, unknown>;
+    observation?: Record<string, unknown>;
+};
+/** Called after each semantic step; failures inside it never fail the drive. */
+export type DriveStepListener = (report: DriveStepReport) => void | Promise<void>;
 export declare function studioBrowserLaunchOptions(headless: boolean): {
     readonly headless: boolean;
     readonly viewport: null;
@@ -19,7 +28,7 @@ export declare function waitForCapturePaint(page: Page): Promise<void>;
 export declare function applyCssViewport(page: Page, dimensions: {
     width: number;
     height: number;
-}): Promise<void>;
+}, deviceScaleFactor?: number): Promise<void>;
 export declare function renderingDiagnostics(page: Page, screenshot: Buffer): Promise<Record<string, unknown>>;
 export declare const VIEW_QUERY: Record<string, Record<string, string>>;
 export declare function viewRequiresRuntimeObservation(view: string): boolean;
@@ -58,6 +67,8 @@ export interface BrowserCaptureRequest {
     timeoutMs?: number;
     observation?: boolean;
     drive?: boolean;
+    /** An Experience drive that will mount an unsaved candidate itself. */
+    awaitsCandidate?: boolean;
     /** Library workspace route (`?workspace=<name>&selection=<stable id>`) instead of a Scene asset. */
     library?: {
         workspace: string;
@@ -65,6 +76,12 @@ export interface BrowserCaptureRequest {
         revision?: string;
         definitionHash?: string;
     };
+    /** Semantic step reports for presence; never part of the request sent to the page. */
+    onStep?: DriveStepListener;
+    /** The presentation render host (`studio_present=1`): the learner stage alone, filling the viewport. */
+    present?: boolean;
+    /** CSS-to-device pixel scale; 0 keeps the display's own. */
+    deviceScaleFactor?: number;
 }
 export interface PreviewObservationRequest extends BrowserCaptureRequest {
     mode?: string;
@@ -96,6 +113,61 @@ export declare class StudioBrowser {
         candidate: Record<string, unknown>;
         candidateHash: string;
     }): Promise<Record<string, unknown>>;
+    driveExperience(request: BrowserCaptureRequest & {
+        scenario: ExperienceScenario;
+        candidate?: unknown;
+        candidateHash?: string;
+    }): Promise<Record<string, unknown>>;
+    /**
+     * Presentation frames (STU-001) of a Scene, from the presentation host.
+     *
+     * Mounts and verifies the candidate exactly as `drive` does, but in the
+     * presentation host (`studio_present=1`) at a viewport of exactly the
+     * output size and device scale 1. `renderBegin` silences the Scene's
+     * sounds into a log and restarts it at 0 s; then for every frame video
+     * time is stepped to `start + k / fps`, the runtime confirms that instant
+     * and a paint, and the whole viewport is captured: one PNG of exactly the
+     * output size, proved before it is kept. Scheduled learner actions happen
+     * at their own seconds. Afterwards the audio log and each logged sound's
+     * canonical bytes are collected for the mixer.
+     */
+    renderFrames(request: BrowserCaptureRequest & {
+        candidate: Record<string, unknown>;
+        candidateHash: string;
+        recipe: RenderRecipe;
+        framesDirectory: string;
+        audioDirectory: string;
+        /** Optional: afterwards, play in real time and sample frame intervals. */
+        measurePlaybackSeconds?: number;
+    }): Promise<Record<string, unknown>>;
+    /**
+     * Presentation frames of an Experience: the canonical Experience player
+     * in the presentation host, driven by video time (`renderBegin`, then
+     * `step` per frame). The master timeline advances with video time only
+     * while it runs, so a learner pause or a wait for a choice stays paused,
+     * as it would in playback; Scenes change, cues fire and sounds start at
+     * the exact video second they would.
+     */
+    renderExperienceFrames(request: BrowserCaptureRequest & {
+        candidate?: unknown;
+        candidateHash?: string;
+        recipe: RenderRecipe;
+        framesDirectory: string;
+        audioDirectory: string;
+    }): Promise<Record<string, unknown>>;
+    private requirePage;
+    /** The page's runtime observation and its one-at-a-time drive channel. */
+    private renderChannel;
+    /** The newest observation once [ready] holds (the page publishes every 150 ms). */
+    private waitForRender;
+    /**
+     * The frame loop both subjects share: actions at their seconds, a step to
+     * each frame's exact second, the subject's own arrival check, a paint,
+     * and a whole-viewport PNG proved to be exactly the output size.
+     */
+    private captureFrames;
+    /** Every logged sound's canonical bytes, written once by key; checked against the logged SHA-256. */
+    private collectAudio;
     /** Local-agent capture from the same browser page and renderer the Maker sees. */
     capturePreview(request: PreviewObservationRequest): Promise<Record<string, unknown>>;
 }