humanish 0.129.0 → 0.131.0

package/dist/analysis/source.js

@@ -0,0 +1,118 @@
+// How analysis reads a run's recorded source: run.json, then each participant's trace one at a time,
+// and the refusals it gives for either.
+import { isCommsReceivingEvidence } from "../comms/receiving-evidence.js";
+import { isReadRunBundleSchema } from "../run/bundle.js";
+import { runIdOf } from "../run/paths.js";
+import { reduceRunTraces, StreamTraceRefusedError, StreamTraceTooLargeError, } from "../run/stream-trace.js";
+import { isRecord } from "../run/type-guards.js";
+import { EVIDENCE_LIMITS } from "./analysis-limits.js";
+/**
+ * A refusal of the run's recorded source. Its message is the code, as every analysis error's is,
+ * and `reason` says which file analysis did not read and why, for the person who asked.
+ */
+export class AnalysisSourceRefusedError extends Error {
+    reason;
+    constructor(code, reason) {
+        super(code);
+        this.reason = reason;
+        this.name = "AnalysisSourceRefusedError";
+    }
+}
+const bytes = (count) => `${count.toLocaleString("en-US")} bytes`;
+const limit = (max) => `${bytes(max)} (${(max / (1024 * 1024)).toLocaleString("en-US", { maximumFractionDigits: 1 })} MiB)`;
+const runFileInvalid = (reason) => new AnalysisSourceRefusedError("ANALYSIS_SOURCE_INVALID", `Analysis did not read run.json: ${reason}`);
+const traceInvalid = (streamId, reason) => new AnalysisSourceRefusedError("ANALYSIS_SOURCE_INVALID", `Analysis did not read participant ${streamId}'s trace: ${reason}`);
+/** The refusal of a run.json of `size` bytes, more than the `max` analysis reads of it. */
+export function runFileTooLarge(size, max) {
+    return new AnalysisSourceRefusedError("ANALYSIS_SOURCE_TOO_LARGE", `Analysis did not read run.json: it holds ${bytes(size)}, more than the ${limit(max)} analysis reads of it. run.json grows with each participant and the study's mission; split the participants across studies to keep it under the limit.`);
+}
+/**
+ * run.json's bundle, when it is a run bundle of the prepared run that analysis reads: at most
+ * EVIDENCE_LIMITS.sourceBytes, at most EVIDENCE_LIMITS.participants participants, and each
+ * participant and event with the fields analysis reads.
+ */
+export function parseAnalysisSource(prepared, bytes) {
+    if (bytes.length > EVIDENCE_LIMITS.sourceBytes)
+        throw runFileTooLarge(bytes.length, EVIDENCE_LIMITS.sourceBytes);
+    const value = JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes));
+    if (!isRecord(value) || !isReadRunBundleSchema(value.schema))
+        throw runFileInvalid("it is not a run bundle of a schema this version of humanish reads.");
+    if (value.runId !== runIdOf(prepared))
+        throw runFileInvalid(`it records another run than ${runIdOf(prepared)}, the run in its folder.`);
+    if (!Array.isArray(value.streams) || !Array.isArray(value.events))
+        throw runFileInvalid("it has no participants or events list.");
+    if (value.streams.length > EVIDENCE_LIMITS.participants)
+        throw runFileInvalid(`it lists ${value.streams.length} participants, more than the ${EVIDENCE_LIMITS.participants} analysis reads; split the participants across studies.`);
+    if (value.commsReceiving !== undefined && !isCommsReceivingEvidence(value.commsReceiving))
+        throw runFileInvalid("its email receiving evidence holds a field or value analysis does not read.");
+    const ids = new Set();
+    for (const [index, stream] of value.streams.entries()) {
+        if (!isRecord(stream) ||
+            typeof stream.id !== "string" ||
+            stream.id.length === 0 ||
+            stream.id.length > 256 ||
+            ids.has(stream.id) ||
+            typeof stream.label !== "string" ||
+            typeof stream.status !== "string") {
+            throw runFileInvalid(`participant ${index + 1} needs an id of 1 to 256 characters no other participant has, a label and a status.`);
+        }
+        ids.add(stream.id);
+        if (stream.actor !== undefined && !isRecord(stream.actor))
+            throw runFileInvalid(`participant ${stream.id}'s actor is not an object.`);
+    }
+    for (const [index, event] of value.events.entries()) {
+        if (!isRecord(event) ||
+            typeof event.id !== "string" ||
+            typeof event.message !== "string" ||
+            typeof event.type !== "string")
+            throw runFileInvalid(`event ${index + 1} needs an id, a type and a message.`);
+    }
+    return value;
+}
+/**
+ * What `keep` takes from each stream of run.json's bundle with the stream's trace loaded, in stream
+ * order. Each trace file is read up to RUN_ARTIFACT_MAX_BYTES, as the loader, verify and the
+ * Observer read it, and is dropped once `keep` returns, so analysis holds one trace at a time
+ * whatever the number of participants. run.json's digest, which the analysis record binds, binds
+ * each trace's sha256. A trace whose index records more than the limit is refused as
+ * ANALYSIS_SOURCE_TOO_LARGE, and any other trace the loader refuses as ANALYSIS_SOURCE_INVALID.
+ */
+export async function readParticipantTraces(prepared, bundle, keep) {
+    try {
+        return await reduceRunTraces(prepared, bundle, (stream, index) => {
+            validateTraceItems(stream);
+            return keep(stream, index);
+        });
+    }
+    catch (error) {
+        // The loader's refusal names the participant, the file, why, and what to do.
+        if (error instanceof StreamTraceRefusedError)
+            throw new AnalysisSourceRefusedError(error instanceof StreamTraceTooLargeError
+                ? "ANALYSIS_SOURCE_TOO_LARGE"
+                : "ANALYSIS_SOURCE_INVALID", error.message);
+        throw error;
+    }
+}
+function validateTraceItems(stream) {
+    const actor = stream.actor;
+    if (actor === undefined)
+        return;
+    if (!isRecord(actor) || !Array.isArray(actor.items))
+        throw traceInvalid(stream.id, "it has no items list.");
+    // The place each id was first seen. A refusal names places, since an id is trace text.
+    const firstPlace = new Map();
+    for (const [index, item] of actor.items.entries()) {
+        if (!isRecord(item) ||
+            typeof item.id !== "string" ||
+            item.id.length === 0 ||
+            item.id.length > 256 ||
+            typeof item.kind !== "string" ||
+            typeof item.title !== "string" ||
+            (item.text !== undefined && typeof item.text !== "string"))
+            throw traceInvalid(stream.id, `item ${index + 1} needs an id of 1 to 256 characters, a kind and a title, and a text that is a string if it has one.`);
+        const first = firstPlace.get(item.id);
+        if (first !== undefined)
+            throw traceInvalid(stream.id, `items ${first + 1} and ${index + 1} share an id, and analysis cites each item by its id.`);
+        firstPlace.set(item.id, index);
+    }
+}

package/dist/evidence/frame-change.d.ts

@@ -0,0 +1,39 @@
+/** A rectangle of the frame, in pixels from its top-left corner. */
+export interface FrameRegion {
+    readonly x: number;
+    readonly y: number;
+    readonly width: number;
+    readonly height: number;
+}
+/**
+ * What one look found, compared with the frame the participant last saw. `settled`: a region of
+ * change held still long enough to wake the participant; `area` bounds it, and `small` is true
+ * when it is at most LARGE_REGION_TILES tiles, which the participant's hint names. `changing`: tiles moved
+ * since the previous look and nothing settled; `lookSoon` when more than LARGE_REGION_TILES moved,
+ * so a large change is confirmed by a quick look. `unchanged`: nothing moved and nothing settled.
+ * `held` on the last two bounds a small region that held still past the hold and is held back by a
+ * ticking tile or a large moving region near it.
+ */
+export type FrameVerdict = {
+    readonly verdict: "unchanged";
+    readonly held?: FrameRegion;
+} | {
+    readonly verdict: "changing";
+    readonly lookSoon: boolean;
+    readonly held?: FrameRegion;
+} | {
+    readonly verdict: "settled";
+    readonly area: FrameRegion;
+    readonly small: boolean;
+};
+/**
+ * Compare later looks at one screen with `reference`, the frame the participant last saw. Call
+ * `next` once per look, in order, with the look's time. Each tile that differs from the reference
+ * keeps the time it last moved, so a moving tile (a spinner) never joins a still region beside
+ * it. Nothing is masked: a large change that stops changing always settles, and a small one does
+ * unless a tile near it keeps ticking. Stillness is sampled at looks, so a blink seen in the same
+ * phase at every look reads as still.
+ */
+export declare function watchFrames(reference: Buffer): {
+    next(frame: Buffer, atMs: number): FrameVerdict;
+};

package/dist/evidence/frame-change.js

@@ -0,0 +1,293 @@
+import { PNG } from "pngjs";
+// Whether a screen changed while a participant waited, and whether the change has stopped
+// changing. The loop's watch (src/actors/computer-use/loop/watch.ts) asks after each look.
+//
+// The constants were chosen on 323 idle screenshot pairs from 203 retained hosted-desktop runs.
+// Regions that changed on their own were the panel clock (2 to 6 tiles, every 60 s), a text caret
+// (2 tiles, about every 0.5 s), Chrome's tab spinner (2 to 4 tiles) and one landing page's
+// rotating tagline (10 to 36 tiles, median 18). Page loads, navigations and app updates changed
+// 94 to 2,598 tiles. A moved list selection changed 35 tiles and a re-rendered
+// block of timestamps 8 to 12, both missed by the frame hash in frame-signature.ts.
+//
+// The rule for small regions was chosen on the wave 9 look corpus: 3 pages recorded for 40 minutes
+// each on hosted desktops, replayed over 84 to 168 look timelines a page. There a small region
+// that held still woke the participant falsely beside a panel clock, a relative timestamp
+// ("Opened 3 minutes ago", whose digit ticks each minute and whose tail shifts when the text
+// grows) and a moving chart. With the conditions below it woke none falsely and missed no event.
+/** The side of a square tile, in pixels: one tile is the unit of change. */
+const TILE_PX = 16;
+/**
+ * A pixel changed when one of its channels differs by more than this (0 to 255). On the wave 9
+ * corpus 32 added a false chart wake, and 48 split a new row's light fill from its text, so a row
+ * read as small regions and was seen 69.9 s late.
+ */
+const PIXEL_TOLERANCE = 24;
+/**
+ * A tile changed when at least this many of its pixels changed. One or two pixels are an
+ * antialiased edge: a spinner edge 2 pixels over the tolerance woke a 900 s watch in wave 7 live
+ * run 3c. On the wave 9 corpus every tile of its status and export labels changed 10 to 69 pixels,
+ * and its new rows (509 and 545 tiles) lost 7 or 8 tiles at 3. 1 and 3 gave the same wakes there.
+ */
+const TILE_MIN_PIXELS = 3;
+/**
+ * A still region of more than this many tiles wakes the participant at once: it is above every
+ * self-changing region measured except 11 of 110 tagline swaps, and below every page change. On
+ * the wave 9 corpus the status label (30 tiles) and new rows were large, and the export label (8
+ * tiles) small.
+ */
+const LARGE_REGION_TILES = 24;
+/**
+ * A still region of at most LARGE_REGION_TILES tiles wakes the participant once every tile in it
+ * has held still for more than this, and no tile within NEAR_TILES of it ticked or moved with a
+ * region of more than MOVING_REGION_TILES tiles in this time. Minute updates (a clock's minutes
+ * digit, "2 min ago") move again within it. On the wave 9 corpus a one-time 8-tile text change was
+ * seen 72.4 s after it landed (median, 5 s looks).
+ */
+const SMALL_REGION_HOLD_MS = 65_000;
+/**
+ * How near, in tiles, a ticking tile or a large moving region holds a small region back. The
+ * relative timestamp's tail ("1 minute ago" to "2 minutes ago") sat 4 tiles from its digit in
+ * wave 7 live run 2a and on the wave 9 corpus, where a distance of 1 left 4 tail wakes in 168
+ * timelines and 4 left none.
+ */
+const NEAR_TILES = 4;
+/**
+ * A tile ticks when its last move came at least this long after the move before it, or when it
+ * has moved once. A minute counter ticks. A spinner tile that moves every few looks does not, so a
+ * label beside a spinner still wakes the participant, at 90 s on the wave 9 synthetic screens.
+ */
+const TICK_GAP_MS = 30_000;
+/**
+ * A tile that moves with a connected moving region of more than this many tiles (a chart, a video)
+ * holds a small region within NEAR_TILES back. A chart's tiles move every few looks, so they never
+ * tick. On the wave 9 corpus, without this condition a chart's tiles woke 3 watches falsely in its
+ * page's 40 minutes, and 259 in its 84 replayed timelines; with it, none. It was not tuned apart
+ * from LARGE_REGION_TILES and has the same value. A spinner of 1 to 4 tiles stays under it.
+ */
+const MOVING_REGION_TILES = 24;
+const UNCHANGED = { verdict: "unchanged" };
+function decode(frame) {
+    try {
+        const image = PNG.sync.read(frame);
+        return image.width > 0 && image.height > 0 ? image : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+/**
+ * Compare later looks at one screen with `reference`, the frame the participant last saw. Call
+ * `next` once per look, in order, with the look's time. Each tile that differs from the reference
+ * keeps the time it last moved, so a moving tile (a spinner) never joins a still region beside
+ * it. Nothing is masked: a large change that stops changing always settles, and a small one does
+ * unless a tile near it keeps ticking. Stillness is sampled at looks, so a blink seen in the same
+ * phase at every look reads as still.
+ */
+export function watchFrames(reference) {
+    const base = decode(reference);
+    // Each tile that differs from the reference, with its pixels at the look it last moved at. Only
+    // these are kept from the previous look, so a watch holds one decoded frame and not two.
+    const differing = new Map();
+    let moves;
+    return {
+        next(frame, atMs) {
+            const current = decode(frame);
+            // A frame that cannot be decoded is compared by its bytes; a frame of another size is a new
+            // screen, one change over the whole frame.
+            if (base === undefined || current === undefined)
+                return frame.equals(reference) ? UNCHANGED : wholeFrame(current ?? base);
+            if (current.width !== base.width || current.height !== base.height)
+                return wholeFrame(current);
+            moves ??= tileMoves(base);
+            return compare(base, current, atMs, differing, moves);
+        },
+    };
+}
+function tileMoves(image) {
+    const columns = Math.ceil(image.width / TILE_PX);
+    const rows = Math.ceil(image.height / TILE_PX);
+    const never = () => new Float64Array(columns * rows).fill(Number.NaN);
+    return { columns, rows, moved: never(), movedBefore: never(), movedLarge: never() };
+}
+function wholeFrame(image) {
+    return {
+        verdict: "settled",
+        area: { x: 0, y: 0, width: image?.width ?? 0, height: image?.height ?? 0 },
+        small: false,
+    };
+}
+function compare(base, current, atMs, differing, moves) {
+    const { columns, rows } = moves;
+    const still = new Set();
+    const movedNow = new Set();
+    let moved = 0;
+    const move = (tile) => {
+        moves.movedBefore[tile] = moves.moved[tile];
+        moves.moved[tile] = atMs;
+        movedNow.add(tile);
+    };
+    for (let row = 0; row < rows; row += 1)
+        for (let column = 0; column < columns; column += 1) {
+            const tile = row * columns + column;
+            const before = differing.get(tile);
+            if (!framesDiffer(base, current, column, row)) {
+                if (before === undefined)
+                    continue;
+                differing.delete(tile);
+                // A move back to the reference is a move: a ticking digit can tick back to its old shape.
+                if (tileDiffers(before, current, column, row))
+                    move(tile);
+                continue;
+            }
+            // A tile that starts differing has moved, whatever the previous look's pixels were.
+            if (before === undefined || tileDiffers(before, current, column, row)) {
+                differing.set(tile, tilePixels(current, column, row));
+                move(tile);
+                moved += 1;
+            }
+            else
+                still.add(tile);
+        }
+    for (const region of connectedRegions(movedNow, columns))
+        if (region.length > MOVING_REGION_TILES)
+            for (const tile of region)
+                moves.movedLarge[tile] = atMs;
+    const regions = connectedRegions(still, columns);
+    const area = (region) => areaOf(region, columns, base.width, base.height);
+    const large = largest(regions.filter((region) => region.length > LARGE_REGION_TILES));
+    if (large !== undefined)
+        return { verdict: "settled", area: area(large), small: false };
+    const heldSmall = regions.filter((region) => region.every((tile) => atMs - moves.moved[tile] > SMALL_REGION_HOLD_MS));
+    const settled = largest(heldSmall.filter((region) => !heldBack(region, moves, atMs)));
+    if (settled !== undefined)
+        return { verdict: "settled", area: area(settled), small: true };
+    const waiting = largest(heldSmall);
+    const held = waiting === undefined ? {} : { held: area(waiting) };
+    if (moved > 0)
+        return { verdict: "changing", lookSoon: moved > LARGE_REGION_TILES, ...held };
+    return { verdict: "unchanged", ...held };
+}
+/**
+ * Whether a tile within NEAR_TILES of the region, outside it, ticked or moved with a large region
+ * within the hold.
+ */
+function heldBack(region, moves, atMs) {
+    const inRegion = new Set(region);
+    // NaN, a tile that never moved, is never recent.
+    const recent = (ms) => atMs - ms <= SMALL_REGION_HOLD_MS;
+    for (const tile of region)
+        for (let dy = -NEAR_TILES; dy <= NEAR_TILES; dy += 1)
+            for (let dx = -NEAR_TILES; dx <= NEAR_TILES; dx += 1) {
+                const column = (tile % moves.columns) + dx;
+                const row = Math.floor(tile / moves.columns) + dy;
+                if (column < 0 || column >= moves.columns || row < 0 || row >= moves.rows)
+                    continue;
+                const near = row * moves.columns + column;
+                const moved = moves.moved[near];
+                if (inRegion.has(near) || !recent(moved))
+                    continue;
+                const before = moves.movedBefore[near];
+                if (Number.isNaN(before) ||
+                    moved - before >= TICK_GAP_MS ||
+                    recent(moves.movedLarge[near]))
+                    return true;
+            }
+    return false;
+}
+/** Whether at least TILE_MIN_PIXELS pixels of one tile differ between two frames of one size. */
+function framesDiffer(a, b, column, row) {
+    const { width, height } = a;
+    const xEnd = Math.min(width, (column + 1) * TILE_PX);
+    const yEnd = Math.min(height, (row + 1) * TILE_PX);
+    let changed = 0;
+    for (let y = row * TILE_PX; y < yEnd; y += 1)
+        for (let x = column * TILE_PX; x < xEnd; x += 1) {
+            const at = (y * width + x) * 4;
+            if ((Math.abs(a.data[at] - b.data[at]) > PIXEL_TOLERANCE ||
+                Math.abs(a.data[at + 1] - b.data[at + 1]) > PIXEL_TOLERANCE ||
+                Math.abs(a.data[at + 2] - b.data[at + 2]) > PIXEL_TOLERANCE) &&
+                ++changed >= TILE_MIN_PIXELS)
+                return true;
+        }
+    return false;
+}
+/** A copy of one tile's RGBA pixels, row by row. */
+function tilePixels(image, column, row) {
+    const xStart = column * TILE_PX;
+    const xEnd = Math.min(image.width, xStart + TILE_PX);
+    const yEnd = Math.min(image.height, (row + 1) * TILE_PX);
+    const pixels = new Uint8Array((xEnd - xStart) * (yEnd - row * TILE_PX) * 4);
+    let to = 0;
+    for (let y = row * TILE_PX; y < yEnd; y += 1) {
+        const from = (y * image.width + xStart) * 4;
+        pixels.set(image.data.subarray(from, from + (xEnd - xStart) * 4), to);
+        to += (xEnd - xStart) * 4;
+    }
+    return pixels;
+}
+/**
+ * Whether at least TILE_MIN_PIXELS pixels of one tile of `image` differ from `pixels`, that tile's
+ * earlier pixels.
+ */
+function tileDiffers(pixels, image, column, row) {
+    const xStart = column * TILE_PX;
+    const xEnd = Math.min(image.width, xStart + TILE_PX);
+    const yEnd = Math.min(image.height, (row + 1) * TILE_PX);
+    const { data } = image;
+    let at = 0;
+    let changed = 0;
+    for (let y = row * TILE_PX; y < yEnd; y += 1)
+        for (let x = xStart; x < xEnd; x += 1, at += 4) {
+            const from = (y * image.width + x) * 4;
+            if ((Math.abs(pixels[at] - data[from]) > PIXEL_TOLERANCE ||
+                Math.abs(pixels[at + 1] - data[from + 1]) > PIXEL_TOLERANCE ||
+                Math.abs(pixels[at + 2] - data[from + 2]) > PIXEL_TOLERANCE) &&
+                ++changed >= TILE_MIN_PIXELS)
+                return true;
+        }
+    return false;
+}
+/** Tiles grouped with every tile of the set next to them, diagonals included. */
+function connectedRegions(tiles, columns) {
+    const seen = new Set();
+    const regions = [];
+    for (const start of tiles) {
+        if (seen.has(start))
+            continue;
+        seen.add(start);
+        const region = [];
+        const queue = [start];
+        for (let tile = queue.pop(); tile !== undefined; tile = queue.pop()) {
+            region.push(tile);
+            const column = tile % columns;
+            for (const dy of [-1, 0, 1])
+                for (const dx of [-1, 0, 1]) {
+                    const neighbourColumn = column + dx;
+                    const neighbour = tile + dy * columns + dx;
+                    if (neighbourColumn < 0 || neighbourColumn >= columns)
+                        continue;
+                    if (!tiles.has(neighbour) || seen.has(neighbour))
+                        continue;
+                    seen.add(neighbour);
+                    queue.push(neighbour);
+                }
+        }
+        regions.push(region);
+    }
+    return regions;
+}
+function largest(regions) {
+    return regions.reduce((best, region) => (best === undefined || region.length > best.length ? region : best), undefined);
+}
+function areaOf(tiles, columns, width, height) {
+    const columnsOf = tiles.map((tile) => tile % columns);
+    const rowsOf = tiles.map((tile) => Math.floor(tile / columns));
+    const x = Math.min(...columnsOf) * TILE_PX;
+    const y = Math.min(...rowsOf) * TILE_PX;
+    return {
+        x,
+        y,
+        width: Math.min(width, (Math.max(...columnsOf) + 1) * TILE_PX) - x,
+        height: Math.min(height, (Math.max(...rowsOf) + 1) * TILE_PX) - y,
+    };
+}

package/dist/feedback/export.js

@@ -17,7 +17,6 @@ import { buildObserverData, OBSERVER_DATA_SCHEMA, observerDataFromSaved, } from
 import { resolveRunPath } from "../run/locate.js";
 import { loadRunTraces, StreamTraceRefusedError } from "../run/stream-trace.js";
 import { verifyRun } from "../verify/verify.js";
-import {} from "../run/bundle.js";
 import { exportRedactedBundle } from "./export-bundle.js";
 import { loadAnalysis } from "../analysis/load.js";
 import { readNotesForSharing } from "../run/note-files.js";

package/dist/observer/data.d.ts

@@ -1,7 +1,7 @@
 import { type ActorEnding } from "../actors/stop-cause.js";
 import type { RunBundle, RunCostSummary, RunEvent, RunSimulation } from "../run/bundle.js";
 import { type RunDisplay } from "../run/display.js";
-import type { RunStream, RunStreamKind } from "../run/streams.js";
+import type { RunStream, RunStreamKind, StoredRunStream } from "../run/streams.js";
 /**
  * The Observer's data. From v2 observer-data.json leaves out each participant's items, and from v3
  * each participant's per-request lists (REQUEST_LISTS in src/run/stream-trace.ts), which no part
@@ -96,7 +96,7 @@ export interface ObserverData {
         artifactRoot: string;
     };
 }
-export interface ObserverStream extends RunStream {
+export interface ObserverStream extends StoredRunStream {
     /** Only the attached server may grant provider-origin access for a live desktop iframe. */
     embed?: NonNullable<RunStream["embed"]> & {
         runtimeDesktop?: true;