humanish 0.123.0 → 0.125.0

package/dist/observer/participant-items.js

@@ -0,0 +1,85 @@
+// The Observer's data in two parts: observer-data.json, the run and every participant without
+// their trace items, and the items themselves. A participant working for hours records thousands
+// of items, so the items are most of the bytes; apart, the saved observer-data.json stays small
+// enough to verify, and the page carries the items compressed. The Observer bundles this module,
+// so it imports types only (docs/decisions/0002-observer-is-one-self-contained-file.md).
+export const OBSERVER_ITEMS_SCHEMA = "humanish.observer-items.v1";
+/** The data as observer-data.json and the page's items carry it. */
+export function splitObserverData(data) {
+    // No prototype, so a stream id such as `__proto__` is an entry like any other.
+    const items = Object.create(null);
+    const streams = data.streams.map((stream) => {
+        const { actor, liveActor, ...rest } = stream;
+        const entry = {};
+        const shell = rest;
+        // Saved projections are read unchecked, and one written by an older version may have no list.
+        if (Array.isArray(stream.artifacts) &&
+            stream.artifacts.some((artifact) => artifact.kind === "screenshot")) {
+            entry.artifacts = stream.artifacts;
+            shell.artifacts = stream.artifacts.filter((artifact) => artifact.kind !== "screenshot");
+            shell.screenshotArtifacts = stream.artifacts.length - shell.artifacts.length;
+        }
+        if (actor !== undefined) {
+            const { items: actorItems, ...summary } = actor;
+            entry.actor = actorItems;
+            shell.actor = summary;
+        }
+        if (liveActor !== undefined) {
+            const { items: liveItems, ...summary } = liveActor;
+            entry.liveActor = liveItems;
+            shell.liveActor = summary;
+        }
+        if (Object.keys(entry).length > 0)
+            items[stream.id] = entry;
+        return shell;
+    });
+    return {
+        shell: { ...data, streams },
+        items: { schema: OBSERVER_ITEMS_SCHEMA, runId: data.run.runId, streams: items },
+    };
+}
+/**
+ * Whether `items` are this run's and hold every participant's items that the data leaves out. The
+ * page reads nothing short of that: participants shown with no steps would say they did nothing.
+ */
+export function itemsCoverShell(shell, items) {
+    if (items.runId !== shell.run.runId)
+        return false;
+    return shell.streams.every((stream) => {
+        const entry = Object.hasOwn(items.streams, stream.id) ? items.streams[stream.id] : undefined;
+        return ((stream.actor === undefined || Array.isArray(entry?.actor)) &&
+            (stream.liveActor === undefined || Array.isArray(entry?.liveActor)) &&
+            (stream.screenshotArtifacts === undefined ||
+                (Array.isArray(entry?.artifacts) &&
+                    entry.artifacts.filter((artifact) => artifact.kind === "screenshot").length ===
+                        stream.screenshotArtifacts)));
+    });
+}
+/** Whether the data leaves out items or screenshot entries that only an items document holds. */
+export function shellNeedsItems(shell) {
+    return shell.streams.some((stream) => stream.actor !== undefined ||
+        stream.liveActor !== undefined ||
+        stream.screenshotArtifacts !== undefined);
+}
+/**
+ * The data with each participant's items back in place. A stream the items do not cover keeps an
+ * empty list, so the page shows the participant with no recorded steps.
+ */
+export function joinObserverData(shell, items) {
+    const byStream = items !== null && items.runId === shell.run.runId ? items.streams : {};
+    return {
+        ...shell,
+        streams: shell.streams.map((stream) => {
+            const entry = Object.hasOwn(byStream, stream.id) ? byStream[stream.id] : undefined;
+            const { actor, liveActor, screenshotArtifacts: _moved, ...rest } = stream;
+            return {
+                ...rest,
+                artifacts: entry?.artifacts ?? stream.artifacts,
+                ...(actor === undefined ? {} : { actor: { ...actor, items: entry?.actor ?? [] } }),
+                ...(liveActor === undefined
+                    ? {}
+                    : { liveActor: { ...liveActor, items: entry?.liveActor ?? [] } }),
+            };
+        }),
+    };
+}

package/dist/observer/pinned-files.d.ts

@@ -17,10 +17,12 @@ export interface PinnedDirectory {
 /**
  * The file's bytes, or null when it cannot be read under the root's checks. With `maxBytes`, a
  * file larger than that is refused from its opened size and the read stops there, so a large file
- * is never held in memory.
+ * is never held in memory. With `length`, only the file's first `length` bytes are read, or all of
+ * a shorter file.
  */
 export declare function readContainedFile(root: PinnedDirectory, filePathInput: string, options?: {
     maxBytes?: number;
+    length?: number;
 }): Promise<Buffer | null>;
 export declare function openContainedFile(root: PinnedDirectory, filePathInput: string): Promise<{
     handle: FileHandle;

package/dist/observer/pinned-files.js

@@ -5,23 +5,26 @@ import { createHash } from "node:crypto";
 import { constants as fsConstants } from "node:fs";
 import { lstat, open, realpath } from "node:fs/promises";
 import path from "node:path";
-import { readOpenedAtMost } from "../run/contained-output.js";
+import { readOpenedAtMost, readOpenedPrefix } from "../run/contained-output.js";
 import { isPathInside, isSafeRunIdSegment } from "../run/paths.js";
 /**
  * The file's bytes, or null when it cannot be read under the root's checks. With `maxBytes`, a
  * file larger than that is refused from its opened size and the read stops there, so a large file
- * is never held in memory.
+ * is never held in memory. With `length`, only the file's first `length` bytes are read, or all of
+ * a shorter file.
  */
 export async function readContainedFile(root, filePathInput, options = {}) {
     const opened = await openContainedFile(root, filePathInput);
     if (!opened)
         return null;
     try {
-        const body = options.maxBytes === undefined
-            ? await opened.handle.readFile()
-            : opened.size > options.maxBytes
-                ? null
-                : await readOpenedAtMost(opened.handle, options.maxBytes);
+        const body = options.length !== undefined
+            ? await readOpenedPrefix(opened.handle, options.length)
+            : options.maxBytes === undefined
+                ? await opened.handle.readFile()
+                : opened.size > options.maxBytes
+                    ? null
+                    : await readOpenedAtMost(opened.handle, options.maxBytes);
         if (body === null)
             return null;
         // A write that landed during the read changed the file's mtime and ctime.

package/dist/observer/render.js

@@ -17,6 +17,7 @@ import { RUN_STATUS_FILE } from "../run/status.js";
 import { verifyRunPrepared } from "../verify/verify.js";
 import { renderObserverHtml } from "./artifact.js";
 import { buildObserverData } from "./data.js";
+import { splitObserverData } from "./participant-items.js";
 import { buildServeSecurityHeaders, hostAllowed, parsePublicOrigin } from "./http.js";
 import { listenOnLoopback } from "./listen.js";
 import { NOTES_PATH } from "../run/note-shape.js";
@@ -116,7 +117,7 @@ export async function renderObserver(cwdInput, runInput, options = {}) {
         verifiedAt: new Date().toISOString(),
         reasons: verified.shareSafety.reasons.map((reason) => reason.code),
     };
-    await writeContainedOutputFile(preparedRunPaths, path.join("observer", "observer-data.json"), `${JSON.stringify(observerData, null, 2)}\n`, "utf8");
+    await writeContainedOutputFile(preparedRunPaths, path.join("observer", "observer-data.json"), `${JSON.stringify(splitObserverData(observerData).shell, null, 2)}\n`, "utf8");
     await writeContainedOutputFile(preparedRunPaths, path.join("observer", "study-analysis.json"), `${JSON.stringify(analysis, null, 2)}\n`, "utf8");
     await writeContainedOutputFile(preparedRunPaths, path.join("observer", "index.html"), renderObserverHtml(observerData, { analysis, notes: await readRunNotes(preparedRunPaths) }), "utf8");
     await validatePreparedRunArtifactPaths(preparedRunPaths);

package/dist/observer/run-routes.js

@@ -16,7 +16,8 @@ import { MAX_NOTE_FILE_BYTES, readRunNotes, RUN_NOTES_DIR, } from "../run/note-f
 import { analysisCostOf, runCost, runCostLabel } from "../run/run-cost.js";
 import { RUN_STATUS_FILE, RUN_STATUS_STALE_MS, isRunStatusRecord, } from "../run/status.js";
 import { renderObserverHtml } from "./artifact.js";
-import { buildObserverData, recordedStreamEmbed, withObserverEndings, } from "./data.js";
+import { splitObserverData } from "./participant-items.js";
+import { buildObserverData, observerDataFromSaved, recordedStreamEmbed, withObserverEndings, } from "./data.js";
 import { buildArtifactSecurityHeaders } from "./http.js";
 import { assertPinnedDirectory, openContainedFile, pinDirectChildDirectory, readContainedFile, relativeToRoot, sha256OfOpenedFile, } from "./pinned-files.js";
 /** The run's artifact paths, only when they are still the pinned directory being served. */
@@ -116,13 +117,19 @@ export async function serveRunPath(runRoot, relativePath, response, runtimeStrea
         }), "text/html; charset=utf-8");
         return;
     }
-    if (cleanedRelativePath === "observer/observer-data.json") {
+    // observer-data.json is the run without its participants' items, as the run saves it. The page
+    // polls observer-feed.json, the same data with the items, so one projection serves each poll.
+    if (cleanedRelativePath === "observer/observer-data.json" ||
+        cleanedRelativePath === "observer/observer-feed.json") {
         const observerData = await observerDataOrReason(runRoot, runtimeStreamUrls);
         if ("unavailable" in observerData) {
             writeResponse(response, 404, `Observer data not found: ${observerData.unavailable}`, "text/plain; charset=utf-8");
             return;
         }
-        writeResponse(response, 200, JSON.stringify(observerData, null, 2), "application/json; charset=utf-8");
+        const { shell, items } = splitObserverData(observerData);
+        writeResponse(response, 200, cleanedRelativePath === "observer/observer-data.json"
+            ? JSON.stringify(shell, null, 2)
+            : JSON.stringify({ data: shell, items }), "application/json; charset=utf-8");
         return;
     }
     if (cleanedRelativePath === "observer/study-analysis.json") {
@@ -158,9 +165,10 @@ export async function serveRunPath(runRoot, relativePath, response, runtimeStrea
 }
 /** The run's trace files, read through its pinned folder. */
 function pinnedTraceReader(runRoot) {
-    return async (relativePath, maxBytes) => {
+    return async (relativePath, maxBytes, length) => {
         const bytes = await readContainedFile(runRoot, path.join(runRoot.physicalPath, relativePath), {
             maxBytes,
+            ...(length === undefined ? {} : { length }),
         }).catch(() => null);
         return bytes === null ? { refused: `${relativePath} could not be read` } : { bytes };
     };
@@ -214,7 +222,10 @@ async function savedObserverData(runRoot, record, runtimeStreamUrls) {
         const observerBytes = await readContainedFile(runRoot, path.join(runRoot.physicalPath, "observer", "observer-data.json"));
         if (!observerBytes)
             throw new Error("observer-data.json unavailable");
-        return withRuntimeStreamUrls(withLocalRunStatus(record, withObserverEndings(JSON.parse(observerBytes.toString("utf8")))), runtimeStreamUrls);
+        const saved = observerDataFromSaved(JSON.parse(observerBytes.toString("utf8")));
+        if (saved === null)
+            throw new Error("observer-data.json holds no participant's items");
+        return withRuntimeStreamUrls(withLocalRunStatus(record, withObserverEndings(saved)), runtimeStreamUrls);
     }
     catch {
         return { unavailable: "The run has written no run.json or Observer data yet." };