@itmetsam/samurai-framework 0.7.0 → 0.8.0

package/dist/project/files-store.js

@@ -0,0 +1,106 @@
+import { constants, createReadStream } from "node:fs";
+import { copyFile, mkdir, readFile, stat } from "node:fs/promises";
+import { createHash } from "node:crypto";
+import path from "node:path";
+/** Files bigger than this aren't stored while recording */
+export const RECORD_LIMIT = 50 * 1024 * 1024;
+const NAME_LIMIT = 120;
+/** A picked file's name as a file name in the files folder: no folders, control characters or leading dots */
+export function cleanFileName(name) {
+    let cleaned = (name.split(/[\\/]/).pop() ?? "")
+        .replace(/[\u0000-\u001f\u007f]/g, "")
+        .trim()
+        .replace(/^\.+/, "")
+        .trim();
+    if (cleaned.length > NAME_LIMIT) {
+        const ext = path.extname(cleaned);
+        const keep = ext.length > 0 && ext.length < 20 ? ext : "";
+        cleaned = cleaned.slice(0, NAME_LIMIT - keep.length) + keep;
+    }
+    return cleaned === "" ? "file" : cleaned;
+}
+/** The files folder: absolute, and relative to the project with forward slashes */
+export function filesFolder(projectDir, filesDir = "files") {
+    const absolute = path.resolve(projectDir, filesDir);
+    return {
+        absolute,
+        relative: path.relative(projectDir, absolute).split(path.sep).join("/"),
+    };
+}
+function hashOf(file) {
+    return new Promise((resolve, reject) => {
+        const hash = createHash("sha256");
+        createReadStream(file)
+            .on("data", (chunk) => hash.update(chunk))
+            .on("end", () => resolve(hash.digest("hex")))
+            .on("error", reject);
+    });
+}
+function sameContent(candidate, size, hash) {
+    return stat(candidate).then(async (info) => info.size !== size ? false : (await hashOf(candidate)) === (await hash()));
+}
+/**
+ * Stores the bytes in `content` in the files folder under their own (cleaned) name, so the page sees at
+ * replay the name it saw while recording. If the name is free the file is copied there; if a file of that
+ * name holds the same content it is reused. If the name is taken by other content, the file goes in a
+ * folder named after the first 8 hex characters of its SHA-256 (`<H>/<name>`; 16, then all 64, on a
+ * collision). Different names with the same content are stored separately. Nothing is overwritten and
+ * nothing is written outside the folder. `content` is left in place. Returns the project-relative path.
+ */
+export async function storeFile(options) {
+    const { absolute, relative } = filesFolder(options.projectDir, options.filesDir);
+    const projectPath = (...parts) => [relative, ...parts].filter((part) => part !== "").join("/");
+    const name = cleanFileName(options.name);
+    const { size } = await stat(options.content);
+    await mkdir(absolute, { recursive: true });
+    let hash;
+    const contentHash = async () => (hash ??= await hashOf(options.content));
+    /** Copies to the target, or reports it reused when it holds the same content; undefined when it holds other content */
+    const place = async (target, relativeParts) => {
+        try {
+            await copyFile(options.content, target, constants.COPYFILE_EXCL);
+            return { path: projectPath(...relativeParts), reused: false };
+        }
+        catch (err) {
+            if (err.code !== "EEXIST")
+                throw err;
+        }
+        return (await sameContent(target, size, contentHash))
+            ? { path: projectPath(...relativeParts), reused: true }
+            : undefined;
+    };
+    const direct = await place(path.join(absolute, name), [name]);
+    if (direct)
+        return direct;
+    const full = await contentHash();
+    for (const length of [8, 16, 64]) {
+        const folder = full.slice(0, length);
+        await mkdir(path.join(absolute, folder), { recursive: true });
+        const stored = await place(path.join(absolute, folder, name), [
+            folder,
+            name,
+        ]);
+        if (stored)
+            return stored;
+    }
+    throw new Error(`${options.name} can't be stored in ${relative}/`);
+}
+/** Whether the project's .gitignore ignores the folder (`files`, `files/`, `/files`, `files/*`, `files/**`) */
+export async function isFolderIgnored(projectDir, relative) {
+    if (relative === "")
+        return false;
+    let text;
+    try {
+        text = await readFile(path.join(projectDir, ".gitignore"), "utf8");
+    }
+    catch {
+        return false;
+    }
+    return text.split(/\r?\n/).some((line) => {
+        const pattern = line
+            .trim()
+            .replace(/^\//, "")
+            .replace(/\/(\*\*?)?$/, "");
+        return pattern === relative;
+    });
+}

package/dist/project/index.d.ts

@@ -1,3 +1,4 @@
 export { ConfigEditError, editConfig, hasConfigPath, propertyText, type ConfigValue, } from "./config-edit.js";
 export { addVariable, ExistsError, VARIABLE_NAME } from "./variables.js";
 export { isGitIgnored, SECRET_NAME, setSecret } from "./secrets-file.js";
+export { cleanFileName, filesFolder, isFolderIgnored, RECORD_LIMIT, storeFile, } from "./files-store.js";

package/dist/project/index.js

@@ -1,3 +1,4 @@
 export { ConfigEditError, editConfig, hasConfigPath, propertyText, } from "./config-edit.js";
 export { addVariable, ExistsError, VARIABLE_NAME } from "./variables.js";
 export { isGitIgnored, SECRET_NAME, setSecret } from "./secrets-file.js";
+export { cleanFileName, filesFolder, isFolderIgnored, RECORD_LIMIT, storeFile, } from "./files-store.js";

package/dist/recorder/apply.js

@@ -1,10 +1,12 @@
 import { applyEdit } from "../steps/edit.js";
 /** Writes a recorder event into the spec source, as an edit of test number `test` */
 export function applyRecorderEvent(source, test, event) {
-    return applyEdit(source, {
-        op: event.op,
-        test,
-        index: event.index,
-        step: event.step,
-    });
+    return event.op === "remove"
+        ? applyEdit(source, { op: "remove", test, index: event.index })
+        : applyEdit(source, {
+            op: event.op,
+            test,
+            index: event.index,
+            step: event.step,
+        });
 }

package/dist/recorder/files-capture.d.ts

@@ -0,0 +1,26 @@
+/** The BiDi sandbox realm the file capture script runs in: page scripts can't reach its listeners or the files it keeps */
+export declare const FILES_SANDBOX = "samurai-files";
+/** `Symbol.for(FILES_KEY)` on the sandbox global holds the hook the recorder reads files through */
+export declare const FILES_KEY = "samurai.files";
+/** A file is read and sent in pieces of this many bytes */
+export declare const CHUNK_BYTES: number;
+/**
+ * Function declaration installed as a preload script in the `samurai-files` sandbox, called as `(send)` with
+ * the recorder's files channel. Top-level documents only. Reports:
+ *
+ * - `{ type: "files", via: "input", target, candidates, files }`: a person changed the files of an
+ *   `<input type=file>` (no files: it was cleared).
+ * - `{ type: "files", via: "drop", target, candidates, files }`: a person dropped files on `target`.
+ * - `{ type: "picker" }`: a file input got a click (the page's `input.click()`, a label, or the input itself),
+ *   which opens the file picker.
+ * - `{ type: "picker-cancel" }`: the picker was closed without choosing.
+ *
+ * `files` is `{ id, name, type, size }[]`. The `File` objects stay here, kept by id, until the recorder reads
+ * them through `globalThis[Symbol.for("samurai.files")].read(id, offset)` (base64 of up to 1 MB) and calls
+ * `release(id)`. A drop's file list can only be read during the event, so it is taken then.
+ * Events a person didn't cause (`isTrusted` false) are ignored, unless `trustSynthetic` (tests only).
+ * A string, so no transpiler helper can end up in it.
+ */
+export declare function filesCaptureSource(options?: {
+    trustSynthetic?: boolean;
+}): string;

package/dist/recorder/files-capture.js

@@ -0,0 +1,92 @@
+// src/recorder/files-capture.ts
+import { CANDIDATES_JS } from "./candidates.js";
+/** The BiDi sandbox realm the file capture script runs in: page scripts can't reach its listeners or the files it keeps */
+export const FILES_SANDBOX = "samurai-files";
+/** `Symbol.for(FILES_KEY)` on the sandbox global holds the hook the recorder reads files through */
+export const FILES_KEY = "samurai.files";
+/** A file is read and sent in pieces of this many bytes */
+export const CHUNK_BYTES = 1024 * 1024;
+/**
+ * Function declaration installed as a preload script in the `samurai-files` sandbox, called as `(send)` with
+ * the recorder's files channel. Top-level documents only. Reports:
+ *
+ * - `{ type: "files", via: "input", target, candidates, files }`: a person changed the files of an
+ *   `<input type=file>` (no files: it was cleared).
+ * - `{ type: "files", via: "drop", target, candidates, files }`: a person dropped files on `target`.
+ * - `{ type: "picker" }`: a file input got a click (the page's `input.click()`, a label, or the input itself),
+ *   which opens the file picker.
+ * - `{ type: "picker-cancel" }`: the picker was closed without choosing.
+ *
+ * `files` is `{ id, name, type, size }[]`. The `File` objects stay here, kept by id, until the recorder reads
+ * them through `globalThis[Symbol.for("samurai.files")].read(id, offset)` (base64 of up to 1 MB) and calls
+ * `release(id)`. A drop's file list can only be read during the event, so it is taken then.
+ * Events a person didn't cause (`isTrusted` false) are ignored, unless `trustSynthetic` (tests only).
+ * A string, so no transpiler helper can end up in it.
+ */
+export function filesCaptureSource(options = {}) {
+    return String.raw `(send) => {
+  if (window !== window.top) return;
+  const KEY = Symbol.for("${FILES_KEY}");
+  if (globalThis[KEY]) globalThis[KEY].dispose();
+
+  ${CANDIDATES_JS}
+
+  const TRUST_SYNTHETIC = ${options.trustSynthetic === true};
+  const trusted = (event) => event.isTrusted || TRUST_SYNTHETIC;
+  const files = new Map();
+  let nextId = 1;
+  const keep = (list) => Array.from(list || []).map((file) => {
+    const id = nextId++;
+    files.set(id, file);
+    return { id, name: file.name, type: file.type, size: file.size };
+  });
+  const isFileInput = (el) =>
+    el instanceof Element && el.tagName === "INPUT" && (el.getAttribute("type") || "").toLowerCase() === "file";
+  const listening = [];
+  const listen = (type, handler) => {
+    window.addEventListener(type, handler, true);
+    listening.push([type, handler]);
+  };
+
+  listen("change", (event) => {
+    const el = event.target;
+    if (!trusted(event) || !isFileInput(el)) return;
+    send({ type: "files", via: "input", target: el, candidates: candidatesOf(el), files: keep(el.files) });
+  });
+  listen("cancel", (event) => {
+    if (trusted(event) && isFileInput(event.target)) send({ type: "picker-cancel" });
+  });
+  // The page opens a picker with input.click(), which is not a trusted click: any click on a file input counts
+  listen("click", (event) => {
+    if (isFileInput(event.target)) send({ type: "picker" });
+  });
+  listen("drop", (event) => {
+    const el = event.target;
+    if (!trusted(event) || inToolbar(event) || !(el instanceof Element)) return;
+    const data = event.dataTransfer;
+    if (!data || Array.from(data.types || []).indexOf("Files") < 0 || data.files.length === 0) return;
+    send({ type: "files", via: "drop", target: el, candidates: candidatesOf(el), files: keep(data.files) });
+  });
+
+  // arrayBuffer() hands back a page-realm buffer the sandbox may not read (Xray wrapper); a data URL is a string
+  const toBase64 = (blob) => new Promise((resolve, reject) => {
+    const reader = new FileReader();
+    reader.onload = () => resolve(String(reader.result).slice(String(reader.result).indexOf(",") + 1));
+    reader.onerror = () => reject(reader.error || new Error("read failed"));
+    reader.readAsDataURL(blob);
+  });
+  globalThis[KEY] = {
+    read: async (id, offset) => {
+      const file = files.get(id);
+      if (!file) throw new Error("No file " + id);
+      return toBase64(file.slice(offset, offset + ${CHUNK_BYTES}));
+    },
+    release: (id) => { files.delete(id); },
+    dispose: () => {
+      for (const [type, handler] of listening.splice(0)) window.removeEventListener(type, handler, true);
+      files.clear();
+      if (globalThis[KEY] && globalThis[KEY].read) delete globalThis[KEY];
+    },
+  };
+}`;
+}

package/dist/recorder/files-reader.d.ts

@@ -0,0 +1,7 @@
+import type { BiDiConnector } from "../transport/bidi-connection.js";
+/** Reads a file the capture script kept, `CHUNK_BYTES` at a time, handing each piece to `onChunk` in order */
+export declare function readPageFile(connector: BiDiConnector, context: string, id: number, size: number, onChunk: (bytes: Buffer) => unknown): Promise<void>;
+/** Lets the capture script drop its reference to a file */
+export declare function releasePageFile(connector: BiDiConnector, context: string, id: number): Promise<void>;
+/** Takes the capture script's listeners off the loaded document (the recording ended) */
+export declare function disposeFilesCapture(connector: BiDiConnector, context: string): Promise<void>;

package/dist/recorder/files-reader.js

@@ -0,0 +1,25 @@
+// src/recorder/files-reader.ts
+import { callFunction } from "../script/call-function.js";
+import { CHUNK_BYTES, FILES_KEY, FILES_SANDBOX } from "./files-capture.js";
+const HOOK = `globalThis[Symbol.for(${JSON.stringify(FILES_KEY)})]`;
+/** Reads a file the capture script kept, `CHUNK_BYTES` at a time, handing each piece to `onChunk` in order */
+export async function readPageFile(connector, context, id, size, onChunk) {
+    for (let offset = 0; offset < size; offset += CHUNK_BYTES) {
+        const base64 = await callFunction(connector, context, `(id, offset) => ${HOOK}.read(id, offset)`, [id, offset], { sandbox: FILES_SANDBOX });
+        await onChunk(Buffer.from(base64, "base64"));
+    }
+}
+/** Lets the capture script drop its reference to a file */
+export async function releasePageFile(connector, context, id) {
+    await callFunction(connector, context, `(id) => { ${HOOK}?.release(id); }`, [id], {
+        sandbox: FILES_SANDBOX,
+        awaitPromise: false,
+    });
+}
+/** Takes the capture script's listeners off the loaded document (the recording ended) */
+export async function disposeFilesCapture(connector, context) {
+    await callFunction(connector, context, `() => { ${HOOK}?.dispose(); }`, [], {
+        sandbox: FILES_SANDBOX,
+        awaitPromise: false,
+    });
+}

package/dist/recorder/index.d.ts

@@ -1,3 +1,3 @@
-export { Recorder, type RecorderOptions, type RecorderEvent, type ToolbarOptions, } from "./recorder.js";
+export { Recorder, type RecorderOptions, type RecorderEvent, type ToolbarOptions, type FileRecording, type MissingFile, missingFileText, } from "./recorder.js";
 export { applyRecorderEvent } from "./apply.js";
 export { recordSpec, pickTest, type RecordSpecOptions, type RecordResult, } from "./session.js";

package/dist/recorder/index.js

@@ -1,3 +1,3 @@
-export { Recorder, } from "./recorder.js";
+export { Recorder, missingFileText, } from "./recorder.js";
 export { applyRecorderEvent } from "./apply.js";
 export { recordSpec, pickTest, } from "./session.js";

package/dist/recorder/normalise.d.ts

@@ -1,5 +1,5 @@
-import type { Expectation, LocatorSpec, Step } from "../steps/model.js";
-/** What the recorder does to the step list: a new step at `index`, or a step rewritten in place */
+import type { Expectation, FileEntry, LocatorSpec, Step } from "../steps/model.js";
+/** What the recorder does to the step list: a new step at `index`, a step rewritten in place, or one taken out */
 export type RecorderEvent = {
     op: "insert";
     index: number;
@@ -8,6 +8,9 @@ export type RecorderEvent = {
     op: "replace";
     index: number;
     step: Step;
+} | {
+    op: "remove";
+    index: number;
 };
 /** A page event whose element already has its locators */
 export type Observation = {
@@ -45,6 +48,21 @@ export type Observation = {
         kind: "env" | "secret";
         name: string;
     };
+}
+/** A file input got a click: the click recorded just before it opened the file picker */
+ | {
+    type: "picker";
+}
+/** The file picker closed without a choice */
+ | {
+    type: "picker-cancel";
+}
+/** Files were set on an input or dropped on an element, and are stored (or listed as missing) */
+ | {
+    type: "files";
+    via: "input" | "drop";
+    locator: LocatorSpec;
+    files: FileEntry[];
 } | {
     type: "wait-idle";
 } | {
@@ -66,6 +84,8 @@ export interface NormaliserOptions {
     /** A navigation within this many ms of the last action is a consequence of it. @default 2000 */
     navigationWindow?: number;
 }
+/** A picker that opens within this many ms of a click was opened by it */
+export declare const PICKER_WINDOW = 1000;
 /** The name a secret gets in `secrets.<NAME>`: the field's name or id, upper snake case */
 export declare function secretNameFor(hint: string | undefined): string;
 export declare function expectationFor(text: string, value: string | undefined): Expectation;
@@ -83,6 +103,10 @@ export declare class Normaliser {
     private lastFill;
     private lastAction;
     private lastStep;
+    /** The last click, so a picker it opened can take its place */
+    private lastClick;
+    /** The click that opened the file picker that is open now */
+    private picker;
     private readonly window;
     constructor(options: NormaliserOptions);
     /** Index the next inserted step gets */

package/dist/recorder/normalise.js

@@ -1,4 +1,6 @@
 const TEXT_LIMIT = 100;
+/** A picker that opens within this many ms of a click was opened by it */
+export const PICKER_WINDOW = 1000;
 /** The name a secret gets in `secrets.<NAME>`: the field's name or id, upper snake case */
 export function secretNameFor(hint) {
     const name = (hint ?? "")
@@ -38,6 +40,7 @@ export class Normaliser {
     }
     insert(step) {
         this.lastStep = step;
+        this.picker = undefined;
         return { op: "insert", index: this.next++, step };
     }
     /** The path (and query) of `url` when it is inside the base URL, else `url` */
@@ -90,12 +93,56 @@ export class Normaliser {
             this.lastFill = undefined;
             return [this.insert(fromToolbar)];
         }
+        // The sandbox reports a picker for any click on a file input, including ones a page script fires:
+        // it is only a hint, and only the very last step, recorded just before, may be replaced or removed
+        // (a page's own input.click() also counts as the opening click, bounded to the last step within PICKER_WINDOW)
+        if (observation.type === "picker") {
+            const click = this.lastClick;
+            this.picker =
+                click && click.index === this.next - 1 && at - click.at <= PICKER_WINDOW
+                    ? click.index
+                    : undefined;
+            return [];
+        }
+        if (observation.type === "picker-cancel") {
+            const index = this.picker;
+            this.picker = undefined;
+            if (index === undefined || index !== this.next - 1)
+                return [];
+            this.next--;
+            this.lastClick = undefined;
+            this.lastStep = undefined;
+            return [{ op: "remove", index }];
+        }
+        if (observation.type === "files") {
+            this.lastFill = undefined;
+            this.lastAction = at;
+            const step = {
+                kind: observation.via === "drop" ? "dropFiles" : "setInputFiles",
+                locator: observation.locator,
+                files: observation.files,
+            };
+            const index = this.picker;
+            if (observation.via === "input" &&
+                index !== undefined &&
+                index === this.next - 1) {
+                this.picker = undefined;
+                this.lastClick = undefined;
+                this.lastStep = step;
+                return [{ op: "replace", index, step }];
+            }
+            return [this.insert(step)];
+        }
         if (observation.type === "click") {
             if (observation.textEntry)
                 return []; // focusing a field: the fill that follows needs no click
             this.lastFill = undefined;
             this.lastAction = at;
-            return [this.insert({ kind: "click", locator: observation.locator })];
+            const events = [
+                this.insert({ kind: "click", locator: observation.locator }),
+            ];
+            this.lastClick = { index: this.next - 1, at };
+            return events;
         }
         if (observation.type === "press") {
             // The page may react by clearing the field: what is typed next is a new fill, not a rewrite of this one

package/dist/recorder/recorder.d.ts

@@ -21,6 +21,24 @@ export type ToolbarOptions = ControllerOptions & {
      */
     onStop?: () => void;
 };
+/** Where uploads made while recording are stored */
+export interface FileRecording {
+    /** The project folder */
+    projectDir: string;
+    /** Relative to the project folder. @default "files" */
+    filesDir?: string;
+    /** A file that isn't stored (over 50 MB, or unreadable): its step points at `path`, which doesn't exist */
+    onMissing?: (file: MissingFile) => void;
+    /** @internal Tests only: also record events no person caused (`isTrusted` false) */
+    trustSynthetic?: boolean;
+}
+export interface MissingFile {
+    /** Project-relative, as the step has it */
+    path: string;
+    name: string;
+    size: number;
+    reason: "too-big" | "unreadable";
+}
 export interface RecorderOptions {
     /** Called for every step the tester produces, in order. Steps get indexes from `at` on */
     onEvent: (event: RecorderEvent) => void;
@@ -36,6 +54,8 @@ export interface RecorderOptions {
     baseURL?: string;
     /** Shows the recorder toolbar in the page (pause, assert, insert, wait, screenshot, stop). Without it, no toolbar */
     toolbar?: ToolbarOptions;
+    /** Records uploads (file inputs and drops) into the project's files folder. Without it, uploads aren't recorded */
+    files?: FileRecording;
 }
 /**
  * Watches the tester use the page and reports steps as they happen. A preload script reports clicks,
@@ -55,9 +75,21 @@ export declare class Recorder {
     private readonly queue;
     private readonly known;
     private stopped;
+    /** Set when stopping begins: files messages after it are not taken */
+    private ending;
     private stopping;
     private controller;
     private constructor();
+    /** The files capture script's channel, handed only to its sandbox */
+    private readonly filesChannel;
+    private filesPreload;
+    private filesRealms;
+    private readonly stored;
+    private readonly missing;
+    /** Files stored or reused in the files folder by this recording, project-relative */
+    get storedFiles(): string[];
+    /** Files whose steps point at a path that wasn't written */
+    get missingFiles(): MissingFile[];
     /** The context's `samurai-toolbar` sandbox realms, as last listed: the only realms toolbar messages are taken from */
     private toolbarRealms;
     /** Toolbar messages waiting for a realm lookup, and those that arrived behind them (kept in order) */
@@ -75,6 +107,7 @@ export declare class Recorder {
      * holds if a page realm ever got the toolbar's channel.
      */
     private readonly onMessage;
+    private listFilesRealms;
     private listToolbarRealms;
     /** A message from the toolbar's own sandbox */
     private fromToolbar;
@@ -91,6 +124,9 @@ export declare class Recorder {
     private emit;
     private locatorFor;
     private handle;
+    private fromFiles;
+    /** Saves one kept file to the files folder; returns the path its step gets */
+    private storePageFile;
     private release;
     /**
      * Pauses capture in the loaded document: clicks and typing are not reported until `unmute`.
@@ -104,3 +140,5 @@ export declare class Recorder {
     stop(): Promise<void>;
     private end;
 }
+/** The warning for a file that isn't saved, as the toolbar and the CLI show it */
+export declare function missingFileText(file: MissingFile, folder: string): string;