ankka 0.15.0 → 0.16.0

package/dist/storage.js

@@ -26,8 +26,35 @@ import { Agent as HttpsAgent, request as httpsRequest } from "node:https";
 import { isAbsolute, join, resolve } from "node:path";
 import { Readable } from "node:stream";
 import { CommandError, ErrorCode } from "./effects/common.js";
+import { subjectProblem } from "./personal.js";
 import { Credentials, EMPTY_PAYLOAD, UNSIGNED_PAYLOAD, encode, hmac, presign, sha256Hex, sign, SignedRequest } from "./sigv4.js";
-import { Instant } from "./time.js";
+import { Duration, Instant } from "./time.js";
+const IMMEDIATE = Object.freeze({ kind: "immediate" });
+/**
+ * Where a service keeps the objects of `subject` (feature 042): `subjects/`, the data subject with every
+ * `/` written `%2F`, and `/`. A data subject cannot hold `%`, so no subject prefix holds another's:
+ * `player`'s is `subjects/player/` and `player/8c1f`'s `subjects/player%2F8c1f/`. A string that is no
+ * data subject is refused (`BAD_REQUEST`).
+ */
+export function subjectPrefix(subject) {
+    const p = subjectProblem(subject);
+    if (p !== undefined)
+        throw badRequest(p);
+    return "subjects/" + subject.replaceAll("/", "%2F") + "/";
+}
+/** `erase` for a store that keeps no versions: list the subject's prefix and delete each object. */
+async function eraseListed(bucket, subject, finality = IMMEDIATE) {
+    const names = [];
+    for await (const e of bucket.list(subjectPrefix(subject)))
+        names.push(e.name);
+    for (const n of names)
+        await bucket.delete(n);
+    return Object.freeze({ count: names.length, finality });
+}
+/** A bucket made of its other calls, with `erase` by listing. */
+function listedBucket(bucket) {
+    return Object.freeze({ ...bucket, erase: (subject) => eraseListed(bucket, subject) });
+}
 // ── The rules and the words ────────────────────────────────────────────────────
 const utf8 = new TextEncoder();
 function shown(text) {
@@ -107,6 +134,10 @@ export const ObjectRules = Object.freeze({
         if (p !== undefined)
             throw badRequest(p);
     },
+    /** Where a service keeps the objects of `subject`: `subjectPrefix`. */
+    subjectPrefix(subject) {
+        return subjectPrefix(subject);
+    },
 });
 /** What the storage client says when it refuses: the runtime's words (`StorageRefusals` in the Scala SDK). */
 export const StorageRefusals = Object.freeze({
@@ -321,7 +352,7 @@ export class InMemoryBuckets {
             return m;
         };
         const publicBase = this.#publicBase;
-        return Object.freeze({
+        return listedBucket({
             name,
             exposure,
             async put(objectName, body, options) {
@@ -538,7 +569,7 @@ export class DirectoryBuckets {
             }
         };
         const address = (objectName) => `${store.#addressBase().replace(/\/$/, "")}${STORAGE_PATH}/${segment}/${addressOf(objectName)}`;
-        return Object.freeze({
+        return listedBucket({
             name,
             exposure,
             async put(objectName, body, options) {
@@ -663,6 +694,16 @@ export const MAX_ADDRESS_SEGMENTS = 16;
 export function addressOf(objectName) {
     return encode(objectName, objectName.split("/").length > MAX_ADDRESS_SEGMENTS);
 }
+/** `ANKKA_S3_SOFT_DELETE_DAYS`: a whole number of days from 1, or unset. */
+function softDeleteOf(env) {
+    const variable = PREFIX + "SOFT_DELETE_DAYS";
+    const text = env[variable]?.trim() ?? "";
+    if (text === "")
+        return undefined;
+    if (!/^[0-9]+$/.test(text) || Number(text) < 1)
+        throw new Error(`${variable} is '${text}'; it is a whole number of days from 1`);
+    return Duration.ofSeconds(Number(text) * 86_400);
+}
 const PREFIX = "ANKKA_S3_";
 function nonEmpty(env, name) {
     const v = env[name];
@@ -754,7 +795,8 @@ export function s3Settings(env = process.env, timeoutMs = 30_000) {
             throw new Error(`bucket '${name}' is public and ${PREFIX}PUBLIC_URL_${suffix} is not set`);
         named.push(Object.freeze({ name, exposure: word, target: Object.freeze({ ...unnamed, bucket }), publicUrl }));
     }
-    return Object.freeze({ unnamed: unnamed && unnamed.bucket !== "" ? unnamed : undefined, named: Object.freeze(named), timeoutMs });
+    const softDelete = softDeleteOf(env);
+    return Object.freeze({ unnamed: unnamed && unnamed.bucket !== "" ? unnamed : undefined, named: Object.freeze(named), timeoutMs, softDelete });
 }
 const httpAgent = new HttpAgent({ keepAlive: true });
 const httpsAgent = new HttpsAgent({ keepAlive: true });
@@ -857,6 +899,30 @@ export function parseListing(xml) {
     const truncated = element(xml, "IsTruncated")?.trim() === "true";
     return { entries, next: truncated ? element(xml, "NextContinuationToken") : undefined };
 }
+/** A `ListObjectVersions` answer: its versions and delete markers, and where the next page starts while it is truncated. */
+export function parseVersions(xml) {
+    const versions = [];
+    let generations = false;
+    for (const m of xml.matchAll(/<(Version|DeleteMarker)>([\s\S]*?)<\/\1>/g)) {
+        const block = m[2];
+        const key = element(block, "Key");
+        if (key === undefined)
+            throw new Error(`a version listing's ${m[1]} has no Key`);
+        const versionId = element(block, "VersionId");
+        const generation = versionId === undefined ? element(block, "Generation") : undefined;
+        if (generation !== undefined)
+            generations = true;
+        const id = (versionId ?? generation)?.trim() ?? "";
+        versions.push([key, id]);
+    }
+    return {
+        versions,
+        truncated: element(xml, "IsTruncated")?.trim() === "true",
+        nextKey: element(xml, "NextKeyMarker"),
+        nextVersion: element(xml, "NextVersionIdMarker") ?? element(xml, "NextGenerationMarker"),
+        generations: generations || element(xml, "NextGenerationMarker") !== undefined,
+    };
+}
 function lastModifiedOf(response) {
     const v = response.headers["last-modified"];
     const d = v ? new Date(v) : undefined;
@@ -870,7 +936,9 @@ export class S3Bucket {
     #publicUrl;
     #timeoutMs;
     #endpoint;
-    constructor(target, name, exposure, publicUrl, timeoutMs = 30_000) {
+    #finality;
+    constructor(target, name, exposure, publicUrl, timeoutMs = 30_000, softDelete) {
+        this.#finality = softDelete === undefined ? IMMEDIATE : Object.freeze({ kind: "after", after: softDelete });
         this.#target = target;
         this.name = name;
         this.exposure = exposure;
@@ -952,6 +1020,69 @@ export class S3Bucket {
             } while (token !== undefined);
         })();
     }
+    /**
+     * Lists every version and delete marker under the subject's prefix (`ListObjectVersions`) and deletes
+     * each by its version id; the count is the objects with a version. A store that lists no versions
+     * (Garage answers `NotImplemented`) is listed plainly and each object deleted. The finality is
+     * `ANKKA_S3_SOFT_DELETE_DAYS`'s, or immediate.
+     */
+    async erase(subject) {
+        const prefix = subjectPrefix(subject);
+        const versions = await this.#versionsUnder(prefix);
+        if (versions === undefined)
+            return eraseListed(this, subject, this.#finality);
+        for (const [key, version] of versions)
+            await this.#deleteVersion(key, version);
+        const objects = new Set(versions.filter(([, v]) => v !== "").map(([k]) => k));
+        return Object.freeze({ count: objects.size, finality: this.#finality });
+    }
+    /** Every version and delete marker under `prefix`, or `undefined` when the store lists none. */
+    async #versionsUnder(prefix) {
+        const found = [];
+        let keyMarker;
+        let versionMarker;
+        let generations = false;
+        for (;;) {
+            const query = [
+                ["prefix", prefix],
+                ["versions", ""],
+            ];
+            if (keyMarker !== undefined)
+                query.push(["key-marker", keyMarker]);
+            // Google pages by generation and names the next one `generation-marker`.
+            if (versionMarker !== undefined)
+                query.push([generations ? "generation-marker" : "version-id-marker", versionMarker]);
+            const { status, response } = await this.#send("GET", `/${this.#target.bucket}`, query);
+            const text = await textOf(response, 64 << 20);
+            if (status === 501 || text.includes("<Code>NotImplemented</Code>"))
+                return undefined;
+            if (status !== 200)
+                throw this.#refused(status, text);
+            let page;
+            try {
+                page = parseVersions(text);
+            }
+            catch (e) {
+                throw StorageRefusals.unreadable(this.#endpoint, e instanceof Error ? e.message : String(e));
+            }
+            found.push(...page.versions);
+            keyMarker = page.nextKey;
+            versionMarker = page.nextVersion;
+            generations = page.generations;
+            if (!page.truncated)
+                return found;
+        }
+    }
+    /**
+     * One version, or a delete marker; `version` empty is the plain delete. The store's `null` version is
+     * deleted as `versionId=null`: a plain delete would only add a marker above it.
+     */
+    async #deleteVersion(objectName, version) {
+        const { status, response } = await this.#send("DELETE", this.#objectPath(objectName), version === "" ? [] : [["versionId", version]]);
+        const text = await textOf(response);
+        if (status !== 200 && status !== 204 && status !== 404)
+            throw this.#refused(status, text);
+    }
     signedUrl(objectName, access, spanSeconds = ObjectRules.DEFAULT_SIGNED_URL_SECONDS) {
         ObjectRules.checkName(objectName);
         let base;
@@ -1005,8 +1136,8 @@ export class S3Buckets {
     #named;
     constructor(settings = s3Settings()) {
         const unnamedExposure = settings.unnamed?.publicEndpoint ? "signed" : "private";
-        this.#unnamed = settings.unnamed && new S3Bucket(settings.unnamed, undefined, unnamedExposure, undefined, settings.timeoutMs);
-        this.#named = new Map(settings.named.map((n) => [n.name, new S3Bucket(n.target, n.name, n.exposure, n.publicUrl, settings.timeoutMs)]));
+        this.#unnamed = settings.unnamed && new S3Bucket(settings.unnamed, undefined, unnamedExposure, undefined, settings.timeoutMs, settings.softDelete);
+        this.#named = new Map(settings.named.map((n) => [n.name, new S3Bucket(n.target, n.name, n.exposure, n.publicUrl, settings.timeoutMs, settings.softDelete)]));
     }
     get default() {
         if (!this.#unnamed)

package/dist/testkit/index.d.ts

@@ -1,3 +1,3 @@
 export { EventSourcedTestKit, EndpointTestKit, Response, type RequestOptions, type Materialised, type SocketRun, type ReplicationMode } from "./unit.ts";
 export { KeyValueTestKit, WorkflowTestKit, ViewTestKit, KeyedViewTestKit, ConsumerTestKit, GraphConsumerTestKit, TimedActionTestKit, AgentTestKit, ScriptedModel, type WorkflowProgress, type ModelResponse, type ModelCall, type GraphChange, } from "./kinds.ts";
-export { AnkkaTestKit, PUBLISHED_SIDECAR, sidecarImage, type AnkkaTestKitOptions, type HttpResponse, type Http, type Beside } from "./integration.ts";
+export { AnkkaTestKit, PUBLISHED_SIDECAR, sidecarImage, type AnkkaTestKitOptions, type HttpResponse, type Http, type Beside, type KitKeyring } from "./integration.ts";

package/dist/testkit/integration.d.ts

@@ -1,6 +1,32 @@
 import type { ComponentClient } from "../client.ts";
 import type { ServiceBuilder } from "../service.ts";
 import { type Buckets } from "../storage.ts";
+/**
+ * One statement that searches every column of every table of the service's database, cast to text,
+ * for each value as written, as base64 (standard and URL-safe, unpadded) and as lower-case hex in the
+ * lower-cased text: the rules of the JVM kit's `PersonalAssertions`, the same forms. Each row is
+ * `<index of the value> <form> <schema>.<table>.<column>`.
+ */
+export declare function personalValueQuery(values: readonly string[]): string;
+/** The JVM kit's failure message for what `personalValueQuery` found, or `undefined` for nothing. */
+export declare function personalValueFailure(values: readonly string[], output: string): string | undefined;
+/**
+ * The keyring of the kit's sidecar (feature 042): the runtime's local keyring, kept in the kit's
+ * database, reached through the sidecar's local console endpoint. `erase` destroys a data subject's key
+ * at once and sends the notice, as an applied erasure request does; `outage` makes the keyring refuse
+ * every request as unavailable, erasures included, until it ends or `recover` ends it. An outage is the
+ * sidecar's: `kit.restart()` ends it.
+ */
+export declare class KitKeyring {
+    #private;
+    constructor(url: () => string);
+    /** Erases `subject` in the service's project now; the notice's sequence, once the sidecar has carried it out (or after ten seconds). Throws in an outage. */
+    erase(subject: string, rerun?: boolean): Promise<number>;
+    /** Refuses every request as unavailable for `seconds`: a subject whose key the sidecar holds is still read and written; one it does not hold is refused, `ErrorCode.Unavailable`. */
+    outage(seconds: number): Promise<void>;
+    /** Ends an outage now. */
+    recover(): Promise<void>;
+}
 export interface AnkkaTestKitOptions {
     /** The sidecar image; `sidecarImage()` by default. */
     readonly image?: string;
@@ -52,6 +78,8 @@ export declare const PUBLISHED_SIDECAR = "ghcr.io/thinkmorestupidless/ankka-side
 export declare function sidecarImage(version?: string): string;
 export declare class AnkkaTestKit {
     #private;
+    /** The sidecar's keyring: erase a data subject, start or end an outage (feature 042). */
+    readonly keyring: KitKeyring;
     /** The component client every handler in `service` uses, pointed at the sidecar's mapped callback port. */
     readonly client: ComponentClient;
     /** The buckets the service holds: what a handler kept, a test reads here, and what a test puts here a handler reads. */
@@ -64,6 +92,13 @@ export declare class AnkkaTestKit {
     get baseUrl(): string;
     /** The JDBC URL of the throwaway database. */
     get jdbcUrl(): string;
+    /**
+     * Fails, naming each table and column, when any of `values` can be read back from the service's
+     * database as written, as base64 (standard and URL-safe) or as hex: what a personal field must never
+     * be (feature 042, FR-030). Every column of every table, cast to text, searched by the kit's Postgres
+     * itself with the JVM kit's rules (`personalValueQuery`).
+     */
+    assertNoPersonalValue(...values: string[]): Promise<void>;
     /** Replaces the sidecar against the same database: every instance is gone from memory, so the next read rebuilds from the journal. */
     restart(): Promise<void>;
     /** Starts another image against this kit's Postgres — the Scala cart, say, to prove the journal is shared. */

package/dist/testkit/integration.js

@@ -21,6 +21,82 @@ import { VERSION } from "../version.js";
 const POSTGRES_IMAGE = "postgres:17-alpine";
 const HTTP_PORT = 9000;
 const CALLBACK_PORT = 9011;
+// The sidecar's local console endpoint, where `kit.keyring` files erasures and outages: named with
+// ANKKA_OBSERVABILITY_ADDRESS so Docker can publish it, where a service run on a machine binds it to
+// loopback on an ephemeral port.
+const LOCAL_ENDPOINT_PORT = 9012;
+/**
+ * One statement that searches every column of every table of the service's database, cast to text,
+ * for each value as written, as base64 (standard and URL-safe, unpadded) and as lower-case hex in the
+ * lower-cased text: the rules of the JVM kit's `PersonalAssertions`, the same forms. Each row is
+ * `<index of the value> <form> <schema>.<table>.<column>`.
+ */
+export function personalValueQuery(values) {
+    const quote = (text) => `'${text.replaceAll("'", "''")}'`;
+    const rows = values.flatMap((value, i) => {
+        const data = Buffer.from(value, "utf8");
+        const forms = {
+            plain: value,
+            base64: data.toString("base64").replace(/=+$/, ""),
+            base64url: data.toString("base64url"),
+            hex: data.toString("hex"),
+        };
+        return Object.entries(forms).map(([name, form]) => `(${i}, ${quote(name)}, ${quote(form)})`);
+    });
+    return ("SELECT f.i || ' ' || f.form_name || ' ' || c.table_schema || '.' || c.table_name || '.' || c.column_name " +
+        `FROM information_schema.columns c CROSS JOIN (VALUES ${rows.join(", ")}) AS f(i, form_name, form) ` +
+        "WHERE c.table_schema NOT IN ('pg_catalog', 'information_schema') " +
+        "AND (xpath('/row/n/text()', query_to_xml(format('SELECT count(*) AS n FROM %I.%I WHERE strpos(%s, %L) > 0', " +
+        "c.table_schema, c.table_name, " +
+        "CASE WHEN f.form_name = 'hex' THEN format('lower(%I::text)', c.column_name) ELSE format('%I::text', c.column_name) END, " +
+        "f.form), false, true, '')))[1]::text::int > 0 " +
+        "ORDER BY c.table_schema, c.table_name, c.ordinal_position, f.i");
+}
+/** The JVM kit's failure message for what `personalValueQuery` found, or `undefined` for nothing. */
+export function personalValueFailure(values, output) {
+    const found = output
+        .split("\n")
+        .filter((line) => line.trim() !== "")
+        .map((line) => {
+        const [i, form, ...where] = line.split(" ");
+        return `  '${values[Number(i)]}' as ${form} in ${where.join(" ")}`;
+    });
+    return found.length === 0 ? undefined : `a personal value can be read back from what the service kept:\n${found.join("\n")}`;
+}
+/**
+ * The keyring of the kit's sidecar (feature 042): the runtime's local keyring, kept in the kit's
+ * database, reached through the sidecar's local console endpoint. `erase` destroys a data subject's key
+ * at once and sends the notice, as an applied erasure request does; `outage` makes the keyring refuse
+ * every request as unavailable, erasures included, until it ends or `recover` ends it. An outage is the
+ * sidecar's: `kit.restart()` ends it.
+ */
+export class KitKeyring {
+    #url;
+    constructor(url) {
+        this.#url = url;
+    }
+    async #post(path, body) {
+        const r = await fetch(this.#url() + path, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify(body) });
+        const text = await r.text();
+        if (r.status !== 200)
+            throw new Error(`${path} answered ${r.status}: ${text}`);
+        return JSON.parse(text);
+    }
+    /** Erases `subject` in the service's project now; the notice's sequence, once the sidecar has carried it out (or after ten seconds). Throws in an outage. */
+    async erase(subject, rerun = false) {
+        return Number((await this.#post("/_ankka/erasures", { subject, rerun })).seq);
+    }
+    /** Refuses every request as unavailable for `seconds`: a subject whose key the sidecar holds is still read and written; one it does not hold is refused, `ErrorCode.Unavailable`. */
+    async outage(seconds) {
+        if (!(seconds > 0))
+            throw new Error("an outage lasts more than 0 seconds; recover() ends one");
+        await this.#post("/_ankka/keyring/outage", { seconds });
+    }
+    /** Ends an outage now. */
+    async recover() {
+        await this.#post("/_ankka/keyring/outage", { seconds: 0 });
+    }
+}
 /** What `http` answers with. */
 export class HttpResponse {
     status;
@@ -148,6 +224,13 @@ export class AnkkaTestKit {
     #server;
     #processPort = 0;
     #http;
+    #localUrl;
+    /** The sidecar's keyring: erase a data subject, start or end an outage (feature 042). */
+    keyring = new KitKeyring(() => {
+        if (!this.#localUrl)
+            throw new Error("the testkit is not started");
+        return this.#localUrl;
+    });
     /** The component client every handler in `service` uses, pointed at the sidecar's mapped callback port. */
     client;
     /** The buckets the service holds: what a handler kept, a test reads here, and what a test puts here a handler reads. */
@@ -228,9 +311,10 @@ export class AnkkaTestKit {
             // A fresh secret key per kit, so the secret store works with no setup; the caller's `env`
             // replaces it, and `ANKKA_SECRET_KEY: ""` starts a service with none.
             ANKKA_SECRET_KEY: randomBytes(32).toString("base64"),
+            ANKKA_OBSERVABILITY_ADDRESS: `0.0.0.0:${LOCAL_ENDPOINT_PORT}`,
             ...this.#env,
         })
-            .withExposedPorts(HTTP_PORT, CALLBACK_PORT)
+            .withExposedPorts(HTTP_PORT, CALLBACK_PORT, LOCAL_ENDPOINT_PORT)
             // Both ports, not only the health route: it answers as soon as HTTP is bound, which can be
             // before the callback server listens, and Docker's port proxy accepts then resets a
             // connection to a port nothing in the container listens on yet — so a handler's first call
@@ -241,6 +325,27 @@ export class AnkkaTestKit {
         this.#sidecar = container;
         this.client.reconnect(`${container.getHost()}:${container.getMappedPort(CALLBACK_PORT)}`);
         this.#http = new Http(`http://${container.getHost()}:${container.getMappedPort(HTTP_PORT)}`);
+        this.#localUrl = `http://${container.getHost()}:${container.getMappedPort(LOCAL_ENDPOINT_PORT)}`;
+    }
+    /**
+     * Fails, naming each table and column, when any of `values` can be read back from the service's
+     * database as written, as base64 (standard and URL-safe) or as hex: what a personal field must never
+     * be (feature 042, FR-030). Every column of every table, cast to text, searched by the kit's Postgres
+     * itself with the JVM kit's rules (`personalValueQuery`).
+     */
+    async assertNoPersonalValue(...values) {
+        if (values.length === 0)
+            return;
+        if (!this.#postgres)
+            throw new Error("the testkit is not started");
+        const r = await this.#postgres.exec(["psql", "-U", "ankka", "-d", "ankka", "-v", "ON_ERROR_STOP=1", "-At", "-c", personalValueQuery(values)]);
+        if (r.exitCode !== 0)
+            throw new Error(`the database could not be searched: ${r.output}`);
+        const failure = personalValueFailure(values, r.stdout ?? r.output);
+        if (failure !== undefined) {
+            const { AssertionError } = await import("node:assert");
+            throw new AssertionError({ message: failure });
+        }
     }
     /** Replaces the sidecar against the same database: every instance is gone from memory, so the next read rebuilds from the journal. */
     async restart() {

package/dist/version.d.ts

@@ -1 +1 @@
-export declare const VERSION = "0.15.0";
+export declare const VERSION = "0.16.0";

package/dist/version.js

@@ -1,2 +1,2 @@
 // Written by scripts/proto.ts from package.json; do not edit. 0.0.0 in the tree, the tag's version in a release.
-export const VERSION = "0.15.0";
+export const VERSION = "0.16.0";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ankka",
-  "version": "0.15.0",
+  "version": "0.16.0",
   "description": "Build services on ankka in TypeScript: entities, views, workflows, agents and endpoints hosted by the ankka sidecar.",
   "license": "Apache-2.0",
   "type": "module",