@interop/was-conformance-suite 0.1.0

package/LICENSE.md

@@ -0,0 +1,20 @@
+MIT License
+
+Copyright (c) 2026 Interop Alliance
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/README.md

@@ -0,0 +1,264 @@
+# WAS Conformance Suite _(@interop/was-conformance-suite)_
+
+[![Node.js CI](https://github.com/interop-alliance/was-conformance-suite/workflows/CI/badge.svg)](https://github.com/interop-alliance/was-conformance-suite/actions?query=workflow%3A%22CI%22)
+[![NPM Version](https://img.shields.io/npm/v/@interop/was-conformance-suite.svg)](https://npm.im/@interop/was-conformance-suite)
+
+> Conformance test suite for
+> [Wallet Attached Storage (WAS)](https://digitalcredentials.github.io/wallet-attached-storage-spec/)
+> servers, runnable as a CLI, in the browser, or as a library.
+
+## Table of Contents
+
+- [Background](#background)
+- [Security](#security)
+- [Install](#install)
+- [Usage](#usage)
+  - [CLI](#cli)
+  - [Web app](#web-app)
+  - [Programmatic API](#programmatic-api)
+  - [Onboarding token](#onboarding-token)
+  - [Optional vs required tests](#optional-vs-required-tests)
+  - [Adding a test](#adding-a-test)
+- [Contribute](#contribute)
+- [License](#license)
+
+## Background
+
+This suite black-box tests any running Wallet Attached Storage server
+implementation over HTTP: spaces, collections, resources, access-control
+policies, change queries, encryption markers, ZCap delegation, and space
+export/import. It never inspects server internals -- point it at a server URL
+(plus an optional onboarding token, if the server gates provisioning) and it
+reports per-test pass/fail results against the
+[WAS spec](https://digitalcredentials.github.io/wallet-attached-storage-spec/).
+
+### The server URL must be byte-identical everywhere
+
+ZCap authorization embeds the invocation target URL (including host and port) in
+signed capabilities, and servers verify it as an exact string match. The URL you
+point this suite at must therefore be **byte-identical** to the URL the server
+believes it is serving on (e.g. its `SERVER_URL` setting). In particular,
+`http://localhost:3002` and `http://127.0.0.1:3002` are _not_ interchangeable --
+a mismatch makes the delegated-access tests fail with 404s even though the
+server is otherwise reachable.
+
+## Security
+
+The suite uses hardcoded, publicly known did:key test seeds (deterministic test
+identities for its "alice"/"bob" actors). Never authorize these identities on a
+production server. Onboarding tokens are only sent as `Authorization: Bearer`
+headers to the server under test.
+
+## Install
+
+- Node.js 24+ is recommended.
+
+### PNPM
+
+To install via PNPM:
+
+```
+pnpm install @interop/was-conformance-suite
+```
+
+### Development
+
+To install locally (for development):
+
+```
+git clone https://github.com/interop-alliance/was-conformance-suite.git
+cd was-conformance-suite
+pnpm install
+```
+
+## Usage
+
+The suite runs the same test registry three ways: from the command line, from a
+hosted web app, or as a library. In every case you provide a WAS server URL and,
+if the server gates provisioning, an [onboarding token](#onboarding-token).
+
+Remember the
+[byte-identical URL constraint](#the-server-url-must-be-byte-identical-everywhere):
+the server URL you pass must exactly match the URL the server is configured to
+serve on.
+
+### CLI
+
+The package ships a single `was-conformance` bin, so you can run it directly
+with `npx` (no install step needed):
+
+```
+npx @interop/was-conformance-suite http://localhost:3002
+```
+
+Or, once installed:
+
+```
+was-conformance http://localhost:3002
+```
+
+Options:
+
+```
+Usage: was-conformance <server-url> [options]
+
+Runs the WAS conformance suite against a running server.
+<server-url> falls back to the TEST_SERVER_URL environment variable.
+
+Options:
+  -t, --token <token>       onboarding token (or TEST_ONBOARDING_TOKEN)
+  -s, --suite <id>          run only the named suite(s), repeatable
+  -g, --grep <pattern>      filter tests by name
+      --include-optional    treat optional tests as required (default: run
+                            them but report as warnings)
+      --skip-optional       do not run optional tests at all
+  -r, --reporter <name>     pretty (default) | json
+      --timeout <ms>        per-test timeout
+      --fail-fast           stop on first failure
+  -h, --help                show this help and exit
+  -V, --version             print the version and exit
+```
+
+The available suite ids (for `--suite`) are:
+
+| id                      | tests                                  |
+| ----------------------- | -------------------------------------- |
+| `changes-query-api`     | Collection changes query profile       |
+| `client-backends`       | WasClient -- BYOS backend registration |
+| `client-delegation`     | WasClient -- Delegation                |
+| `client-export-import`  | WasClient -- Export / Import           |
+| `client-resources`      | WasClient -- Resources                 |
+| `client-spaces`         | WasClient -- Spaces & Collections      |
+| `collection-api`        | Collections API                        |
+| `encryption-marker-api` | Encryption marker API                  |
+| `policy-api`            | Access-control policy API              |
+| `resource-api`          | Resource API                           |
+| `server`                | Server                                 |
+| `spaces-api`            | Spaces                                 |
+
+#### Environment-variable fallbacks
+
+Two settings can be supplied via the environment instead of on the command line:
+
+- `TEST_SERVER_URL` -- used when the `<server-url>` positional is omitted.
+- `TEST_ONBOARDING_TOKEN` -- used when `--token` is omitted.
+
+```
+TEST_SERVER_URL=http://localhost:3002 TEST_ONBOARDING_TOKEN=secret was-conformance
+```
+
+#### Exit codes
+
+- `0` -- all required tests passed.
+- `1` -- one or more conformance (required-test) failures.
+- `2` -- usage error, or the server could not be reached.
+
+Optional-test failures do not affect the exit code unless `--include-optional`
+promotes them (see [below](#optional-vs-required-tests)).
+
+#### JSON reporter for CI
+
+For machine-readable output, use the JSON reporter:
+
+```
+was-conformance http://localhost:3002 --reporter json
+```
+
+It writes a structured report to stdout while preserving the same exit-code
+semantics, so a CI job can both assert on the exit code and archive the report.
+
+### Web app
+
+A hosted runner is available at
+<https://interop-alliance.github.io/was-conformance-suite/>. Paste in a server
+URL and an optional onboarding token, and the suite runs **fully client-side in
+your browser** -- the token is used only to talk to the server under test and
+never leaves the browser.
+
+Because the run happens in the browser, the server under test must allow
+[CORS](https://developer.mozilla.org/docs/Web/HTTP/CORS) from the web app's
+origin. If a run is diagnosed as "server unreachable from a browser (network or
+CORS)", use the [CLI](#cli) instead, which is not subject to browser CORS
+restrictions.
+
+You can prefill the server URL field with the `?server=` query parameter, e.g.
+`.../was-conformance-suite/?server=http://localhost:3002`.
+
+To run the web app locally during development:
+
+```
+pnpm web:dev
+```
+
+### Programmatic API
+
+Import `runConformance` (and, if you want to select a subset, `suites`) and read
+the returned `RunReport`:
+
+```ts
+import { runConformance, suites } from '@interop/was-conformance-suite'
+
+const report = await runConformance({
+  serverUrl: 'http://localhost:3002',
+  onboardingToken: 'secret', // omit or null if provisioning is open
+  suites, // optional: defaults to the full registry
+  onEvent: event => {
+    if (event.type === 'test-end') {
+      console.log(event.result.status, event.result.name)
+    }
+  }
+})
+
+console.log(`conformant: ${report.conformant}`)
+console.log(report.counts) // { total, pass, fail, skip, optionalFail }
+```
+
+`report.conformant` is `true` when no required test failed. See
+[`src/harness/types.ts`](src/harness/types.ts) for the full `RunReport`,
+`RunEvent`, and `TestResult` shapes.
+
+### Onboarding token
+
+Some WAS servers gate space provisioning behind a shared-secret onboarding
+token. When configured, the suite sends it as an `Authorization: Bearer` header
+on provisioning requests. If a server does not gate provisioning, space creation
+is open and no token is needed -- omit `--token` / `TEST_ONBOARDING_TOKEN` and
+the `onboardingToken` option.
+
+### Optional vs required tests
+
+Some tests are tagged **optional**: they reflect reference-server-flavored
+behavior that is not clearly mandated by the spec. By default optional tests
+still run, but their failures are reported as warnings and do **not** affect the
+exit code. Two flags change this:
+
+- `--include-optional` promotes optional tests to required, so their failures
+  count toward the exit code.
+- `--skip-optional` does not run optional tests at all.
+
+### Adding a test
+
+Tests live in `src/suites/*.ts` as plain data. A suite is a `Suite` object with
+an array of `TestCase` entries; each test has:
+
+- a stable `id` slug (e.g. `spaces.create-post`), unique within its suite;
+- a human-readable `name`;
+- a `run(ctx, state)` function that receives the test context (server URL, test
+  actors, and the provisioning/utility helpers) and throws on failure.
+
+Register a new suite by adding it to the registry in
+[`src/suites/index.ts`](src/suites/index.ts). Write assertions through the local
+assert module ([`src/harness/assert.ts`](src/harness/assert.ts)) so they work
+unchanged in both Node and the browser.
+
+## Contribute
+
+PRs accepted. See [CONTRIBUTING.md](CONTRIBUTING.md) for editor setup (Prettier,
+ESLint, and EditorConfig) and how it maps to CI.
+
+If editing the Readme, please conform to the
+[standard-readme](https://github.com/RichardLitt/standard-readme) specification.
+
+## License
+
+[MIT License](LICENSE.md) © 2026 Interop Alliance. </content> </invoke>

package/dist/cli/index.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+export {};
+//# sourceMappingURL=index.d.ts.map

package/dist/cli/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/cli/index.ts"],"names":[],"mappings":""}

package/dist/cli/index.js

@@ -0,0 +1,69 @@
+#!/usr/bin/env node
+/*!
+ * Copyright (c) 2026 Interop Alliance. All rights reserved.
+ */
+/**
+ * The `was-conformance` bin entry: reads the tool and client versions from the
+ * installed packages, binds real process I/O, and delegates to the testable
+ * `main()`. All logic lives in `./main.js`.
+ */
+import { readFileSync } from 'node:fs';
+import { createRequire } from 'node:module';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { runConformance } from '../index.js';
+import { suites } from '../suites/index.js';
+import { main } from './main.js';
+/** Reads this tool's own version from its package.json (../../ from dist/cli). */
+function readToolVersion() {
+    try {
+        const pkgPath = join(dirname(fileURLToPath(import.meta.url)), '../../package.json');
+        return JSON.parse(readFileSync(pkgPath, 'utf8')).version ?? '0.0.0';
+    }
+    catch {
+        return '0.0.0';
+    }
+}
+/**
+ * Best-effort `@interop/was-client` version: its package.json is not exported,
+ * so resolve the entry module and walk up to the owning package manifest.
+ */
+function readClientVersion() {
+    try {
+        const require = createRequire(import.meta.url);
+        let dir = dirname(require.resolve('@interop/was-client'));
+        for (let i = 0; i < 8; i++) {
+            try {
+                const pkg = JSON.parse(readFileSync(join(dir, 'package.json'), 'utf8'));
+                if (pkg.name === '@interop/was-client') {
+                    return pkg.version;
+                }
+            }
+            catch {
+                // Keep walking up to the package root.
+            }
+            const parent = dirname(dir);
+            if (parent === dir) {
+                break;
+            }
+            dir = parent;
+        }
+    }
+    catch {
+        // No client version available; the JSON reporter simply omits it.
+    }
+    return undefined;
+}
+const exitCode = await main(process.argv.slice(2), {
+    runConformance,
+    suites,
+    fetch: globalThis.fetch,
+    stdout: text => process.stdout.write(text),
+    stderr: text => process.stderr.write(text),
+    env: process.env,
+    isTTY: Boolean(process.stdout.isTTY),
+    version: readToolVersion(),
+    clientVersion: readClientVersion()
+});
+process.exitCode = exitCode;
+//# sourceMappingURL=index.js.map

package/dist/cli/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/cli/index.ts"],"names":[],"mappings":";AACA;;GAEG;AACH;;;;GAIG;AACH,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAA;AACtC,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAA;AAC3C,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AACzC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAA;AACxC,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAC5C,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAA;AAC3C,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,kFAAkF;AAClF,SAAS,eAAe;IACtB,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,IAAI,CAClB,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EACvC,oBAAoB,CACrB,CAAA;QACD,OAAO,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,IAAI,OAAO,CAAA;IACrE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,OAAO,CAAA;IAChB,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,SAAS,iBAAiB;IACxB,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAC9C,IAAI,GAAG,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,qBAAqB,CAAC,CAAC,CAAA;QACzD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,EAAE,cAAc,CAAC,EAAE,MAAM,CAAC,CAAC,CAAA;gBACvE,IAAI,GAAG,CAAC,IAAI,KAAK,qBAAqB,EAAE,CAAC;oBACvC,OAAO,GAAG,CAAC,OAAO,CAAA;gBACpB,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,uCAAuC;YACzC,CAAC;YACD,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAA;YAC3B,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;gBACnB,MAAK;YACP,CAAC;YACD,GAAG,GAAG,MAAM,CAAA;QACd,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,kEAAkE;IACpE,CAAC;IACD,OAAO,SAAS,CAAA;AAClB,CAAC;AAED,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;IACjD,cAAc;IACd,MAAM;IACN,KAAK,EAAE,UAAU,CAAC,KAAK;IACvB,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;IAC1C,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;IAC1C,GAAG,EAAE,OAAO,CAAC,GAAG;IAChB,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC;IACpC,OAAO,EAAE,eAAe,EAAE;IAC1B,aAAa,EAAE,iBAAiB,EAAE;CACnC,CAAC,CAAA;AAEF,OAAO,CAAC,QAAQ,GAAG,QAAQ,CAAA"}

package/dist/cli/main.d.ts

@@ -0,0 +1,64 @@
+import { runConformance } from '../index.js';
+import type { Suite } from '../harness/types.js';
+/** The I/O and collaborators the CLI needs, injectable for tests. */
+export interface CliDeps {
+    runConformance: typeof runConformance;
+    /** The suite registry to draw from (default: the full registry). */
+    suites: Array<Suite<any>>;
+    /** Connectivity probe; any resolved response counts as reachable. */
+    fetch: typeof fetch;
+    stdout: (text: string) => void;
+    stderr: (text: string) => void;
+    env: Record<string, string | undefined>;
+    /** Whether stdout is a TTY, used (with NO_COLOR) to gate ANSI color. */
+    isTTY: boolean;
+    /** This tool's version, read from its package by the bin wrapper. */
+    version: string;
+    /** The `@interop/was-client` version, if cheaply resolvable. */
+    clientVersion?: string;
+}
+/** A validated run configuration, produced by `parseArgv`. */
+interface RunConfig {
+    serverUrl: string;
+    onboardingToken: string | null;
+    suiteIds: string[];
+    grep: string | null;
+    includeOptional: boolean;
+    skipOptional: boolean;
+    reporter: 'pretty' | 'json';
+    timeoutMs: number | null;
+    failFast: boolean;
+}
+type ParseResult = {
+    kind: 'run';
+    config: RunConfig;
+} | {
+    kind: 'help';
+} | {
+    kind: 'version';
+} | {
+    kind: 'usage';
+    message: string;
+};
+/**
+ * Parses argv (the tokens after `node script`) into a `ParseResult`, resolving
+ * env fallbacks. Pure: it never touches the network or the suite registry, so
+ * arg handling is unit-testable on its own.
+ *
+ * @param argv {string[]} the raw argument tokens
+ * @param env {object} the environment (for TEST_SERVER_URL / token fallbacks)
+ * @returns {ParseResult}
+ */
+export declare function parseArgv(argv: string[], env: Record<string, string | undefined>): ParseResult;
+/**
+ * The CLI entry point, minus process wiring. Returns the intended exit code:
+ * 0 = all required tests passed, 1 = conformance failures, 2 = usage error or
+ * unreachable server.
+ *
+ * @param argv {string[]} argument tokens (after `node script`)
+ * @param deps {CliDeps} injected I/O and collaborators
+ * @returns {Promise<number>} the exit code
+ */
+export declare function main(argv: string[], deps: CliDeps): Promise<number>;
+export {};
+//# sourceMappingURL=main.d.ts.map

package/dist/cli/main.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"main.d.ts","sourceRoot":"","sources":["../../src/cli/main.ts"],"names":[],"mappings":"AAUA,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAG5C,OAAO,KAAK,EAAuB,KAAK,EAAE,MAAM,qBAAqB,CAAA;AAErE,qEAAqE;AACrE,MAAM,WAAW,OAAO;IACtB,cAAc,EAAE,OAAO,cAAc,CAAA;IACrC,oEAAoE;IACpE,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAA;IACzB,qEAAqE;IACrE,KAAK,EAAE,OAAO,KAAK,CAAA;IACnB,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAA;IAC9B,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAA;IAC9B,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,CAAA;IACvC,wEAAwE;IACxE,KAAK,EAAE,OAAO,CAAA;IACd,qEAAqE;IACrE,OAAO,EAAE,MAAM,CAAA;IACf,gEAAgE;IAChE,aAAa,CAAC,EAAE,MAAM,CAAA;CACvB;AAED,8DAA8D;AAC9D,UAAU,SAAS;IACjB,SAAS,EAAE,MAAM,CAAA;IACjB,eAAe,EAAE,MAAM,GAAG,IAAI,CAAA;IAC9B,QAAQ,EAAE,MAAM,EAAE,CAAA;IAClB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;IACnB,eAAe,EAAE,OAAO,CAAA;IACxB,YAAY,EAAE,OAAO,CAAA;IACrB,QAAQ,EAAE,QAAQ,GAAG,MAAM,CAAA;IAC3B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAA;IACxB,QAAQ,EAAE,OAAO,CAAA;CAClB;AAED,KAAK,WAAW,GACZ;IAAE,IAAI,EAAE,KAAK,CAAC;IAAC,MAAM,EAAE,SAAS,CAAA;CAAE,GAClC;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,GAChB;IAAE,IAAI,EAAE,SAAS,CAAA;CAAE,GACnB;IAAE,IAAI,EAAE,OAAO,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAAA;AAoBtC;;;;;;;;GAQG;AACH,wBAAgB,SAAS,CACvB,IAAI,EAAE,MAAM,EAAE,EACd,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,CAAC,GACtC,WAAW,CAyGb;AAyFD;;;;;;;;GAQG;AACH,wBAAsB,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAyEzE"}