@heroiclands/package-build 23.4.0 → 23.5.0

package/container.mjs

@@ -380,6 +380,216 @@ export function needsFoundryAccount({
     return !licensed;
 }
 
+/** The S3 image: MinIO images cannot be pulled everywhere, SeaweedFS can. */
+export const S3_IMAGE = "chrislusf/seaweedfs:3.80";
+
+/** The name the S3 server answers to on the shared network, and in `awsConfig`. */
+export const S3_ALIAS = "s3.localhost";
+
+/** The port the S3 server listens on and publishes. */
+export const S3_PORT = 8333;
+
+/** The bucket the S3 server is created with. */
+export const S3_BUCKET = "sohl-e2e";
+
+/** The consumer's S3 identities, relative to the repository root. */
+export const S3_IDENTITIES_FILE = "cypress/s3/s3-identities.json";
+
+/** The consumer's Foundry `awsConfig`, relative to the repository root. */
+export const S3_AWS_CONFIG_FILE = "cypress/s3/aws-e2e.json";
+
+/**
+ * Whether the S3 server is asked for: `FOUNDRYVTT_CONTAINER_S3` set to `1` or
+ * `true`. Anything else, including unset, leaves the container as it is.
+ *
+ * @param {NodeJS.ProcessEnv} [env] - Environment to read.
+ * @returns {boolean} Whether to run the S3 server.
+ */
+export function s3Requested(env = process.env) {
+    return ["1", "true"].includes(env.FOUNDRYVTT_CONTAINER_S3?.trim().toLowerCase() ?? "");
+}
+
+/**
+ * Everything the S3 opt-in does, as data.
+ *
+ * @typedef {object} S3Plan
+ * @property {string} network - The user-defined network the Foundry container
+ *   and the S3 server share.
+ * @property {string} server - The S3 server's container name.
+ * @property {string} identities - Absolute path of the identities file.
+ * @property {string} awsConfig - Absolute path of the Foundry `awsConfig`.
+ * @property {string} awsConfigName - The file's name under the data root's
+ *   `Config/`, and the value of `FOUNDRY_AWS_CONFIG`.
+ */
+
+/**
+ * The S3 plan for a container, or `null` when the opt-in is off.
+ *
+ * @param {object} opts
+ * @param {string} opts.name - The Foundry container's name.
+ * @param {string} opts.rootDir - The consumer repository root, which holds the
+ *   identities and the `awsConfig`.
+ * @param {NodeJS.ProcessEnv} [opts.env] - Environment to read.
+ * @returns {S3Plan|null} The plan.
+ */
+export function resolveS3({ name, rootDir, env = process.env }) {
+    if (!s3Requested(env)) return null;
+    return {
+        network: `${name}-net`,
+        server: `${name}-s3`,
+        identities: path.resolve(rootDir, S3_IDENTITIES_FILE),
+        awsConfig: path.resolve(rootDir, S3_AWS_CONFIG_FILE),
+        awsConfigName: path.basename(S3_AWS_CONFIG_FILE),
+    };
+}
+
+/**
+ * The `docker run` argument vector for the S3 server.
+ *
+ * @param {S3Plan} plan - The S3 plan.
+ * @returns {string[]} Arguments after `docker`.
+ */
+export function s3RunArgs(plan) {
+    return [
+        "run",
+        "--detach",
+        "--name",
+        plan.server,
+        "--network",
+        plan.network,
+        "--network-alias",
+        S3_ALIAS,
+        "--publish",
+        `${S3_PORT}:${S3_PORT}`,
+        "--volume",
+        `${plan.identities}:/etc/seaweedfs/s3.json:ro`,
+        S3_IMAGE,
+        "server",
+        "-dir=/data",
+        "-s3",
+        `-s3.port=${S3_PORT}`,
+        "-s3.config=/etc/seaweedfs/s3.json",
+        "-s3.allowedOrigins=*",
+    ];
+}
+
+/**
+ * The `docker exec` argument vector that runs one `weed shell` command in the
+ * S3 server.
+ *
+ * @param {S3Plan} plan - The S3 plan.
+ * @param {string} command - The shell command, such as `s3.bucket.list`.
+ * @returns {string[]} Arguments after `docker`.
+ */
+export function s3ShellArgs(plan, command) {
+    return [
+        "exec",
+        plan.server,
+        "sh",
+        "-c",
+        `echo "${command}" | weed shell -master=localhost:9333`,
+    ];
+}
+
+/**
+ * Start the S3 server on its network and create the bucket, unless it is
+ * already running, and copy the `awsConfig` into the data root.
+ *
+ * @param {object} opts
+ * @param {S3Plan} opts.plan - The S3 plan.
+ * @param {string} opts.dataRoot - The Foundry user-data root.
+ * @param {(message: string) => void} [opts.log] - Progress reporting.
+ * @param {(args: string[]) => number} [opts.run] - Runs docker, returning the
+ *   exit status.
+ * @param {(args: string[]) => string} [opts.capture] - Runs docker, returning
+ *   its stdout, or `""` on failure.
+ * @param {(ms: number) => void} [opts.sleep] - Waits.
+ * @param {number} [opts.attempts] - How often to ask the server whether it is
+ *   ready.
+ * @throws {Error} When an input file is missing or the server never answers.
+ */
+export function ensureS3({
+    plan,
+    dataRoot,
+    log = () => {},
+    run = runDocker,
+    capture = captureDocker,
+    sleep = (ms) => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms),
+    attempts = 60,
+}) {
+    for (const file of [plan.identities, plan.awsConfig]) {
+        if (!fs.existsSync(file)) {
+            throw new Error(`FOUNDRYVTT_CONTAINER_S3 needs ${file}, which does not exist.`);
+        }
+    }
+    const configDir = path.join(dataRoot, "Config");
+    fs.mkdirSync(configDir, { recursive: true });
+    fs.copyFileSync(plan.awsConfig, path.join(configDir, plan.awsConfigName));
+
+    if (
+        !capture(["network", "ls", "--filter", `name=^${plan.network}$`, "--format", "{{.Name}}"])
+    ) {
+        run(["network", "create", plan.network]);
+    }
+    const present = capture([
+        "ps",
+        "-a",
+        "--filter",
+        `name=^${plan.server}$`,
+        "--format",
+        "{{.Names}}",
+    ]);
+    if (present.split("\n").includes(plan.server)) {
+        const running = capture([
+            "ps",
+            "--filter",
+            `name=^${plan.server}$`,
+            "--format",
+            "{{.Names}}",
+        ]);
+        if (!running.split("\n").includes(plan.server)) run(["start", plan.server]);
+        return;
+    }
+    log(`Starting S3 server '${plan.server}' (${S3_IMAGE}) as ${S3_ALIAS}:${S3_PORT}`);
+    const status = run(s3RunArgs(plan));
+    if (status !== 0) throw new Error(`Starting the S3 server failed with status ${status}.`);
+    // `weed shell` exits 0 whether or not the filer is up; it names the filers
+    // it found once there is one.
+    for (let attempt = 0; ; attempt++) {
+        if (capture(s3ShellArgs(plan, "s3.bucket.list")).includes("filers: [")) break;
+        if (attempt >= attempts) {
+            throw new Error(`The S3 server '${plan.server}' did not become ready.`);
+        }
+        sleep(1000);
+    }
+    run(s3ShellArgs(plan, `s3.bucket.create -name ${S3_BUCKET}`));
+}
+
+/**
+ * Remove the S3 server and its network, tolerating either being absent.
+ *
+ * @param {S3Plan} plan - The S3 plan.
+ * @param {(args: string[]) => number} [run] - Runs docker.
+ * @param {(args: string[]) => string} [capture] - Runs docker for its stdout.
+ */
+export function removeS3(plan, run = runDocker, capture = captureDocker) {
+    const present = capture([
+        "ps",
+        "-a",
+        "--filter",
+        `name=^${plan.server}$`,
+        "--format",
+        "{{.Names}}",
+    ]);
+    if (present.split("\n").includes(plan.server)) {
+        run(["stop", plan.server]);
+        run(["rm", plan.server]);
+    }
+    if (capture(["network", "ls", "--filter", `name=^${plan.network}$`, "--format", "{{.Name}}"])) {
+        run(["network", "rm", plan.network]);
+    }
+}
+
 /**
  * The full `docker run` argument vector for a stage's container.
  *
@@ -399,6 +609,11 @@ export function needsFoundryAccount({
  * @param {string|null} [opts.licenseKey] - Dedicated licence key.
  * @param {boolean} [opts.account] - Whether to pass `FOUNDRY_USERNAME` and
  *   `FOUNDRY_PASSWORD` through; see {@link needsFoundryAccount}.
+ * @param {string|null} [opts.network] - A user-defined network to attach the
+ *   container to as it is created, so it can reach the S3 server before Foundry
+ *   starts.
+ * @param {string|null} [opts.awsConfig] - Name of the `awsConfig` file under the
+ *   data root's `Config/`, passed as `FOUNDRY_AWS_CONFIG`.
  * @param {string|null} [opts.user] - `--user` value to run the container as.
  *   `felddy/foundryvtt` runs Foundry as user 1000 and ignores `FOUNDRY_UID` and
  *   `FOUNDRY_GID`; under rootless Docker, container root is the host user, so
@@ -417,6 +632,8 @@ export function dockerRunArgs({
     world = null,
     licenseKey = null,
     account = true,
+    network = null,
+    awsConfig = null,
     user = null,
 }) {
     const args = [
@@ -444,6 +661,8 @@ export function dockerRunArgs({
     if (version) args.push("-e", `FOUNDRY_VERSION=${version}`);
     if (world !== null) args.push("-e", `FOUNDRY_WORLD=${world}`);
     if (licenseKey) args.push("-e", `FOUNDRY_LICENSE_KEY=${licenseKey}`);
+    if (network) args.push("--network", network);
+    if (awsConfig) args.push("-e", `FOUNDRY_AWS_CONFIG=${awsConfig}`);
     if (user?.trim()) args.push("--user", user.trim());
     args.push(image);
     return args;
@@ -660,6 +879,7 @@ export function clearStaleLock(dataRoot, log = () => {}) {
  * @property {string|null} licenseKey
  * @property {string|null} cacheDir
  * @property {string|null} user - From `FOUNDRYVTT_CONTAINER_USER`.
+ * @property {S3Plan|null} s3 - Set by `FOUNDRYVTT_CONTAINER_S3`.
  */
 
 /**
@@ -680,9 +900,10 @@ export function resolveContainer({ stage, config, env = process.env }) {
         e2eStage: config.e2eStage,
     });
     const port = resolveStagePort(stage, { env, stages });
+    const name = containerName(config.packageId, stage, config.containerName);
     return {
         stage,
-        name: containerName(config.packageId, stage, config.containerName),
+        name,
         port,
         image: resolveImage({
             env,
@@ -696,6 +917,7 @@ export function resolveContainer({ stage, config, env = process.env }) {
         licenseKey: resolveLicenseKey(stage, env),
         cacheDir: env.FOUNDRYVTT_CACHE?.trim() || null,
         user: env.FOUNDRYVTT_CONTAINER_USER?.trim() || null,
+        s3: resolveS3({ name, rootDir: config.rootDir, env }),
     };
 }
 
@@ -710,10 +932,16 @@ export function resolveContainer({ stage, config, env = process.env }) {
  * @returns {number} The exit status.
  */
 export function startContainer(container, { dataRoot, env = process.env, log = () => {} }) {
+    // Before Foundry starts, so its S3 client finds the server on the first try.
+    if (container.s3) ensureS3({ plan: container.s3, dataRoot, log });
     if (containerExists(container.name)) {
         // Nothing is running against this data root — we are about to start it
         // — so any lock present was left by a previous crash.
         if (!containerExists(container.name, true)) clearStaleLock(dataRoot, log);
+        if (container.s3) {
+            // Fails harmlessly when the container is already on the network.
+            captureDocker(["network", "connect", container.s3.network, container.name]);
+        }
         log(`Starting existing container '${container.name}' → ${container.url}`);
         return runDocker(["start", container.name]);
     }
@@ -743,6 +971,8 @@ export function startContainer(container, { dataRoot, env = process.env, log = (
                 version: container.version,
                 licenseKey: container.licenseKey ?? env.FOUNDRY_LICENSE_KEY?.trim() ?? null,
             }),
+            network: container.s3?.network ?? null,
+            awsConfig: container.s3?.awsConfigName ?? null,
             user: container.user,
         }),
     );
@@ -818,6 +1048,7 @@ export function containerAction({ action, stage, config, env = process.env, log
         }
         case "rm":
             removeContainer(container.name, log);
+            if (container.s3) removeS3(container.s3);
             return 0;
         case "status":
             return runDocker(["ps", "-a", "--filter", `name=^${container.name}$`]);

package/content-config.mjs

@@ -69,7 +69,7 @@ import YAML from "yaml";
 // Leaves with no local imports of their own, so naming them here cannot close
 // a cycle around a consumer's config file (see `engine/pack-config.mjs`).
 import { ADDRESS_SEGMENT_PATTERN, isAddressSegment } from "./engine/address-charset.mjs";
-import { ASSET_TYPE_NAMES } from "./engine/asset-types.mjs";
+import { ASSET_TYPE_NAMES, shippingLicensesProblem } from "./engine/asset-types.mjs";
 import { isReservedPackage } from "./engine/packages.mjs";
 import { EMPTY_ICON_REGISTRY, checkIconRegistry } from "./engine/content-icons.mjs";
 import { MAP_TYPES, NO_PACK, PACK_BY_TYPE } from "./engine/ids.mjs";
@@ -2453,6 +2453,10 @@ function normalizePackageBuild(value) {
                 "reads, and that package validates what is inside it",
         );
     }
+    // Read by the asset reference pass as well as by the packaging commands, so
+    // a malformed list is refused wherever the configuration loads.
+    const problem = shippingLicensesProblem(value.shippingLicenses);
+    if (problem) fail("packageBuild.shippingLicenses", problem);
     return /** @type {Readonly<PackageBuildSection>} */ (deepFreeze(structuredClone(value)));
 }
 

package/docs/api.md

@@ -891,31 +891,67 @@ Emitting this package's content index. Every content build already walks the who
 
 The asset types — `icon`, `image` and `audio` — and the three roots they are walked from. An asset is addressed exactly as a note is; what differs is that a `.webp` carries no frontmatter and has nowhere to say what it is, so the root supplies the type and the list of roots is closed.
 
-| Export             | Signature                       | Returns                                                      | Use it when                                                                                                             |
-| ------------------ | ------------------------------- | ------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------- |
-| `ASSET_TYPES`      | `const ASSET_TYPES`             | `readonly AssetType[]` — type, root, extensions, description | walking the roots, or presenting the vocabulary                                                                         |
-| `ASSET_TYPE_NAMES` | `const ASSET_TYPE_NAMES`        | `ReadonlySet<string>`                                        | admitting the asset types into a type vocabulary an address is parsed against                                           |
-| `ASSET_SYSTEM`     | `const ASSET_SYSTEM`            | `string` — always `none`                                     | writing an asset address, whose `<system>` segment is a property of the type rather than of the reference               |
-| `IMAGE_EXTENSIONS` | `const IMAGE_EXTENSIONS`        | `readonly string[]` — lowercase, dot-led                     | deciding whether a file under a picture root is an asset at all                                                         |
-| `AUDIO_EXTENSIONS` | `const AUDIO_EXTENSIONS`        | `readonly string[]` — lowercase, dot-led                     | the same question for `audio/`                                                                                          |
-| `isAssetType`      | `isAssetType(type)`             | `boolean`                                                    | scoping a rewrite rule, which may substitute a file and nothing else                                                    |
-| `assetTypeOfRoot`  | `assetTypeOfRoot(root)`         | `AssetType \| undefined`                                     | reading a root directory name back to the type it declares                                                              |
-| `isAssetShortcode` | `isAssetShortcode(shortcode)`   | `boolean`                                                    | telling an addressable filename from one that carries a version string, a hyphen or a date stamp                        |
-| `imageDimensions`  | `imageDimensions(absPath, ext)` | `{width, height} \| undefined`                               | reading a raster file's own pixel size from its header, which an SVG carries none of and a partial file fails closed on |
+| Export                    | Signature                        | Returns                                                          | Use it when                                                                                                             |
+| ------------------------- | -------------------------------- | ---------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- |
+| `ASSET_TYPES`             | `const ASSET_TYPES`              | `readonly AssetType[]` — type, root, extensions, description     | walking the roots, or presenting the vocabulary                                                                         |
+| `ASSET_TYPE_NAMES`        | `const ASSET_TYPE_NAMES`         | `ReadonlySet<string>`                                            | admitting the asset types into a type vocabulary an address is parsed against                                           |
+| `ASSET_SYSTEM`            | `const ASSET_SYSTEM`             | `string` — always `none`                                         | writing an asset address, whose `<system>` segment is a property of the type rather than of the reference               |
+| `IMAGE_EXTENSIONS`        | `const IMAGE_EXTENSIONS`         | `readonly string[]` — lowercase, dot-led                         | deciding whether a file under a picture root is an asset at all                                                         |
+| `AUDIO_EXTENSIONS`        | `const AUDIO_EXTENSIONS`         | `readonly string[]` — lowercase, dot-led                         | the same question for `audio/`                                                                                          |
+| `isAssetType`             | `isAssetType(type)`              | `boolean`                                                        | scoping a rewrite rule, which may substitute a file and nothing else                                                    |
+| `assetTypeOfRoot`         | `assetTypeOfRoot(root)`          | `AssetType \| undefined`                                         | reading a root directory name back to the type it declares                                                              |
+| `isAssetShortcode`        | `isAssetShortcode(shortcode)`    | `boolean`                                                        | telling an addressable filename from one that carries a version string, a hyphen or a date stamp                        |
+| `imageDimensions`         | `imageDimensions(absPath, ext)`  | `{width, height} \| undefined`                                   | reading a raster file's own pixel size from its header, which an SVG carries none of and a partial file fails closed on |
+| `DEPLOYED_ASSET_TYPES`    | `const DEPLOYED_ASSET_TYPES`     | `readonly AssetType[]` — `icon` and `image`                      | naming the types a location holds flat, which audio is not                                                              |
+| `isDeployedSuffix`        | `isDeployedSuffix(suffix)`       | `boolean`                                                        | asking whether a path inside `assets/` sits under a picture root, and so deploys flat                                   |
+| `deployedSuffix`          | `deployedSuffix(suffix)`         | `string` — `<icons\|images>/<basename>`, or the suffix unchanged | turning a source path inside `assets/` into the path the file has in the package, on the CDN and in every location      |
+| `shippingLicensesProblem` | `shippingLicensesProblem(value)` | `string` — a clause, or `""`                                     | checking a `packageBuild.shippingLicenses` value: a list of non-empty, distinct strings                                 |
 
 ### `engine.assetIndex`
 
-The asset record: one line of the content index per addressable file. The record is emitted by the package holding the bytes, so its `path` is that package's path and each consumer joins its own root onto it — Foundry its data directory, the website its CDN prefix, the book its asset base.
-
-| Export                      | Signature                                                       | Returns                       | Use it when                                                                                                                                             |
-| --------------------------- | --------------------------------------------------------------- | ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `ASSET_ROLES`               | `const ASSET_ROLES`                                             | `readonly string[]`           | naming the closed set of roles a picture may declare                                                                                                    |
-| `ASSET_RECORD_FIELDS`       | `const ASSET_RECORD_FIELDS`                                     | `readonly AssetRecordField[]` | reading what an `asset` block carries, and where each field comes from                                                                                  |
-| `PROVENANCE_KEYS`           | `const PROVENANCE_KEYS`                                         | `ReadonlySet<string>`         | validating a provenance file, whose keys are derived from the record's own fields                                                                       |
-| `REQUIRED_PROVENANCE_KEYS`  | `const REQUIRED_PROVENANCE_KEYS`                                | `ReadonlySet<string>`         | the keys a provenance file must state — a record resolves wholesale, so one omitting them leaves every file it covers with no rights holder or no terms |
-| `PROVENANCE_FILE`           | `const PROVENANCE_FILE`                                         | `string` — `provenance.yaml`  | naming the file a directory records provenance for its subtree in                                                                                       |
-| `PROVENANCE_SIDECAR_SUFFIX` | `const PROVENANCE_SIDECAR_SUFFIX`                               | `string` — `.yaml`            | naming a per-file record, which replaces an inherited one wholesale                                                                                     |
-| `collectAssetRecords`       | `collectAssetRecords(assetsBase, { contentPackage, problems })` | `Array<Record<string, any>>`  | reading a package's asset roots into index records, without walking its notes                                                                           |
+The asset record: one line of the content index per addressable file. The record is emitted by the package holding the bytes, so its `path` is that package's source path inside `assets/`, which the book stages from; Foundry and the website serve the deployed layout derived from it (`engine.assetShipping`). Every asset is covered by a provenance record that states `attribution`, `license` and `ai`, and a file no record covers is a finding.
+
+| Export                      | Signature                                                       | Returns                       | Use it when                                                                                                                                                                  |
+| --------------------------- | --------------------------------------------------------------- | ----------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `ASSET_ROLES`               | `const ASSET_ROLES`                                             | `readonly string[]`           | naming the closed set of roles a picture may declare                                                                                                                         |
+| `ASSET_RECORD_FIELDS`       | `const ASSET_RECORD_FIELDS`                                     | `readonly AssetRecordField[]` | reading what an `asset` block carries, and where each field comes from                                                                                                       |
+| `PROVENANCE_KEYS`           | `const PROVENANCE_KEYS`                                         | `ReadonlySet<string>`         | validating a provenance file, whose keys are derived from the record's own fields                                                                                            |
+| `REQUIRED_PROVENANCE_KEYS`  | `const REQUIRED_PROVENANCE_KEYS`                                | `ReadonlySet<string>`         | the keys a provenance file must state — `attribution`, `license` and `ai`; a record resolves wholesale, so one omitting them leaves every file it covers without that answer |
+| `PROVENANCE_FILE`           | `const PROVENANCE_FILE`                                         | `string` — `provenance.yaml`  | naming the file a directory records provenance for its subtree in                                                                                                            |
+| `PROVENANCE_SIDECAR_SUFFIX` | `const PROVENANCE_SIDECAR_SUFFIX`                               | `string` — `.yaml`            | naming a per-file record, which replaces an inherited one wholesale                                                                                                          |
+| `collectAssetRecords`       | `collectAssetRecords(assetsBase, { contentPackage, problems })` | `Array<Record<string, any>>`  | reading a package's asset roots into index records, without walking its notes                                                                                                |
+
+### `engine.assetShipping`
+
+The deployed layout of a package's pictures, and which of them ship. A Foundry package holds its icons and images flat — `icons/<basename>`, `images/<basename>` — and the stage copies a picture only when its provenance says `ai: false` and its licence is on the shipping list. Its asset set, `asset-sets/default.json`, lists what it ships. The selection is one function, so the stage and any command publishing what the stage leaves out read the same answer.
+
+| Export                    | Signature                                                                                          | Returns                                                                         | Use it when                                                                                                                     |
+| ------------------------- | -------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- |
+| `SHIPPING_LICENSES`       | `const SHIPPING_LICENSES`                                                                          | `readonly string[]`                                                             | reading the licences a shipped picture may carry, exactly as provenance spells them                                             |
+| `SHIPPED_SET_FILE`        | `const SHIPPED_SET_FILE`                                                                           | `string` — `asset-sets/default.json`                                            | naming the staged package's own asset set, relative to the package root                                                         |
+| `STAND_IN_SHORTCODE`      | `const STAND_IN_SHORTCODE`                                                                         | `string` — `missing`                                                            | naming the shortcode of a package's stand-in picture                                                                            |
+| `DEPLOYED_ASSET_TYPES`    | `const DEPLOYED_ASSET_TYPES`                                                                       | `readonly AssetType[]`                                                          | re-exported from `engine.assetTypes`                                                                                            |
+| `deployedSuffix`          | `deployedSuffix(suffix)`                                                                           | `string`                                                                        | re-exported from `engine.assetTypes`                                                                                            |
+| `deployedAssetPath`       | `deployedAssetPath(record)`                                                                        | `string \| undefined` — `<icons\|images>/<basename>`                            | reading the deployed path of one asset record; `undefined` for audio                                                            |
+| `exclusionReason`         | `exclusionReason(record, additions?)`                                                              | `string` — a clause, or `""`                                                    | saying why a picture does not ship                                                                                              |
+| `shipsAsset`              | `shipsAsset(record, additions?)`                                                                   | `boolean`                                                                       | asking whether one picture ships                                                                                                |
+| `standInAddress`          | `standInAddress(pkg, type)`                                                                        | `string` — `<package>-none-<type>-missing`                                      | naming the stand-in an excluded picture compiles to                                                                             |
+| `selectStagedAssets`      | `selectStagedAssets(records, additions?)`                                                          | `{staged, excluded}` — both in address order, each excluded one with its reason | splitting a package's pictures into what ships and what does not                                                                |
+| `shippedSetName`          | `shippedSetName(title)`                                                                            | `string` — `<title> — shipped art`                                              | naming the staged package's own asset set                                                                                       |
+| `assetsManifest`          | `assetsManifest(staged, stageDir)`                                                                 | `{address, source, target, size}[]`                                             | listing staged files as the shipped set does, each `source` relative to `asset-sets/`, sizes measured in the stage              |
+| `stagePackageAssets`      | `stagePackageAssets({ assetsBase, contentPackage, stageDir, title, transform, shippingLicenses })` | `{staged, excluded, findings, manifest}`                                        | staging a package's shipped pictures flat and writing `asset-sets/default.json`; nothing is staged when `findings` is non-empty |
+| `rowsStagingPictureRoots` | `rowsStagingPictureRoots(rows, { rootDir, assetsBase })`                                           | `{index, from, root}[]`                                                         | finding `packageBuild.assets` rows that would copy a picture root around the filter                                             |
+
+### `engine.assetReferences`
+
+Every asset reference in a compiled document states the asset's address: an `<img>` carries `data-asset`, and a path field's address sits in `flags.<system>.assets` on the document holding it, nested to mirror the field's path. A reference to a picture its package does not ship compiles to that package's stand-in and keeps the excluded address.
+
+| Export                    | Signature                                                                | Returns                                   | Use it when                                                                                            |
+| ------------------------- | ------------------------------------------------------------------------ | ----------------------------------------- | ------------------------------------------------------------------------------------------------------ |
+| `ASSET_PATH_FIELDS`       | `const ASSET_PATH_FIELDS`                                                | `Record<string, AssetPathField[]>`        | reading which fields of which document types name a picture, and where each records its address        |
+| `assetReferenceLookup`    | `assetReferenceLookup(linkIndex, config)`                                | `{byPath, byAddress}`                     | indexing every picture a compiled document can name, local and fetched, by Foundry path and by address |
+| `annotateHtml`            | `annotateHtml(html, lookup, report)`                                     | `string`                                  | marking each `<img>` in one HTML string with its address                                               |
+| `annotateAssetReferences` | `annotateAssetReferences(doc, documentType, { lookup, system, report })` | `object` — the document, changed in place | stating every reference's address on a finished document and drawing stand-ins                         |
 
 ### `engine.packages`
 
@@ -1719,36 +1755,48 @@ console.log(dataEnvVar("dev"));
 // -> "FOUNDRYVTT_DEV_DATA"
 ```
 
-| Export                     | Signature                                                                                                  | Returns                        | Use it when                                                                                                                                                                                 |
-| -------------------------- | ---------------------------------------------------------------------------------------------------------- | ------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `DEFAULT_STAGE_PORTS`      | `const DEFAULT_STAGE_PORTS`                                                                                | —                              | reading the host port each conventional stage (`dev`/`qa`/`prod`/`test`) publishes by default, chosen so all four can run at once                                                           |
-| `CONTAINER_PORT`           | `const CONTAINER_PORT`                                                                                     | —                              | the fixed port Foundry listens on inside the container                                                                                                                                      |
-| `CACHE_MOUNT`              | `const CACHE_MOUNT`                                                                                        | —                              | the path a host-provided download cache is mounted at inside the container                                                                                                                  |
-| `CONTAINER_ACTIONS`        | `const CONTAINER_ACTIONS`                                                                                  | —                              | the closed set of actions `package-build container` accepts                                                                                                                                 |
-| `dataEnvVar`               | `dataEnvVar(stage)`                                                                                        | `string`                       | deriving the environment variable that names a stage's Foundry data root, without hand-tabulating one per stage                                                                             |
-| `containerName`            | `containerName(packageId, stage, base = null)`                                                             | `string`                       | computing the stable container name a stage runs under, so a signed Foundry licence (bound to the hostname) survives a recreate; pass `base` to share one container/licence across packages |
-| `resolveStagePort`         | `resolveStagePort(stage, { env, stages } = {})`                                                            | `number`                       | resolving the host port for a stage, honoring `FOUNDRYVTT_<STAGE>_PORT` over a declared or conventional default                                                                             |
-| `resolveFoundryVersion`    | `resolveFoundryVersion(stage, { env, stages, compatibilityMinimum, e2eStage = "test" } = {})`              | `string\|null`                 | pinning the exact Foundry build a stage runs, deriving the end-to-end stage's pin from the package's own `compatibility.minimum` so the floor is actually exercised                         |
-| `resolveImage`             | `resolveImage({ env, image, version, compatibilityMinimum } = {})`                                         | `string`                       | resolving the `felddy/foundryvtt` image reference a run uses — the pinned build's own tag when a build is pinned                                                                            |
-| `resolveWorld`             | `resolveWorld(stage, { env, stages } = {})`                                                                | `string\|null`                 | resolving which world a stage auto-launches — `""` declares "never auto-launch", distinct from `null` (leave `FOUNDRY_WORLD` alone)                                                         |
-| `resolveLicenseKey`        | `resolveLicenseKey(stage, env = process.env)`                                                              | `string\|null`                 | finding a stage's dedicated Foundry licence key, since Foundry is single-seat and two stages running at once need two keys                                                                  |
-| `resolveDataRoot`          | `resolveDataRoot(stage, { env } = {})`                                                                     | `string`                       | resolving the local Foundry data root path for a stage, checked for what a bind mount needs                                                                                                 |
-| `passthroughEnv`           | `passthroughEnv(env = process.env)`                                                                        | `[string, string][]`           | collecting the image's own `FOUNDRY_*`/`CONTAINER_*` environment pairs to pass through (deliberately excludes `CONTAINER_CACHE`, which `dockerRunArgs` sets itself)                         |
-| `needsFoundryAccount`      | `needsFoundryAccount({ cacheDir, dataRoot, version, licenseKey, fileExists })`                             | `boolean`                      | deciding whether the image needs `FOUNDRY_USERNAME`/`FOUNDRY_PASSWORD`: only when the pinned build's archive is not cached, or the licence must come from the account                       |
-| `dockerRunArgs`            | `dockerRunArgs({ name, image, port, dataRoot, env, cacheDir, version, world, licenseKey, account, user })` | `string[]`                     | building the full `docker run` argument vector for a stage's container; `account: false` withholds the foundryvtt.com account                                                               |
-| `runDocker`                | `runDocker(args)`                                                                                          | `number`                       | running the `docker` CLI with inherited stdio and getting back its exit status                                                                                                              |
-| `captureDocker`            | `captureDocker(args)`                                                                                      | `string`                       | running `docker` and capturing trimmed stdout, tolerating failure (returns `""`)                                                                                                            |
-| `dataRootScriptArgs`       | `dataRootScriptArgs({ image, dataRoot, script, args, user })`                                              | `string[]`                     | building the `docker run` vector for a throwaway container that runs a shell script against a stage's data root, mounted at `/data`                                                         |
-| `runDataRootScript`        | `runDataRootScript(opts)`                                                                                  | `string`                       | running that script and getting its stdout, throwing on a non-zero exit; the way to remove or hand over files that the Foundry container's user owns                                        |
-| `containerTreeRemover`     | `containerTreeRemover({ container, runScript })`                                                           | `(tree, { dataRoot }) => void` | removing a tree beneath a stage's data root that the Foundry container's user owns, as root in a throwaway container; the remover `package-build deploy` passes to `deployStage`            |
-| `containerRemovalCommand`  | `containerRemovalCommand({ image, dataRoot, tree })`                                                       | `string`                       | writing out the `docker run` command that removes such a tree, for a person to paste                                                                                                        |
-| `containerExists`          | `containerExists(name, runningOnly = false)`                                                               | `boolean`                      | checking whether a container with exactly this name exists (optionally, only counting a running one)                                                                                        |
-| `runningFoundryContainers` | `runningFoundryContainers(except = "")`                                                                    | `string[]`                     | listing HeroicLands-convention Foundry containers currently running, to warn about a licence clash before a run starts                                                                      |
-| `clearStaleLock`           | `clearStaleLock(dataRoot, log = () => {})`                                                                 | —                              | removing a data-root lock left behind by a container that did not shut down cleanly; only safe to call while the container is stopped                                                       |
-| `resolveContainer`         | `resolveContainer({ stage, config, env = process.env })`                                                   | `ResolvedContainer`            | resolving all of a stage's container settings in one call, from the package-build configuration                                                                                             |
-| `startContainer`           | `startContainer(container, { dataRoot, env, log })`                                                        | `number`                       | starting a stage's container, creating it first if it does not exist                                                                                                                        |
-| `removeContainer`          | `removeContainer(name, log = () => {})`                                                                    | —                              | stopping and removing a container, tolerating "not running" and "no such container"                                                                                                         |
-| `containerAction`          | `containerAction({ action, stage, config, env, log })`                                                     | `number`                       | performing one `CONTAINER_ACTIONS` action for a stage end to end                                                                                                                            |
+| Export                     | Signature                                                                                                                      | Returns                        | Use it when                                                                                                                                                                                 |
+| -------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | ------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `DEFAULT_STAGE_PORTS`      | `const DEFAULT_STAGE_PORTS`                                                                                                    | —                              | reading the host port each conventional stage (`dev`/`qa`/`prod`/`test`) publishes by default, chosen so all four can run at once                                                           |
+| `CONTAINER_PORT`           | `const CONTAINER_PORT`                                                                                                         | —                              | the fixed port Foundry listens on inside the container                                                                                                                                      |
+| `CACHE_MOUNT`              | `const CACHE_MOUNT`                                                                                                            | —                              | the path a host-provided download cache is mounted at inside the container                                                                                                                  |
+| `CONTAINER_ACTIONS`        | `const CONTAINER_ACTIONS`                                                                                                      | —                              | the closed set of actions `package-build container` accepts                                                                                                                                 |
+| `dataEnvVar`               | `dataEnvVar(stage)`                                                                                                            | `string`                       | deriving the environment variable that names a stage's Foundry data root, without hand-tabulating one per stage                                                                             |
+| `containerName`            | `containerName(packageId, stage, base = null)`                                                                                 | `string`                       | computing the stable container name a stage runs under, so a signed Foundry licence (bound to the hostname) survives a recreate; pass `base` to share one container/licence across packages |
+| `resolveStagePort`         | `resolveStagePort(stage, { env, stages } = {})`                                                                                | `number`                       | resolving the host port for a stage, honoring `FOUNDRYVTT_<STAGE>_PORT` over a declared or conventional default                                                                             |
+| `resolveFoundryVersion`    | `resolveFoundryVersion(stage, { env, stages, compatibilityMinimum, e2eStage = "test" } = {})`                                  | `string\|null`                 | pinning the exact Foundry build a stage runs, deriving the end-to-end stage's pin from the package's own `compatibility.minimum` so the floor is actually exercised                         |
+| `resolveImage`             | `resolveImage({ env, image, version, compatibilityMinimum } = {})`                                                             | `string`                       | resolving the `felddy/foundryvtt` image reference a run uses — the pinned build's own tag when a build is pinned                                                                            |
+| `resolveWorld`             | `resolveWorld(stage, { env, stages } = {})`                                                                                    | `string\|null`                 | resolving which world a stage auto-launches — `""` declares "never auto-launch", distinct from `null` (leave `FOUNDRY_WORLD` alone)                                                         |
+| `resolveLicenseKey`        | `resolveLicenseKey(stage, env = process.env)`                                                                                  | `string\|null`                 | finding a stage's dedicated Foundry licence key, since Foundry is single-seat and two stages running at once need two keys                                                                  |
+| `resolveDataRoot`          | `resolveDataRoot(stage, { env } = {})`                                                                                         | `string`                       | resolving the local Foundry data root path for a stage, checked for what a bind mount needs                                                                                                 |
+| `passthroughEnv`           | `passthroughEnv(env = process.env)`                                                                                            | `[string, string][]`           | collecting the image's own `FOUNDRY_*`/`CONTAINER_*` environment pairs to pass through (deliberately excludes `CONTAINER_CACHE`, which `dockerRunArgs` sets itself)                         |
+| `needsFoundryAccount`      | `needsFoundryAccount({ cacheDir, dataRoot, version, licenseKey, fileExists })`                                                 | `boolean`                      | deciding whether the image needs `FOUNDRY_USERNAME`/`FOUNDRY_PASSWORD`: only when the pinned build's archive is not cached, or the licence must come from the account                       |
+| `dockerRunArgs`            | `dockerRunArgs({ name, image, port, dataRoot, env, cacheDir, version, world, licenseKey, account, network, awsConfig, user })` | `string[]`                     | building the full `docker run` argument vector for a stage's container; `account: false` withholds the foundryvtt.com account                                                               |
+| `runDocker`                | `runDocker(args)`                                                                                                              | `number`                       | running the `docker` CLI with inherited stdio and getting back its exit status                                                                                                              |
+| `captureDocker`            | `captureDocker(args)`                                                                                                          | `string`                       | running `docker` and capturing trimmed stdout, tolerating failure (returns `""`)                                                                                                            |
+| `dataRootScriptArgs`       | `dataRootScriptArgs({ image, dataRoot, script, args, user })`                                                                  | `string[]`                     | building the `docker run` vector for a throwaway container that runs a shell script against a stage's data root, mounted at `/data`                                                         |
+| `runDataRootScript`        | `runDataRootScript(opts)`                                                                                                      | `string`                       | running that script and getting its stdout, throwing on a non-zero exit; the way to remove or hand over files that the Foundry container's user owns                                        |
+| `containerTreeRemover`     | `containerTreeRemover({ container, runScript })`                                                                               | `(tree, { dataRoot }) => void` | removing a tree beneath a stage's data root that the Foundry container's user owns, as root in a throwaway container; the remover `package-build deploy` passes to `deployStage`            |
+| `containerRemovalCommand`  | `containerRemovalCommand({ image, dataRoot, tree })`                                                                           | `string`                       | writing out the `docker run` command that removes such a tree, for a person to paste                                                                                                        |
+| `containerExists`          | `containerExists(name, runningOnly = false)`                                                                                   | `boolean`                      | checking whether a container with exactly this name exists (optionally, only counting a running one)                                                                                        |
+| `runningFoundryContainers` | `runningFoundryContainers(except = "")`                                                                                        | `string[]`                     | listing HeroicLands-convention Foundry containers currently running, to warn about a licence clash before a run starts                                                                      |
+| `clearStaleLock`           | `clearStaleLock(dataRoot, log = () => {})`                                                                                     | —                              | removing a data-root lock left behind by a container that did not shut down cleanly; only safe to call while the container is stopped                                                       |
+| `S3_IMAGE`                 | `const S3_IMAGE`                                                                                                               | `string`                       | reading the S3 server's image                                                                                                                                                               |
+| `S3_ALIAS`                 | `const S3_ALIAS`                                                                                                               | `string`                       | reading the name the S3 server answers to on the network                                                                                                                                    |
+| `S3_PORT`                  | `const S3_PORT`                                                                                                                | `number`                       | reading the port the S3 server publishes                                                                                                                                                    |
+| `S3_BUCKET`                | `const S3_BUCKET`                                                                                                              | `string`                       | reading the bucket the S3 server is created with                                                                                                                                            |
+| `S3_IDENTITIES_FILE`       | `const S3_IDENTITIES_FILE`                                                                                                     | `string`                       | reading the repository-relative SeaweedFS identities file the S3 opt-in takes                                                                                                               |
+| `S3_AWS_CONFIG_FILE`       | `const S3_AWS_CONFIG_FILE`                                                                                                     | `string`                       | reading the repository-relative Foundry `awsConfig` the S3 opt-in takes                                                                                                                     |
+| `s3Requested`              | `s3Requested(env = process.env)`                                                                                               | `boolean`                      | asking whether `FOUNDRYVTT_CONTAINER_S3` opts in                                                                                                                                            |
+| `resolveS3`                | `resolveS3({ name, rootDir, env })`                                                                                            | `S3Plan\|null`                 | planning the network, server and files of the S3 opt-in; `null` when off                                                                                                                    |
+| `s3RunArgs`                | `s3RunArgs(plan)`                                                                                                              | `string[]`                     | building the S3 server's `docker run` vector                                                                                                                                                |
+| `s3ShellArgs`              | `s3ShellArgs(plan, command)`                                                                                                   | `string[]`                     | building the `docker exec` vector running one `weed shell` command                                                                                                                          |
+| `ensureS3`                 | `ensureS3({ plan, dataRoot, log, run, capture, sleep, attempts })`                                                             | `void`                         | starting the network, server and bucket and copying the `awsConfig` into the data root                                                                                                      |
+| `removeS3`                 | `removeS3(plan, run, capture)`                                                                                                 | `void`                         | removing the S3 server and its network                                                                                                                                                      |
+| `resolveContainer`         | `resolveContainer({ stage, config, env = process.env })`                                                                       | `ResolvedContainer`            | resolving all of a stage's container settings in one call, from the package-build configuration                                                                                             |
+| `startContainer`           | `startContainer(container, { dataRoot, env, log })`                                                                            | `number`                       | starting a stage's container, creating it first if it does not exist                                                                                                                        |
+| `removeContainer`          | `removeContainer(name, log = () => {})`                                                                                        | —                              | stopping and removing a container, tolerating "not running" and "no such container"                                                                                                         |
+| `containerAction`          | `containerAction({ action, stage, config, env, log })`                                                                         | `number`                       | performing one `CONTAINER_ACTIONS` action for a stage end to end                                                                                                                            |
 
 ## `./coverage`
 

package/docs/authoring/assets.md

@@ -26,16 +26,31 @@ The embed's label is alternative text. `![[icon-harbor|]]` is decorative. `size=
 
 ## Ownership and source records
 
-Record provenance for the assets you ship. Put `provenance.yaml` in an asset directory to describe files there and below, or put a YAML sidecar beside one asset to give that file its own record. A sidecar takes precedence over a directory record for that asset. An asset with no record still enters the index with blank provenance fields. Keep attribution with the authored asset so movement through the tree does not lose its source or license. See the [asset record reference](../reference/format-details.md#the-asset-record) for the accepted keys, inheritance, and examples.
+Every asset is covered by a provenance record. Put `provenance.yaml` in an asset directory to describe files there and below, or put a YAML sidecar named for the whole filename — `anvil.svg.yaml` — beside one asset to give that file its own record. A sidecar replaces the directory record for that asset; otherwise the nearest `provenance.yaml` in the file's directory or one above it, up to the type root, answers. Each file resolves on its own, so a directory may hold files its `provenance.yaml` covers beside files a sidecar answers for. A file no record covers is an error at the file. Keep attribution with the authored asset so movement through the tree does not lose its source or license. See the [asset record reference](../reference/format-details.md#the-asset-record) for the accepted keys, inheritance, and examples.
+
+Every record states `attribution`, `license` and `ai` — whether a machine made the file, as the YAML boolean `true` or `false`. A record that leaves `ai` out is an error at the record; it is never read as `false`.
+
+```yaml
+attribution: Tom Rodriguez
+license: CC-BY-SA-4.0
+ai: false
+```
 
 A directory of portraits also states what the pictures in it are for:
 
 ```yaml
 attribution: Tom Rodriguez
 license: CC-BY-SA-4.0
+ai: false
 role: portrait
 ```
 
+## What the Foundry package ships
+
+The repository keeps every picture; the Foundry package carries a filtered copy, laid out flat. A picture ships when its record says `ai: false` and its licence is one of `LicenseRef-PublicDomain`, `CC0-1.0`, `CC0`, `CC-BY-3.0`, `CC-BY-4.0`, `CC-BY-SA-4.0` or `LicenseRef-NounProject-Pro`, together with any licence the package adds under `packageBuild.shippingLicenses`. A shipped picture sits at `icons/<basename>` or `images/<basename>` at the package root, whatever directories it is kept in here, and the package's asset set, `asset-sets/default.json`, lists what it ships.
+
+A note may still name a picture the package does not ship. Its compiled document draws the package's stand-in instead — `icons/missing.<ext>` or `images/missing.<ext>`, which you provide as an ordinary human-made picture licensed to ship, addressed `<package>-none-icon-missing` and `<package>-none-image-missing` — and keeps the real picture's address, so a world that holds the real file can point back at it. A package that leaves out a picture a note names, and ships no stand-in of that type, fails the build at the note. See [what a package ships](../reference/format-details.md#what-a-package-ships).
+
 **A hero image is 1792×768.** `data.banner` fills one fixed strip wherever it is drawn, so every banner is cut to that size and a picture of any other size is a finding naming both sizes — resize the picture to match. That is the one size any art field requires.
 
 `data.banner` reads in one of three states: