@stigmer/server 3.22.1 → 3.23.0

package/src/domain/skill/transfer/slots.ts

@@ -1,9 +1,18 @@
 /**
- * Skill artifact upload slots — ports pkg/domain/skill/transfer/slots.go.
- * The in-memory registry of single-use upload capabilities plus the
- * staging directory their bytes land in (#675).
+ * Skill artifact upload slots — the local blob driver's staging mechanism,
+ * ported from pkg/domain/skill/transfer/slots.go: an in-memory registry of
+ * single-use upload capabilities plus the staging files their bytes land
+ * in (#675).
  *
- * In-memory is deliberate: the OSS server is single-instance (the same
+ * A slot is keyed by the STAGING KEY the domain names (`skills/staging/
+ * <hex>.zip`, transfer/staging.ts), and its file is that key resolved
+ * under the blob driver's root — so the driver's own download(key) reads
+ * what the lane received and its delete(key) retires the bytes, exactly
+ * as a bucket driver's would. The registry therefore knows nothing of
+ * consumption: it reserves, receives, and sweeps; the domain reads and
+ * deletes through the driver, one path for every backend.
+ *
+ * In-memory is deliberate: the local driver is single-instance (the same
  * assumption the in-process router transport and SQLite store already
  * make), and a slot is worthless across restarts anyway — its bytes live
  * in the staging directory, which is swept on boot.
@@ -11,20 +20,16 @@
  * Sentinel error classes replace Go's sentinel error values so the HTTP
  * handler maps registry failures onto honest status codes without string
  * matching; their message texts are Go's, verbatim (they ride wire-visible
- * copy: push's "artifact_upload_ref not usable: %v" and the handler's 400
- * body).
+ * copy: the handler's 404, 409 and 400 bodies).
  *
  * Proven by __tests__/slots.test.ts (injected clock for expiry) and the
  * conformance suite's transfer-lane tests.
  */
-import { randomBytes } from "node:crypto";
 import fs from "node:fs";
 import path from "node:path";
 import { pipeline } from "node:stream/promises";
 import type { Readable } from "node:stream";
 
-import { REF_BYTE_LEN, REF_PREFIX } from "../constants.js";
-
 /** Go errSlotUnknown — never existed, expired, or swept. */
 export class SlotUnknownError extends Error {
   constructor() {
@@ -41,18 +46,12 @@ export class SlotConsumedError extends Error {
   }
 }
 
-/** Go errSlotEmpty — minted but never (successfully) uploaded to. */
-export class SlotEmptyError extends Error {
-  constructor() {
-    super("upload reference has no uploaded bytes");
-    this.name = "SlotEmptyError";
-  }
-}
-
-/** Go errSizeMismatch — the body disagreed with the minted declaration. */
+/** Go errSizeMismatch — the body disagreed with the reserved declaration. */
 export class SizeMismatchError extends Error {
   constructor(received: number, declared: number) {
-    super(`upload size mismatch: received ${received} bytes, declared ${declared}`);
+    super(
+      `upload size mismatch: received ${received} bytes, declared ${declared}`,
+    );
     this.name = "SizeMismatchError";
   }
 }
@@ -63,65 +62,76 @@ interface Slot {
   uploaded: boolean;
 }
 
+/**
+ * The one file-name shape a staging key may carry past its prefix: the
+ * domain's hex reference plus the archive suffix. Keys are server-minted,
+ * never client-supplied, so this is an invariant check, not input
+ * validation — a violation is a programming error and throws.
+ */
+const STAGED_FILE_NAME = /^[0-9a-f]+\.zip$/;
+
 export class UploadSlots {
   private readonly slots = new Map<string, Slot>();
   private readonly stagingDir: string;
-  private readonly ttlMs: number;
-  private readonly maxSize: number;
   /** Injectable for expiry tests, mirroring Go's `now` field. */
   private readonly now: () => number;
 
   /**
-   * Creates the registry and prepares the staging directory. Any file
-   * already present is an orphan from a previous process (the registry
-   * that knew about it died with that process), so the directory is
-   * emptied — this is also the crash-recovery story for uploads that
-   * never reached their push.
+   * Creates the registry over `root` (the local blob driver's root) and
+   * prepares the staging directory `root/stagingPrefix`. Any file already
+   * present is an orphan from a previous process (the registry that knew
+   * about it died with that process), so the directory is emptied — this
+   * is also the crash-recovery story for uploads that never reached their
+   * push.
    */
   constructor(
-    stagingDir: string,
-    ttlMs: number,
-    maxSize: number,
+    private readonly root: string,
+    private readonly stagingPrefix: string,
+    private readonly ttlMs: number,
+    private readonly maxSize: number,
     now: () => number = Date.now,
   ) {
-    fs.rmSync(stagingDir, { recursive: true, force: true });
-    fs.mkdirSync(stagingDir, { recursive: true, mode: 0o700 });
-    this.stagingDir = stagingDir;
-    this.ttlMs = ttlMs;
-    this.maxSize = maxSize;
+    this.stagingDir = path.join(root, stagingPrefix);
+    fs.rmSync(this.stagingDir, { recursive: true, force: true });
+    fs.mkdirSync(this.stagingDir, { recursive: true, mode: 0o700 });
     this.now = now;
   }
 
   /**
-   * Reserves an upload slot for an artifact of declaredSize bytes and
-   * returns its single-use reference + TTL. The caller has already
+   * Reserves an upload slot for an artifact of declaredSize bytes at the
+   * staging key; returns the TTL granted. The caller has already
    * authorized the request and validated declaredSize against the skill
-   * size limit; this guard is the registry's own invariant.
+   * size limit; the bound here is the registry's own invariant.
    */
-  mint(declaredSize: number): { ref: string; ttlMs: number } {
+  reserve(key: string, declaredSize: number): { ttlMs: number } {
     if (declaredSize <= 0 || declaredSize > this.maxSize) {
-      throw new Error(`declared size ${declaredSize} outside (0, ${this.maxSize}]`);
+      throw new Error(
+        `declared size ${declaredSize} outside (0, ${this.maxSize}]`,
+      );
+    }
+    this.stagePath(key);
+    if (this.slots.has(key)) {
+      throw new Error(`staging key ${key} is already reserved`);
     }
-    const ref = REF_PREFIX + randomBytes(REF_BYTE_LEN).toString("hex");
     this.sweep();
-    this.slots.set(ref, {
+    this.slots.set(key, {
       declaredSize,
       expiresAtMs: this.now() + this.ttlMs,
       uploaded: false,
     });
-    return { ref, ttlMs: this.ttlMs };
+    return { ttlMs: this.ttlMs };
   }
 
   /**
    * Streams an upload's body into the slot's staging file. The body must
-   * match the size declared at mint time exactly: a shorter body means a
+   * match the size declared at reservation exactly: a shorter body means a
    * truncated transfer, a longer one means the client lied — both reject
    * rather than staging bytes that would fail (or worse, surprise)
-   * validation later. The staged file only becomes consumable once this
-   * resolves.
+   * validation later. The staged file only becomes readable through the
+   * driver once this resolves.
    */
-  async receive(ref: string, body: Readable): Promise<void> {
-    const slot = this.slots.get(ref);
+  async receive(key: string, body: Readable): Promise<void> {
+    const slot = this.slots.get(key);
     if (slot === undefined || this.now() > slot.expiresAtMs) {
       throw new SlotUnknownError();
     }
@@ -130,9 +140,16 @@ export class UploadSlots {
     }
     const declared = slot.declaredSize;
 
-    const filePath = this.stagePath(ref);
+    const filePath = this.stagePath(key);
     let written = 0;
     try {
+      // The driver's delete(key) prunes directories it empties, so the
+      // staging directory may be gone between two uploads; recreate it
+      // rather than fail the second one.
+      await fs.promises.mkdir(path.dirname(filePath), {
+        recursive: true,
+        mode: 0o700,
+      });
       // Consume at most declared+1 bytes: seeing the extra byte proves the
       // body exceeds the declaration without buffering an unbounded stream
       // (Go's io.LimitReader(declared+1) + written != declared check).
@@ -157,7 +174,7 @@ export class UploadSlots {
 
     // Re-check after the write: the slot may have expired mid-upload
     // (Go re-checks under the lock for the same reason).
-    const current = this.slots.get(ref);
+    const current = this.slots.get(key);
     if (current === undefined || this.now() > current.expiresAtMs) {
       await fs.promises.rm(filePath, { force: true });
       throw new SlotUnknownError();
@@ -166,66 +183,35 @@ export class UploadSlots {
   }
 
   /**
-   * Returns the staged bytes for ref and retires the slot — an upload
-   * reference is strictly single-use. Push calls this when it sees
-   * artifact_upload_ref; whatever happens downstream (validation failure
-   * included), the slot is gone and the client must re-mint to retry.
-   */
-  async consume(ref: string): Promise<Uint8Array> {
-    const slot = this.slots.get(ref);
-    if (slot === undefined || this.now() > slot.expiresAtMs) {
-      throw new SlotUnknownError();
-    }
-    if (!slot.uploaded) {
-      throw new SlotEmptyError();
-    }
-    this.slots.delete(ref);
-
-    const filePath = this.stagePath(ref);
-    try {
-      const data = await fs.promises.readFile(filePath);
-      return data;
-    } catch (error) {
-      throw new Error(
-        `failed to read staged artifact: ${error instanceof Error ? error.message : String(error)}`,
-      );
-    } finally {
-      await fs.promises.rm(filePath, { force: true });
-    }
-  }
-
-  /**
-   * Drops expired slots and their staged files. Called from mint, which
-   * bounds the registry: it can hold at most the slots minted within one
+   * Drops expired slots and their staged files. Called from reserve, which
+   * bounds the registry: it can hold at most the slots reserved within one
    * TTL window.
    */
   private sweep(): void {
     const nowMs = this.now();
-    for (const [ref, slot] of this.slots) {
+    for (const [key, slot] of this.slots) {
       if (nowMs > slot.expiresAtMs) {
-        this.slots.delete(ref);
-        fs.rmSync(this.stagePath(ref), { force: true });
+        this.slots.delete(key);
+        fs.rmSync(this.stagePath(key), { force: true });
       }
     }
   }
 
   /**
-   * The staging file name for a reference. Public since O5: the local
-   * driver's presigned-PUT arm maps refs onto driver staging keys
-   * (boot/compose.ts), and that mapping must come from HERE — a
-   * re-declared "<ref>.zip" in the composition would drift from stagePath
-   * with nothing to catch it.
-   */
-  stagedFileName(ref: string): string {
-    return `${ref}.zip`;
-  }
-
-  /**
-   * Maps a reference to its staging file. refs are server-generated hex
-   * (never client-supplied paths), so simple joining is safe.
+   * Maps a staging key to its file: the key resolved under the driver's
+   * root, which is how the driver's download(key) will look for it. The
+   * key must carry the staging prefix and a plain hex file name — anything
+   * else is a caller bug, refused before it touches the filesystem.
    */
-  private stagePath(ref: string): string {
-    return path.join(this.stagingDir, this.stagedFileName(ref));
+  private stagePath(key: string): string {
+    if (!key.startsWith(this.stagingPrefix)) {
+      throw new Error(`staging key ${key} is outside ${this.stagingPrefix}`);
+    }
+    const fileName = key.slice(this.stagingPrefix.length);
+    if (!STAGED_FILE_NAME.test(fileName)) {
+      throw new Error(`staging key ${key} does not name a staged archive`);
+    }
+    return path.join(this.root, key);
   }
 }
 
@@ -238,13 +224,16 @@ function limitBytes(
   limit: number,
   onBytes: (count: number) => void,
 ): (source: AsyncIterable<Buffer>) => AsyncIterable<Buffer> {
-  return async function* (source: AsyncIterable<Buffer>): AsyncIterable<Buffer> {
+  return async function* (
+    source: AsyncIterable<Buffer>,
+  ): AsyncIterable<Buffer> {
     let remaining = limit;
     for await (const chunk of source) {
       if (remaining <= 0) {
         return;
       }
-      const slice = chunk.length <= remaining ? chunk : chunk.subarray(0, remaining);
+      const slice =
+        chunk.length <= remaining ? chunk : chunk.subarray(0, remaining);
       remaining -= slice.length;
       onBytes(slice.length);
       yield slice;

package/src/domain/skill/transfer/staging.ts

@@ -0,0 +1,191 @@
+/**
+ * Archive staging — how an artifact above the gRPC message cap travels,
+ * on every blob driver, through one code path (#675, stigmer#1219).
+ *
+ * The domain owns the two identities and the mapping between them: the
+ * wire reference a client is handed (`sau_<hex>`, the capability the
+ * push presents) and the staging key the bytes land under in the blob
+ * store (`skills/staging/<hex>.zip`, the surface a sweep targets). A
+ * driver is asked only to sign or stage a PUT for a key it is given, to
+ * read the key back, and to delete it — so the local driver rides the
+ * transfer lane's slot registry and a bucket driver signs a PUT straight
+ * to the bucket, and nothing above this module can tell which.
+ *
+ * Consume reads then deletes: the delete is what keeps a reference
+ * single-use on a bucket, whose signed URL is otherwise repeatable within
+ * its TTL. The delete is best effort and loud — the bytes are already in
+ * hand, a replay of the same reference is a content-addressed no-op, and
+ * the bucket lifecycle rule on the staging prefix sweeps what the delete
+ * missed. On the local driver the same delete retires the staged file the
+ * slot registry would otherwise sweep at expiry.
+ *
+ * A reference the driver cannot read — malformed, never uploaded,
+ * expired, already consumed — is one condition on every driver
+ * ("unknown or expired"): a bucket cannot distinguish them, so neither
+ * does the wire.
+ *
+ * Proven by __tests__/staging.test.ts over a fake driver and over the
+ * real local driver with its slot registry; the conformance suite's
+ * transfer-lane block pins the wire behaviour on both editions.
+ */
+import { randomBytes } from "node:crypto";
+
+import type { ArtifactStorage } from "../../../artifactstorage/artifact-storage.js";
+import { ArtifactStorageNotFoundError } from "../../../artifactstorage/artifact-storage.js";
+import type { Logger } from "../../../boot/logger.js";
+import {
+  DEFAULT_SLOT_TTL_MS,
+  DOWNLOAD_URL_TTL_MS,
+  MAX_ZIP_SIZE,
+  REF_BYTE_LEN,
+  REF_PREFIX,
+  STAGING_KEY_PREFIX,
+} from "../constants.js";
+import { SlotUnknownError } from "./slots.js";
+
+/** The hex the reference and the staging key share; 128 bits, lowercase. */
+const HEX = new RegExp(`^[0-9a-f]{${REF_BYTE_LEN * 2}}$`);
+const STAGED_ARCHIVE_SUFFIX = ".zip";
+
+/** The shared hex of a well-formed reference or key, else undefined. */
+function hexBetween(
+  value: string,
+  prefix: string,
+  suffix: string,
+): string | undefined {
+  if (!value.startsWith(prefix) || !value.endsWith(suffix)) {
+    return undefined;
+  }
+  const hex = value.slice(prefix.length, value.length - suffix.length);
+  return HEX.test(hex) ? hex : undefined;
+}
+
+/** A fresh upload reference: the prefix and 128 random bits, hex. */
+export function newUploadRef(): string {
+  return REF_PREFIX + randomBytes(REF_BYTE_LEN).toString("hex");
+}
+
+/**
+ * The staging key a reference's bytes land under, or undefined when the
+ * reference is not one this server could have minted. Refs arrive from
+ * clients (the push request, the lane's URL path), so the shape is
+ * checked before it becomes a storage key.
+ */
+export function stagingKeyOf(ref: string): string | undefined {
+  const hex = hexBetween(ref, REF_PREFIX, "");
+  return hex === undefined
+    ? undefined
+    : `${STAGING_KEY_PREFIX}${hex}${STAGED_ARCHIVE_SUFFIX}`;
+}
+
+/**
+ * The reference for a staging key this module minted. Keys reach here from
+ * the domain's own mint, never from a client, so a foreign shape is a
+ * programming error and throws.
+ */
+export function uploadRefOf(stagingKey: string): string {
+  const hex = hexBetween(stagingKey, STAGING_KEY_PREFIX, STAGED_ARCHIVE_SUFFIX);
+  if (hex === undefined) {
+    throw new Error(`${stagingKey} is not a staging key this domain minted`);
+  }
+  return REF_PREFIX + hex;
+}
+
+/** What createArtifactUploadUrl hands the client. */
+export interface StagedUpload {
+  /** The single-use reference the push presents. */
+  readonly ref: string;
+  /** Accepts one HTTP PUT of exactly the declared byte count. */
+  readonly url: string;
+  /** How long the reference and the URL stay valid. */
+  readonly ttlMs: number;
+}
+
+/** What getArtifactDownloadUrl hands the client. */
+export interface DownloadCapability {
+  readonly url: string;
+  /** The floor of the URL's validity (DOWNLOAD_URL_TTL_MS). */
+  readonly ttlMs: number;
+}
+
+/** The staging port both archive kinds (skills, plugins) push through. */
+export interface ArchiveStaging {
+  /** Mints a reference and the URL its bytes are PUT to. */
+  mint(declaredSizeBytes: number): Promise<StagedUpload>;
+  /**
+   * Reads the staged bytes for a reference and retires it. Throws
+   * SlotUnknownError when the driver holds nothing under the reference;
+   * any other failure is the driver's and propagates as an infrastructure
+   * fault.
+   */
+  consume(ref: string): Promise<Uint8Array>;
+  /** A time-limited download URL for a stored archive key. */
+  downloadUrl(storageKey: string): Promise<DownloadCapability>;
+}
+
+export function newArchiveStaging(
+  driver: ArtifactStorage,
+  logger: Logger,
+): ArchiveStaging {
+  return {
+    async mint(declaredSizeBytes) {
+      // The controllers refuse over-limit declarations with the limit in
+      // the message before reaching here; this is the module's own
+      // invariant, so no driver is ever asked to stage more than the
+      // archive ceiling.
+      if (declaredSizeBytes <= 0 || declaredSizeBytes > MAX_ZIP_SIZE) {
+        throw new Error(
+          `declared size ${declaredSizeBytes} outside (0, ${MAX_ZIP_SIZE}]`,
+        );
+      }
+      const ref = newUploadRef();
+      const key = stagingKeyOf(ref);
+      if (key === undefined) {
+        throw new Error(`minted reference ${ref} has no staging key`);
+      }
+      const presigned = await driver.presignPut(
+        key,
+        declaredSizeBytes,
+        DEFAULT_SLOT_TTL_MS,
+      );
+      return { ref, url: presigned.url, ttlMs: presigned.ttlMs };
+    },
+
+    async consume(ref) {
+      const key = stagingKeyOf(ref);
+      if (key === undefined) {
+        throw new SlotUnknownError();
+      }
+      let data: Uint8Array;
+      try {
+        // The allocation is bounded by construction: a bucket accepts only
+        // the signed Content-Length, the slot registry only the reserved
+        // size, and both were checked against MAX_ZIP_SIZE at mint.
+        data = await driver.download(key);
+      } catch (error) {
+        if (error instanceof ArtifactStorageNotFoundError) {
+          throw new SlotUnknownError();
+        }
+        throw error;
+      }
+      try {
+        await driver.delete(key);
+      } catch (error) {
+        logger.error("failed to delete consumed staged artifact", {
+          stagingKey: key,
+          error: error instanceof Error ? error.message : String(error),
+        });
+      }
+      return data;
+    },
+
+    async downloadUrl(storageKey) {
+      const url = await driver.getSignedUrl(
+        storageKey,
+        DOWNLOAD_URL_TTL_MS,
+        "",
+      );
+      return { url, ttlMs: DOWNLOAD_URL_TTL_MS };
+    },
+  };
+}

package/src/pipeline/steps/resolve-artifact-source.ts

@@ -8,18 +8,22 @@
  * are indistinguishable past this point.
  *
  * Skills carried this step alone (the Go ResolveArtifactSource port);
- * plugins are pushed the same two ways over the same upload slots (one
- * upload surface, as boot/compose.ts wires it), so the step lives
- * here and each kind names its lane-absent sentence and the request
- * fields it reads.
+ * plugins are pushed the same two ways over the same staging port (one
+ * upload surface, as boot/compose.ts wires it), so the step lives here
+ * and each kind names its lane-absent sentence and the request fields it
+ * reads. The staging port (domain/skill/transfer/staging.ts) hides which
+ * blob driver holds the bytes: the step never sees a slot or a bucket.
  *
  * The unusable-reference arm is INVALID_ARGUMENT, not a server fault: an
- * unknown, expired, already consumed, or minted-but-never-uploaded
- * reference all mean the client must re-mint and re-upload.
+ * unknown, expired, already consumed, or never-uploaded reference all mean
+ * the client must re-mint and re-upload. Any other failure reading the
+ * staged bytes is the driver's — an infrastructure fault the executor
+ * answers as a sanitized Internal, never dressed as the client's mistake.
  */
 import type { DescMessage, MessageShape } from "@bufbuild/protobuf";
 
-import type { UploadSlots } from "../../domain/skill/transfer/slots.js";
+import { SlotUnknownError } from "../../domain/skill/transfer/slots.js";
+import type { ArchiveStaging } from "../../domain/skill/transfer/staging.js";
 import { failedPreconditionError, invalidArgumentError } from "../errors.js";
 import type { PipelineStep } from "../pipeline.js";
 import type { RequestContext } from "../request-context.js";
@@ -38,7 +42,7 @@ export interface ArtifactSourceBinding<Desc extends DescMessage> {
 }
 
 export function newResolveArtifactSourceStep<Desc extends DescMessage>(
-  slots: UploadSlots | undefined,
+  staging: ArchiveStaging | undefined,
   binding: ArtifactSourceBinding<Desc>,
 ): PipelineStep<Desc> {
   return {
@@ -49,16 +53,19 @@ export function newResolveArtifactSourceStep<Desc extends DescMessage>(
         ctx.set(ARTIFACT_BYTES_KEY, artifact);
         return;
       }
-      if (slots === undefined) {
+      if (staging === undefined) {
         throw failedPreconditionError(binding.laneNotConfigured);
       }
       let data: Uint8Array;
       try {
-        data = await slots.consume(artifactUploadRef);
+        data = await staging.consume(artifactUploadRef);
       } catch (error) {
-        throw invalidArgumentError(
-          `artifact_upload_ref not usable: ${error instanceof Error ? error.message : String(error)} — request a new upload URL via createArtifactUploadUrl`,
-        );
+        if (error instanceof SlotUnknownError) {
+          throw invalidArgumentError(
+            `artifact_upload_ref not usable: ${error.message} — request a new upload URL via createArtifactUploadUrl`,
+          );
+        }
+        throw error;
       }
       ctx.set(ARTIFACT_BYTES_KEY, data);
     },

package/src/transport/registry/__tests__/lanes.test.ts

@@ -38,7 +38,7 @@ beforeAll(async () => {
   // test previously composed against the DEFAULT paths, which meant
   // opening the developer's real ~/.stigmer/stigmer.db — and with the
   // skill domain's boot-time staging wipe (#8) it would now also clear
-  // ~/.stigmer/storage/skills-staging. Tests never touch the home dir.
+  // ~/.stigmer/storage/skills/staging. Tests never touch the home dir.
   testDir = mkdtempSync(path.join(tmpdir(), "registry-lanes-test-"));
   const config = loadConfig({
     STIGMER_MODEL_REGISTRY_REFRESH: "off",