@stigmer/server 3.22.1 → 3.23.0

package/src/boot/config.ts

@@ -121,9 +121,10 @@ export interface ServerConfig {
   readonly oauthRedirectUri: string;
   /**
    * Skill artifact storage root (STORAGE_PATH; Go defaultStoragePath
-   * ~/.stigmer/storage). Artifacts live at {storagePath}/skills/,
-   * upload staging at {storagePath}/skills-staging/ — byte-identical to
-   * Go's layout, so a Go-written directory is served in place at cutover.
+   * ~/.stigmer/storage). Artifacts live at {storagePath}/skills/ —
+   * byte-identical to Go's layout, so a Go-written directory is served in
+   * place at cutover — and upload staging at {storagePath}/skills/staging/,
+   * the prefix the domain names (domain/skill/constants.ts).
    */
   readonly storagePath: string;
   /**

package/src/boot/skill-transfer-origin.ts

@@ -0,0 +1,76 @@
+/**
+ * Whether the skill transfer lane's origin can be reached by the callers
+ * it is minted for.
+ *
+ * On the local blob driver the server itself carries artifact bytes over
+ * HTTP, and every upload and download capability URL it mints is rendered
+ * from `SKILL_TRANSFER_BASE_URL`, which defaults to the server's own
+ * loopback. That default is right for a single-operator install reached
+ * on the machine it runs on, and silently wrong for a hosted one: the
+ * server boots, serves, and passes every in-process check while handing
+ * the CLI, the SDKs and the sandbox runner a host none of them can reach —
+ * every push above the gRPC cap fails on the PUT and every sandbox mount
+ * of a skill with supporting files loses them (stigmer#1219). No test
+ * inside the process can see it, so the process says it at boot.
+ *
+ * The hosted signal is the require-authentication posture (boot/compose.ts
+ * combines its two sources): a server that requires credentials serves
+ * callers it does not share a machine with. A bucket driver signs URLs
+ * straight to its bucket and never renders the base URL, so the verdict
+ * is moot there and says so rather than warning about a knob nothing
+ * reads.
+ *
+ * Proven by __tests__/skill-transfer-origin.test.ts.
+ */
+
+/** Hosts that name the process's own machine and nothing beyond it. */
+const LOOPBACK_HOSTS = new Set([
+  "localhost",
+  "127.0.0.1",
+  "[::1]",
+  "0.0.0.0",
+  "[::]",
+]);
+
+export type SkillTransferOriginVerdict =
+  /** The base URL names a host other than the server's own loopback. */
+  | { readonly kind: "reachable" }
+  /** Loopback on a single-operator install: the default, and correct. */
+  | { readonly kind: "loopback-local" }
+  /** Loopback on a server that requires authentication: minted URLs are dead on arrival. */
+  | { readonly kind: "loopback-hosted"; readonly baseUrl: string }
+  /** The blob driver signs its own URLs; the base URL is never rendered. */
+  | { readonly kind: "unused" };
+
+export interface SkillTransferOriginInputs {
+  /** The configured or defaulted `SKILL_TRANSFER_BASE_URL`. */
+  readonly baseUrl: string;
+  /** Whether the local driver relays artifact bytes through this server. */
+  readonly relaysThroughServer: boolean;
+  /** The composed require-authentication posture. */
+  readonly requireAuthentication: boolean;
+}
+
+export function assessSkillTransferOrigin(
+  inputs: SkillTransferOriginInputs,
+): SkillTransferOriginVerdict {
+  if (!inputs.relaysThroughServer) {
+    return { kind: "unused" };
+  }
+  if (!namesLoopback(inputs.baseUrl)) {
+    return { kind: "reachable" };
+  }
+  return inputs.requireAuthentication
+    ? { kind: "loopback-hosted", baseUrl: inputs.baseUrl }
+    : { kind: "loopback-local" };
+}
+
+function namesLoopback(baseUrl: string): boolean {
+  try {
+    return LOOPBACK_HOSTS.has(new URL(baseUrl).hostname);
+  } catch {
+    // An unparseable base URL renders into every capability URL as-is; it
+    // is not loopback, and the failure it causes is loud on the first mint.
+    return false;
+  }
+}

package/src/domain/plugin/controller.ts

@@ -106,8 +106,10 @@ import type { VersionHistoryBinding } from "../../pipeline/steps/version-history
 import { newAuthorizeVisibilityTransitionStep } from "../../pipeline/steps/visibility-gates.js";
 import { metadataOf } from "../../pipeline/steps/shapes.js";
 import type { Store } from "../../store/interface.js";
-import { uploadUrl } from "../skill/transfer/handler.js";
-import type { UploadSlots } from "../skill/transfer/slots.js";
+import type {
+  ArchiveStaging,
+  StagedUpload,
+} from "../skill/transfer/staging.js";
 import { TRANSFER_LANE_NOT_CONFIGURED } from "./constants.js";
 import type { PluginMaterializerProvider } from "./materialize/ports.js";
 import { findMembers } from "./members.js";
@@ -140,12 +142,6 @@ import {
 } from "./push.js";
 import { pluginSearchExtractor } from "./search-extractor.js";
 
-/** The transfer lane's controller-side pair; the slots are the skill lane's (one upload surface). */
-export interface PluginTransferLaneDeps {
-  readonly slots: UploadSlots;
-  readonly baseUrl: string;
-}
-
 export interface PluginControllerDeps {
   readonly store: Store;
   readonly logger: Logger;
@@ -153,7 +149,8 @@ export interface PluginControllerDeps {
   readonly authorizationLifecycle: ResourceAuthorizationLifecycle | undefined;
   readonly artifactStorage: ContentAddressedArchiveStore;
   readonly materializerProvider: PluginMaterializerProvider;
-  readonly transferLane?: PluginTransferLaneDeps;
+  /** The skill lane's staging port (one upload surface for every archive); absent, the lane answers FailedPrecondition. */
+  readonly staging?: ArchiveStaging;
 }
 
 /** Registers both plugin services on the router (routes stage). */
@@ -208,7 +205,7 @@ async function push(
     .addStep(newValidateProtoStep())
     .addStep(
       newResolveArtifactSourceStep<typeof PushPluginRequestSchema>(
-        deps.transferLane?.slots,
+        deps.staging,
         {
           source: (input) => ({
             artifact: input.artifact,
@@ -274,16 +271,17 @@ async function push(
 // ─── createArtifactUploadUrl ─────────────────────────────────────────────
 
 /**
- * Mints a short-lived, single-use upload URL over the shared slots; an
- * over-limit declaration is refused with the limit BEFORE any bytes move.
+ * Mints a short-lived, single-use upload URL through the shared staging
+ * port; an over-limit declaration is refused with the limit BEFORE any
+ * bytes move.
  */
 async function createArtifactUploadUrl(
   deps: PluginControllerDeps,
   req: CreatePluginArtifactUploadUrlRequest,
   ctx: HandlerContext,
 ): Promise<PluginArtifactUploadUrl> {
-  const lane = deps.transferLane;
-  if (lane === undefined) {
+  const staging = deps.staging;
+  if (staging === undefined) {
     throw failedPreconditionError(TRANSFER_LANE_NOT_CONFIGURED);
   }
   const reqCtx = new RequestContext(
@@ -310,14 +308,14 @@ async function createArtifactUploadUrl(
       `plugin archive size ${req.sizeBytes} bytes exceeds the ${MAX_ZIP_SIZE}-byte (100MB) archive limit`,
     );
   }
-  let minted: { ref: string; ttlMs: number };
+  let minted: StagedUpload;
   try {
-    minted = lane.slots.mint(Number(req.sizeBytes));
+    minted = await staging.mint(Number(req.sizeBytes));
   } catch (error) {
     throw internalError(error, "failed to mint upload reference");
   }
   return create(PluginArtifactUploadUrlSchema, {
-    url: uploadUrl(lane.baseUrl, minted.ref),
+    url: minted.url,
     artifactUploadRef: minted.ref,
     ttlSeconds: Math.trunc(minted.ttlMs / 1000),
   });

package/src/domain/skill/__tests__/skill.test.ts

@@ -434,7 +434,7 @@ describe("transfer lane (#675) — mint → PUT → push-by-ref → download", (
     );
   });
 
-  it("a ref whose slot was minted but never uploaded is rejected at push", async () => {
+  it("a ref whose slot was minted but never uploaded is rejected at push — the one condition every driver can answer", async () => {
     const minted = await command.createArtifactUploadUrl({ org: ORG, sizeBytes: 1024n });
     const err = await expectCode(
       command.push({ org: ORG, artifactUploadRef: minted.artifactUploadRef }),
@@ -442,7 +442,7 @@ describe("transfer lane (#675) — mint → PUT → push-by-ref → download", (
       "empty slot",
     );
     expect(err.rawMessage).toBe(
-      "artifact_upload_ref not usable: upload reference has no uploaded bytes — request a new upload URL via createArtifactUploadUrl",
+      "artifact_upload_ref not usable: upload reference unknown or expired — request a new upload URL via createArtifactUploadUrl",
     );
   });
 
@@ -499,7 +499,9 @@ describe("transfer lane (#675) — mint → PUT → push-by-ref → download", (
     const minted = await query.getArtifactDownloadUrl({
       artifactStorageKey: pushed.status!.artifactStorageKey,
     });
-    expect(minted.ttlSeconds).toBe(0);
+    // The floor of the URL's validity, the same on every driver; the local
+    // lane's content-hash URL simply outlives it.
+    expect(minted.ttlSeconds).toBe(3600);
     expect(minted.sizeBytes).toBe(BigInt(artifact.length));
 
     const resp = await fetch(minted.url);

package/src/domain/skill/__tests__/slots.test.ts

@@ -1,28 +1,39 @@
 /**
- * Pins the upload-slot registry against Go's slots_test.go: mint bounds,
- * ref shape, TTL expiry (injected clock), exact-declared-size enforcement
- * (short AND long bodies), strict single-use consume, staged-file cleanup,
- * expiry sweep on mint, and the boot-time staging wipe.
+ * Pins the upload-slot registry against Go's slots_test.go, keyed by the
+ * domain's staging key: reservation bounds and key shape, TTL expiry
+ * (injected clock), exact-declared-size enforcement (short AND long
+ * bodies), single upload per key, staged-file placement under the driver
+ * root, expiry sweep on reservation, the boot-time staging wipe, and the
+ * staging directory's recreation after the driver's delete pruned it.
  */
-import { existsSync, mkdtempSync, mkdirSync, readdirSync, rmSync, writeFileSync } from "node:fs";
+import {
+  existsSync,
+  mkdtempSync,
+  mkdirSync,
+  readdirSync,
+  readFileSync,
+  rmSync,
+  writeFileSync,
+} from "node:fs";
 import { tmpdir } from "node:os";
 import path from "node:path";
 import { Readable } from "node:stream";
 
 import { afterEach, beforeEach, describe, expect, it } from "vitest";
 
+import { STAGING_KEY_PREFIX } from "../constants.js";
 import {
   SizeMismatchError,
   SlotConsumedError,
-  SlotEmptyError,
   SlotUnknownError,
   UploadSlots,
 } from "../transfer/slots.js";
+import { newUploadRef, stagingKeyOf } from "../transfer/staging.js";
 
 const TTL_MS = 15 * 60 * 1000;
 const MAX_SIZE = 100 * 1024 * 1024;
 
-let dir: string;
+let root: string;
 let stagingDir: string;
 let nowMs: number;
 let slots: UploadSlots;
@@ -31,101 +42,143 @@ function body(text: string): Readable {
   return Readable.from([Buffer.from(text)]);
 }
 
+/** A fresh, well-formed staging key — what the domain's mint hands the registry. */
+function freshKey(): string {
+  const key = stagingKeyOf(newUploadRef());
+  if (key === undefined) throw new Error("a minted ref always has a key");
+  return key;
+}
+
 beforeEach(() => {
-  dir = mkdtempSync(path.join(tmpdir(), "skill-slots-test-"));
-  stagingDir = path.join(dir, "skills-staging");
+  root = mkdtempSync(path.join(tmpdir(), "skill-slots-test-"));
+  stagingDir = path.join(root, STAGING_KEY_PREFIX);
   nowMs = 1_000_000;
-  slots = new UploadSlots(stagingDir, TTL_MS, MAX_SIZE, () => nowMs);
+  slots = new UploadSlots(
+    root,
+    STAGING_KEY_PREFIX,
+    TTL_MS,
+    MAX_SIZE,
+    () => nowMs,
+  );
 });
 
 afterEach(() => {
-  rmSync(dir, { recursive: true, force: true });
+  rmSync(root, { recursive: true, force: true });
 });
 
 describe("UploadSlots", () => {
-  it("mints sau_-prefixed 128-bit hex refs with the registry TTL", () => {
-    const { ref, ttlMs } = slots.mint(1024);
-    expect(ref).toMatch(/^sau_[0-9a-f]{32}$/);
-    expect(ttlMs).toBe(TTL_MS);
+  it("reserves a key with the registry TTL", () => {
+    expect(slots.reserve(freshKey(), 1024)).toEqual({ ttlMs: TTL_MS });
   });
 
-  it("rejects out-of-bounds declarations at mint", () => {
-    expect(() => slots.mint(0)).toThrow("declared size 0 outside (0, 104857600]");
-    expect(() => slots.mint(-1)).toThrow(`declared size -1 outside (0, ${MAX_SIZE}]`);
-    expect(() => slots.mint(MAX_SIZE + 1)).toThrow(
+  it("rejects out-of-bounds declarations at reservation", () => {
+    expect(() => slots.reserve(freshKey(), 0)).toThrow(
+      "declared size 0 outside (0, 104857600]",
+    );
+    expect(() => slots.reserve(freshKey(), -1)).toThrow(
+      `declared size -1 outside (0, ${MAX_SIZE}]`,
+    );
+    expect(() => slots.reserve(freshKey(), MAX_SIZE + 1)).toThrow(
       `declared size ${MAX_SIZE + 1} outside (0, ${MAX_SIZE}]`,
     );
   });
 
-  it("round-trips: mint → receive → consume returns the exact bytes and retires the slot", async () => {
-    const { ref } = slots.mint(5);
-    await slots.receive(ref, body("hello"));
-    const data = await slots.consume(ref);
-    expect(Buffer.from(data).toString()).toBe("hello");
-    // Strictly single-use: the slot AND its staged file are gone.
-    await expect(slots.consume(ref)).rejects.toThrow(SlotUnknownError);
-    expect(readdirSync(stagingDir)).toEqual([]);
+  it("refuses a key outside the staging prefix or not shaped like a staged archive", () => {
+    expect(() => slots.reserve("skills/deadbeef.zip", 1)).toThrow(
+      `staging key skills/deadbeef.zip is outside ${STAGING_KEY_PREFIX}`,
+    );
+    expect(() =>
+      slots.reserve(`${STAGING_KEY_PREFIX}../escape.zip`, 1),
+    ).toThrow("does not name a staged archive");
+    expect(() => slots.reserve(`${STAGING_KEY_PREFIX}notes.txt`, 1)).toThrow(
+      "does not name a staged archive",
+    );
+  });
+
+  it("refuses to reserve the same key twice", () => {
+    const key = freshKey();
+    slots.reserve(key, 1);
+    expect(() => slots.reserve(key, 1)).toThrow(
+      `staging key ${key} is already reserved`,
+    );
+  });
+
+  it("stages a received body at the key under the root — where the driver reads it", async () => {
+    const key = freshKey();
+    slots.reserve(key, 5);
+    await slots.receive(key, body("hello"));
+    expect(readFileSync(path.join(root, key), "utf8")).toBe("hello");
   });
 
-  it("rejects receive on an unknown ref", async () => {
-    await expect(slots.receive("sau_nope", body("x"))).rejects.toThrow(SlotUnknownError);
+  it("rejects receive on an unreserved key", async () => {
+    await expect(slots.receive(freshKey(), body("x"))).rejects.toThrow(
+      SlotUnknownError,
+    );
   });
 
-  it("rejects receive on an expired ref", async () => {
-    const { ref } = slots.mint(1);
+  it("rejects receive on an expired key", async () => {
+    const key = freshKey();
+    slots.reserve(key, 1);
     nowMs += TTL_MS + 1;
-    await expect(slots.receive(ref, body("x"))).rejects.toThrow(SlotUnknownError);
+    await expect(slots.receive(key, body("x"))).rejects.toThrow(
+      SlotUnknownError,
+    );
   });
 
-  it("rejects a second upload to the same ref", async () => {
-    const { ref } = slots.mint(1);
-    await slots.receive(ref, body("x"));
-    await expect(slots.receive(ref, body("x"))).rejects.toThrow(SlotConsumedError);
+  it("rejects a second upload to the same key", async () => {
+    const key = freshKey();
+    slots.reserve(key, 1);
+    await slots.receive(key, body("x"));
+    await expect(slots.receive(key, body("x"))).rejects.toThrow(
+      SlotConsumedError,
+    );
   });
 
   it("rejects a short body, naming both sizes, and stages nothing", async () => {
-    const { ref } = slots.mint(10);
-    await expect(slots.receive(ref, body("short"))).rejects.toThrow(
+    const key = freshKey();
+    slots.reserve(key, 10);
+    await expect(slots.receive(key, body("short"))).rejects.toThrow(
       "upload size mismatch: received 5 bytes, declared 10",
     );
-    await expect(slots.receive(ref, body("short"))).rejects.toBeInstanceOf(SizeMismatchError);
+    await expect(slots.receive(key, body("short"))).rejects.toBeInstanceOf(
+      SizeMismatchError,
+    );
     expect(readdirSync(stagingDir)).toEqual([]);
   });
 
   it("rejects an over-declaration body without buffering it whole (declared+1 proof)", async () => {
-    const { ref } = slots.mint(3);
-    await expect(slots.receive(ref, body("longer than three"))).rejects.toThrow(
+    const key = freshKey();
+    slots.reserve(key, 3);
+    await expect(slots.receive(key, body("longer than three"))).rejects.toThrow(
       "upload size mismatch: received 4 bytes, declared 3",
     );
   });
 
-  it("rejects consume on a minted-but-never-uploaded ref", async () => {
-    const { ref } = slots.mint(1);
-    await expect(slots.consume(ref)).rejects.toThrow(SlotEmptyError);
-  });
-
-  it("rejects consume on an expired ref", async () => {
-    const { ref } = slots.mint(1);
-    await slots.receive(ref, body("x"));
-    nowMs += TTL_MS + 1;
-    await expect(slots.consume(ref)).rejects.toThrow(SlotUnknownError);
-  });
-
-  it("sweeps expired slots and their staged files on mint", async () => {
-    const { ref } = slots.mint(1);
-    await slots.receive(ref, body("x"));
+  it("sweeps expired slots and their staged files on reservation", async () => {
+    const key = freshKey();
+    slots.reserve(key, 1);
+    await slots.receive(key, body("x"));
     expect(readdirSync(stagingDir)).toHaveLength(1);
     nowMs += TTL_MS + 1;
-    slots.mint(1);
+    slots.reserve(freshKey(), 1);
     expect(readdirSync(stagingDir)).toEqual([]);
   });
 
+  it("recreates the staging directory when the driver's delete pruned it between uploads", async () => {
+    const key = freshKey();
+    slots.reserve(key, 1);
+    rmSync(stagingDir, { recursive: true, force: true });
+    await slots.receive(key, body("x"));
+    expect(readFileSync(path.join(root, key), "utf8")).toBe("x");
+  });
+
   it("wipes the staging directory at construction (orphans are unreachable)", () => {
-    const orphanDir = path.join(dir, "staging-with-orphans");
+    const orphanRoot = path.join(root, "orphan-root");
+    const orphanDir = path.join(orphanRoot, STAGING_KEY_PREFIX);
     mkdirSync(orphanDir, { recursive: true });
-    const orphan = path.join(orphanDir, "sau_orphan.zip");
+    const orphan = path.join(orphanDir, "deadbeef.zip");
     writeFileSync(orphan, "stale bytes");
-    new UploadSlots(orphanDir, TTL_MS, MAX_SIZE);
+    new UploadSlots(orphanRoot, STAGING_KEY_PREFIX, TTL_MS, MAX_SIZE);
     expect(existsSync(orphan)).toBe(false);
     expect(readdirSync(orphanDir)).toEqual([]);
   });