@stigmer/server 3.18.0 → 3.19.0

package/src/domain/mcpserver/complete-endpoint-auth.ts

@@ -0,0 +1,277 @@
+/**
+ * CompleteEndpointAuth — the McpServer create and update chains' step that
+ * makes a URL enough: a server saved with only an `http.url` is asked once
+ * whether it wants OAuth, and when it does, the spec is completed so Sign
+ * in exists from the moment the row does.
+ *
+ * Why the domain and not the plugin. A plugin is one producer of servers
+ * among three (a plugin install, `stigmer apply` of a YAML, the console's
+ * form), and every producer's server runs these two chains. A step here
+ * reaches all three; a behaviour in the plugin's materialisation would
+ * make the same YAML come alive from a plugin and stay dead by hand.
+ *
+ * What the step reads: the author's shape only. An `http` server whose
+ * `auth` is unset, whose `env` is empty, and whose headers carry neither a
+ * `${VAR}` placeholder nor an `Authorization` header. Anything else is the
+ * author speaking about authentication (a declared token variable, a
+ * pasted literal token, an `auth` block, an `oauth_app_ref`) and the step
+ * leaves the state exactly as sent.
+ *
+ * What the step writes, and only on an OAuth challenge: the least that
+ * makes Sign in possible and the page honest. `auth.target_env_var`
+ * (`<SLUG>_ACCESS_TOKEN`), the secret `env` declaration for it, the
+ * `Authorization: Bearer ${VAR}` header, `auth.oauth_only` (the reading the
+ * runner's "requires OAuth" sentence and the console's credential hook
+ * already apply to a 401 challenge: manual token entry is a dead end), and
+ * the provenance label `stigmer.ai/mcp-auth: endpoint`, vouched through
+ * the server-stamped mechanism so GuardReservedLabels admits it for an
+ * ordinary caller. Not written: `discovery_url` (the author's override;
+ * Sign in resolves the login server from `http.url` every time through
+ * the RFC 9728 walk) and `scope_hints` (initiate reads the login server's
+ * `scopes_supported` when hints are empty). A cached fact drifts; the
+ * endpoint restates these on every Sign in.
+ *
+ * Every other outcome (a 2xx, a non-OAuth 401, any other status, no
+ * answer) leaves the spec as sent and writes no status: `ConnectStatus`
+ * records connect operations, and a probe is not one. The best-effort
+ * connect that follows an apply then runs for these servers as before.
+ *
+ * Carry-over on update, three questions in order:
+ *
+ *   1. Is the author speaking about auth? An incoming `auth`, `env`,
+ *      placeholder or `Authorization` header that is NOT exactly the trio
+ *      this step derived on the existing row means yes: the state is left
+ *      as sent and the label is dropped (a removal passes the guard by
+ *      design). An exact echo of the derived trio (the console's per-field
+ *      save re-sends the whole spec) is not the author speaking: the label
+ *      is kept and nothing is probed.
+ *   2. Is it the same endpoint? A changed URL with the author's shape is
+ *      probed again.
+ *   3. Has it already answered? The author's shape at the same URL on a
+ *      row carrying the label reuses the existing completion with no
+ *      network: a plugin re-push and a `stigmer apply` re-run cost nothing
+ *      and change nothing.
+ *
+ * The token-path default applies to every producer: whenever
+ * `auth.target_env_var` is set (by hand, by a plugin overlay, or by this
+ * step), the variable is declared in `env` if it is not, and the
+ * `Authorization: Bearer ${VAR}` header is added if no header references
+ * it. The chain is `auth.target_env_var` names the variable, `env` declares
+ * it (connect and execution inject a grant's token only into DECLARED
+ * variables), and a header sends it; break one link and no token is sent.
+ * An `auth` block written without the other two links used to sit silently
+ * dead; this makes it work.
+ *
+ * The probe: one complete `initialize` to `http.url` with the author's
+ * literal headers, through the slice's egress-guarded `outboundFetch`,
+ * under a 3 s deadline that covers DNS (the save waits on it; a hosted
+ * endpoint answers in well under a second, an unreachable host fails at
+ * once, and only a host that accepts and stalls reaches the ceiling). No
+ * tool is called, no metadata document is fetched, no redirect is
+ * followed. A thrown probe is logged and read as unreachable; the save
+ * never fails on it.
+ *
+ * Placement: after BuildNewState (create) or BuildUpdateState (update),
+ * immediately before GuardReservedLabels, the lineage step's law: the
+ * labels the guard diffs are the ones this step stamped.
+ *
+ * Proven by __tests__/complete-endpoint-auth.test.ts and the McpServer
+ * conformance suite's URL-only arms (CONFORMANCE_TARGET=local).
+ */
+import { create } from "@bufbuild/protobuf";
+
+import type { OutboundFetch } from "@stigmer/outbound/egress";
+import { probeEndpointAuth } from "@stigmer/outbound/mcp-oauth";
+import { EnvVarDeclarationSchema } from "@stigmer/protos/ai/stigmer/agentic/environment/v1/spec_pb";
+import type { McpServer, McpServerSchema } from "@stigmer/protos/ai/stigmer/agentic/mcpserver/v1/api_pb";
+import { McpServerAuthSchema } from "@stigmer/protos/ai/stigmer/agentic/mcpserver/v1/spec_pb";
+import type { McpServerSpec } from "@stigmer/protos/ai/stigmer/agentic/mcpserver/v1/spec_pb";
+
+import type { Logger } from "../../boot/logger.js";
+import { MCP_AUTH_ENDPOINT, MCP_AUTH_LABEL } from "../../pipeline/apiresource-labels.js";
+import type { PipelineStep } from "../../pipeline/pipeline.js";
+import type { RequestContext } from "../../pipeline/request-context.js";
+import { EXISTING_RESOURCE_KEY } from "../../pipeline/steps/load-existing.js";
+import { recordServerStampedReservedLabels } from "../../pipeline/steps/server-stamped-reserved-labels.js";
+
+/** The save waits on the probe; a real endpoint answers well within this, DNS included. */
+export const ENDPOINT_PROBE_TIMEOUT_MS = 3_000;
+
+/** How the control plane names itself in the handshake's `clientInfo`. */
+const PROBE_CLIENT_NAME = "stigmer-server";
+
+/** The suffix of the token variable the step declares for a completed server. */
+export const ACCESS_TOKEN_VAR_SUFFIX = "_ACCESS_TOKEN";
+
+const AUTHORIZATION_HEADER = "authorization";
+
+export interface CompleteEndpointAuthDeps {
+  readonly outboundFetch: OutboundFetch;
+  readonly logger: Logger;
+}
+
+export function newCompleteEndpointAuthStep(deps: CompleteEndpointAuthDeps): PipelineStep<typeof McpServerSchema> {
+  return {
+    name: "CompleteEndpointAuth",
+    async execute(ctx: RequestContext<typeof McpServerSchema>): Promise<void> {
+      const server = ctx.newState;
+      const spec = server.spec;
+      if (spec === undefined) return;
+
+      applyTokenPathDefault(spec, server.metadata?.slug ?? "");
+
+      const existing = ctx.get(EXISTING_RESOURCE_KEY) as McpServer | undefined;
+      const derived = existing !== undefined ? derivedCompletionOf(existing) : undefined;
+
+      const url = authorShapeUrl(spec);
+      if (url === undefined) {
+        // The author speaks about auth. An exact echo of what this step
+        // derived is the console re-sending what it loaded: keep the label.
+        // Anything else is the author's own; the label goes.
+        if (derived !== undefined && echoesDerived(spec, derived)) {
+          stamp(ctx, server);
+        } else {
+          dropLabel(server);
+        }
+        return;
+      }
+
+      if (derived !== undefined && derived.url === url) {
+        applyCompletion(spec, derived.variable, derived.description);
+        stamp(ctx, server);
+        return;
+      }
+
+      const outcome = await probe(url, spec, deps);
+      if (outcome !== "oauth") {
+        dropLabel(server);
+        return;
+      }
+      const variable = accessTokenVariable(server.metadata?.slug ?? "");
+      applyCompletion(spec, variable, tokenDescription(server.metadata?.slug ?? ""));
+      stamp(ctx, server);
+    },
+  };
+}
+
+/**
+ * The URL when the spec is the author's shape (an `http` server that says
+ * nothing about authentication); undefined otherwise.
+ */
+function authorShapeUrl(spec: McpServerSpec): string | undefined {
+  if (spec.serverType.case !== "http") return undefined;
+  if (spec.auth !== undefined) return undefined;
+  if (Object.keys(spec.env).length > 0) return undefined;
+  for (const [name, value] of Object.entries(spec.serverType.value.headers)) {
+    if (name.toLowerCase() === AUTHORIZATION_HEADER) return undefined;
+    if (value.includes("${")) return undefined;
+  }
+  return spec.serverType.value.url;
+}
+
+/** The completion an earlier save derived, read from a row that carries the label. */
+interface DerivedCompletion {
+  readonly url: string;
+  readonly variable: string;
+  readonly description: string;
+}
+
+function derivedCompletionOf(existing: McpServer): DerivedCompletion | undefined {
+  if (existing.metadata?.labels[MCP_AUTH_LABEL] !== MCP_AUTH_ENDPOINT) return undefined;
+  const spec = existing.spec;
+  if (spec === undefined || spec.serverType.case !== "http") return undefined;
+  const variable = spec.auth?.targetEnvVar ?? "";
+  if (variable === "") return undefined;
+  return { url: spec.serverType.value.url, variable, description: spec.env[variable]?.description ?? "" };
+}
+
+/** Whether the incoming spec carries exactly the trio the step derived, and nothing more about auth. */
+function echoesDerived(spec: McpServerSpec, derived: DerivedCompletion): boolean {
+  if (spec.serverType.case !== "http" || spec.serverType.value.url !== derived.url) return false;
+  const auth = spec.auth;
+  if (auth === undefined) return false;
+  if (auth.targetEnvVar !== derived.variable || !auth.oauthOnly) return false;
+  if ((auth.oauthAppRef?.slug ?? "") !== "" || auth.discoveryUrl !== "" || auth.scopeHints.length > 0 || auth.tokenLifetimeHint !== "") return false;
+  const envKeys = Object.keys(spec.env);
+  if (envKeys.length !== 1 || envKeys[0] !== derived.variable) return false;
+  const declaration = spec.env[derived.variable];
+  if (declaration === undefined || !declaration.isSecret || declaration.optional) return false;
+  const headers = Object.entries(spec.serverType.value.headers);
+  const authorization = headers.filter(([name]) => name.toLowerCase() === AUTHORIZATION_HEADER);
+  if (authorization.length !== 1 || authorization[0]?.[1] !== bearerHeader(derived.variable)) return false;
+  return headers.every(([name, value]) => name.toLowerCase() === AUTHORIZATION_HEADER || !value.includes("${"));
+}
+
+function applyCompletion(spec: McpServerSpec, variable: string, description: string): void {
+  spec.auth = create(McpServerAuthSchema, { targetEnvVar: variable, oauthOnly: true });
+  spec.env[variable] = create(EnvVarDeclarationSchema, { isSecret: true, optional: false, description });
+  applyTokenPathDefault(spec, "");
+}
+
+/**
+ * Whenever `auth.target_env_var` names a variable, the variable is declared
+ * (secret, required) if the author did not, and the header that sends it is
+ * added if no header references it; a declaration or a header the author
+ * wrote is left as written. Only an `http` server carries headers; a stdio
+ * server's token reaches it through the declared variable alone.
+ */
+function applyTokenPathDefault(spec: McpServerSpec, slug: string): void {
+  const variable = spec.auth?.targetEnvVar ?? "";
+  if (variable === "") return;
+  if (spec.env[variable] === undefined) {
+    spec.env[variable] = create(EnvVarDeclarationSchema, { isSecret: true, optional: false, description: tokenDescription(slug) });
+  }
+  if (spec.serverType.case !== "http") return;
+  const headers = spec.serverType.value.headers;
+  const reference = `\${${variable}}`;
+  if (Object.values(headers).some((value) => value.includes(reference))) return;
+  headers["Authorization"] = bearerHeader(variable);
+}
+
+function bearerHeader(variable: string): string {
+  return `Bearer \${${variable}}`;
+}
+
+/** `<SLUG>_ACCESS_TOKEN`, the slug upper-cased with every non-alphanumeric run as one underscore. */
+export function accessTokenVariable(slug: string): string {
+  const stem = slug.toUpperCase().replace(/[^A-Z0-9]+/g, "_").replace(/^_+|_+$/g, "");
+  return `${stem === "" ? "MCP" : stem}${ACCESS_TOKEN_VAR_SUFFIX}`;
+}
+
+function tokenDescription(slug: string): string {
+  return `OAuth access token for the '${slug}' MCP server; Sign in fills it`;
+}
+
+function stamp(ctx: RequestContext<typeof McpServerSchema>, server: McpServer): void {
+  if (server.metadata === undefined) return;
+  server.metadata.labels[MCP_AUTH_LABEL] = MCP_AUTH_ENDPOINT;
+  recordServerStampedReservedLabels(ctx, MCP_AUTH_LABEL);
+}
+
+function dropLabel(server: McpServer): void {
+  if (server.metadata === undefined) return;
+  delete server.metadata.labels[MCP_AUTH_LABEL];
+}
+
+async function probe(url: string, spec: McpServerSpec, deps: CompleteEndpointAuthDeps): Promise<"oauth" | "other"> {
+  const headers = spec.serverType.case === "http" ? spec.serverType.value.headers : {};
+  try {
+    const outcome = await probeEndpointAuth(url, headers, {
+      fetchImpl: deps.outboundFetch,
+      timeoutMs: ENDPOINT_PROBE_TIMEOUT_MS,
+      clientName: PROBE_CLIENT_NAME,
+    });
+    if (outcome.kind === "unreachable") {
+      deps.logger.debug("MCP endpoint probe got no answer; saving the spec as sent", { url, error: outcome.error });
+    }
+    return outcome.kind === "oauth" ? "oauth" : "other";
+  } catch (error) {
+    // probeEndpointAuth never throws by contract; a throw here is a defect
+    // in the fetch it was handed, and a save must not fail on it.
+    deps.logger.warn("MCP endpoint probe threw; saving the spec as sent", {
+      url,
+      error: error instanceof Error ? error.message : String(error),
+    });
+    return "other";
+  }
+}

package/src/domain/mcpserver/complete-oauth-connect.ts

@@ -131,6 +131,7 @@ export async function completeOAuthConnect(
       pendingState.clientId,
       pendingState.clientSecret,
       pendingState.tokenAuthMethod,
+      deps.outboundFetch,
     );
   } catch (error) {
     throw unavailableError(

package/src/domain/mcpserver/connect.ts

@@ -14,6 +14,7 @@
  * Proven by mcpserver-connect.conformance.test.ts
  * (CONFORMANCE_TARGET=local-execution) and __tests__/connect.test.ts.
  */
+import type { OutboundFetch } from "@stigmer/outbound/egress";
 import { create } from "@bufbuild/protobuf";
 import type { MessageInitShape } from "@bufbuild/protobuf";
 import { Code, ConnectError } from "@connectrpc/connect";
@@ -218,6 +219,14 @@ export interface McpServerConnectDeps {
   readonly pendingOAuthStates: PendingOAuthStateStore;
   readonly secretService: SecretService;
   readonly oauthRedirectUri: string;
+  /**
+   * The ONE fetch this slice dials user-supplied URLs with: the MCP
+   * endpoint probed at save time and the login server reached on Sign in
+   * (discovery, registration, the authorize preflight, token exchange and
+   * refresh). Composed at the root under the edition's egress policy
+   * (`drivers.outboundEgress`); no module here holds a `fetch` of its own.
+   */
+  readonly outboundFetch: OutboundFetch;
 }
 
 /**
@@ -1001,6 +1010,7 @@ export async function refreshOAuthTokenIfNeeded(
       clientSecret,
       tokenAuthMethod,
       deps.logger,
+      deps.outboundFetch,
     );
   } catch (error) {
     throw failedPreconditionError(
@@ -1185,23 +1195,19 @@ export async function startBestEffortConnect(
     CONNECT_TIMEOUT.ms + BEST_EFFORT_CONNECT_GET_BUFFER_MS,
   );
   if (!outcome.ok) {
-    deps.logger.warn(
-      "Best-effort connect workflow did not complete (non-fatal)",
-      {
-        workflow_id: run.workflowId,
-        mcp_server_id: mcpServerId,
-        failure_kind: outcome.failure.kind,
-      },
-    );
-    await persistConnectFailure(
-      deps.store,
+    // The same mapping the blocking lane and the async settle apply, so
+    // the page of a freshly applied server says what the runner found
+    // ("requires OAuth", "Missing Cursor credential") rather than that a
+    // background task did not complete. ConnectStatus promises the lanes
+    // record alike; this arm once recorded a generic sentence instead.
+    const failure = mapConnectFailure(
       deps.logger,
-      mcpServerId,
-      internalError(
-        new Error(`connect run failed: ${outcome.failure.kind}`),
-        "best-effort connect did not complete",
-      ),
+      mcpServer,
+      run.workflowId,
+      outcome.failure,
+      CONNECT_TIMEOUT,
     );
+    await persistConnectFailure(deps.store, deps.logger, mcpServerId, failure);
     return;
   }
 

package/src/domain/mcpserver/controller.ts

@@ -107,6 +107,10 @@ import { completeOAuthConnect } from "./complete-oauth-connect.js";
 import { connect, startBestEffortConnect } from "./connect.js";
 import type { McpServerConnectDeps } from "./connect.js";
 import { disconnectOAuth } from "./disconnect-oauth.js";
+import {
+  newCompleteEndpointAuthStep,
+  type CompleteEndpointAuthDeps,
+} from "./complete-endpoint-auth.js";
 import { getOAuthGrantStatus } from "./get-oauth-grant-status.js";
 import { initiateOAuthConnect } from "./initiate-oauth-connect.js";
 import { mcpServerSearchExtractor } from "./search-extractor.js";
@@ -202,6 +206,11 @@ function kindOf(ctx: HandlerContext): ApiResourceKind {
   return ctx.values.get(apiResourceKindKey);
 }
 
+/** The endpoint-auth step's slice of the deps: the guarded fetch the connect slice holds. */
+function endpointAuthDeps(deps: McpServerControllerDeps): CompleteEndpointAuthDeps {
+  return { outboundFetch: deps.connect.outboundFetch, logger: deps.logger };
+}
+
 /** Create — chain per Go buildCreatePipeline. */
 async function createMcpServer(
   deps: McpServerControllerDeps,
@@ -227,6 +236,10 @@ async function createMcpServer(
     .addStep(newCheckDuplicateStep(deps.store))
     .addStep(newBuildNewStateStep())
     .addStep(newGuardPublicVisibilityStep(deps.authorizer))
+    // A URL-only server is asked once whether it wants OAuth and completed
+    // when it does; immediately before GuardReservedLabels because the
+    // provenance label it stamps is one the guard diffs.
+    .addStep(newCompleteEndpointAuthStep(endpointAuthDeps(deps)))
     .addStep(newGuardReservedLabelsStep(deps.authorizer))
     .addStep(newNormalizeReferencesStep())
     .addStep(newPersistStep(deps.store))
@@ -275,6 +288,9 @@ async function update(
     // STORED labels and the plugin row's existence).
     .addStep(newGuardPluginManagedStep(deps.store))
     .addStep(newBuildUpdateStateStep())
+    // Reads the existing row LoadExisting set, to reuse a completion or to
+    // tell an author's own auth from the derived one (its header).
+    .addStep(newCompleteEndpointAuthStep(endpointAuthDeps(deps)))
     .addStep(newGuardReservedLabelsStep(deps.authorizer))
     .addStep(newValidateDefaultEnabledToolsStep())
     .addStep(newNormalizeReferencesStep())

package/src/domain/mcpserver/initiate-oauth-connect.ts

@@ -45,7 +45,7 @@ import { tokenAuthMethodFromSpec } from "./connect.js";
 import type { McpServerConnectDeps } from "./connect.js";
 import { generatePkce } from "./oauth/pkce.js";
 import type { PkcePair } from "./oauth/pkce.js";
-import { discoverAuthorizationServer } from "./oauth/discovery.js";
+import { discoverAtIssuer, discoverForResource } from "./oauth/discovery.js";
 import { registerClient } from "./oauth/dcr.js";
 import { preflightAuthorize } from "./oauth/preflight.js";
 import type { AuthorizeRejection } from "./oauth/preflight.js";
@@ -172,15 +172,15 @@ async function initiateDcr(
   stateParam: string,
 ): Promise<InitiateResult> {
   // Resolve the URL for OAuth authorization server discovery. Priority:
-  // auth.discovery_url > http.url — discovery_url enables DCR for stdio
-  // servers that have no HTTP URL.
-  let serverUrl = mcpServer.spec?.auth?.discoveryUrl ?? "";
-  if (serverUrl === "") {
-    const serverType = mcpServer.spec?.serverType;
-    if (serverType?.case === "http") {
-      serverUrl = serverType.value.url;
-    }
-  }
+  // auth.discovery_url > http.url — discovery_url is the author naming the
+  // login server itself (and the only route for a stdio server, which has
+  // no HTTP URL) and is read as an issuer; http.url is the protected
+  // resource, whose login server the RFC 9728 walk finds
+  // (oauth/discovery.ts).
+  const discoveryUrl = mcpServer.spec?.auth?.discoveryUrl ?? "";
+  const serverType = mcpServer.spec?.serverType;
+  const resourceUrl = serverType?.case === "http" ? serverType.value.url : "";
+  const serverUrl = discoveryUrl !== "" ? discoveryUrl : resourceUrl;
   if (serverUrl === "") {
     throw failedPreconditionError(
       `DCR requires a discoverable URL. MCP server '${mcpServer.metadata?.id ?? ""}' has no http.url and no auth.discovery_url. ` +
@@ -190,7 +190,10 @@ async function initiateDcr(
 
   let metadata;
   try {
-    metadata = await discoverAuthorizationServer(serverUrl);
+    metadata =
+      discoveryUrl !== ""
+        ? await discoverAtIssuer(discoveryUrl, deps.outboundFetch)
+        : await discoverForResource(resourceUrl, deps.outboundFetch);
   } catch (error) {
     throw failedPreconditionError(
       `OAuth authorization server discovery failed for ${serverUrl}: ${error instanceof Error ? error.message : String(error)}`,
@@ -198,8 +201,15 @@ async function initiateDcr(
   }
 
   if (metadata.registrationEndpoint === "") {
+    // A login server without RFC 7591 registration cannot take a client
+    // Stigmer mints on the spot; the sentence keeps its opening (the
+    // conformance suite's pin before this clause joined it) and tells the
+    // user the one thing that helps: an OAuth app registered with the
+    // vendor, referenced from the server's definition.
     throw failedPreconditionError(
-      `MCP server at ${serverUrl} does not advertise a registration_endpoint for DCR`,
+      `MCP server at ${serverUrl} does not advertise a registration_endpoint for DCR: ` +
+        `${loginServerHost(metadata, serverUrl)} does not allow automatic client registration, so this server needs ` +
+        "an OAuth app registered with the vendor and referenced from its definition (auth.oauth_app_ref)",
     );
   }
 
@@ -210,6 +220,7 @@ async function initiateDcr(
       metadata.registrationEndpoint,
       deps.oauthRedirectUri,
       clientName,
+      deps.outboundFetch,
     );
   } catch (error) {
     throw failedPreconditionError(
@@ -239,7 +250,7 @@ async function initiateDcr(
   // Fail-open by contract: only a definite rejection blocks initiate.
   let rejection: AuthorizeRejection | undefined;
   try {
-    rejection = await preflightAuthorize(authUrl);
+    rejection = await preflightAuthorize(authUrl, deps.outboundFetch);
   } catch (probeError) {
     deps.logger.debug("authorize pre-flight probe inconclusive; proceeding", {
       mcp_server_id: mcpServer.metadata?.id ?? "",
@@ -276,6 +287,19 @@ async function initiateDcr(
   };
 }
 
+/** The login server a user must register with: its metadata's issuer, else its authorization endpoint, else the URL we asked. */
+function loginServerHost(metadata: { issuer: string; authorizationEndpoint: string }, serverUrl: string): string {
+  for (const candidate of [metadata.issuer, metadata.authorizationEndpoint, serverUrl]) {
+    try {
+      const host = new URL(candidate).host;
+      if (host !== "") return host;
+    } catch {
+      // Not a URL; try the next candidate.
+    }
+  }
+  return serverUrl;
+}
+
 async function initiateVendorOAuth(
   deps: McpServerConnectDeps,
   mcpServer: McpServer,