@stigmer/server 3.14.0-dev.20260910084630

package/src/observability/rpc-metrics.ts

@@ -0,0 +1,203 @@
+/**
+ * The gRPC request metrics the serving chain records — Java's
+ * `stigmer.grpc.request.count` and `stigmer.grpc.request.duration`
+ * (`GrpcRequestMetricsInterceptor`, cloud `backend/libs/java/grpc/
+ * grpc-request`), names and labels verbatim: `rpc.service` (the
+ * fully-qualified service name), `rpc.method` (the bare method name) and
+ * `rpc.grpc.status_code` (the gRPC status NAME — `OK`, `NOT_FOUND`,
+ * `CANCELLED` with two Ls — never Connect's PascalCase enum key, which the
+ * SigNoz rules would not match). Three consumers read the pair under the
+ * dotted-to-underscored spelling: `telemetry-heartbeat-lost` (the absence
+ * of the count series for ten minutes), `grpc-error-rate-spike` (the
+ * share of `status_code != OK`) and `grpc-latency-regression-p95` (the
+ * duration histogram's buckets), plus the `gRPC RED` dashboard grouped by
+ * every label. Convergence entry 20260909.02.
+ *
+ * This module is library instrumentation, on `@opentelemetry/api` alone:
+ * instruments are created lazily on first use against the GLOBAL meter
+ * provider and are no-ops until a host registers one, so every call site
+ * records unconditionally (the OSS runner's otel-metrics.ts shape; the
+ * cloud composition's registries are its siblings). Lazy creation is what
+ * makes the host's boot order irrelevant — the API has no proxy meter
+ * provider, so an instrument created before registration would stay a
+ * no-op forever. Two facts a host must know:
+ *
+ *   - The counter is BORN AT ZERO by `initRpcMetrics` (Java's constructor
+ *     `add(0)`, the heartbeat's premise, cloud#255): an attribute-less
+ *     series exists from the first post-boot export, so the absence rule's
+ *     ten-minute window measures the export pipeline, not an RPC-free lull
+ *     after a restart. The histogram is deliberately NOT pre-registered —
+ *     a `record(0)` is a real sample that would pollute the percentiles,
+ *     and no rule watches its absence.
+ *   - A host that installs this package from its own `node_modules` runs a
+ *     second copy of `@opentelemetry/api`. The two share one global only
+ *     while this copy's minor version does not exceed the registering
+ *     copy's (`isCompatible`, `internal/semver.ts`); a lockfile bump here
+ *     ahead of the host's makes these instruments silently inert. The
+ *     cloud composition pins the same `^1.9.0` and proves the sharing with
+ *     a composed test on every pin.
+ *
+ * `metricsExportPosture` lets the boot say which of the two states it is
+ * in, so "nothing exported" and "nothing instrumented" stay
+ * distinguishable in the log (the guidelines' explicit-optional-
+ * infrastructure rule).
+ */
+import { Code } from "@connectrpc/connect";
+import type { Counter, Histogram } from "@opentelemetry/api";
+
+const METER_NAME = "stigmer-server";
+
+/** `rpc.grpc.status_code` values: the gRPC status names, Java's `Status.Code.name()`. */
+export type GrpcStatusName =
+  | "OK"
+  | "CANCELLED"
+  | "UNKNOWN"
+  | "INVALID_ARGUMENT"
+  | "DEADLINE_EXCEEDED"
+  | "NOT_FOUND"
+  | "ALREADY_EXISTS"
+  | "PERMISSION_DENIED"
+  | "RESOURCE_EXHAUSTED"
+  | "FAILED_PRECONDITION"
+  | "ABORTED"
+  | "OUT_OF_RANGE"
+  | "UNIMPLEMENTED"
+  | "INTERNAL"
+  | "UNAVAILABLE"
+  | "DATA_LOSS"
+  | "UNAUTHENTICATED";
+
+export const GRPC_STATUS_OK: GrpcStatusName = "OK";
+
+/**
+ * Connect's error code → the gRPC status name. Connect's `Code` enum has no
+ * OK member (success is the absence of an error), so the table covers the
+ * sixteen error codes and `GRPC_STATUS_OK` names success. Byte-exact
+ * against grpc-java's `Status.Code`; the one spelling trap is CANCELLED.
+ */
+export function grpcStatusName(code: Code): GrpcStatusName {
+  switch (code) {
+    case Code.Canceled:
+      return "CANCELLED";
+    case Code.Unknown:
+      return "UNKNOWN";
+    case Code.InvalidArgument:
+      return "INVALID_ARGUMENT";
+    case Code.DeadlineExceeded:
+      return "DEADLINE_EXCEEDED";
+    case Code.NotFound:
+      return "NOT_FOUND";
+    case Code.AlreadyExists:
+      return "ALREADY_EXISTS";
+    case Code.PermissionDenied:
+      return "PERMISSION_DENIED";
+    case Code.ResourceExhausted:
+      return "RESOURCE_EXHAUSTED";
+    case Code.FailedPrecondition:
+      return "FAILED_PRECONDITION";
+    case Code.Aborted:
+      return "ABORTED";
+    case Code.OutOfRange:
+      return "OUT_OF_RANGE";
+    case Code.Unimplemented:
+      return "UNIMPLEMENTED";
+    case Code.Internal:
+      return "INTERNAL";
+    case Code.Unavailable:
+      return "UNAVAILABLE";
+    case Code.DataLoss:
+      return "DATA_LOSS";
+    case Code.Unauthenticated:
+      return "UNAUTHENTICATED";
+    default: {
+      const exhaustive: never = code;
+      throw new Error(`unknown Connect code ${String(exhaustive)}`);
+    }
+  }
+}
+
+export interface RpcInstruments {
+  /** stigmer.grpc.request.duration {rpc.service, rpc.method, rpc.grpc.status_code} — ms, fractional. */
+  readonly requestDuration: Histogram;
+  /** stigmer.grpc.request.count — same labels; the heartbeat and error-rate series. */
+  readonly requestCount: Counter;
+}
+
+// The PROMISE is memoized (not the result) so concurrent first callers
+// share one creation — two racing registries would double-create every
+// instrument.
+let instrumentsPromise: Promise<RpcInstruments> | null = null;
+
+export function getRpcInstruments(): Promise<RpcInstruments> {
+  instrumentsPromise ??= createInstruments();
+  return instrumentsPromise;
+}
+
+async function createInstruments(): Promise<RpcInstruments> {
+  const api = await import("@opentelemetry/api");
+  const meter = api.metrics.getMeter(METER_NAME);
+  return {
+    requestDuration: meter.createHistogram("stigmer.grpc.request.duration", {
+      unit: "ms",
+      description: "Duration of gRPC server requests in milliseconds",
+    }),
+    requestCount: meter.createCounter("stigmer.grpc.request.count", {
+      description: "Total number of gRPC server requests",
+    }),
+  };
+}
+
+/**
+ * Fire-and-forget record of one finished RPC — sync so the interceptor
+ * never awaits instrument resolution on the request path. Both
+ * instruments take the same attribute set, Java's single `close` record.
+ */
+export function recordRpc(
+  service: string,
+  method: string,
+  status: GrpcStatusName,
+  durationMs: number,
+): void {
+  void getRpcInstruments().then((registry) => {
+    const attributes = {
+      "rpc.service": service,
+      "rpc.method": method,
+      "rpc.grpc.status_code": status,
+    };
+    registry.requestDuration.record(durationMs, attributes);
+    registry.requestCount.add(1, attributes);
+  });
+}
+
+/**
+ * Births the count series at zero, attribute-less (Java's constructor
+ * posture — no fake `rpc.*` labels on a point that counts nothing).
+ * Awaited by the boot before the port binds so a boot-time provider
+ * exports the series before any traffic.
+ */
+export async function initRpcMetrics(): Promise<void> {
+  const registry = await getRpcInstruments();
+  registry.requestCount.add(0);
+}
+
+/**
+ * Whether a meter provider is registered for these instruments to record
+ * into. `exporting` is the host's doing (the cloud composition's
+ * bootstrap, or any `setGlobalMeterProvider` before the first record);
+ * `inert` is the OSS default until an export bootstrap exists. Read
+ * through the public API only: the no-op meter is a singleton the API
+ * exposes, so identity against it is the supported check.
+ */
+export type MetricsExportPosture = "exporting" | "inert";
+
+export async function metricsExportPosture(): Promise<MetricsExportPosture> {
+  const api = await import("@opentelemetry/api");
+  return api.metrics.getMeter(METER_NAME) === api.createNoopMeter()
+    ? "inert"
+    : "exporting";
+}
+
+/** Test seam: clears the memoized registry. */
+export function resetRpcInstruments(): void {
+  instrumentsPromise = null;
+}

package/src/pipeline/__tests__/apiresource-labels.test.ts

@@ -0,0 +1,67 @@
+/**
+ * Pins the reserved-label predicates (apiresource-labels.ts, the TS twin
+ * of backend/libs/go/apiresource/labels.go): isDefaultInstance is true
+ * ONLY for the exact DEFAULT_INSTANCE_LABEL key with the exact "true"
+ * value — any other value is inert (matching cloud's "true".equals(...))
+ * and missing metadata is undefined-safe.
+ */
+import { create } from "@bufbuild/protobuf";
+import { describe, expect, it } from "vitest";
+
+import { ApiResourceMetadataSchema } from "@stigmer/protos/ai/stigmer/commons/apiresource/metadata_pb";
+
+import {
+  DEFAULT_INSTANCE_LABEL,
+  RESERVED_LABEL_TRUE,
+  isDefaultInstance,
+} from "../apiresource-labels.js";
+
+function metadataWithLabels(labels: Record<string, string>) {
+  return create(ApiResourceMetadataSchema, {
+    id: "ain_test",
+    name: "Test Instance",
+    org: "acme",
+    labels,
+  });
+}
+
+describe("isDefaultInstance", () => {
+  it("is true for the exact label key and the exact 'true' value", () => {
+    expect(
+      isDefaultInstance(
+        metadataWithLabels({ [DEFAULT_INSTANCE_LABEL]: RESERVED_LABEL_TRUE }),
+      ),
+    ).toBe(true);
+  });
+
+  it("is false for any other value of the label — other values are inert", () => {
+    expect(
+      isDefaultInstance(
+        metadataWithLabels({ [DEFAULT_INSTANCE_LABEL]: "false" }),
+      ),
+    ).toBe(false);
+    expect(
+      isDefaultInstance(
+        metadataWithLabels({ [DEFAULT_INSTANCE_LABEL]: "TRUE" }),
+      ),
+    ).toBe(false);
+    expect(
+      isDefaultInstance(metadataWithLabels({ [DEFAULT_INSTANCE_LABEL]: "" })),
+    ).toBe(false);
+  });
+
+  it("is false when the label is missing entirely", () => {
+    expect(isDefaultInstance(metadataWithLabels({}))).toBe(false);
+    expect(
+      isDefaultInstance(
+        metadataWithLabels({
+          "stigmer.ai/system-managed": RESERVED_LABEL_TRUE,
+        }),
+      ),
+    ).toBe(false);
+  });
+
+  it("is false (not a crash) for undefined metadata", () => {
+    expect(isDefaultInstance(undefined)).toBe(false);
+  });
+});

package/src/pipeline/__tests__/chain.test.ts

@@ -0,0 +1,199 @@
+/**
+ * Interceptor-chain behavior tests, exercised through the REAL stigmer
+ * service descriptors over the in-process router transport (the SP-B path,
+ * so these tests double as proof that in-process calls get full pipeline
+ * treatment — the Go bufconn parity property).
+ *
+ * Pinned behaviors:
+ *   - protovalidate refuses an invalid Agent BEFORE the handler runs,
+ *     with InvalidArgument (D2 §2);
+ *   - a valid request reaches the handler with the apiresource kind
+ *     injected from the service option (interceptor.go parity);
+ *   - the logging tiers (grpc lib server.go:295-345): success info,
+ *     health success debug, NotFound debug, InvalidArgument warn, other
+ *     errors error — and the REAL error message is logged while the wire
+ *     carries whatever the handler threw.
+ */
+import {
+  Code,
+  ConnectError,
+  createClient,
+  createRouterTransport,
+} from "@connectrpc/connect";
+import { create } from "@bufbuild/protobuf";
+import { AgentSchema } from "@stigmer/protos/ai/stigmer/agentic/agent/v1/api_pb";
+import { AgentCommandController } from "@stigmer/protos/ai/stigmer/agentic/agent/v1/command_pb";
+import { ApiResourceKind } from "@stigmer/protos/ai/stigmer/commons/apiresource/apiresourcekind/api_resource_kind_pb";
+import {
+  Health,
+  HealthCheckResponse_ServingStatus as ServingStatus,
+} from "@stigmer/protos/grpc/health/v1/health_pb";
+import { describe, expect, it } from "vitest";
+
+import { createLogger } from "../../boot/logger.js";
+import type { LogFields } from "../../boot/logger.js";
+import { buildInterceptorChain } from "../chain.js";
+import { createVerifierChainInterceptor } from "../interceptors/auth.js";
+import { apiResourceKindKey } from "../interceptors/apiresource.js";
+
+const VALID_AGENT = {
+  apiVersion: "agentic.stigmer.ai/v1",
+  kind: "Agent",
+  metadata: { name: "chain test agent" },
+  spec: {
+    instructions: "You are a helpful test agent used by interceptor tests.",
+  },
+} as const;
+
+interface CapturedLine {
+  level: string;
+  message: string;
+  [key: string]: unknown;
+}
+
+function testHarness(handlers: {
+  create?: (context: { values: { get<T>(key: { id: symbol }): T } }) => unknown;
+}) {
+  const lines: CapturedLine[] = [];
+  const logger = createLogger({
+    level: "debug",
+    pretty: false,
+    write: (line) => lines.push(JSON.parse(line) as CapturedLine),
+  });
+  const transport = createRouterTransport(
+    (router) => {
+      router.service(AgentCommandController, {
+        create: (request, context) =>
+          (handlers.create?.(context) as never) ?? create(AgentSchema, request),
+        apply: (request) => request,
+        update: (request) => request,
+        updateVisibility: () => create(AgentSchema, VALID_AGENT),
+        delete: () => create(AgentSchema, VALID_AGENT),
+      });
+      router.service(Health, {
+        check: () => ({ status: ServingStatus.SERVING }),
+        list: () => ({ statuses: {} }),
+        watch: async function* () {},
+      });
+    },
+    // The serving-shape identity source with zero verifiers — the OSS
+    // default posture (O2): every request resolves to trusted-local.
+    {
+      router: {
+        interceptors: buildInterceptorChain(
+          logger,
+          createVerifierChainInterceptor([], [], logger),
+        ),
+      },
+    },
+  );
+  return { transport, lines };
+}
+
+function rpcLines(lines: CapturedLine[]): CapturedLine[] {
+  return lines.filter((line) => line.message.startsWith("rpc"));
+}
+
+describe("interceptor chain over the in-process transport", () => {
+  it("refuses an invalid Agent with InvalidArgument before the handler runs", async () => {
+    let handlerRan = false;
+    const { transport, lines } = testHarness({
+      create: () => {
+        handlerRan = true;
+        return create(AgentSchema, VALID_AGENT);
+      },
+    });
+
+    const failure = await createClient(AgentCommandController, transport)
+      .create({})
+      .then(() => null)
+      .catch((error: unknown) => ConnectError.from(error));
+
+    expect(failure?.code).toBe(Code.InvalidArgument);
+    expect(failure?.rawMessage).toContain("api_version");
+    expect(handlerRan, "validation must reject at the boundary").toBe(false);
+
+    // InvalidArgument is the client's mistake: warn tier, never error.
+    const logged = rpcLines(lines);
+    expect(logged).toHaveLength(1);
+    expect(logged[0]).toMatchObject({
+      level: "warn",
+      message: "rpc client error",
+    });
+  });
+
+  it("passes a valid Agent through and injects the service's api_resource_kind", async () => {
+    let observedKind: ApiResourceKind | undefined;
+    const { transport, lines } = testHarness({
+      create: (context) => {
+        observedKind = context.values.get(apiResourceKindKey);
+        return create(AgentSchema, VALID_AGENT);
+      },
+    });
+
+    await createClient(AgentCommandController, transport).create(VALID_AGENT);
+
+    expect(
+      observedKind,
+      "AgentCommandController carries option api_resource_kind = agent",
+    ).toBe(ApiResourceKind.agent);
+    const logged = rpcLines(lines);
+    expect(logged[0]).toMatchObject({
+      level: "info",
+      message: "rpc completed",
+    });
+  });
+
+  it("logs health-service successes at debug, not info (polled chatter)", async () => {
+    const { transport, lines } = testHarness({});
+
+    await createClient(Health, transport).check({});
+
+    const logged = rpcLines(lines);
+    expect(logged[0]).toMatchObject({
+      level: "debug",
+      message: "rpc completed",
+      procedure: "/grpc.health.v1.Health/Check",
+    });
+  });
+
+  it("logs NotFound at debug with the real error message", async () => {
+    const { transport, lines } = testHarness({
+      create: () => {
+        throw new ConnectError('agent "ghost" not found', Code.NotFound);
+      },
+    });
+
+    const failure = await createClient(AgentCommandController, transport)
+      .create(VALID_AGENT)
+      .then(() => null)
+      .catch((error: unknown) => ConnectError.from(error));
+
+    expect(failure?.code).toBe(Code.NotFound);
+    const logged = rpcLines(lines);
+    expect(logged[0]).toMatchObject({
+      level: "debug",
+      message: "rpc returned not found",
+    });
+    expect(String(logged[0]?.["error"])).toContain('agent "ghost" not found');
+  });
+
+  it("logs unexpected failures at error tier", async () => {
+    const { transport, lines } = testHarness({
+      create: () => {
+        throw new Error("sqlite exploded");
+      },
+    });
+
+    const failure = await createClient(AgentCommandController, transport)
+      .create(VALID_AGENT)
+      .then(() => null)
+      .catch((error: unknown) => ConnectError.from(error));
+
+    // ConnectRPC surfaces an uncaught non-ConnectError as Internal.
+    expect(failure?.code).toBe(Code.Internal);
+    const logged = rpcLines(lines);
+    expect(logged[0]).toMatchObject({ level: "error", message: "rpc failed" });
+    expect(String(logged[0]?.["error"])).toContain("sqlite exploded");
+  });
+});

package/src/pipeline/__tests__/pipeline.test.ts

@@ -0,0 +1,141 @@
+/**
+ * Pins the pipeline executor's contract (Go pipeline.go / error.go):
+ * ordered execution, halt-on-first-error, and the #478 wire sanitization —
+ * a ConnectError passes through with its clean domain message; a plain
+ * error reaches the wire as Internal/"internal server error" with the real
+ * cause kept server-side. Also pins RequestContext's input immutability.
+ */
+import { testCallerIdentity } from "./support.js";
+import { Code, ConnectError } from "@connectrpc/connect";
+import { describe, expect, it } from "vitest";
+
+import { OrganizationSchema } from "@stigmer/protos/ai/stigmer/tenancy/organization/v1/api_pb";
+import { create } from "@bufbuild/protobuf";
+
+import type { Logger } from "../../boot/logger.js";
+import { INTERNAL_FALLBACK_MESSAGE, newPipeline } from "../pipeline.js";
+import type { PipelineStep } from "../pipeline.js";
+import { LOADED_EXECUTION_KEY, RequestContext } from "../request-context.js";
+
+const silentLogger: Logger = {
+  debug() {},
+  info() {},
+  warn() {},
+  error() {},
+};
+
+function orgContext(): RequestContext<typeof OrganizationSchema> {
+  return new RequestContext(
+    OrganizationSchema,
+    create(OrganizationSchema, {
+      apiVersion: "tenancy.stigmer.ai/v1",
+      kind: "Organization",
+      metadata: { name: "Acme" },
+    }),
+    testCallerIdentity(),
+  );
+}
+
+function step(
+  name: string,
+  fn: (order: string[]) => void,
+  order: string[],
+): PipelineStep<typeof OrganizationSchema> {
+  return {
+    name,
+    execute() {
+      fn(order);
+    },
+  };
+}
+
+describe("pipeline execution", () => {
+  it("runs steps in order and stops on the first error", async () => {
+    const order: string[] = [];
+    const pipeline = newPipeline<typeof OrganizationSchema>(
+      "test",
+      silentLogger,
+    )
+      .addStep(step("First", (o) => o.push("first"), order))
+      .addStep({
+        name: "Boom",
+        execute() {
+          throw new ConnectError("nope", Code.FailedPrecondition);
+        },
+      })
+      .addStep(step("Never", (o) => o.push("never"), order))
+      .build();
+
+    await expect(pipeline.execute(orgContext())).rejects.toThrow("nope");
+    expect(order).toEqual(["first"]);
+  });
+
+  it("preserves a ConnectError's code and clean message on the wire", async () => {
+    const pipeline = newPipeline<typeof OrganizationSchema>(
+      "test",
+      silentLogger,
+    )
+      .addStep({
+        name: "Reject",
+        execute() {
+          throw new ConnectError(
+            "Organization already exists: slug 'acme'",
+            Code.AlreadyExists,
+          );
+        },
+      })
+      .build();
+
+    const error = await pipeline.execute(orgContext()).catch((e: unknown) => e);
+    expect(error).toBeInstanceOf(ConnectError);
+    expect((error as ConnectError).code).toBe(Code.AlreadyExists);
+    // No "pipeline step X failed" prefix reaches the wire (Go GRPCStatus).
+    expect((error as ConnectError).rawMessage).toBe(
+      "Organization already exists: slug 'acme'",
+    );
+  });
+
+  it("sanitizes a plain error to Internal/'internal server error' with the cause kept server-side (#478)", async () => {
+    const raw = new Error(
+      "SQLITE_IOERR: /home/user/.stigmer/stigmer.db is corrupt",
+    );
+    const pipeline = newPipeline<typeof OrganizationSchema>(
+      "test",
+      silentLogger,
+    )
+      .addStep({
+        name: "Leaky",
+        execute() {
+          throw raw;
+        },
+      })
+      .build();
+
+    const error = await pipeline.execute(orgContext()).catch((e: unknown) => e);
+    expect(error).toBeInstanceOf(ConnectError);
+    expect((error as ConnectError).code).toBe(Code.Internal);
+    expect((error as ConnectError).rawMessage).toBe(INTERNAL_FALLBACK_MESSAGE);
+    expect((error as ConnectError).rawMessage).not.toContain("stigmer.db");
+    expect((error as ConnectError).cause).toBe(raw);
+  });
+});
+
+describe("RequestContext", () => {
+  it("clones the input into newState so steps never mutate the request", () => {
+    const ctx = orgContext();
+    ctx.newState.metadata!.slug = "mutated";
+    expect(ctx.input.metadata?.slug).toBe("");
+    expect(ctx.newState.metadata?.slug).toBe("mutated");
+  });
+
+  it("passes inter-step values by key", () => {
+    const ctx = orgContext();
+    expect(ctx.get("missing")).toBeUndefined();
+    ctx.set("shouldCreate", true);
+    expect(ctx.get("shouldCreate")).toBe(true);
+  });
+
+  it("pins the loaded-execution context key (C4 Stage 3): both lifecycle chains stamp it and extension gate steps read it", () => {
+    expect(LOADED_EXECUTION_KEY).toBe("loadedExecution");
+  });
+});