@opengeni/api-router 3.0.1 → 3.0.2-canary.38894290271001

package/src/routes/analytics-consent.ts

@@ -0,0 +1,123 @@
+// Content-free analytics consent beacon.
+//
+// When a person answers the web console's optional-analytics banner, the
+// browser reports only the closed decision (`granted` or `denied`). Each
+// accepted report increments `opengeni_analytics_consent_total{decision}`, so
+// product reports can state how much of the audience the consent-gated
+// analytics providers never see. The body carries no identifier, URL, cookie,
+// or user content, the browser sends it with `credentials: "omit"`, and the
+// route writes no log line. It admits anonymous callers because the banner is
+// answered before sign-in too. Any HTTP client that omits `Origin` can still
+// send reports up to the admission ceiling, so read the counter as a ratio of
+// decisions, not as a count of people.
+import type { Settings } from "@opengeni/config";
+import {
+  ANALYTICS_CONSENT_DECISIONS,
+  ANALYTICS_CONSENT_PATH,
+  ANALYTICS_CONSENT_REPORT_MAX_BYTES,
+  type AnalyticsConsentDecision,
+} from "@opengeni/contracts/analytics-consent-report";
+import type { Observability } from "@opengeni/observability";
+import type { Context, Hono } from "hono";
+import { bodyLimit } from "hono/body-limit";
+import { z } from "zod";
+
+import { isAllowedBrowserOrigin } from "../http/cors";
+import { createKeyedAdmission, type KeyedAdmission } from "../http/keyed-admission";
+
+export { ANALYTICS_CONSENT_DECISIONS, ANALYTICS_CONSENT_PATH, type AnalyticsConsentDecision };
+
+const REJECTION_REASONS = ["invalid", "too_large", "origin", "rate_limited"] as const;
+type RejectionReason = (typeof REJECTION_REASONS)[number];
+
+export const AnalyticsConsentReport = z
+  .object({ decision: z.enum(ANALYTICS_CONSENT_DECISIONS) })
+  .strict();
+export type AnalyticsConsentReport = z.infer<typeof AnalyticsConsentReport>;
+
+const CONSENT_METRIC = {
+  name: "opengeni_analytics_consent_total",
+  help: "Answers to the web console's optional-analytics banner, by closed decision. Admission is bounded per API process, so this is a lower bound.",
+} as const;
+const REJECTIONS_METRIC = {
+  name: "opengeni_analytics_consent_reports_rejected_total",
+  help: "Analytics consent reports the API refused, by closed reason.",
+} as const;
+
+export type AnalyticsConsentAdmission = KeyedAdmission<AnalyticsConsentDecision>;
+
+export function parseAnalyticsConsentReport(body: string | null): AnalyticsConsentReport | null {
+  if (
+    body === null ||
+    new TextEncoder().encode(body).byteLength > ANALYTICS_CONSENT_REPORT_MAX_BYTES
+  ) {
+    return null;
+  }
+  let value: unknown;
+  try {
+    value = JSON.parse(body);
+  } catch {
+    return null;
+  }
+  const parsed = AnalyticsConsentReport.safeParse(value);
+  return parsed.success ? parsed.data : null;
+}
+
+/**
+ * The exact beacon request. The app-wide request-body limit skips it because
+ * the route enforces its own small limit on the streamed body.
+ */
+export function isAnalyticsConsentReportRequest(method: string, pathname: string): boolean {
+  return method === "POST" && pathname === ANALYTICS_CONSENT_PATH;
+}
+
+export function registerAnalyticsConsentRoutes(
+  app: Hono,
+  deps: {
+    observability: Observability;
+    settings: Pick<Settings, "corsAllowOriginRegex" | "publicBaseUrl" | "webBaseUrl">;
+    admission?: AnalyticsConsentAdmission;
+  },
+): void {
+  const { observability, settings } = deps;
+  // Consent answers are rare (once per browser), so a small burst and a slow
+  // refill still cover a launch-day spike on each API process.
+  const admission =
+    deps.admission ??
+    createKeyedAdmission<AnalyticsConsentDecision>({ capacity: 60, refillPerSecond: 1 });
+  // Publish every finite series at zero so the first answer after a deploy is
+  // an increase from a baseline rather than a series appearing from nothing.
+  for (const decision of ANALYTICS_CONSENT_DECISIONS) {
+    observability.incrementCounter({ ...CONSENT_METRIC, labels: { decision }, amount: 0 });
+  }
+  for (const reason of REJECTION_REASONS) {
+    observability.incrementCounter({ ...REJECTIONS_METRIC, labels: { reason }, amount: 0 });
+  }
+  const reject = (c: Context, reason: RejectionReason, status: 400 | 403 | 413 | 429) => {
+    observability.incrementCounter({ ...REJECTIONS_METRIC, labels: { reason } });
+    c.header("cache-control", "no-store");
+    return c.body(null, status);
+  };
+
+  app.post(
+    ANALYTICS_CONSENT_PATH,
+    bodyLimit({
+      maxSize: ANALYTICS_CONSENT_REPORT_MAX_BYTES,
+      onError: (c) => reject(c, "too_large", 413),
+    }),
+    async (c) => {
+      c.header("cache-control", "no-store");
+      // Browsers always send Origin on this POST, so a foreign page cannot
+      // inflate the counter through its visitors' browsers.
+      const origin = c.req.header("origin");
+      if (origin !== undefined && !isAllowedBrowserOrigin(origin, settings)) {
+        return reject(c, "origin", 403);
+      }
+      const report = parseAnalyticsConsentReport(await c.req.text().catch(() => null));
+      if (!report) return reject(c, "invalid", 400);
+      if (!admission.admit(report.decision)) return reject(c, "rate_limited", 429);
+      observability.incrementCounter({ ...CONSENT_METRIC, labels: { decision: report.decision } });
+      return c.body(null, 204);
+    },
+  );
+}

package/src/routes/api-integrations.ts

@@ -7,6 +7,7 @@ import {
 } from "@opengeni/capabilities";
 import {
   ApiIntegrationPreview,
+  OrganizationIntegrationDeniedError,
   ListIntegrationDefinitionsResponse,
   ApiIntegrationUninstallPreview,
   InstallApiIntegrationRequest,
@@ -17,12 +18,19 @@ import {
   UninstallApiIntegrationResult,
   type AccessGrant,
 } from "@opengeni/contracts";
-import { requireAccessGrant, type ApiRouteDeps } from "@opengeni/core";
 import {
+  requireAccessGrant,
+  integrationSourceForOrganizationPolicy,
+  type ApiRouteDeps,
+} from "@opengeni/core";
+import { withOrganizationIntegrationPolicyFence } from "@opengeni/db/organization-integration-policy";
+import {
+  ApiIntegrationConnectionReferenceError,
   ApiIntegrationInstallationVersionConflictError,
   buildConnectionTokenResolver,
   getApiIntegrationUninstallPreview,
   getConnectionMetadata,
+  getApiIntegrationReconciliationSnapshot,
   installApiIntegration,
   IntegrationFacetBindingOwnershipConflictError,
   IntegrationFacetBindingVersionConflictError,
@@ -40,6 +48,8 @@ import {
   type ApiIntegrationConnectionDescriptor,
   type ResolvedApiIntegrationPreview,
 } from "../integrations/api-integrations";
+import { ApiHttpError } from "../http/api-error";
+import { parseRequestJson } from "../http/request-body";
 
 export type ApiIntegrationRouteOverrides = Readonly<{ fetchImpl?: FetchLike }>;
 
@@ -81,6 +91,14 @@ export function validatedIntegrationInstallInput(
     throw new HTTPException(422, {
       message: "Choose a Personal Connection before installing for yourself.",
     });
+  if (payload.autoApprovedTools?.length) {
+    const selected = new Set(payload.allowedTools ?? resolved.preview.tools.map((tool) => tool.id));
+    const outside = payload.autoApprovedTools.find((tool) => !selected.has(tool));
+    if (outside)
+      throw new HTTPException(422, {
+        message: `autoApprovedTools must name selected Integration tools: ${outside.slice(0, 200)}`,
+      });
+  }
   return {
     accountId: grant.accountId,
     workspaceId,
@@ -108,12 +126,63 @@ export function validatedIntegrationInstallInput(
       : {}),
     requiredScopes: resolved.requiredScopes,
     ownership: resolved.preview.connectionOwnership === "personal" ? "subject" : "workspace",
-    ...(payload.allowedTools ? { allowedTools: payload.allowedTools } : {}),
+    ...(payload.allowedTools
+      ? { allowedTools: validatedAllowedToolIds(payload.allowedTools, resolved.revision.tools) }
+      : {}),
+    ...(payload.autoApprovedTools ? { autoApprovedTools: payload.autoApprovedTools } : {}),
     facetDefinitions: integrationFacetDefinitions(resolved.preview.definitionId),
     revision: resolved.revision,
   };
 }
 
+const MAX_REPORTED_TOOL_IDS = 50;
+
+/**
+ * `allowedTools` names tools by their preview `id`, the stable identity the
+ * install persists. Reject anything else as a client error that lists the
+ * valid ids; a caller that sent a preview `operationKey` is told which id it
+ * maps to rather than having it silently reinterpreted.
+ */
+export function validatedAllowedToolIds(
+  allowedTools: readonly string[],
+  tools: readonly { id: string; operationKey: string }[],
+): string[] {
+  const ids = new Set(tools.map((tool) => tool.id));
+  const unknown = [
+    ...new Set(allowedTools.map((tool) => tool.trim()).filter((tool) => !ids.has(tool))),
+  ];
+  if (unknown.length === 0) return [...allowedTools];
+  const operationKeyMatches = unknown.flatMap((value) => {
+    const tool = tools.find((candidate) => candidate.operationKey === value);
+    return tool ? [{ operationKey: value, id: tool.id }] : [];
+  });
+  const validToolIds = tools.map((tool) => tool.id);
+  const shownUnknown = unknown.slice(0, 10);
+  const hint =
+    operationKeyMatches.length > 0
+      ? ` allowedTools takes tool ids, not operationKeys: ${operationKeyMatches
+          .slice(0, 5)
+          .map((match) => `${match.operationKey} -> ${match.id}`)
+          .join(", ")}.`
+      : "";
+  throw new ApiHttpError(422, {
+    code: "validation_failed",
+    message: `allowedTools contains unknown tool ids: ${shownUnknown.join(", ")}${
+      unknown.length > shownUnknown.length
+        ? `, and ${unknown.length - shownUnknown.length} more`
+        : ""
+    }. Use the \`id\` of each tool in the Integration preview.${hint}`,
+    retryable: false,
+    details: {
+      code: "unknown_integration_tools",
+      unknownTools: shownUnknown,
+      operationKeyMatches: operationKeyMatches.slice(0, MAX_REPORTED_TOOL_IDS),
+      validToolIds: validToolIds.slice(0, MAX_REPORTED_TOOL_IDS),
+      validToolIdCount: validToolIds.length,
+    },
+  });
+}
+
 export function registerApiIntegrationRoutes(
   app: Hono,
   deps: ApiRouteDeps,
@@ -205,24 +274,37 @@ export function registerApiIntegrationRoutes(
   app.post("/v1/workspaces/:workspaceId/integrations/preview", async (c) => {
     const workspaceId = c.req.param("workspaceId");
     const grant = await requireAccessGrant(c, deps, workspaceId, "workspace:read");
-    const payload = PreviewApiIntegrationRequest.parse(await c.req.json());
+    const payload = await parseRequestJson(c, PreviewApiIntegrationRequest);
     const resolved = await resolveForRoute({
       deps,
       transport,
+      accountId: grant.accountId,
       workspaceId,
       subjectId: grant.subjectId,
       payload,
     });
-    return c.json(ApiIntegrationPreview.parse(resolved.preview));
+    const credentialWarnings = await connectionCredentialPlacementWarnings(
+      deps,
+      workspaceId,
+      grant.subjectId,
+      resolved.preview,
+    );
+    return c.json(
+      ApiIntegrationPreview.parse({
+        ...resolved.preview,
+        warnings: [...resolved.preview.warnings, ...credentialWarnings].slice(0, 32),
+      }),
+    );
   });
 
   app.post("/v1/workspaces/:workspaceId/integrations/install", async (c) => {
     const workspaceId = c.req.param("workspaceId");
     const grant = await requireAccessGrant(c, deps, workspaceId, "capabilities:manage");
-    const payload = InstallApiIntegrationRequest.parse(await c.req.json());
+    const payload = await parseRequestJson(c, InstallApiIntegrationRequest);
     const resolved = await resolveForRoute({
       deps,
       transport,
+      accountId: grant.accountId,
       workspaceId,
       subjectId: grant.subjectId,
       payload,
@@ -234,6 +316,11 @@ export function registerApiIntegrationRoutes(
         payload.expectedInstanceVersion === undefined ? 201 : 200,
       );
     } catch (error) {
+      if (error instanceof ApiIntegrationConnectionReferenceError) {
+        throw new HTTPException(error.reason === "not_found" ? 404 : 422, {
+          message: error.reason === "not_found" ? "connection not found" : error.message,
+        });
+      }
       if (
         error instanceof IntegrationFacetBindingVersionConflictError ||
         error instanceof IntegrationFacetBindingVersionRequiredError ||
@@ -274,7 +361,7 @@ export function registerApiIntegrationRoutes(
       const grant = await requireAccessGrant(c, deps, workspaceId, "capabilities:manage");
       const capabilityId = decodeURIComponent(c.req.param("capabilityId"));
       const instanceKey = decodeURIComponent(c.req.param("instanceKey"));
-      const payload = UninstallApiIntegrationRequest.parse(await c.req.json());
+      const payload = await parseRequestJson(c, UninstallApiIntegrationRequest);
       try {
         return c.json(
           UninstallApiIntegrationResult.parse(
@@ -308,21 +395,45 @@ export function registerApiIntegrationRoutes(
 export async function resolveForRoute(input: {
   deps: ApiRouteDeps;
   transport: ReturnType<typeof createPinnedIntegrationTransport>;
+  accountId: string;
   workspaceId: string;
   subjectId: string;
   payload: PreviewApiIntegrationRequest | InstallApiIntegrationRequest;
 }): ReturnType<typeof resolveApiIntegrationPreview> {
+  const payload = structuredClone(input.payload);
+  const preparation = await withOrganizationIntegrationPolicyFence(
+    input.deps.db,
+    input,
+    async (tx, policy) => {
+      try {
+        return { source: integrationSourceForOrganizationPolicy(policy, payload.source) };
+      } catch (error) {
+        if (!(error instanceof OrganizationIntegrationDeniedError)) throw error;
+        const install = InstallApiIntegrationRequest.safeParse(payload);
+        if (install.success) {
+          const resolved = await storedReconciliationPreview(
+            { ...input, deps: { ...input.deps, db: tx } },
+            install.data,
+          );
+          if (resolved) return { resolved };
+        }
+        throw error;
+      }
+    },
+  );
+  if (preparation.resolved) return preparation.resolved;
+  const source = preparation.source!;
   try {
-    const connection = input.payload.connectionId
+    const connection = payload.connectionId
       ? await requireVisibleConnection(
           input.deps,
           input.workspaceId,
           input.subjectId,
-          input.payload.connectionId,
+          payload.connectionId,
         )
       : null;
     return await resolveApiIntegrationPreview({
-      source: input.payload.source,
+      source,
       connection: connectionDescriptor(connection),
       transport: input.transport,
       authority: {
@@ -353,6 +464,154 @@ export async function resolveForRoute(input: {
   }
 }
 
+async function storedReconciliationPreview(
+  input: { deps: ApiRouteDeps; accountId: string; workspaceId: string; subjectId: string },
+  payload: InstallApiIntegrationRequest,
+): Promise<ResolvedApiIntegrationPreview | null> {
+  const snapshot = await getApiIntegrationReconciliationSnapshot(input.deps.db, {
+    accountId: input.accountId,
+    workspaceId: input.workspaceId,
+    subjectId: input.subjectId,
+    source: payload.source,
+    expectedRevisionId: payload.expectedRevisionId,
+    expectedContentSha256: payload.expectedContentSha256,
+    ...(payload.connectionId !== undefined ? { connectionId: payload.connectionId } : {}),
+    ...(payload.instanceKey !== undefined ? { instanceKey: payload.instanceKey } : {}),
+    ...(payload.allowedTools !== undefined ? { allowedTools: payload.allowedTools } : {}),
+  });
+  if (!snapshot) return null;
+  const { runtime, baseServerId, name, provider, requiredScopes } = snapshot;
+  const connection = payload.connectionId
+    ? await requireVisibleConnection(
+        input.deps,
+        input.workspaceId,
+        input.subjectId,
+        payload.connectionId,
+      )
+    : null;
+  const scheme = runtime.authScheme;
+  const auth = !apiIntegrationRequiresConnection(scheme)
+    ? { kind: "none" }
+    : scheme.kind === "oauth2"
+      ? { kind: "oauth2", providerDomain: runtime.providerDomain, scopes: requiredScopes }
+      : scheme.kind === "api_key"
+        ? {
+            kind: "api_key",
+            providerDomain: runtime.providerDomain,
+            carrier: scheme.carrier,
+            name: scheme.name,
+          }
+        : scheme.kind === "http"
+          ? { kind: "http", providerDomain: runtime.providerDomain, scheme: scheme.scheme }
+          : null;
+  if (!auth) return null;
+  return {
+    preview: ApiIntegrationPreview.parse({
+      source: payload.source,
+      definitionId: runtime.definitionId,
+      definitionProvenance: runtime.definitionProvenance,
+      protocol: runtime.protocol,
+      capabilityId: runtime.capabilityId,
+      pluginKey: runtime.pluginKey,
+      serverId: baseServerId,
+      name,
+      description: runtime.description,
+      provider,
+      providerDomain: runtime.providerDomain,
+      baseUrl: runtime.baseUrl,
+      sourceUrl: runtime.sourceUrl,
+      revisionId: runtime.revision.id,
+      contentSha256: runtime.revision.contentSha256,
+      auth,
+      connectionId: connection?.id ?? null,
+      connectionOwnership: connection ? (connection.subjectId ? "personal" : "workspace") : null,
+      tools: runtime.revision.tools.map((tool) => ({
+        id: tool.id,
+        operationKey: tool.operationKey,
+        name: tool.name,
+        description: tool.description,
+        safety: tool.safety,
+        approvalMode: tool.approvalMode,
+        deprecated: tool.deprecated,
+      })),
+      warnings: [],
+    }),
+    revision: runtime.revision,
+    provider,
+    requiredScopes,
+    authScheme: runtime.authScheme,
+  };
+}
+
+/**
+ * Advisory preview check that the selected credential Connection delivers
+ * its secret where the API description says the provider reads it. Compares
+ * carrier and name only; never returns or logs a credential value. Install is
+ * unchanged: descriptions are often imprecise, so a mismatch warns instead of
+ * refusing a Connection the provider may actually accept.
+ */
+async function connectionCredentialPlacementWarnings(
+  deps: ApiRouteDeps,
+  workspaceId: string,
+  subjectId: string,
+  preview: ApiIntegrationPreview,
+): Promise<string[]> {
+  if (!preview.connectionId || (preview.auth.kind !== "api_key" && preview.auth.kind !== "http")) {
+    return [];
+  }
+  const connection = await getConnectionMetadata(
+    deps.db,
+    workspaceId,
+    preview.connectionId,
+    subjectId,
+  );
+  if (!connection || connection.kind === "oauth2") return [];
+  let result: Awaited<ReturnType<ReturnType<typeof buildConnectionTokenResolver>>>;
+  try {
+    result = await buildConnectionTokenResolver(
+      deps.db,
+      deps.settings,
+    )({
+      workspaceId,
+      ...(connection.subjectId ? { subjectId } : {}),
+      serverId: `preview_${connection.id}`,
+      connectionRef: {
+        connectionId: connection.id,
+        providerDomain: connection.providerDomain,
+        kind: connection.kind,
+        subjectScope: connection.subjectId ? "subject" : "workspace",
+      },
+      destinationUrl: preview.baseUrl,
+      credentialTarget: "http_api",
+    });
+  } catch {
+    return [];
+  }
+  if (result.status === "auth_needed") return [];
+  const expected =
+    preview.auth.kind === "api_key"
+      ? { carrier: preview.auth.carrier, name: preview.auth.name }
+      : { carrier: "header" as const, name: "Authorization" };
+  const placements =
+    result.placements ??
+    Object.keys(result.headers).map((name) => ({ carrier: "header" as const, name }));
+  const matches = placements.some(
+    (placement) =>
+      placement.carrier === expected.carrier &&
+      (placement.carrier === "header"
+        ? placement.name.toLowerCase() === expected.name.toLowerCase()
+        : placement.name === expected.name),
+  );
+  if (matches) return [];
+  const actual = placements
+    .slice(0, 3)
+    .map((placement) => `${placement.carrier} "${placement.name.slice(0, 64)}"`)
+    .join(", ");
+  return [
+    `The API description expects the credential in ${expected.carrier} "${expected.name.slice(0, 64)}", but the selected Connection sends it in ${actual}. Calls may be rejected as unauthenticated.`,
+  ];
+}
+
 async function requireVisibleConnection(
   deps: ApiRouteDeps,
   workspaceId: string,