melchizedek-agents 0.17.0 → 0.19.0

package/db/migrations/0013_tool_credentials.sql

@@ -0,0 +1,260 @@
+-- ============================================================================
+-- 0013_tool_credentials.sql — third-party OAuth tokens held for tools (ADR 0072)
+-- ============================================================================
+-- A tool that acts for an end user against a third-party API (a calendar, a
+-- repository host, a CRM) needs that user's delegated access, not the
+-- server's own key. melchizedek_tool_credentials holds one token set per
+-- app, end user and provider:
+--
+--   app_name   the app the run pins (the root syndicate's memory namespace)
+--   user_id    the A2A server's scope key, as sessions and tasks store it
+--   provider   the provider's name as the tool asks for it ('github')
+--   scopes     what the user granted
+--   access_token_enc, refresh_token_enc
+--              ciphertext only, in the engine's envelope ('mzc1.<key id>.…',
+--              AES-256-GCM by default, lib/tools/credentialCipher.ts). The
+--              CHECK refuses anything else, so a plaintext token cannot be
+--              written by mistake. The key never reaches the database.
+--   key_id     which key sealed the row, so a wrong key fails closed by name
+--   expires_at when the access token stops working (NULL: not stated)
+--   version    bumped by every write; a refresh writes only over the version
+--              it read, so two instances refreshing at once keep one result
+--
+-- The server's role reads and writes it; no API role can (RLS on, grants
+-- revoked, as db/hardening.sql repeats). melchizedek_erase_scope is replaced
+-- with the 0011 function plus a `credentials` store, so one erase of a user
+-- also removes their tokens (ADR 0020 item 7). Idempotent.
+-- ============================================================================
+
+CREATE TABLE IF NOT EXISTS melchizedek_tool_credentials (
+  app_name          TEXT        NOT NULL,
+  user_id           TEXT        NOT NULL,
+  provider          TEXT        NOT NULL,
+  scopes            TEXT[]      NOT NULL DEFAULT '{}',
+  access_token_enc  TEXT        NOT NULL CHECK (access_token_enc ~ '^mzc1[.]'),
+  refresh_token_enc TEXT                 CHECK (refresh_token_enc IS NULL OR refresh_token_enc ~ '^mzc1[.]'),
+  key_id            TEXT        NOT NULL,
+  expires_at        TIMESTAMPTZ,
+  version           INTEGER     NOT NULL DEFAULT 1,
+  created_at        TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  updated_at        TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  refreshed_at      TIMESTAMPTZ,
+  PRIMARY KEY (app_name, user_id, provider)
+);
+-- Erase finds a user's rows across apps.
+CREATE INDEX IF NOT EXISTS idx_melchizedek_tool_credentials_user
+  ON melchizedek_tool_credentials (user_id);
+
+ALTER TABLE melchizedek_tool_credentials ENABLE ROW LEVEL SECURITY;
+DO $$
+BEGIN
+  REVOKE ALL ON melchizedek_tool_credentials FROM PUBLIC;
+  IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'anon') THEN
+    REVOKE ALL ON melchizedek_tool_credentials FROM anon;
+  END IF;
+  IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'authenticated') THEN
+    REVOKE ALL ON melchizedek_tool_credentials FROM authenticated;
+  END IF;
+  IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'service_role') THEN
+    GRANT SELECT, INSERT, UPDATE, DELETE ON melchizedek_tool_credentials TO service_role;
+  END IF;
+END $$;
+
+-- One erase covers the credentials too: the 0011 function, plus the
+-- `credentials` store at the end.
+CREATE OR REPLACE FUNCTION melchizedek_erase_scope(
+  p_scope_key      text,
+  p_namespace      text    DEFAULT NULL,
+  p_include_nested boolean DEFAULT false
+) RETURNS TABLE(store text, deleted bigint)
+LANGUAGE plpgsql
+SECURITY INVOKER
+SET search_path = public
+AS $$
+DECLARE
+  -- LIKE pattern for scopes nested beneath this one, with the scope's own
+  -- wildcard characters escaped.
+  nested_like text := replace(replace(replace(p_scope_key, '\', '\\'), '%', '\%'), '_', '\_') || '/%';
+  contexts    text[];
+  kept        text[];
+  traces      text[];
+  n           bigint;
+BEGIN
+  IF p_scope_key IS NULL OR btrim(p_scope_key) = '' THEN
+    RAISE EXCEPTION 'melchizedek_erase_scope: scope_key is required';
+  END IF;
+
+  -- The conversations (context ids) being erased. With a namespace, they are
+  -- the namespace's own session rows; without one, every row of the scope.
+  -- adk_sessions.id is '<app_name>:<user_id>:<context id>'.
+  SELECT coalesce(array_agg(DISTINCT substr(s.id, length(s.app_name) + length(s.user_id) + 3)), '{}')
+    INTO contexts
+  FROM adk_sessions s
+  WHERE (s.user_id = p_scope_key OR (p_include_nested AND s.user_id LIKE nested_like))
+    AND (p_namespace IS NULL OR s.app_name = p_namespace);
+
+  -- With a namespace: the conversations still live in another namespace,
+  -- which are the only ones a namespace erase keeps. Sub-agent rows share
+  -- their conversation's id, so excluding `contexts` keeps them with it.
+  SELECT coalesce(array_agg(DISTINCT substr(s.id, length(s.app_name) + length(s.user_id) + 3)), '{}')
+    INTO kept
+  FROM adk_sessions s
+  WHERE p_namespace IS NOT NULL
+    AND (s.user_id = p_scope_key OR (p_include_nested AND s.user_id LIKE nested_like))
+    AND s.app_name <> p_namespace
+    AND NOT (substr(s.id, length(s.app_name) + length(s.user_id) + 3) = ANY(contexts));
+
+  -- Ledger traces of the turns being erased, collected before anything is deleted.
+  IF to_regclass('public.adk_turns') IS NOT NULL THEN
+    EXECUTE $q$
+      SELECT coalesce(array_agg(DISTINCT trace_id), '{}') FROM adk_turns
+      WHERE (user_id = $1 OR ($2 AND user_id LIKE $3))
+        AND ($4 IS NULL OR session_id IS NULL OR NOT (session_id = ANY($5)))
+    $q$ INTO traces USING p_scope_key, p_include_nested, nested_like, p_namespace, kept;
+  ELSE
+    traces := '{}';
+  END IF;
+
+  -- Memory facts. user_key is '<namespace>/<scope>' and a namespace never
+  -- contains '/', so the scope is exactly what follows the first slash.
+  DELETE FROM adk_memory_facts f
+  WHERE (p_namespace IS NULL OR split_part(f.user_key, '/', 1) = p_namespace)
+    AND (substr(f.user_key, strpos(f.user_key, '/') + 1) = p_scope_key
+         OR (p_include_nested AND substr(f.user_key, strpos(f.user_key, '/') + 1) LIKE nested_like));
+  GET DIAGNOSTICS n = ROW_COUNT;
+  store := 'memory_facts'; deleted := n; RETURN NEXT;
+
+  -- Sessions: every row of the scope in those conversations, whatever app
+  -- name ADK gave it (subagents run under their own agent name).
+  DELETE FROM adk_sessions s
+  WHERE (s.user_id = p_scope_key OR (p_include_nested AND s.user_id LIKE nested_like))
+    AND substr(s.id, length(s.app_name) + length(s.user_id) + 3) = ANY(contexts);
+  GET DIAGNOSTICS n = ROW_COUNT;
+  store := 'sessions'; deleted := n; RETURN NEXT;
+
+  -- The ledger, when installed.
+  IF to_regclass('public.adk_turns') IS NOT NULL THEN
+    EXECUTE $q$
+      DELETE FROM adk_turns
+      WHERE (user_id = $1 OR ($2 AND user_id LIKE $3))
+        AND ($4 IS NULL OR session_id IS NULL OR NOT (session_id = ANY($5)))
+    $q$ USING p_scope_key, p_include_nested, nested_like, p_namespace, kept;
+    GET DIAGNOSTICS n = ROW_COUNT;
+  ELSE
+    n := 0;
+  END IF;
+  store := 'turns'; deleted := n; RETURN NEXT;
+
+  IF to_regclass('public.adk_telemetry') IS NOT NULL THEN
+    EXECUTE 'DELETE FROM adk_telemetry WHERE trace_id = ANY($1)' USING traces;
+    GET DIAGNOSTICS n = ROW_COUNT;
+  ELSE
+    n := 0;
+  END IF;
+  store := 'spans'; deleted := n; RETURN NEXT;
+
+  IF to_regclass('public.adk_payloads') IS NOT NULL THEN
+    EXECUTE 'DELETE FROM adk_payloads WHERE trace_id = ANY($1)' USING traces;
+    GET DIAGNOSTICS n = ROW_COUNT;
+  ELSE
+    n := 0;
+  END IF;
+  store := 'payloads'; deleted := n; RETURN NEXT;
+
+  IF to_regclass('public.adk_verdicts') IS NOT NULL THEN
+    EXECUTE 'DELETE FROM adk_verdicts WHERE trace_id = ANY($1)' USING traces;
+    GET DIAGNOSTICS n = ROW_COUNT;
+  ELSE
+    n := 0;
+  END IF;
+  store := 'verdicts'; deleted := n; RETURN NEXT;
+
+  IF EXISTS (
+    SELECT 1 FROM information_schema.columns
+    WHERE table_schema = 'public' AND table_name = 'adk_labels' AND column_name = 'trace_id'
+  ) THEN
+    EXECUTE 'DELETE FROM adk_labels WHERE trace_id = ANY($1)' USING traces;
+    GET DIAGNOSTICS n = ROW_COUNT;
+  ELSE
+    n := 0;
+  END IF;
+  store := 'labels'; deleted := n; RETURN NEXT;
+
+  IF to_regclass('public.adk_a2a_tasks') IS NOT NULL THEN
+    EXECUTE $q$
+      DELETE FROM adk_a2a_tasks
+      WHERE (owner = $1 OR ($2 AND owner LIKE $3))
+        AND ($4 IS NULL OR context_id IS NULL OR NOT (context_id = ANY($5)))
+    $q$ USING p_scope_key, p_include_nested, nested_like, p_namespace, kept;
+    GET DIAGNOSTICS n = ROW_COUNT;
+  ELSE
+    n := 0;
+  END IF;
+  store := 'tasks'; deleted := n; RETURN NEXT;
+
+  -- Memory ingestion markers (migration 0007): how far each conversation was
+  -- distilled. They carry the scope's key and its conversation ids.
+  IF to_regclass('public.melchizedek_memory_ingest') IS NOT NULL THEN
+    EXECUTE $q$
+      DELETE FROM melchizedek_memory_ingest m
+      WHERE ($1 IS NULL OR split_part(m.user_key, '/', 1) = $1)
+        AND (substr(m.user_key, strpos(m.user_key, '/') + 1) = $2
+             OR ($3 AND substr(m.user_key, strpos(m.user_key, '/') + 1) LIKE $4))
+    $q$ USING p_namespace, p_scope_key, p_include_nested, nested_like;
+    GET DIAGNOSTICS n = ROW_COUNT;
+  ELSE
+    n := 0;
+  END IF;
+  store := 'memory_markers'; deleted := n; RETURN NEXT;
+
+  -- The task tools' list and job queue (migration 0009), per caller across
+  -- every syndicate: erased with the whole scope, kept by a namespace erase.
+  n := 0;
+  IF p_namespace IS NULL AND to_regclass('public.melchizedek_tasks') IS NOT NULL THEN
+    EXECUTE $q$
+      DELETE FROM melchizedek_tasks WHERE owner = $1 OR ($2 AND owner LIKE $3)
+    $q$ USING p_scope_key, p_include_nested, nested_like;
+    GET DIAGNOSTICS n = ROW_COUNT;
+    EXECUTE $q$
+      DELETE FROM melchizedek_task_owners WHERE owner = $1 OR ($2 AND owner LIKE $3)
+    $q$ USING p_scope_key, p_include_nested, nested_like;
+  END IF;
+  store := 'task_tools'; deleted := n; RETURN NEXT;
+
+  -- Third-party credentials held for the scope's tools (this migration).
+  -- They are filed per app (the memory namespace the run pins), so a
+  -- namespace erase removes that app's and keeps the others; an erase
+  -- without one removes every app's.
+  IF to_regclass('public.melchizedek_tool_credentials') IS NOT NULL THEN
+    EXECUTE $q$
+      DELETE FROM melchizedek_tool_credentials c
+      WHERE ($1 IS NULL OR c.app_name = $1)
+        AND (c.user_id = $2 OR ($3 AND c.user_id LIKE $4))
+    $q$ USING p_namespace, p_scope_key, p_include_nested, nested_like;
+    GET DIAGNOSTICS n = ROW_COUNT;
+  ELSE
+    n := 0;
+  END IF;
+  store := 'credentials'; deleted := n; RETURN NEXT;
+END;
+$$;
+
+-- No API role may execute it (db/hardening.sql repeats this for every
+-- melchizedek function); the server's role gets it back.
+DO $$
+BEGIN
+  REVOKE ALL ON FUNCTION melchizedek_erase_scope(text, text, boolean) FROM PUBLIC;
+  IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'anon') THEN
+    REVOKE ALL ON FUNCTION melchizedek_erase_scope(text, text, boolean) FROM anon;
+  END IF;
+  IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'authenticated') THEN
+    REVOKE ALL ON FUNCTION melchizedek_erase_scope(text, text, boolean) FROM authenticated;
+  END IF;
+  IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'service_role') THEN
+    GRANT EXECUTE ON FUNCTION melchizedek_erase_scope(text, text, boolean) TO service_role;
+  END IF;
+END $$;
+
+INSERT INTO melchizedek_schema_version (version, name)
+VALUES (13, '0013_tool_credentials')
+ON CONFLICT (version) DO NOTHING;

package/dist/lib/a2a/app.d.ts

@@ -8,6 +8,10 @@
  * Middleware order, and why:
  *   1. /healthz, /readyz — unauthenticated, so a load balancer or
  *      Kubernetes probe needs no secret. They reveal nothing but liveness.
+ *   1b. The OAuth consent callback, when `toolCredentials.consent` is set
+ *      (ADR 0085): a provider redirects the person's browser to it, so it
+ *      cannot carry the bearer. Its own rate limit; the single-use state
+ *      nonce admits it.
  *   2. Failed-auth limiter — counts only 401s per IP, so the shared secret
  *      cannot be guessed online at whatever rate the host allows.
  *   3. Bearer check (when a secret is configured), constant-time.
@@ -22,24 +26,53 @@
  * the handler cache, the rate-limiter counters. Sessions and memory are
  * durable when Supabase is configured. Run one replica, or put sticky
  * routing in front, until the task store is durable.
+ *
+ * Stores (ADR 0080): `storage` takes a session store and a memory service
+ * with either face, the engine's or ADK's. In-process sessions are the
+ * engine's InProcessSessionService. A `resolveModel` may still return an ADK
+ * model instance: its type is CompileOptions'.
  */
-import type { Express, Request } from 'express';
+import type { Express, Request, RequestHandler } from 'express';
 import type { Store } from 'express-rate-limit';
 import type { AgentCard } from '@a2a-js/sdk';
 import type { TaskStore } from '@a2a-js/sdk/server';
-import type { BaseLlm, BaseMemoryService, BaseSessionService } from '@google/adk';
+import type { CompileOptions } from '../compile.ts';
+import type { EitherSessionService } from '../runtime/adkSessionBridge.ts';
+import type { EitherMemoryService } from '../runtime/adkMemoryBridge.ts';
 import type { SyndicateYamlConfig } from '../loadSyndicate.ts';
 import type { ProviderEndpoint, ProviderId } from '../models/registry.ts';
 import type { TurnLock } from './turnLock.ts';
 import type { RlsHardeningStatus } from '../storage/rlsStatus.ts';
+import type { AuditSink } from '../observability/audit.ts';
+/** One end user's concurrent tasks, unless maxConcurrentPerScope says otherwise (ADR 0039). */
+export declare const DEFAULT_MAX_CONCURRENT_PER_SCOPE = 4;
 import type { EraseCounts } from '../memory/erase.ts';
 import type { Embedder, MemoryExtractor } from '../memory/providers.ts';
 import type { A2AContext } from './executor.ts';
 import type { IdentityScheme } from './identity.ts';
 import type { Policy } from './policy.ts';
 import type { TaskRecord } from '../observability/metrics.ts';
+import type { OAuthConsent, ToolCredentials } from '../tools/oauthConsent.ts';
 /** Agent ids address a config (file or "registry:<id>"): a safe charset only. */
 export declare function isValidAgentId(agentId: string): boolean;
+/**
+ * The consent callback (ADR 0085): the provider's redirect, carrying `code`
+ * and `state` (or `error`). It never reads a redirect URI from the request,
+ * never logs the query, and answers a page that carries no value. The
+ * response is never cached and sends no referrer, since its URL holds the
+ * code.
+ */
+export declare function consentCallback(consent: Pick<OAuthConsent, 'complete'>, opts?: {
+    resolveRequest?: A2AAppOptions['resolveRequest'];
+    /** The server's bearer secret, when it has one: the authenticator is consulted only behind it. */
+    serverSecret?: string;
+    /** Refuse a callback whose request the authenticator does not accept. Default true (ADR 0085). */
+    requireCallerIdentity?: boolean;
+    log?: (m: string) => void;
+    warn?: (m: string) => void;
+}): RequestHandler;
+/** What a model resolver returns, as the compiler takes it. */
+type ResolvedModel = ReturnType<NonNullable<CompileOptions['resolveModel']>>;
 export interface A2AAppOptions {
     /** The syndicate served at /a2a/*: a YAML name or "registry:<id>". */
     defaultSyndicate: string;
@@ -55,8 +88,10 @@ export interface A2AAppOptions {
      * agent id (default: in-process).
      */
     storage?: {
-        sessionService: BaseSessionService;
-        memoryService?: BaseMemoryService;
+        /** The engine's SessionService or ADK's BaseSessionService (ADR 0080). */
+        sessionService: EitherSessionService;
+        /** The engine's MemoryService or ADK's BaseMemoryService. */
+        memoryService?: EitherMemoryService;
         taskStore?: (agentId: string) => TaskStore;
         /** Erase everything stored for a scope (DELETE /memory). Without it the route answers 501. */
         erase?: (scopeKey: string, options: {
@@ -75,6 +110,8 @@ export interface A2AAppOptions {
          * fatal with `requireHardenedDb`, a warning otherwise.
          */
         rlsHardening?: () => Promise<RlsHardeningStatus>;
+        /** Appends audit events (postgresStorage supplies one; ADR 0042). */
+        audit?: AuditSink;
         /**
          * One turn at a time per conversation across instances (postgresStorage
          * supplies an advisory lock). Default: a lock in this process.
@@ -150,6 +187,35 @@ export interface A2AAppOptions {
     };
     /** Adopter routes, mounted after authentication and before the A2A routes. */
     routes?: (app: Express) => void;
+    /**
+     * Tool credentials for every turn (ADR 0072): the sealed per-user store,
+     * and the OAuth consent step (ADR 0085, lib/tools/oauthConsent.ts). With
+     * `consent`, a call whose provider the user has not granted ends its task
+     * input-required with a `consent_request` data part, and the server mounts
+     * the consent callback (GET at the path of the consent's configured
+     * redirect URI): it completes the authorization-code flow with PKCE and
+     * stores the grant. The callback is reached by the person's browser, so it
+     * sits before the bearer check; it has its own rate limit, and when the
+     * request carries a credential the authenticator accepts, the caller must
+     * be the user the flow is for.
+     */
+    toolCredentials?: ToolCredentials & {
+        /** Callback requests per window per client IP. Default 30 per 15 minutes. */
+        callbackLimit?: {
+            windowMs: number;
+            max: number;
+        };
+        /**
+         * Refuse a callback whose request the authenticator does not accept.
+         * Default true: the browser that completes the grant must carry the
+         * flow's user's identity (a session cookie or a gateway header that
+         * resolveRequest reads), so a forwarded authorization link cannot link
+         * someone else's account (ADR 0085). false lets the state nonce alone
+         * bind the flow to its user, for a deployment whose browsers carry no
+         * identity and that accepts that risk.
+         */
+        requireCallerIdentity?: boolean;
+    };
     /** Refuse to start when Supabase hardening is missing (public deployments). */
     requireHardenedDb?: boolean;
     /** Wall-clock budget per task, ms. 0 = none. */
@@ -162,6 +228,15 @@ export interface A2AAppOptions {
     streamText?: boolean;
     /** Concurrent tasks across all agents. 0 = unlimited. */
     maxConcurrentTasks?: number;
+    /**
+     * Where audit events go: failed authentications, task outcomes, erasures
+     * (ADR 0042). Default: the storage's sink when it has one, else none.
+     */
+    audit?: AuditSink;
+    /** Concurrent tasks for one scope (one end user). Default 4; 0 = unlimited. */
+    maxConcurrentPerScope?: number;
+    /** Concurrent tasks for one authenticated caller. Default 0 (unlimited). */
+    maxConcurrentPerCaller?: number;
     /** Task submissions (POST) per window per client IP. */
     rateLimit?: {
         windowMs: number;
@@ -192,9 +267,11 @@ export interface A2AAppOptions {
     /**
      * Model resolution per request. Default: lib/models/registry.ts with the
      * caller's X-API-Key scoped to its X-Provider. Override to route through
-     * your own gateway or credential store.
+     * your own gateway or credential store. Returns what
+     * CompileOptions.resolveModel returns: a model id, or a model instance
+     * (an ADK BaseLlm while the ADK runtime ships).
      */
-    resolveModel?: (modelName: string | undefined, ctx: A2AContext) => string | BaseLlm | undefined;
+    resolveModel?: (modelName: string | undefined, ctx: A2AContext) => ResolvedModel;
     /** Bindings applied at every config load (e.g. current_date). */
     bindings?: () => Record<string, string>;
     log?: (message: string) => void;
@@ -232,6 +309,12 @@ export interface A2AApp {
     /** Stop admitting tasks, wait up to `graceMs` for running ones, cancel the rest.
      *  Resolves with the number of tasks that had to be canceled. */
     shutdown(graceMs: number): Promise<number>;
+    /**
+     * Fail `/readyz` from now on while still serving every request, so a load
+     * balancer stops routing here before the listener closes. The first step of
+     * a graceful stop; `shutdown()` is the last.
+     */
+    markUnready(): void;
 }
 /**
  * Build the agent card for one syndicate at one route prefix, in the A2A 1.0
@@ -258,3 +341,4 @@ export declare function compileAgentCard(config: SyndicateYamlConfig, opts: {
  */
 export declare function memoryAppName(cfg: SyndicateYamlConfig): string;
 export declare function createA2AApp(options: A2AAppOptions): Promise<A2AApp>;
+export {};