@huynle/hub 0.7.4 → 0.8.0

package/src/central/workspaces/checkpoints.ts

@@ -0,0 +1,316 @@
+// Durable supervisor checkpoints (HWA-11; contract: Workspaces `POST /workspaces/:id/checkpoints {cursor}` in
+// docs/plans/2026-10-30-hub-workspaces-artifacts-contracts.md, SupervisorCheckpoint and
+// WORKSPACE_LIMITS.checkpointsPerSupervisor in packages/protocol/src/hub-workspace.ts):
+//
+//   POST /api/agent/workspaces/:id/checkpoints  {cursor, supervisorMemberID?, batchID?}   workspaces:manage
+//   GET  /api/agent/workspaces/:id/checkpoints?supervisorMemberID=                        workspaces:read
+//
+// A checkpoint says "this supervisor has processed the workspace's events up to `cursor`" (an agent-events cursor,
+// hev1_…, as GET /api/agent/events and the batched digests of coordinator.ts hand out). The supervisor defaults to
+// the workspace's master. Each record also pins what the acknowledgment was made against: the supervisor's member
+// generation and session, the workspace and supervision-graph revisions (the snapshot revision), the digest batch
+// it acknowledges (batch identity, so an uncertain delivery can be resolved, see coordinator.ts) and the unresolved
+// work under that supervisor (open / in_progress / blocked assignments of the members it supervises: IDs and a
+// count only, never titles). HWA-12's recovery reads these to resume a replaced master.
+//
+// Who: users and app tokens by scope. Session tokens are refused (contract clarification HWA-06: membership grants
+// nothing; a session reading or writing its own workspace through its token is an open question owned by the user).
+//
+// Durability: every acknowledgment is written (one version-checked aggregate per supervisor) and then the
+// StateStore is flushed BEFORE the answer, so the standalone file store's debounce cannot lose an acknowledged
+// checkpoint. A failed flush is an error to the caller (nothing is claimed); a repeat flushes again.
+//
+// Ordering and idempotency: within one journal epoch a cursor older than the latest checkpoint is a 409 conflict;
+// the same cursor with the same batch again answers the existing record (200). A cursor of another epoch (the
+// standalone journal restarted, or a cluster journal was reset) is not comparable and is recorded as given. A
+// cursor beyond the journal's latest position, another user's or a forged one is 400 invalid_cursor.
+//
+// Storage: StateStore collection `workspace-checkpoints`, key `<workspaceID>.<supervisorMemberID>`, holding the
+// newest `perSupervisor` (contract: 50) records; older ones are dropped. No journal event is appended (the compact
+// journal is unchanged). Rollback: older hubs never read the collection.
+import { randomBytes } from "node:crypto"
+import {
+  CONTRACT_VERSION,
+  ID_PREFIXES,
+  type IdKind,
+  isContractID,
+  LIST_LIMIT_MAX,
+  type SupervisorCheckpoint,
+  WORKSPACE_LIMITS,
+  WORKSPACE_SCOPES,
+  type HubWorkspace,
+  type WorkspaceMember,
+} from "../../protocol"
+import { type CursorKeys, decodeCursor, encodeCursor, parsePosition } from "../agent/event-cursor"
+import type { AgentEvents } from "../agent/events"
+import { AgentError } from "../agent/errors"
+import type { AgentRoute } from "../agent/routes"
+import type { Clock, StateStore } from "../backend/types"
+import type { MembershipStore } from "./members"
+import type { WorkspaceStore } from "./store"
+import type { SupervisionStore } from "./supervision"
+import { AGENT_JSON_BODY_MAX, boundedText } from "../agent/body"
+
+export const WORKSPACE_CHECKPOINTS = "workspace-checkpoints"
+/** Digest batch IDs (coordinator.ts mints them). */
+export const BATCH_ID = /^dig_[0-9a-f]{20}$/
+export const newBatchID = (): string => `dig_${randomBytes(10).toString("hex")}`
+/** Assignment IDs a checkpoint lists as unresolved (the count is always exact). */
+export const UNRESOLVED_IDS_MAX = 50
+/** Version-checked write attempts before a contended acknowledgment gives up. */
+const ATTEMPTS = 16
+const PREFIX = "/api/agent/workspaces"
+const NO_STORE = { "cache-control": "no-store" }
+const [READ_SCOPE, MANAGE_SCOPE] = WORKSPACE_SCOPES
+const OPEN_STATUSES: ReadonlySet<string> = new Set(["open", "in_progress", "blocked"])
+
+export interface CheckpointLimits {
+  readonly perSupervisor: number
+}
+
+/** A checkpoint as stored: the contract record plus what it was made against. `position` never leaves the hub. */
+export interface StoredCheckpoint extends SupervisorCheckpoint {
+  /** The journal position the cursor names. */
+  readonly position: string
+  readonly sessionID: string
+  /** The supervisor member's generation when it acknowledged. */
+  readonly generation: number
+  /** The digest batch it acknowledges, when it names one. */
+  readonly batchID?: string
+  /** The snapshot revision: the workspace record's and the supervision graph's. */
+  readonly snapshot: { readonly workspaceRevision: number; readonly supervisionRevision: number }
+  /** Not-closed assignments of the members this supervisor supervises. */
+  readonly unresolved: { readonly count: number; readonly assignmentIDs: readonly string[]; readonly truncated: boolean }
+}
+
+/** Cursor handling over the agent journal: decode (tenant-bound), encode and the latest position. */
+export interface CheckpointCursors {
+  decode(userID: string, cursor: string): string | undefined
+  encode(userID: string, position: string): string
+  latest(userID: string): Promise<string>
+}
+
+export function journalCursors(keys: CursorKeys, events: Pick<AgentEvents, "latest">): CheckpointCursors {
+  return {
+    decode: (userID, cursor) => decodeCursor(keys, userID, cursor),
+    encode: (userID, position) => encodeCursor(keys, userID, position),
+    latest: (userID) => events.latest(userID),
+  }
+}
+
+export interface CheckpointStoreDeps {
+  readonly state: StateStore
+  readonly workspaces: Pick<WorkspaceStore, "require">
+  readonly members: Pick<MembershipStore, "members" | "assignments">
+  readonly supervision: Pick<SupervisionStore, "graph">
+  readonly cursors: CheckpointCursors
+  readonly now?: Clock
+  readonly newID?: (kind: IdKind) => string
+  readonly limits?: Partial<CheckpointLimits>
+  /** Called after a durable acknowledgment (coordinator.ts resolves the batch it names). Failures are reported. */
+  readonly onAck?: (userID: string, workspaceID: string, supervisorMemberID: string, batchID: string | undefined) => Promise<void>
+}
+
+interface CheckpointDoc {
+  readonly v: 1
+  readonly workspaceID: string
+  readonly supervisorMemberID: string
+  /** Oldest first. */
+  readonly items: readonly StoredCheckpoint[]
+}
+
+const isObj = (v: unknown): v is Record<string, unknown> => typeof v === "object" && v !== null && !Array.isArray(v)
+export const checkpointKey = (workspaceID: string, memberID: string) => `${workspaceID}.${memberID}`
+
+/** Links as the scope walk needs them (live: both ends current). */
+export interface ScopeLink {
+  readonly memberID: string
+  readonly supervisorMemberID: string
+}
+
+/**
+ * The members a supervisor is responsible for: the master supervises every other current member; anyone else its
+ * subtree of live supervision links. undefined when the member supervises nobody and is not the master.
+ */
+export function supervisedMembers(workspace: Pick<HubWorkspace, "masterMemberID">, current: readonly WorkspaceMember[], links: readonly ScopeLink[], supervisorMemberID: string): Set<string> | undefined {
+  const ids = new Set(current.map((m) => m.id))
+  if (!ids.has(supervisorMemberID)) return undefined
+  if (workspace.masterMemberID === supervisorMemberID) return new Set([...ids].filter((id) => id !== supervisorMemberID))
+  const children = new Map<string, string[]>()
+  for (const l of links) if (ids.has(l.memberID) && ids.has(l.supervisorMemberID)) children.set(l.supervisorMemberID, [...(children.get(l.supervisorMemberID) ?? []), l.memberID])
+  if (!children.has(supervisorMemberID)) return undefined
+  const out = new Set<string>()
+  const queue = [supervisorMemberID]
+  while (queue.length) for (const child of children.get(queue.shift()!) ?? []) if (child !== supervisorMemberID && !out.has(child)) out.add(child), queue.push(child)
+  return out
+}
+
+function readDoc(raw: unknown, workspaceID: string, memberID: string): CheckpointDoc {
+  if (raw === undefined) return { v: 1, workspaceID, supervisorMemberID: memberID, items: [] }
+  if (isObj(raw) && typeof raw.v === "number" && raw.v > CONTRACT_VERSION)
+    throw new AgentError("unsupported", "These checkpoints were written by a newer hub", { hint: "Upgrade this hub", extra: { feature: WORKSPACE_CHECKPOINTS } })
+  if (!isObj(raw) || raw.v !== CONTRACT_VERSION || raw.workspaceID !== workspaceID || !Array.isArray(raw.items)) throw new AgentError("internal", "Stored checkpoints are malformed")
+  return raw as unknown as CheckpointDoc
+}
+
+/** -1, 0, 1 for comparable positions (same epoch); undefined across epochs. */
+function compare(a: string, b: string): number | undefined {
+  const x = parsePosition(a)
+  const y = parsePosition(b)
+  if (!x || !y || x.epoch !== y.epoch) return undefined
+  return Math.sign(x.n - y.n)
+}
+
+export class CheckpointStore {
+  readonly limits: CheckpointLimits
+  private readonly now: Clock
+  private readonly newID: (kind: IdKind) => string
+
+  constructor(private readonly deps: CheckpointStoreDeps) {
+    this.limits = { perSupervisor: WORKSPACE_LIMITS.checkpointsPerSupervisor, ...deps.limits }
+    this.now = deps.now ?? Date.now
+    this.newID = deps.newID ?? ((kind) => `${ID_PREFIXES[kind]}_${randomBytes(12).toString("hex")}`)
+  }
+
+  /** Records an acknowledgment and makes it durable before resolving. `created: false` for a repeat. */
+  async ack(userID: string, workspaceID: string, input: unknown): Promise<{ checkpoint: StoredCheckpoint; created: boolean }> {
+    if (!isObj(input)) throw new AgentError("invalid", "body must be an object")
+    for (const key of Object.keys(input)) if (!["cursor", "supervisorMemberID", "batchID"].includes(key)) throw new AgentError("invalid", `unknown field ${/^[A-Za-z0-9_-]{1,40}$/.test(key) ? key : "(unprintable)"}`)
+    const { cursor, batchID } = input
+    if (typeof cursor !== "string" || cursor === "") throw new AgentError("invalid", "cursor is required: the agent-events cursor you processed up to")
+    if (input.supervisorMemberID !== undefined && !isContractID("member", input.supervisorMemberID)) throw new AgentError("invalid", "supervisorMemberID must be a workspace member ID")
+    if (batchID !== undefined && (typeof batchID !== "string" || !BATCH_ID.test(batchID))) throw new AgentError("invalid", "batchID must be a digest batch ID (dig_…)")
+    const position = this.deps.cursors.decode(userID, cursor)
+    if (position === undefined) throw new AgentError("invalid_cursor", "That cursor is not one this hub gave you")
+    if (compare(position, await this.deps.cursors.latest(userID)) === 1) throw new AgentError("invalid_cursor", "That cursor is beyond the journal")
+
+    const workspace = await this.deps.workspaces.require(userID, workspaceID)
+    if (workspace.state === "archived") throw new AgentError("invalid", `Workspace ${workspaceID} is archived`, { hint: "Restore it first" })
+    const memberID = (input.supervisorMemberID as string | undefined) ?? workspace.masterMemberID
+    if (memberID === undefined) throw new AgentError("invalid", "The workspace has no master: name the supervisorMemberID")
+    const views = await this.deps.members.members(userID, workspaceID, { includeLeft: true })
+    const supervisor = views.find((v) => v.member.id === memberID)?.member
+    if (!supervisor) throw new AgentError("not_found", `No member ${memberID} in workspace ${workspaceID}`)
+    if (supervisor.leftAt !== undefined) throw new AgentError("invalid", `Member ${memberID} left the workspace`)
+    const graph = await this.deps.supervision.graph(userID, workspaceID)
+    const live = graph.links.filter((l) => l.member === "current" && l.supervisor === "current").map((l) => l.link)
+    const scope = supervisedMembers(workspace, views.filter((v) => v.current).map((v) => v.member), live, memberID)
+    if (!scope) throw new AgentError("invalid", `Member ${memberID} supervises nobody`, { hint: "Checkpoints belong to the master or a member with supervised members" })
+    const unresolved = await this.unresolved(userID, workspaceID, scope)
+
+    const key = checkpointKey(workspaceID, memberID)
+    let answer: { checkpoint: StoredCheckpoint; created: boolean } | undefined
+    for (let attempt = 0; attempt < ATTEMPTS && !answer; attempt++) {
+      const stored = await this.deps.state.get(userID, WORKSPACE_CHECKPOINTS, key)
+      const doc = readDoc(stored?.value, workspaceID, memberID)
+      const last = doc.items.at(-1)
+      if (last) {
+        const order = compare(position, last.position)
+        if (order === -1)
+          throw new AgentError("conflict", "A later checkpoint is already acknowledged", { hint: "Acknowledge a newer cursor", extra: { currentCursor: last.cursor } })
+        if (order === 0 && last.batchID === batchID) {
+          answer = { checkpoint: last, created: false }
+          break
+        }
+      }
+      const checkpoint: StoredCheckpoint = {
+        v: 1,
+        id: this.newID("checkpoint"),
+        workspaceID,
+        supervisorMemberID: memberID,
+        cursor,
+        at: this.now(),
+        position,
+        sessionID: supervisor.sessionID,
+        generation: supervisor.generation,
+        ...(batchID !== undefined ? { batchID: batchID as string } : {}),
+        snapshot: { workspaceRevision: workspace.revision, supervisionRevision: graph.revision },
+        unresolved,
+      }
+      const next: CheckpointDoc = { v: 1, workspaceID, supervisorMemberID: memberID, items: [...doc.items, checkpoint].slice(-this.limits.perSupervisor) }
+      if ((await this.deps.state.put(userID, WORKSPACE_CHECKPOINTS, key, next, { ifVersion: stored?.version ?? 0 })).ok) answer = { checkpoint, created: true }
+    }
+    if (!answer) throw new Error(`state update of ${WORKSPACE_CHECKPOINTS}/${key} kept conflicting`)
+    // Durable before the answer: the standalone store debounces writes.
+    await this.deps.state.flush?.()
+    if (this.deps.onAck)
+      await this.deps.onAck(userID, workspaceID, memberID, answer.checkpoint.batchID).catch((cause) => console.error(`[hub] checkpoint batch resolve failed: ${(cause as Error).message}`))
+    return answer
+  }
+
+  /** A supervisor's retained checkpoints, newest first; without one, the latest of each supervisor. */
+  async list(userID: string, workspaceID: string, supervisorMemberID?: string): Promise<StoredCheckpoint[]> {
+    await this.deps.workspaces.require(userID, workspaceID)
+    if (supervisorMemberID !== undefined) {
+      if (!isContractID("member", supervisorMemberID)) throw new AgentError("invalid", "supervisorMemberID must be a workspace member ID")
+      const doc = readDoc((await this.deps.state.get(userID, WORKSPACE_CHECKPOINTS, checkpointKey(workspaceID, supervisorMemberID)))?.value, workspaceID, supervisorMemberID)
+      return [...doc.items].reverse()
+    }
+    const out: StoredCheckpoint[] = []
+    for (const { key, value } of await this.deps.state.list(userID, WORKSPACE_CHECKPOINTS)) {
+      if (!key.startsWith(`${workspaceID}.`)) continue
+      const last = readDoc(value, workspaceID, key.slice(workspaceID.length + 1)).items.at(-1)
+      if (last) out.push(last)
+    }
+    return out.sort((a, b) => b.at - a.at)
+  }
+
+  /** The supervisor's latest checkpoint (no workspace check: internal callers already read it). */
+  async latest(userID: string, workspaceID: string, supervisorMemberID: string): Promise<StoredCheckpoint | undefined> {
+    const raw = (await this.deps.state.get(userID, WORKSPACE_CHECKPOINTS, checkpointKey(workspaceID, supervisorMemberID)))?.value
+    return readDoc(raw, workspaceID, supervisorMemberID).items.at(-1)
+  }
+
+  private async unresolved(userID: string, workspaceID: string, scope: ReadonlySet<string>): Promise<StoredCheckpoint["unresolved"]> {
+    const ids: string[] = []
+    let count = 0
+    let cursor: string | undefined
+    do {
+      const page = await this.deps.members.assignments(userID, workspaceID, { limit: LIST_LIMIT_MAX, ...(cursor ? { cursor } : {}) })
+      for (const { assignment } of page.items) {
+        if (!OPEN_STATUSES.has(assignment.status) || !scope.has(assignment.memberID)) continue
+        count++
+        if (ids.length < UNRESOLVED_IDS_MAX) ids.push(assignment.id)
+      }
+      cursor = page.cursor
+    } while (cursor)
+    return { count, assignmentIDs: ids, truncated: count > ids.length }
+  }
+}
+
+/** The public form: without the internal journal position. */
+const publicCheckpoint = ({ position: _position, ...rest }: StoredCheckpoint) => rest
+
+async function bodyOf(request: Request): Promise<unknown> {
+  const text = await boundedText(request, AGENT_JSON_BODY_MAX)
+  try {
+    return JSON.parse(text === "" ? "{}" : text) as unknown
+  } catch {
+    throw new AgentError("invalid", "Body must be JSON")
+  }
+}
+
+export function checkpointRoutes(deps: { readonly checkpoints: CheckpointStore; readonly audit?: { audit(entry: { actor: string; action: string; target: string; detail?: string }): unknown } }): AgentRoute[] {
+  return [
+    {
+      method: "POST",
+      path: `${PREFIX}/:id/checkpoints`,
+      allow: { scope: MANAGE_SCOPE },
+      async handle(context) {
+        const { checkpoint, created } = await deps.checkpoints.ack(context.user.id, context.params.id!, await bodyOf(context.request))
+        if (created) await deps.audit?.audit({ actor: context.user.id, action: "workspace.checkpoint", target: checkpoint.workspaceID, detail: context.actor.kind })
+        return Response.json({ checkpoint: publicCheckpoint(checkpoint) }, { status: created ? 201 : 200, headers: NO_STORE })
+      },
+    },
+    {
+      method: "GET",
+      path: `${PREFIX}/:id/checkpoints`,
+      allow: { scope: READ_SCOPE },
+      async handle(context) {
+        const member = context.url.searchParams.get("supervisorMemberID") || undefined
+        const items = await deps.checkpoints.list(context.user.id, context.params.id!, member)
+        return Response.json({ checkpoints: items.map(publicCheckpoint) }, { headers: NO_STORE })
+      },
+    },
+  ]
+}