@huynle/hub 0.7.3 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +229 -0
- package/Dockerfile +7 -2
- package/README.md +153 -24
- package/package.json +5 -3
- package/packages/claude-code/package.json +1 -1
- package/packages/claude-code/src/daemon.ts +8 -1
- package/packages/claude-code/src/launcher.ts +29 -9
- package/packages/claude-code/src/profile.ts +173 -0
- package/packages/claude-code/src/sessions.ts +22 -4
- package/packages/cli/dist/hub-cli.js +6474 -3470
- package/packages/cli/package.json +1 -1
- package/packages/cli/src/artifact-jobs-mcp.ts +226 -0
- package/packages/cli/src/artifacts-mcp.ts +160 -0
- package/packages/cli/src/bootstrap/enrollment/enroll.ts +20 -8
- package/packages/cli/src/bootstrap/enrollment/policy.ts +125 -12
- package/packages/cli/src/bootstrap/enrollment/status.ts +3 -2
- package/packages/cli/src/cli.ts +95 -13
- package/packages/cli/src/format.ts +48 -10
- package/packages/cli/src/http.ts +1 -0
- package/packages/cli/src/intents.ts +23 -0
- package/packages/cli/src/login.ts +20 -6
- package/packages/cli/src/mcp.ts +380 -9
- package/packages/cli/src/messaging.ts +3 -1
- package/packages/cli/src/setup/commands.ts +3 -2
- package/packages/cli/src/setup/runtime.ts +3 -2
- package/packages/cli/src/team-intake-mcp.ts +158 -0
- package/packages/cli/src/teams-mcp.ts +403 -0
- package/packages/cli/src/watchlist.ts +135 -0
- package/packages/codex/package.json +1 -1
- package/packages/codex/src/profile.ts +163 -0
- package/packages/codex/src/sessions.ts +12 -0
- package/packages/codex/src/spoke.ts +19 -1
- package/packages/node/package.json +1 -1
- package/packages/node/src/config.ts +96 -2
- package/packages/node/src/opencode-warm.ts +325 -0
- package/packages/node/src/run.ts +27 -1
- package/packages/preset/package.json +1 -1
- package/packages/preset/src/profile.ts +34 -0
- package/packages/preset/src/spoke.ts +4 -0
- package/packages/protocol/package.json +1 -1
- package/packages/protocol/src/artifacts.ts +142 -0
- package/packages/protocol/src/concierge.ts +366 -0
- package/packages/protocol/src/contracts.ts +104 -0
- package/packages/protocol/src/graph.ts +70 -0
- package/packages/protocol/src/hub-workspace.ts +201 -0
- package/packages/protocol/src/index.ts +169 -6
- package/packages/protocol/src/launch.ts +22 -0
- package/packages/protocol/src/profiles.ts +301 -0
- package/packages/protocol/src/teams.ts +204 -0
- package/packages/runtime-openshell/package.json +1 -1
- package/packages/runtime-sdk/package.json +1 -1
- package/packages/sdk/package.json +1 -1
- package/packages/sdk/src/address.ts +50 -2
- package/packages/sdk/src/artifact-jobs.ts +194 -0
- package/packages/sdk/src/artifacts.ts +187 -0
- package/packages/sdk/src/client.ts +226 -5
- package/packages/sdk/src/concierge.ts +88 -0
- package/packages/sdk/src/errors.ts +3 -1
- package/packages/sdk/src/index.ts +5 -0
- package/packages/sdk/src/team-intake.ts +122 -0
- package/packages/sdk/src/teams.ts +346 -0
- package/packages/sdk/src/types.ts +548 -1
- package/packages/spoke-sdk/package.json +1 -1
- package/packages/spoke-sdk/src/index.ts +1 -0
- package/packages/spoke-sdk/src/profile-apply.ts +245 -0
- package/src/central/actions.ts +7 -0
- package/src/central/agent/acting-routes.ts +2 -1
- package/src/central/agent/acting-switch.ts +2 -1
- package/src/central/agent/acting.ts +215 -10
- package/src/central/agent/app-tokens.ts +3 -2
- package/src/central/agent/audit.ts +17 -0
- package/src/central/agent/body.ts +163 -0
- package/src/central/agent/decisions.ts +69 -2
- package/src/central/agent/delivery.ts +95 -15
- package/src/central/agent/directory-routes.ts +37 -7
- package/src/central/agent/directory.ts +70 -14
- package/src/central/agent/errors.ts +5 -0
- package/src/central/agent/events-routes.ts +21 -4
- package/src/central/agent/events.ts +11 -3
- package/src/central/agent/form-check.ts +85 -0
- package/src/central/agent/grant-requests.ts +2 -1
- package/src/central/agent/grant-routes.ts +4 -3
- package/src/central/agent/grants.ts +29 -2
- package/src/central/agent/idempotency.ts +133 -0
- package/src/central/agent/intent-routes.ts +116 -0
- package/src/central/agent/intents.ts +505 -0
- package/src/central/agent/interaction-graph-routes.ts +212 -0
- package/src/central/agent/interaction-graph.ts +865 -0
- package/src/central/agent/launch-engine.ts +16 -2
- package/src/central/agent/launch-routes.ts +6 -1
- package/src/central/agent/launch-spec.ts +14 -1
- package/src/central/agent/message-routes.ts +2 -1
- package/src/central/agent/messages.ts +27 -2
- package/src/central/agent/names.ts +12 -2
- package/src/central/agent/read-routes.ts +86 -19
- package/src/central/agent/routes.ts +176 -5
- package/src/central/agent/webhook-routes.ts +2 -1
- package/src/central/agent-data.ts +4 -0
- package/src/central/artifacts/access.ts +44 -0
- package/src/central/artifacts/blob-store.ts +113 -0
- package/src/central/artifacts/blob-support.ts +98 -0
- package/src/central/artifacts/events.ts +27 -0
- package/src/central/artifacts/host-caps.ts +162 -0
- package/src/central/artifacts/host.ts +315 -0
- package/src/central/artifacts/idempotency.ts +109 -0
- package/src/central/artifacts/jobs-routes.ts +773 -0
- package/src/central/artifacts/jobs.ts +344 -0
- package/src/central/artifacts/maintenance.ts +103 -0
- package/src/central/artifacts/routes.ts +360 -0
- package/src/central/artifacts/store.ts +776 -0
- package/src/central/artifacts/workspaces.ts +31 -0
- package/src/central/backend/file-blobs.ts +377 -0
- package/src/central/backend/index.ts +12 -3
- package/src/central/backend/memory-blobs.ts +116 -0
- package/src/central/backend/postgres-blobs.ts +513 -0
- package/src/central/backend/postgres-migrate.ts +33 -0
- package/src/central/backend/postgres-state.ts +19 -0
- package/src/central/backend/types.ts +8 -0
- package/src/central/concierge/activity.ts +142 -0
- package/src/central/concierge/arm.ts +63 -0
- package/src/central/concierge/board-render.ts +190 -0
- package/src/central/concierge/board-results.ts +169 -0
- package/src/central/concierge/board-routes.ts +339 -0
- package/src/central/concierge/board.ts +249 -0
- package/src/central/concierge/context.ts +221 -0
- package/src/central/concierge/entitlements.ts +13 -0
- package/src/central/concierge/guard.ts +120 -0
- package/src/central/concierge/inbound.ts +89 -0
- package/src/central/concierge/index.ts +84 -0
- package/src/central/concierge/notes.ts +41 -0
- package/src/central/concierge/notifier.ts +986 -0
- package/src/central/concierge/prompt.ts +130 -0
- package/src/central/concierge/proposals.ts +262 -0
- package/src/central/concierge/push-remap.ts +49 -0
- package/src/central/concierge/routes.ts +62 -0
- package/src/central/concierge/schedule.ts +113 -0
- package/src/central/concierge/service.ts +243 -0
- package/src/central/concierge/session-routes.ts +69 -0
- package/src/central/concierge/settings.ts +217 -0
- package/src/central/concierge/supervisor.ts +740 -0
- package/src/central/concierge/tiers.ts +260 -0
- package/src/central/concierge/watchlist-routes.ts +277 -0
- package/src/central/concierge/watchlist.ts +273 -0
- package/src/central/config.ts +7 -1
- package/src/central/http.ts +633 -50
- package/src/central/mcp-http.ts +10 -21
- package/src/central/profiles/apply.ts +333 -0
- package/src/central/profiles/content.ts +174 -0
- package/src/central/profiles/events.ts +40 -0
- package/src/central/profiles/idempotency.ts +14 -0
- package/src/central/profiles/import.ts +139 -0
- package/src/central/profiles/routes.ts +356 -0
- package/src/central/profiles/store.ts +597 -0
- package/src/central/profiles/usage-ports.ts +120 -0
- package/src/central/profiles/usage.ts +364 -0
- package/src/central/push.ts +7 -0
- package/src/central/registry.ts +43 -6
- package/src/central/store.ts +4 -0
- package/src/central/teams/collections.ts +34 -0
- package/src/central/teams/discovery.ts +346 -0
- package/src/central/teams/draft-edits.ts +46 -0
- package/src/central/teams/events.ts +46 -0
- package/src/central/teams/intake-routes.ts +100 -0
- package/src/central/teams/intake.ts +772 -0
- package/src/central/teams/policy.ts +231 -0
- package/src/central/teams/routes.ts +516 -0
- package/src/central/teams/seats.ts +272 -0
- package/src/central/teams/staffing.ts +888 -0
- package/src/central/teams/store.ts +667 -0
- package/src/central/teams/validation.ts +256 -0
- package/src/central/workspaces/board.ts +408 -0
- package/src/central/workspaces/checkpoints.ts +316 -0
- package/src/central/workspaces/coordinator.ts +841 -0
- package/src/central/workspaces/events.ts +85 -0
- package/src/central/workspaces/lifecycle.ts +93 -0
- package/src/central/workspaces/master.ts +490 -0
- package/src/central/workspaces/members.ts +632 -0
- package/src/central/workspaces/routes.ts +354 -0
- package/src/central/workspaces/sidechats-routes.ts +410 -0
- package/src/central/workspaces/sidechats.ts +482 -0
- package/src/central/workspaces/store.ts +571 -0
- package/src/central/workspaces/supervision.ts +355 -0
- package/src/index.ts +97 -15
- package/src/spoke/concierge-agent.ts +33 -0
- package/src/spoke/deliver.ts +22 -4
- package/src/spoke/fork-revert.ts +142 -0
- package/src/spoke/hub-rpc.ts +41 -5
- package/src/spoke/hub-tool.ts +383 -18
- package/src/spoke/listing.ts +47 -4
- package/src/spoke/profile-apply.ts +298 -0
- package/src/spoke/session-context.ts +176 -0
- package/src/spoke/spoke.ts +319 -20
- package/src/spoke/timeline.ts +6 -1
- package/src/spoke/tracker.ts +23 -1
- package/src/spoke/who-teams.ts +81 -0
- package/src/tui-messaging.ts +32 -18
- package/web/dist/assets/ArtifactPage-BS0gVkfu.js +1 -0
- package/web/dist/assets/ArtifactPane-PLYHNs6s.js +1 -0
- package/web/dist/assets/ArtifactWorkbench-9eTj1CeW.css +1 -0
- package/web/dist/assets/ArtifactWorkbench-LkDyJ5Dv.js +6 -0
- package/web/dist/assets/Library-CtxRqTjo.js +2 -0
- package/web/dist/assets/Library-DYsFj_5c.css +1 -0
- package/web/dist/assets/ProfilesDestination-BYiHKLmc.js +1 -0
- package/web/dist/assets/TeamsPage-BwjsAAUC.js +1 -0
- package/web/dist/assets/Watchtower-C4swrWVs.js +2 -0
- package/web/dist/assets/Watchtower-CTgqHk5g.css +1 -0
- package/web/dist/assets/index-DaH5GUdV.js +31 -0
- package/web/dist/assets/index-WWuKfWxL.css +1 -0
- package/web/dist/assets/live-BUofsQpz.css +1 -0
- package/web/dist/assets/live-YyTjE47g.js +5 -0
- package/web/dist/assets/pane-layout-CKC4I00k.js +1 -0
- package/web/dist/assets/plural-Bspvoqfq.js +1 -0
- package/web/dist/assets/renderer2d-CX5j3NcD.js +1 -0
- package/web/dist/assets/renderer3d-BXIVOvfi.js +5393 -0
- package/web/dist/assets/safe-area-DfPsOdWR.js +1 -0
- package/web/dist/assets/source-CvUHTc3O.js +1 -0
- package/web/dist/assets/ui-C9Igxzjm.js +1 -0
- package/web/dist/index.html +5 -2
- package/web/dist/sw.js +22 -2
- package/web/dist/assets/index-Dm8OmkaO.css +0 -1
- package/web/dist/assets/index-FUJbj_Du.js +0 -20
|
@@ -0,0 +1,841 @@
|
|
|
1
|
+
// Batched supervisor delivery (HWA-11; contract: Workspaces checkpoints and "Contract clarifications" HWA-06/HWA-10
|
|
2
|
+
// in docs/plans/2026-10-30-hub-workspaces-artifacts-contracts.md). Supervisors (the workspace master, and any member
|
|
3
|
+
// with supervised members) learn what changed under them through the EXISTING messaging/wake path: one hub message
|
|
4
|
+
// per batch to the supervisor session. Nothing here reads anything through a session token, and no new grant exists.
|
|
5
|
+
//
|
|
6
|
+
// What a supervisor hears: compact journal events (canonical agent journal, events.ts) about the members it is
|
|
7
|
+
// responsible for (checkpoints.ts supervisedMembers: the master every other current member, anyone else its
|
|
8
|
+
// supervision subtree): session state and blockers of their sessions, their assignments' status, membership and
|
|
9
|
+
// supervision changes, and master replacement. Lines carry IDs, enums and numbers only, rendered from the already
|
|
10
|
+
// text-free events; never titles, criteria, message text or blocker summaries (the supervisor's session token can't
|
|
11
|
+
// read the workspace routes, so a digest must not leak what they hold). Message, launch and artifact families are
|
|
12
|
+
// never read.
|
|
13
|
+
//
|
|
14
|
+
// Self-trigger and digest loops: a supervisor never hears about its own session, its own membership or messages
|
|
15
|
+
// (delivering a digest wakes the supervisor and produces message and session events; none come back to it). Since
|
|
16
|
+
// supervision is acyclic, a chain of wake-ups only travels up the tree and dies out at the master; the hourly
|
|
17
|
+
// budgets cap it in any case. The sender is a fixed non-user party (coordinatorActor) so digests never reset the
|
|
18
|
+
// recipient's message hop chain the way a human turn would.
|
|
19
|
+
//
|
|
20
|
+
// Batching and waking (RP-07 #10: supervisors were woken for "no action" turns): three kinds of lines.
|
|
21
|
+
// - Priority (a blocker opened, a session blocked or erroring, an assignment blocked, a member leaving, a master
|
|
22
|
+
// replaced): next pass, urgent (steered into a running turn), listed first, waking the supervisor.
|
|
23
|
+
// - Wake-worthy (an assignment closed: done or cancelled): debounced by wakeDebounceMs (at most wakeMaxDelayMs),
|
|
24
|
+
// waking the supervisor, not steered; whatever routine lines are held go along.
|
|
25
|
+
// - Routine (session state, assignment progress, membership/supervision/workspace changes, calm snapshots):
|
|
26
|
+
// debounced by debounceMs (at most maxDelayMs) and sent with wake:false: admitted for the supervisor's next turn,
|
|
27
|
+
// never starting one (an idle supervisor is not woken; the digest folds into its next real wake).
|
|
28
|
+
// - Snapshots are routine unless urgent: a priority line was folded into them (dropped by an overflow, lost to a
|
|
29
|
+
// reset, or in an abandoned priority batch) or they show a member blocked, erroring or with open blockers. An
|
|
30
|
+
// urgent snapshot wakes and steers like a priority batch (priority budget while there is one).
|
|
31
|
+
// - Repeated trouble (RF-05): a session blocked or erroring again for a reason the supervisor was woken about within
|
|
32
|
+
// blockFoldMs (same session; same blocker kind, or the same blocked/error state; recorded in the ledger's
|
|
33
|
+
// `alerts` when that batch was delivered with a wake the hub carried out, `woke`, or acknowledged) is a routine line, not a priority one: a member asking the
|
|
34
|
+
// user one permission after another wakes its supervisor once per window, not per prompt. Another kind, another
|
|
35
|
+
// member, or the window passed (measured from the last wake about it) wakes again; snapshots fold the same way.
|
|
36
|
+
// Resolutions (blocker.closed, the recovery) were routine already.
|
|
37
|
+
// State lines (a session's state, an assignment's status) are collapsed per subject while held (only the latest
|
|
38
|
+
// counts), except that a held priority line is never replaced: an error or block that already cleared still goes
|
|
39
|
+
// out, followed by the recovery. A batch is cut only from what the supervisor does not know yet: a state equal to
|
|
40
|
+
// the last one delivered to it (ledger `facts`, from deltas and the latest snapshot; bounded by FACTS_MAX) is
|
|
41
|
+
// dropped unless a priority line about the same subject goes with it, and a batch with nothing left is skipped (no
|
|
42
|
+
// message, no budget). Presence (session.added / session.removed, subject `p:<session>`, RF-08) is a state line too:
|
|
43
|
+
// only the latest held presence of a session counts, and it is dropped when it equals the last delivered one
|
|
44
|
+
// (snapshots record each member's): a removed → added flap (a spoke restart or reconnect) is not news, a session
|
|
45
|
+
// really gone, or back after a reported removal, is (routine; a member leaving is workspace.member.left, priority,
|
|
46
|
+
// never collapsed). Held lines are never compared with each other: a duplicate event must not hide a real change.
|
|
47
|
+
// Blockers and other event lines are never deduplicated.
|
|
48
|
+
// Budgets per supervisor and rolling hour: batchesPerHour routine batches and priorityPerHour priority ones (a
|
|
49
|
+
// priority batch over its budget uses the routine one, then waits); a batch holds at most linesPerBatch lines (the
|
|
50
|
+
// rest is counted). Held lines are kept up to bufferLines; beyond that they are dropped, counted, and the next batch
|
|
51
|
+
// is a snapshot instead.
|
|
52
|
+
//
|
|
53
|
+
// Snapshots replace deltas when the delta can't be trusted: a new supervisor without a usable checkpoint, a cursor
|
|
54
|
+
// the journal no longer covers (expired, trimmed, or the standalone hub restarted: the memory journal gets a new
|
|
55
|
+
// epoch), a buffer overflow, or an abandoned batch. A snapshot lists the supervised members (session, role,
|
|
56
|
+
// presence, state, blocker count) and their unresolved assignments (IDs and status), with the workspace and
|
|
57
|
+
// supervision revisions.
|
|
58
|
+
//
|
|
59
|
+
// Delivery truthfulness (at-least-once, never exactly-once): every batch has an identity (dig_…) and a sequence
|
|
60
|
+
// number and is stored in the supervisor's ledger BEFORE it is sent. A send is first claimed in the ledger
|
|
61
|
+
// (version-checked, so two pods never both claim it); the outcome is recorded after. A claim found unrecorded after
|
|
62
|
+
// claimGraceMs (the sender crashed or was deposed mid-send) is uncertain and resent with the SAME batch ID; so is a
|
|
63
|
+
// send that failed in a way that may have stored the message. Refusals that store nothing (rate limited, recipient
|
|
64
|
+
// unknown or unsupported) are retried too. The text says "attempt n" and asks the supervisor to skip a batch ID it
|
|
65
|
+
// already handled. After maxAttempts the batch is abandoned and the next batch is a snapshot that says so. A
|
|
66
|
+
// checkpoint naming the batch (POST /workspaces/:id/checkpoints {batchID}) resolves it: no more retries.
|
|
67
|
+
//
|
|
68
|
+
// Revocation: a supervisor that departs, loses its last supervised member, or whose workspace is archived has its
|
|
69
|
+
// ledger deleted on the next pass, pending batch included: nothing more is sent to it.
|
|
70
|
+
//
|
|
71
|
+
// Leadership and parity: passes run on the `workspace-coordinator` leader (http.ts); `isLeader` is checked before
|
|
72
|
+
// each user and each send, and every ledger write is version-checked, so a deposed leader cannot double-advance a
|
|
73
|
+
// ledger. Ledgers live in the StateStore (collection `workspace-coordinator`, key `<workspaceID>.<memberID>`, the
|
|
74
|
+
// file store standalone, Postgres in a cluster) and survive restarts, session and launch expiry. A per-user scan
|
|
75
|
+
// marker (`workspace-coordinator-scan`) skips users whose journal did not move and who have nothing pending.
|
|
76
|
+
// Rollback: older hubs never read these collections.
|
|
77
|
+
import {
|
|
78
|
+
LIST_LIMIT_MAX,
|
|
79
|
+
type HubWorkspace,
|
|
80
|
+
type WorkspaceMember,
|
|
81
|
+
} from "../../protocol"
|
|
82
|
+
import type { AppActor } from "../agent/actor"
|
|
83
|
+
import { AgentError } from "../agent/errors"
|
|
84
|
+
import type { AgentEvents, EventRecord, EventsSince } from "../agent/events"
|
|
85
|
+
import { updateVersioned } from "../agent/versioned"
|
|
86
|
+
import { realTimers } from "../backend/memory-bus"
|
|
87
|
+
import type { Clock, StateStore, Timers } from "../backend/types"
|
|
88
|
+
import type { BoardSession, BoardSessions } from "./board"
|
|
89
|
+
import { type CheckpointCursors, type CheckpointStore, checkpointKey, newBatchID, supervisedMembers } from "./checkpoints"
|
|
90
|
+
import type { MembershipStore } from "./members"
|
|
91
|
+
import type { WorkspaceStore } from "./store"
|
|
92
|
+
import type { SupervisionStore } from "./supervision"
|
|
93
|
+
|
|
94
|
+
export const WORKSPACE_COORDINATOR = "workspace-coordinator"
|
|
95
|
+
export const WORKSPACE_COORDINATOR_SCAN = "workspace-coordinator-scan"
|
|
96
|
+
const SCAN_KEY = "journal"
|
|
97
|
+
/** How often the leader looks at the journal. */
|
|
98
|
+
export const COORDINATOR_TICK_MS = 10_000
|
|
99
|
+
/** The fixed sender of digests: an app party no real token can be (app token IDs are 16 hex digits). */
|
|
100
|
+
export const COORDINATOR_TOKEN = "workspace-coordinator"
|
|
101
|
+
const HOUR_MS = 60 * 60_000
|
|
102
|
+
const LINE_MAX = 200
|
|
103
|
+
const HISTORY_MAX = 20
|
|
104
|
+
/** Last reported states kept per supervisor (for skipping digests that tell nothing new). */
|
|
105
|
+
const FACTS_MAX = 400
|
|
106
|
+
|
|
107
|
+
export interface CoordinatorLimits {
|
|
108
|
+
/** Quiet period before routine lines go out (queued, without a wake). */
|
|
109
|
+
readonly debounceMs: number
|
|
110
|
+
/** Routine lines go out at most this long after the first one. */
|
|
111
|
+
readonly maxDelayMs: number
|
|
112
|
+
/** Quiet period before wake-worthy, non-urgent lines (an assignment closed) go out, waking the supervisor. */
|
|
113
|
+
readonly wakeDebounceMs: number
|
|
114
|
+
/** Wake-worthy lines go out at most this long after the first one. */
|
|
115
|
+
readonly wakeMaxDelayMs: number
|
|
116
|
+
/** Routine batches (snapshots included) per supervisor per rolling hour. */
|
|
117
|
+
readonly batchesPerHour: number
|
|
118
|
+
/** Priority batches per supervisor per rolling hour (over it they use the routine budget). */
|
|
119
|
+
readonly priorityPerHour: number
|
|
120
|
+
/** Lines listed in one batch. */
|
|
121
|
+
readonly linesPerBatch: number
|
|
122
|
+
/** Lines held per supervisor while waiting; beyond it they are dropped and a snapshot follows. */
|
|
123
|
+
readonly bufferLines: number
|
|
124
|
+
/** Send attempts of one batch before it is abandoned. */
|
|
125
|
+
readonly maxAttempts: number
|
|
126
|
+
/** Waits before each retry (the last repeats). */
|
|
127
|
+
readonly retryMs: readonly number[]
|
|
128
|
+
/** A send not answered within this long is given up on as uncertain (must stay below claimGraceMs). */
|
|
129
|
+
readonly sendTimeoutMs: number
|
|
130
|
+
/** A claimed but unrecorded send is in flight elsewhere for this long; after it, uncertain. */
|
|
131
|
+
readonly claimGraceMs: number
|
|
132
|
+
/** Supervisors served per user per pass. */
|
|
133
|
+
readonly supervisorsPerUser: number
|
|
134
|
+
/**
|
|
135
|
+
* A member blocked again for a reason the supervisor was woken about this recently (same session, same blocker
|
|
136
|
+
* kind or blocked/error state) is reported routinely instead of waking it again (RF-05).
|
|
137
|
+
*/
|
|
138
|
+
readonly blockFoldMs: number
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
export const COORDINATOR_LIMITS: CoordinatorLimits = {
|
|
142
|
+
debounceMs: 2 * 60_000,
|
|
143
|
+
maxDelayMs: 15 * 60_000,
|
|
144
|
+
wakeDebounceMs: 30_000,
|
|
145
|
+
wakeMaxDelayMs: 5 * 60_000,
|
|
146
|
+
batchesPerHour: 12,
|
|
147
|
+
priorityPerHour: 12,
|
|
148
|
+
linesPerBatch: 40,
|
|
149
|
+
bufferLines: 200,
|
|
150
|
+
maxAttempts: 6,
|
|
151
|
+
retryMs: [30_000, 2 * 60_000, 10 * 60_000, 30 * 60_000],
|
|
152
|
+
sendTimeoutMs: 60_000,
|
|
153
|
+
claimGraceMs: 2 * 60_000,
|
|
154
|
+
supervisorsPerUser: 400,
|
|
155
|
+
blockFoldMs: 15 * 60_000,
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/** Hands one digest to the messaging path (production: Delivery.send as coordinatorActor). */
|
|
159
|
+
export type DigestSend = (
|
|
160
|
+
userID: string,
|
|
161
|
+
sessionID: string,
|
|
162
|
+
text: string,
|
|
163
|
+
options: { readonly wake: boolean; readonly urgent: boolean },
|
|
164
|
+
) => Promise<{ readonly messageID: string; readonly delivery?: string; readonly woke?: boolean }>
|
|
165
|
+
|
|
166
|
+
/** Whether a sent digest woke its supervisor: the hub says so (`woke`), else only a turn or a steer counts. */
|
|
167
|
+
const woke = (result: { readonly delivery?: string; readonly woke?: boolean }): boolean => result.woke ?? (result.delivery === "turn" || result.delivery === "steer")
|
|
168
|
+
|
|
169
|
+
/** The party digests come from: never the user (a user message would reset hop chains and imply a human). */
|
|
170
|
+
export const coordinatorActor = (userID: string): AppActor => ({ kind: "app", userID, tokenID: COORDINATOR_TOKEN, scopes: [], label: "workspace digest" })
|
|
171
|
+
|
|
172
|
+
/** Refusals that store nothing: retrying cannot duplicate. Anything else may have stored the message. */
|
|
173
|
+
const REFUSED: ReadonlySet<string> = new Set(["rate_limited", "not_found", "unsupported", "forbidden", "invalid", "gone"])
|
|
174
|
+
|
|
175
|
+
export interface CoordinatorDeps {
|
|
176
|
+
readonly state: StateStore
|
|
177
|
+
readonly events: Pick<AgentEvents, "since" | "latest">
|
|
178
|
+
readonly cursors: Pick<CheckpointCursors, "encode">
|
|
179
|
+
readonly workspaces: Pick<WorkspaceStore, "list">
|
|
180
|
+
readonly members: Pick<MembershipStore, "members" | "assignments">
|
|
181
|
+
readonly supervision: Pick<SupervisionStore, "graph">
|
|
182
|
+
/** Where a new ledger resumes (the supervisor's latest checkpoint); without it new ledgers snapshot. */
|
|
183
|
+
readonly checkpoints?: Pick<CheckpointStore, "latest">
|
|
184
|
+
/** Session facts for snapshots (the board's lookup: directory and catalog, never a spoke). */
|
|
185
|
+
readonly sessions?: BoardSessions
|
|
186
|
+
readonly send: DigestSend
|
|
187
|
+
readonly users: () => Iterable<string> | Promise<Iterable<string>>
|
|
188
|
+
/** Leadership (cluster: the leader loop); checked before each user and each send. Default: always. */
|
|
189
|
+
readonly isLeader?: () => boolean
|
|
190
|
+
readonly owner?: string
|
|
191
|
+
readonly now?: Clock
|
|
192
|
+
readonly timers?: Timers
|
|
193
|
+
readonly tickMs?: number
|
|
194
|
+
readonly limits?: Partial<CoordinatorLimits>
|
|
195
|
+
readonly newBatchID?: () => string
|
|
196
|
+
readonly log?: (message: string) => void
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
interface Line {
|
|
200
|
+
/** Priority: goes out on the next pass, steered, waking the supervisor. */
|
|
201
|
+
readonly p: boolean
|
|
202
|
+
readonly text: string
|
|
203
|
+
/** Wakes the supervisor (an assignment closed), after the shorter wake debounce; not steered. */
|
|
204
|
+
readonly w?: true
|
|
205
|
+
/** A state line: the subject it is about (later lines replace it) and the state it reports. */
|
|
206
|
+
readonly k?: string
|
|
207
|
+
readonly f?: string
|
|
208
|
+
/** Further states a snapshot line reports (a member's presence next to its state): recorded like k/f. */
|
|
209
|
+
readonly x?: Facts
|
|
210
|
+
/** A priority line about trouble of a session: its reasons (`<session> blocker:<kind>`, `<session> state:<state>`). */
|
|
211
|
+
readonly b?: readonly string[]
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
/** Last reported state per subject (`s:<session>`, `p:<session>` presence, `a:<assignment>`). */
|
|
215
|
+
type Facts = Readonly<Record<string, string>>
|
|
216
|
+
|
|
217
|
+
export interface DigestBatch {
|
|
218
|
+
readonly id: string
|
|
219
|
+
readonly seq: number
|
|
220
|
+
readonly kind: "delta" | "snapshot"
|
|
221
|
+
/** Snapshots: why (and how many lines were dropped). */
|
|
222
|
+
readonly reason?: SnapshotReason
|
|
223
|
+
readonly dropped?: number
|
|
224
|
+
/** Which budget it uses (priority batches are urgent). */
|
|
225
|
+
readonly budget: "priority" | "routine"
|
|
226
|
+
/** Whether it starts a turn (absent in batches stored by older hubs: they woke). */
|
|
227
|
+
readonly wake?: boolean
|
|
228
|
+
/** The states it reports; a snapshot's replace the ledger's on delivery, a delta's are merged. */
|
|
229
|
+
readonly facts?: Facts
|
|
230
|
+
/** The trouble reasons it wakes the supervisor about (Line.b of its priority lines). */
|
|
231
|
+
readonly alerts?: readonly string[]
|
|
232
|
+
readonly lines: readonly string[]
|
|
233
|
+
readonly more: number
|
|
234
|
+
/** The journal position it covers up to (its cursor). */
|
|
235
|
+
readonly position: string
|
|
236
|
+
readonly cutAt: number
|
|
237
|
+
readonly state: "ready" | "sending" | "retry"
|
|
238
|
+
readonly attempts: number
|
|
239
|
+
readonly nextAt?: number
|
|
240
|
+
readonly claimedBy?: string
|
|
241
|
+
readonly claimedAt?: number
|
|
242
|
+
/** Why the last attempt did not land. */
|
|
243
|
+
readonly last?: "uncertain" | "refused"
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
export type SnapshotReason = "new" | "reset" | "overflow" | "abandoned"
|
|
247
|
+
|
|
248
|
+
/** A snapshot owed instead of a delta; `urgent` when priority news was folded into it (it wakes and steers). */
|
|
249
|
+
interface OwedSnapshot {
|
|
250
|
+
readonly reason: SnapshotReason
|
|
251
|
+
readonly dropped: number
|
|
252
|
+
readonly urgent?: true
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
const urgentIf = (urgent: boolean): { urgent?: true } => (urgent ? { urgent: true } : {})
|
|
256
|
+
|
|
257
|
+
export interface DigestOutcome {
|
|
258
|
+
readonly batchID: string
|
|
259
|
+
readonly seq: number
|
|
260
|
+
readonly kind: DigestBatch["kind"]
|
|
261
|
+
readonly at: number
|
|
262
|
+
readonly attempts: number
|
|
263
|
+
readonly outcome: "delivered" | "acknowledged" | "abandoned"
|
|
264
|
+
readonly messageID?: string
|
|
265
|
+
readonly delivery?: string
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/** A supervisor's durable delivery state. */
|
|
269
|
+
export interface SupervisorLedger {
|
|
270
|
+
readonly v: 1
|
|
271
|
+
readonly workspaceID: string
|
|
272
|
+
readonly supervisorMemberID: string
|
|
273
|
+
readonly sessionID: string
|
|
274
|
+
readonly generation: number
|
|
275
|
+
/** Journal position scanned up to. */
|
|
276
|
+
readonly position: string
|
|
277
|
+
/** A snapshot is owed instead of a delta. */
|
|
278
|
+
readonly snapshot?: OwedSnapshot
|
|
279
|
+
readonly buffer: readonly Line[]
|
|
280
|
+
readonly firstAt?: number
|
|
281
|
+
readonly lastAt?: number
|
|
282
|
+
readonly batch?: DigestBatch
|
|
283
|
+
/** Sends in the last hour (budgets). */
|
|
284
|
+
readonly window: readonly { readonly at: number; readonly budget: DigestBatch["budget"] }[]
|
|
285
|
+
readonly history: readonly DigestOutcome[]
|
|
286
|
+
readonly seq: number
|
|
287
|
+
/** The states delivered so far (bounded): a digest that would only repeat them is skipped. */
|
|
288
|
+
readonly facts?: Facts
|
|
289
|
+
/** Trouble reasons the supervisor was woken about → when (bounded): repeats within blockFoldMs are folded. */
|
|
290
|
+
readonly alerts?: Readonly<Record<string, number>>
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
interface Context {
|
|
294
|
+
readonly userID: string
|
|
295
|
+
readonly workspace: HubWorkspace
|
|
296
|
+
readonly supervisor: WorkspaceMember
|
|
297
|
+
readonly master: boolean
|
|
298
|
+
readonly scope: ReadonlySet<string>
|
|
299
|
+
/** Supervised members' sessions → member IDs. */
|
|
300
|
+
readonly sessions: ReadonlyMap<string, string>
|
|
301
|
+
readonly current: readonly { member: WorkspaceMember; presence: string }[]
|
|
302
|
+
readonly supervisionRevision: number
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
type Since = (position: string) => Promise<EventsSince>
|
|
306
|
+
|
|
307
|
+
const without = <T extends object, K extends keyof T>(value: T, ...keys: K[]): Omit<T, K> => {
|
|
308
|
+
const out = { ...value }
|
|
309
|
+
for (const key of keys) delete out[key]
|
|
310
|
+
return out
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
const OPEN: ReadonlySet<string> = new Set(["open", "in_progress", "blocked"])
|
|
314
|
+
const SESSION_TYPES: ReadonlySet<string> = new Set(["session.changed", "blocker.opened", "blocker.closed", "session.added", "session.removed"])
|
|
315
|
+
|
|
316
|
+
function render(p: boolean, type: string, ids: readonly (string | undefined)[], data: Readonly<Record<string, string | number | boolean>>, extra: Omit<Line, "p" | "text"> = {}): Line {
|
|
317
|
+
const kv = Object.entries(data).map(([k, v]) => `${k}=${String(v)}`)
|
|
318
|
+
const text = [p ? "!" : "-", type, ...ids.filter((id): id is string => !!id), ...kv].join(" ")
|
|
319
|
+
return { p, text: text.length > LINE_MAX ? `${text.slice(0, LINE_MAX - 1)}…` : text, ...extra }
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
/** Assignment statuses that close it: the supervisor is woken (its worker finished or gave up). */
|
|
323
|
+
const CLOSED: ReadonlySet<unknown> = new Set(["done", "cancelled"])
|
|
324
|
+
/** Session states that need the supervisor (priority lines, urgent snapshots). */
|
|
325
|
+
const URGENT_STATES: ReadonlySet<unknown> = new Set(["blocked", "error"])
|
|
326
|
+
const sessionFact = (state: unknown) => `state=${String(state)}`
|
|
327
|
+
/** A session's presence (session.added: online, session.removed: offline), its own subject `p:<session>`. */
|
|
328
|
+
const presenceFact = (online: boolean) => `presence=${online ? "online" : "offline"}`
|
|
329
|
+
const PRESENCE: ReadonlySet<string> = new Set(["session.added", "session.removed"])
|
|
330
|
+
const assignmentFact = (status: unknown, memberID: string | undefined) => `status=${String(status)} member=${memberID ?? "?"}`
|
|
331
|
+
|
|
332
|
+
/** The line a journal record is for this supervisor, or undefined (not its business, or its own doing). */
|
|
333
|
+
export function digestLine(record: Pick<EventRecord, "event" | "family">, ctx: Pick<Context, "workspace" | "supervisor" | "master" | "scope" | "sessions">): Line | undefined {
|
|
334
|
+
const { event, family } = record
|
|
335
|
+
const s = event.subject
|
|
336
|
+
const d = event.data
|
|
337
|
+
const me = ctx.supervisor
|
|
338
|
+
if (s.sessionID === me.sessionID || s.senderID === me.sessionID || s.memberID === me.id) return undefined
|
|
339
|
+
if (family === "session") {
|
|
340
|
+
if (!SESSION_TYPES.has(event.type) || !s.sessionID) return undefined
|
|
341
|
+
const memberID = ctx.sessions.get(s.sessionID)
|
|
342
|
+
if (!memberID) return undefined
|
|
343
|
+
const p = event.type === "blocker.opened" || (event.type === "session.changed" && URGENT_STATES.has(d.state))
|
|
344
|
+
const state =
|
|
345
|
+
event.type === "session.changed"
|
|
346
|
+
? { k: `s:${s.sessionID}`, f: sessionFact(d.state) }
|
|
347
|
+
: PRESENCE.has(event.type)
|
|
348
|
+
? { k: `p:${s.sessionID}`, f: presenceFact(event.type === "session.added") }
|
|
349
|
+
: {}
|
|
350
|
+
const reason = event.type === "blocker.opened" ? `blocker:${String(d.kind)}` : `state:${String(d.state)}`
|
|
351
|
+
return render(p, event.type, [s.sessionID, memberID], d, { ...state, ...(p ? { b: [`${s.sessionID} ${reason}`] } : {}) })
|
|
352
|
+
}
|
|
353
|
+
if (family !== "manage" || s.workspaceID !== ctx.workspace.id) return undefined
|
|
354
|
+
const inScope = s.memberID !== undefined && ctx.scope.has(s.memberID)
|
|
355
|
+
const ids = [s.memberID, s.sessionID, s.assignmentID, s.supervisorMemberID]
|
|
356
|
+
switch (event.type) {
|
|
357
|
+
case "workspace.assignment.changed":
|
|
358
|
+
if (!inScope || !s.assignmentID) return undefined
|
|
359
|
+
return render(d.status === "blocked", event.type, ids, d, { k: `a:${s.assignmentID}`, f: assignmentFact(d.status, s.memberID), ...(CLOSED.has(d.status) ? { w: true as const } : {}) })
|
|
360
|
+
case "workspace.member.left":
|
|
361
|
+
return inScope || ctx.master ? render(true, event.type, ids, d) : undefined
|
|
362
|
+
case "workspace.master.replaced":
|
|
363
|
+
return render(true, event.type, ids, d)
|
|
364
|
+
case "workspace.supervision.changed":
|
|
365
|
+
return inScope || s.supervisorMemberID === me.id || ctx.master ? render(false, event.type, ids, d) : undefined
|
|
366
|
+
case "workspace.member.joined":
|
|
367
|
+
case "workspace.updated":
|
|
368
|
+
case "workspace.sidechat.created":
|
|
369
|
+
return ctx.master || inScope ? render(false, event.type, ids, d) : undefined
|
|
370
|
+
default:
|
|
371
|
+
return undefined
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
|
|
375
|
+
/** The ledger's facts once `batch` reached the supervisor: a snapshot's replace them, a delta's are merged (bounded). */
|
|
376
|
+
function reported(ledger: SupervisorLedger, batch: DigestBatch): Facts {
|
|
377
|
+
const base: Record<string, string> = batch.kind === "snapshot" ? {} : { ...ledger.facts }
|
|
378
|
+
for (const [key, fact] of Object.entries(batch.facts ?? {})) {
|
|
379
|
+
delete base[key] // re-inserted last: the oldest facts are dropped first
|
|
380
|
+
base[key] = fact
|
|
381
|
+
}
|
|
382
|
+
const keys = Object.keys(base)
|
|
383
|
+
for (const key of keys.slice(0, Math.max(0, keys.length - FACTS_MAX))) delete base[key]
|
|
384
|
+
return base
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
/** The ledger's alerts once `batch` woke the supervisor at `at`: expired ones dropped, the batch's added (bounded). */
|
|
388
|
+
function withAlerts(ledger: SupervisorLedger, batch: DigestBatch, at: number, foldMs: number): Readonly<Record<string, number>> {
|
|
389
|
+
const out: Record<string, number> = {}
|
|
390
|
+
for (const [key, when] of Object.entries(ledger.alerts ?? {})) if (at - when < foldMs) out[key] = when
|
|
391
|
+
for (const key of batch.alerts ?? []) {
|
|
392
|
+
delete out[key] // re-inserted last: the oldest are dropped first
|
|
393
|
+
out[key] = at
|
|
394
|
+
}
|
|
395
|
+
const keys = Object.keys(out)
|
|
396
|
+
for (const key of keys.slice(0, Math.max(0, keys.length - FACTS_MAX))) delete out[key]
|
|
397
|
+
return out
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
export class SupervisorCoordinator {
|
|
401
|
+
readonly limits: CoordinatorLimits
|
|
402
|
+
private readonly deps: CoordinatorDeps
|
|
403
|
+
private readonly now: Clock
|
|
404
|
+
private readonly timers: Timers
|
|
405
|
+
private readonly owner: string
|
|
406
|
+
private readonly log: (message: string) => void
|
|
407
|
+
private readonly newBatchID: () => string
|
|
408
|
+
private running: Promise<void> | undefined
|
|
409
|
+
private timer: unknown
|
|
410
|
+
private started = false
|
|
411
|
+
private stopped = false
|
|
412
|
+
|
|
413
|
+
constructor(deps: CoordinatorDeps) {
|
|
414
|
+
this.deps = deps
|
|
415
|
+
this.limits = { ...COORDINATOR_LIMITS, ...deps.limits }
|
|
416
|
+
this.now = deps.now ?? Date.now
|
|
417
|
+
this.timers = deps.timers ?? realTimers
|
|
418
|
+
this.owner = deps.owner ?? `coordinator-${process.pid}`
|
|
419
|
+
this.log = deps.log ?? ((message) => console.error(message))
|
|
420
|
+
this.newBatchID = deps.newBatchID ?? newBatchID
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
/** Runs a pass every tickMs until stop() (the leader calls it when elected). */
|
|
424
|
+
start(): void {
|
|
425
|
+
this.stopped = false
|
|
426
|
+
if (this.started) return
|
|
427
|
+
this.started = true
|
|
428
|
+
this.schedule()
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
/** Stops scheduling and sending (deposed or shutting down); a pass in flight sends nothing more. */
|
|
432
|
+
stop(): void {
|
|
433
|
+
this.stopped = true
|
|
434
|
+
this.started = false
|
|
435
|
+
if (this.timer !== undefined) this.timers.clear(this.timer)
|
|
436
|
+
this.timer = undefined
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
/** One pass over every user (passes never overlap). */
|
|
440
|
+
tick(): Promise<void> {
|
|
441
|
+
this.running ??= this.pass().finally(() => {
|
|
442
|
+
this.running = undefined
|
|
443
|
+
})
|
|
444
|
+
return this.running
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
/** A supervisor's ledger (tests, HWA-12 recovery). */
|
|
448
|
+
async ledger(userID: string, workspaceID: string, supervisorMemberID: string): Promise<SupervisorLedger | undefined> {
|
|
449
|
+
return (await this.deps.state.get<SupervisorLedger>(userID, WORKSPACE_COORDINATOR, checkpointKey(workspaceID, supervisorMemberID)))?.value
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
/** A checkpoint named a batch: if it is still pending, it is resolved (no more retries). */
|
|
453
|
+
async acknowledged(userID: string, workspaceID: string, supervisorMemberID: string, batchID: string | undefined): Promise<void> {
|
|
454
|
+
if (!batchID) return
|
|
455
|
+
await updateVersioned<SupervisorLedger>(this.deps.state, userID, WORKSPACE_COORDINATOR, checkpointKey(workspaceID, supervisorMemberID), (ledger) => {
|
|
456
|
+
if (!ledger?.batch || ledger.batch.id !== batchID) return undefined
|
|
457
|
+
return { ...without(ledger, "batch"), facts: reported(ledger, ledger.batch), alerts: this.alerted(ledger, ledger.batch), history: this.outcome(ledger, ledger.batch, "acknowledged") }
|
|
458
|
+
})
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
// ---------- internals ----------
|
|
462
|
+
|
|
463
|
+
private alerted(ledger: SupervisorLedger, batch: DigestBatch, at = this.now()): Readonly<Record<string, number>> {
|
|
464
|
+
return withAlerts(ledger, batch, at, this.limits.blockFoldMs)
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
/** A priority line about trouble the supervisor was woken about within blockFoldMs: routine instead (RF-05). */
|
|
468
|
+
private fold(line: Line, ledger: SupervisorLedger, t: number): Line {
|
|
469
|
+
if (!line.p || !line.b?.length) return line
|
|
470
|
+
const alerts = ledger.alerts ?? {}
|
|
471
|
+
const known = line.b.every((key) => alerts[key] !== undefined && t - alerts[key]! < this.limits.blockFoldMs)
|
|
472
|
+
return known ? { ...without(line, "b"), p: false, text: line.text.replace(/^! /, "- ") } : line
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
private leading(): boolean {
|
|
476
|
+
return !this.stopped && (this.deps.isLeader?.() ?? true)
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
private schedule(): void {
|
|
480
|
+
this.timer = this.timers.set(() => {
|
|
481
|
+
this.timer = undefined
|
|
482
|
+
void this.tick().finally(() => {
|
|
483
|
+
if (this.started) this.schedule()
|
|
484
|
+
})
|
|
485
|
+
}, this.deps.tickMs ?? COORDINATOR_TICK_MS)
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
private async pass(): Promise<void> {
|
|
489
|
+
if (!this.leading()) return
|
|
490
|
+
let users: string[]
|
|
491
|
+
try {
|
|
492
|
+
users = [...(await this.deps.users())]
|
|
493
|
+
} catch (cause) {
|
|
494
|
+
this.log(`[hub] workspace coordinator: users failed: ${(cause as Error).message}`)
|
|
495
|
+
return
|
|
496
|
+
}
|
|
497
|
+
for (const userID of users) {
|
|
498
|
+
if (!this.leading()) return
|
|
499
|
+
try {
|
|
500
|
+
await this.user(userID)
|
|
501
|
+
} catch (cause) {
|
|
502
|
+
this.log(`[hub] workspace coordinator: pass failed: ${(cause as Error).message}`)
|
|
503
|
+
}
|
|
504
|
+
}
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
private async user(userID: string): Promise<void> {
|
|
508
|
+
const { state } = this.deps
|
|
509
|
+
const latest = await this.deps.events.latest(userID)
|
|
510
|
+
const ledgers = await state.list<SupervisorLedger>(userID, WORKSPACE_COORDINATOR)
|
|
511
|
+
const marker = (await state.get<{ position: string }>(userID, WORKSPACE_COORDINATOR_SCAN, SCAN_KEY))?.value
|
|
512
|
+
const busy = ledgers.some(({ value }) => value.batch !== undefined || value.buffer.length > 0 || value.snapshot !== undefined)
|
|
513
|
+
if (marker?.position === latest && !busy) return
|
|
514
|
+
|
|
515
|
+
const memo = new Map<string, Promise<EventsSince>>()
|
|
516
|
+
const since: Since = (position) => {
|
|
517
|
+
let result = memo.get(position)
|
|
518
|
+
if (!result) memo.set(position, (result = this.deps.events.since(userID, position)))
|
|
519
|
+
return result
|
|
520
|
+
}
|
|
521
|
+
const active: HubWorkspace[] = []
|
|
522
|
+
let cursor: string | undefined
|
|
523
|
+
do {
|
|
524
|
+
const page = await this.deps.workspaces.list(userID, { state: "active", limit: LIST_LIMIT_MAX, ...(cursor ? { cursor } : {}) })
|
|
525
|
+
active.push(...page.items)
|
|
526
|
+
cursor = page.cursor
|
|
527
|
+
} while (cursor)
|
|
528
|
+
|
|
529
|
+
const served = new Set<string>()
|
|
530
|
+
const examined = new Set<string>()
|
|
531
|
+
let count = 0
|
|
532
|
+
for (const workspace of active) {
|
|
533
|
+
if (count >= this.limits.supervisorsPerUser) {
|
|
534
|
+
this.log(`[hub] workspace coordinator: over ${this.limits.supervisorsPerUser} supervisors for one user; the rest wait`)
|
|
535
|
+
break
|
|
536
|
+
}
|
|
537
|
+
examined.add(workspace.id)
|
|
538
|
+
const views = await this.deps.members.members(userID, workspace.id)
|
|
539
|
+
const graph = await this.deps.supervision.graph(userID, workspace.id)
|
|
540
|
+
const links = graph.links.filter((l) => l.member === "current" && l.supervisor === "current").map((l) => l.link)
|
|
541
|
+
const current = views.map((v) => v.member)
|
|
542
|
+
const supervisors = new Set<string>([...(workspace.masterMemberID ? [workspace.masterMemberID] : []), ...links.map((l) => l.supervisorMemberID)])
|
|
543
|
+
for (const memberID of supervisors) {
|
|
544
|
+
const supervisor = current.find((m) => m.id === memberID)
|
|
545
|
+
const scope = supervisor && supervisedMembers(workspace, current, links, memberID)
|
|
546
|
+
if (!supervisor || !scope) continue
|
|
547
|
+
if (count++ >= this.limits.supervisorsPerUser) break
|
|
548
|
+
served.add(checkpointKey(workspace.id, memberID))
|
|
549
|
+
const sessions = new Map(current.filter((m) => scope.has(m.id)).map((m) => [m.sessionID, m.id]))
|
|
550
|
+
const ctx: Context = { userID, workspace, supervisor, master: workspace.masterMemberID === memberID, scope, sessions, current: views, supervisionRevision: graph.revision }
|
|
551
|
+
try {
|
|
552
|
+
await this.step(ctx, latest, since)
|
|
553
|
+
} catch (cause) {
|
|
554
|
+
this.log(`[hub] workspace coordinator: supervisor step failed: ${(cause as Error).message}`)
|
|
555
|
+
}
|
|
556
|
+
if (!this.leading()) return
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
// Revocation: ledgers of workspaces no longer active, or of members that no longer supervise.
|
|
560
|
+
const activeIDs = new Set(active.map((w) => w.id))
|
|
561
|
+
for (const { key, value } of ledgers) {
|
|
562
|
+
if (served.has(key)) continue
|
|
563
|
+
if (activeIDs.has(value.workspaceID) && !examined.has(value.workspaceID)) continue
|
|
564
|
+
await state.delete(userID, WORKSPACE_COORDINATOR, key)
|
|
565
|
+
}
|
|
566
|
+
await state.put(userID, WORKSPACE_COORDINATOR_SCAN, SCAN_KEY, { position: latest })
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
/** A new ledger: after the supervisor's checkpoint when it has one, else a snapshot from now. */
|
|
570
|
+
private async fresh(ctx: Context, latest: string): Promise<SupervisorLedger> {
|
|
571
|
+
const { supervisor, workspace } = ctx
|
|
572
|
+
const checkpoint = await this.deps.checkpoints?.latest(ctx.userID, workspace.id, supervisor.id)
|
|
573
|
+
const resume = checkpoint && checkpoint.generation === supervisor.generation && checkpoint.sessionID === supervisor.sessionID
|
|
574
|
+
return {
|
|
575
|
+
v: 1,
|
|
576
|
+
workspaceID: workspace.id,
|
|
577
|
+
supervisorMemberID: supervisor.id,
|
|
578
|
+
sessionID: supervisor.sessionID,
|
|
579
|
+
generation: supervisor.generation,
|
|
580
|
+
position: resume ? checkpoint.position : latest,
|
|
581
|
+
...(resume ? {} : { snapshot: { reason: "new" as const, dropped: 0 } }),
|
|
582
|
+
buffer: [],
|
|
583
|
+
window: [],
|
|
584
|
+
history: [],
|
|
585
|
+
seq: 0,
|
|
586
|
+
}
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
private async step(ctx: Context, latest: string, since: Since): Promise<void> {
|
|
590
|
+
const { state } = this.deps
|
|
591
|
+
const key = checkpointKey(ctx.workspace.id, ctx.supervisor.id)
|
|
592
|
+
const stored = await state.get<SupervisorLedger>(ctx.userID, WORKSPACE_COORDINATOR, key)
|
|
593
|
+
const t = this.now()
|
|
594
|
+
let ledger = stored?.value
|
|
595
|
+
if (!ledger || ledger.sessionID !== ctx.supervisor.sessionID || ledger.generation !== ctx.supervisor.generation) ledger = await this.fresh(ctx, latest)
|
|
596
|
+
const pending = ledger.batch
|
|
597
|
+
// Someone may be sending it right now (another pod before failover, or a pass in flight).
|
|
598
|
+
if (pending?.state === "sending" && (pending.claimedAt ?? 0) + this.limits.claimGraceMs > t) return
|
|
599
|
+
|
|
600
|
+
let next = await this.scan(ledger, ctx, since)
|
|
601
|
+
next = { ...next, window: next.window.filter((w) => w.at > t - HOUR_MS) }
|
|
602
|
+
if (!next.batch) {
|
|
603
|
+
const budget = this.due(next, t)
|
|
604
|
+
if (budget) next = await this.cut(next, ctx, budget, t)
|
|
605
|
+
}
|
|
606
|
+
const batch = next.batch
|
|
607
|
+
const sendable = batch !== undefined && (batch.state === "ready" || batch.state === "sending" || (batch.state === "retry" && (batch.nextAt ?? 0) <= t))
|
|
608
|
+
let claimed: DigestBatch | undefined
|
|
609
|
+
if (sendable) {
|
|
610
|
+
claimed = {
|
|
611
|
+
...batch,
|
|
612
|
+
state: "sending",
|
|
613
|
+
attempts: batch.attempts + 1,
|
|
614
|
+
claimedBy: this.owner,
|
|
615
|
+
claimedAt: t,
|
|
616
|
+
// A claim nobody recorded: the earlier send may or may not have landed.
|
|
617
|
+
...(batch.state === "sending" ? { last: "uncertain" as const } : {}),
|
|
618
|
+
}
|
|
619
|
+
next = { ...next, batch: claimed }
|
|
620
|
+
}
|
|
621
|
+
if (!this.leading()) return
|
|
622
|
+
const put = await state.put(ctx.userID, WORKSPACE_COORDINATOR, key, next, { ifVersion: stored?.version ?? 0 })
|
|
623
|
+
if (!put.ok || !claimed) return
|
|
624
|
+
await this.attempt(ctx, key, next, claimed)
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
/** Reads the journal after the ledger's position into its buffer (or owes a snapshot). */
|
|
628
|
+
private async scan(ledger: SupervisorLedger, ctx: Context, since: Since): Promise<SupervisorLedger> {
|
|
629
|
+
const result = await since(ledger.position)
|
|
630
|
+
if ("reset" in result) {
|
|
631
|
+
const position = await this.deps.events.latest(ctx.userID)
|
|
632
|
+
const urgent = ledger.snapshot?.urgent === true || ledger.buffer.some((l) => l.p)
|
|
633
|
+
return { ...without(ledger, "firstAt", "lastAt"), position, buffer: [], snapshot: { reason: "reset", dropped: ledger.snapshot?.dropped ?? 0, ...urgentIf(urgent) } }
|
|
634
|
+
}
|
|
635
|
+
if (!result.records.length) return ledger
|
|
636
|
+
let { snapshot, firstAt, lastAt } = ledger
|
|
637
|
+
let buffer: Line[] = [...ledger.buffer]
|
|
638
|
+
const t = this.now()
|
|
639
|
+
for (const record of result.records) {
|
|
640
|
+
const raw = digestLine(record, ctx)
|
|
641
|
+
if (!raw) continue
|
|
642
|
+
const line = this.fold(raw, ledger, t)
|
|
643
|
+
if (snapshot) {
|
|
644
|
+
if (snapshot.reason === "overflow") snapshot = { ...snapshot, dropped: snapshot.dropped + 1 }
|
|
645
|
+
if (line.p) snapshot = { ...snapshot, urgent: true } // folded in: the snapshot wakes and steers
|
|
646
|
+
continue
|
|
647
|
+
}
|
|
648
|
+
// A state line replaces the routine one held for the same subject (only the latest state matters), never a
|
|
649
|
+
// held priority line: an error or a block that cleared meanwhile is still news, followed by the recovery.
|
|
650
|
+
// Presence (p:<session>) too: a removed → added flap leaves only the latest, which cut() drops when it equals
|
|
651
|
+
// what the supervisor was last told (RF-08), never by comparing with a held line (duplicates exist).
|
|
652
|
+
if (line.k !== undefined) buffer = buffer.filter((held) => held.k !== line.k || held.p)
|
|
653
|
+
if (buffer.length >= this.limits.bufferLines) {
|
|
654
|
+
snapshot = { reason: "overflow", dropped: 1, ...urgentIf(line.p || buffer.some((l) => l.p)) }
|
|
655
|
+
buffer = []
|
|
656
|
+
firstAt = lastAt = undefined
|
|
657
|
+
continue
|
|
658
|
+
}
|
|
659
|
+
buffer.push(line)
|
|
660
|
+
firstAt ??= record.event.at
|
|
661
|
+
lastAt = record.event.at
|
|
662
|
+
}
|
|
663
|
+
const position = result.records.at(-1)!.position
|
|
664
|
+
const base = without(ledger, "firstAt", "lastAt", "snapshot")
|
|
665
|
+
return { ...base, position, buffer, ...(snapshot ? { snapshot } : {}), ...(firstAt !== undefined && buffer.length ? { firstAt, lastAt: lastAt! } : {}) }
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
/** Whether a batch is due now, and the budget it may use; undefined when not due or over budget. */
|
|
669
|
+
private due(ledger: SupervisorLedger, t: number): DigestBatch["budget"] | undefined {
|
|
670
|
+
const routine = ledger.window.filter((w) => w.budget === "routine").length < this.limits.batchesPerHour
|
|
671
|
+
const priority = ledger.window.filter((w) => w.budget === "priority").length < this.limits.priorityPerHour
|
|
672
|
+
if (ledger.snapshot?.urgent) return priority ? "priority" : routine ? "routine" : undefined
|
|
673
|
+
if (ledger.snapshot) return routine ? "routine" : undefined
|
|
674
|
+
if (!ledger.buffer.length) return undefined
|
|
675
|
+
if (ledger.buffer.some((l) => l.p)) return priority ? "priority" : routine ? "routine" : undefined
|
|
676
|
+
const wake = ledger.buffer.some((l) => l.w)
|
|
677
|
+
const quiet = t - (ledger.lastAt ?? t) >= (wake ? this.limits.wakeDebounceMs : this.limits.debounceMs)
|
|
678
|
+
const late = t - (ledger.firstAt ?? t) >= (wake ? this.limits.wakeMaxDelayMs : this.limits.maxDelayMs)
|
|
679
|
+
return (quiet || late) && routine ? "routine" : undefined
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
/** Turns the buffer (or an owed snapshot) into the next batch. */
|
|
683
|
+
private async cut(ledger: SupervisorLedger, ctx: Context, budget: DigestBatch["budget"], t: number): Promise<SupervisorLedger> {
|
|
684
|
+
const base = { ...without(ledger, "firstAt", "lastAt", "snapshot"), buffer: [] as Line[] }
|
|
685
|
+
let listed: Line[]
|
|
686
|
+
let more = 0
|
|
687
|
+
let wake: boolean
|
|
688
|
+
if (ledger.snapshot) {
|
|
689
|
+
if (ctx.scope.size === 0) return base // nothing to tell yet
|
|
690
|
+
const all = (await this.snapshotLines(ctx)).map((l) => this.fold(l, ledger, t))
|
|
691
|
+
listed = all.slice(0, this.limits.linesPerBatch)
|
|
692
|
+
more = all.length - listed.length
|
|
693
|
+
// A snapshot is routine (queued) unless priority news was folded into it or it shows a member blocked,
|
|
694
|
+
// erroring or with open blockers: then it wakes and steers, using the priority budget while there is one.
|
|
695
|
+
wake = ledger.snapshot.urgent === true || all.some((l) => l.p)
|
|
696
|
+
if (wake && budget === "routine" && ledger.window.filter((w) => w.budget === "priority").length < this.limits.priorityPerHour) budget = "priority"
|
|
697
|
+
} else {
|
|
698
|
+
// Only what the supervisor does not know yet: a state line repeating the last delivered state is dropped,
|
|
699
|
+
// unless a priority line about the same subject goes out with it (then it reports the recovery).
|
|
700
|
+
const known = ledger.facts ?? {}
|
|
701
|
+
const urgentSubjects = new Set(ledger.buffer.flatMap((l) => (l.p && l.k !== undefined ? [l.k] : [])))
|
|
702
|
+
const news = ledger.buffer.filter((l) => l.p || l.k === undefined || urgentSubjects.has(l.k) || known[l.k] !== l.f)
|
|
703
|
+
if (!news.length) return base // nothing material: skipped, no budget used
|
|
704
|
+
const ordered = [...news.filter((l) => l.p), ...news.filter((l) => !l.p)]
|
|
705
|
+
listed = ordered.slice(0, this.limits.linesPerBatch)
|
|
706
|
+
more = ordered.length - listed.length
|
|
707
|
+
wake = listed.some((l) => l.p || l.w)
|
|
708
|
+
}
|
|
709
|
+
const lines = listed.map((l) => l.text)
|
|
710
|
+
const facts = Object.fromEntries(listed.flatMap((l) => [...(l.k !== undefined && l.f !== undefined ? [[l.k, l.f] as const] : []), ...Object.entries(l.x ?? {})]))
|
|
711
|
+
const alerts = [...new Set(listed.flatMap((l) => (l.p ? (l.b ?? []) : [])))]
|
|
712
|
+
const seq = ledger.seq + 1
|
|
713
|
+
const batch: DigestBatch = {
|
|
714
|
+
id: this.newBatchID(),
|
|
715
|
+
seq,
|
|
716
|
+
kind: ledger.snapshot ? "snapshot" : "delta",
|
|
717
|
+
...(ledger.snapshot ? { reason: ledger.snapshot.reason, dropped: ledger.snapshot.dropped } : {}),
|
|
718
|
+
budget,
|
|
719
|
+
wake,
|
|
720
|
+
facts,
|
|
721
|
+
...(alerts.length ? { alerts } : {}),
|
|
722
|
+
lines,
|
|
723
|
+
more,
|
|
724
|
+
position: ledger.position,
|
|
725
|
+
cutAt: t,
|
|
726
|
+
state: "ready",
|
|
727
|
+
attempts: 0,
|
|
728
|
+
}
|
|
729
|
+
return { ...base, seq, batch }
|
|
730
|
+
}
|
|
731
|
+
|
|
732
|
+
/** The snapshot, line by line; member states and open assignments are also the states it reports. */
|
|
733
|
+
private async snapshotLines(ctx: Context): Promise<Line[]> {
|
|
734
|
+
const supervised = ctx.current.filter((v) => ctx.scope.has(v.member.id))
|
|
735
|
+
const facts: ReadonlyMap<string, BoardSession> = this.deps.sessions ? await this.deps.sessions(ctx.userID, supervised.map((v) => v.member.sessionID)) : new Map()
|
|
736
|
+
const open: Line[] = []
|
|
737
|
+
let cursor: string | undefined
|
|
738
|
+
do {
|
|
739
|
+
const page = await this.deps.members.assignments(ctx.userID, ctx.workspace.id, { limit: LIST_LIMIT_MAX, ...(cursor ? { cursor } : {}) })
|
|
740
|
+
for (const { assignment: a } of page.items) {
|
|
741
|
+
if (OPEN.has(a.status) && ctx.scope.has(a.memberID)) open.push({ p: false, text: `assignment ${a.id} ${a.memberID} ${a.status}`, k: `a:${a.id}`, f: assignmentFact(a.status, a.memberID) })
|
|
742
|
+
}
|
|
743
|
+
cursor = page.cursor
|
|
744
|
+
} while (cursor)
|
|
745
|
+
const head: Line = { p: false, text: `workspace ${ctx.workspace.id} revision=${ctx.workspace.revision} supervision=${ctx.supervisionRevision} members=${supervised.length} unresolved=${open.length}` }
|
|
746
|
+
const memberLines = supervised.map(({ member, presence }): Line => {
|
|
747
|
+
const fact = facts.get(member.sessionID)
|
|
748
|
+
const blockers = fact?.blockers ? `blockers=${fact.blockers.length}` : "blockers=?"
|
|
749
|
+
const text = `member ${member.id} ${member.sessionID} ${member.role} ${presence}${fact ? ` state=${fact.state}` : ""} ${blockers}`
|
|
750
|
+
const urgent = fact !== undefined && (URGENT_STATES.has(fact.state) || (fact.blockers?.length ?? 0) > 0)
|
|
751
|
+
const reasons = urgent
|
|
752
|
+
? [...new Set([...(URGENT_STATES.has(fact.state) ? [`state:${fact.state}`] : []), ...(fact.blockers ?? []).map((b) => `blocker:${b.kind}`)])].map((r) => `${member.sessionID} ${r}`)
|
|
753
|
+
: []
|
|
754
|
+
const x = presence === "online" || presence === "offline" ? { x: { [`p:${member.sessionID}`]: presenceFact(presence === "online") } } : {}
|
|
755
|
+
return fact ? { p: urgent, text, k: `s:${member.sessionID}`, f: sessionFact(fact.state), ...x, ...(reasons.length ? { b: reasons } : {}) } : { p: false, text, ...x }
|
|
756
|
+
})
|
|
757
|
+
return [head, ...memberLines, ...open]
|
|
758
|
+
}
|
|
759
|
+
|
|
760
|
+
/** Sends a claimed batch and records the outcome (only if the ledger still holds that batch). */
|
|
761
|
+
private async attempt(ctx: Context, key: string, ledger: SupervisorLedger, batch: DigestBatch): Promise<void> {
|
|
762
|
+
if (!this.leading()) return // the claim stays: after claimGraceMs the next leader resends it
|
|
763
|
+
const text = this.text(ctx, ledger, batch)
|
|
764
|
+
let result: { messageID: string; delivery?: string; woke?: boolean } | undefined
|
|
765
|
+
let failure: "uncertain" | "refused" | undefined
|
|
766
|
+
try {
|
|
767
|
+
// Routine batches are admitted for the supervisor's next turn, never starting one (RP-07 #10).
|
|
768
|
+
result = await this.bounded(this.deps.send(ctx.userID, ctx.supervisor.sessionID, text, { wake: batch.wake ?? true, urgent: batch.budget === "priority" }))
|
|
769
|
+
} catch (cause) {
|
|
770
|
+
failure = cause instanceof AgentError && REFUSED.has(cause.code) ? "refused" : "uncertain"
|
|
771
|
+
this.log(`[hub] workspace coordinator: digest ${batch.id} attempt ${batch.attempts} ${failure}: ${(cause as Error).message}`)
|
|
772
|
+
}
|
|
773
|
+
const t = this.now()
|
|
774
|
+
await updateVersioned<SupervisorLedger>(this.deps.state, ctx.userID, WORKSPACE_COORDINATOR, key, (current) => {
|
|
775
|
+
if (!current?.batch || current.batch.id !== batch.id) return undefined // resolved by a checkpoint, or revoked
|
|
776
|
+
const sent = { ...batch, ...(failure ? { last: failure } : {}) }
|
|
777
|
+
const window = failure === "refused" ? current.window : [...current.window, { at: t, budget: batch.budget }]
|
|
778
|
+
// The fold window starts only with a wake the hub carried out (not over the wake budget or hop cap).
|
|
779
|
+
const alerts = result && woke(result) ? this.alerted(current, batch, t) : this.alerted(current, { ...batch, alerts: [] }, t)
|
|
780
|
+
if (!failure) return { ...without(current, "batch"), window, facts: reported(current, batch), alerts, history: this.outcome(current, sent, "delivered", t, result) }
|
|
781
|
+
if (batch.attempts >= this.limits.maxAttempts) {
|
|
782
|
+
return {
|
|
783
|
+
...without(current, "batch", "firstAt", "lastAt", "facts"),
|
|
784
|
+
window,
|
|
785
|
+
buffer: [],
|
|
786
|
+
snapshot: { reason: "abandoned" as const, dropped: 0, ...urgentIf(batch.budget === "priority" || batch.wake === true || current.buffer.some((l) => l.p)) },
|
|
787
|
+
history: this.outcome(current, sent, "abandoned", t),
|
|
788
|
+
}
|
|
789
|
+
}
|
|
790
|
+
const wait = this.limits.retryMs[Math.min(batch.attempts - 1, this.limits.retryMs.length - 1)]!
|
|
791
|
+
return { ...current, window, batch: { ...without(sent, "claimedBy", "claimedAt"), state: "retry", nextAt: t + wait } }
|
|
792
|
+
})
|
|
793
|
+
}
|
|
794
|
+
|
|
795
|
+
/** The send, or a rejection after sendTimeoutMs (it may still land: the caller treats that as uncertain). */
|
|
796
|
+
private bounded<T>(send: Promise<T>): Promise<T> {
|
|
797
|
+
let handle: unknown
|
|
798
|
+
const timeout = new Promise<never>((_, reject) => {
|
|
799
|
+
handle = this.timers.set(() => reject(new Error(`no answer within ${this.limits.sendTimeoutMs} ms`)), this.limits.sendTimeoutMs)
|
|
800
|
+
})
|
|
801
|
+
return Promise.race([send, timeout]).finally(() => this.timers.clear(handle))
|
|
802
|
+
}
|
|
803
|
+
|
|
804
|
+
private outcome(ledger: SupervisorLedger, batch: DigestBatch, outcome: DigestOutcome["outcome"], at = this.now(), result?: { messageID: string; delivery?: string }): DigestOutcome[] {
|
|
805
|
+
const entry: DigestOutcome = {
|
|
806
|
+
batchID: batch.id,
|
|
807
|
+
seq: batch.seq,
|
|
808
|
+
kind: batch.kind,
|
|
809
|
+
at,
|
|
810
|
+
attempts: batch.attempts,
|
|
811
|
+
outcome,
|
|
812
|
+
...(result ? { messageID: result.messageID, ...(result.delivery ? { delivery: result.delivery } : {}) } : {}),
|
|
813
|
+
}
|
|
814
|
+
return [...ledger.history, entry].slice(-HISTORY_MAX)
|
|
815
|
+
}
|
|
816
|
+
|
|
817
|
+
private text(ctx: Context, ledger: SupervisorLedger, batch: DigestBatch): string {
|
|
818
|
+
const priority = batch.kind === "delta" ? batch.lines.filter((l) => l.startsWith("!")).length : 0
|
|
819
|
+
const what =
|
|
820
|
+
batch.kind === "delta"
|
|
821
|
+
? `delta · ${batch.lines.length + batch.more} updates${priority ? ` (${priority} priority)` : ""}`
|
|
822
|
+
: `snapshot (${
|
|
823
|
+
{
|
|
824
|
+
new: "first digest",
|
|
825
|
+
reset: "the event journal no longer covers your last cursor (expired or the hub restarted); earlier updates may be missing",
|
|
826
|
+
overflow: `buffer full: ${batch.dropped ?? 0} updates dropped; this snapshot replaces them`,
|
|
827
|
+
abandoned: "an earlier batch was abandoned after repeated failed sends and may not have arrived",
|
|
828
|
+
}[batch.reason ?? "new"]
|
|
829
|
+
})`
|
|
830
|
+
const previous = ledger.history.at(-1)
|
|
831
|
+
const delivery = batch.wake === false ? " · routine: queued for your next turn, no turn was started for it" : ""
|
|
832
|
+
return [
|
|
833
|
+
`[hub] workspace digest · automated, not from the user · workspace ${ctx.workspace.id} · you supervise as ${ctx.supervisor.id}${delivery}`,
|
|
834
|
+
`batch ${batch.id} #${batch.seq} · ${what} · attempt ${batch.attempts} · at-least-once: if you already handled ${batch.id}, skip this copy${previous ? ` · previous ${previous.batchID} ${previous.outcome}` : ""}`,
|
|
835
|
+
...batch.lines,
|
|
836
|
+
...(batch.more ? [`… ${batch.more} more updates not listed`] : []),
|
|
837
|
+
`cursor ${this.deps.cursors.encode(ctx.userID, batch.position)}`,
|
|
838
|
+
"No reply needed. IDs only: the user's workspace board has details.",
|
|
839
|
+
].join("\n")
|
|
840
|
+
}
|
|
841
|
+
}
|