@huynle/hub 0.7.4 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (221) hide show
  1. package/CHANGELOG.md +217 -0
  2. package/Dockerfile +7 -2
  3. package/README.md +148 -19
  4. package/package.json +5 -3
  5. package/packages/claude-code/package.json +1 -1
  6. package/packages/claude-code/src/daemon.ts +8 -1
  7. package/packages/claude-code/src/launcher.ts +29 -9
  8. package/packages/claude-code/src/profile.ts +173 -0
  9. package/packages/claude-code/src/sessions.ts +22 -4
  10. package/packages/cli/dist/hub-cli.js +6474 -3470
  11. package/packages/cli/package.json +1 -1
  12. package/packages/cli/src/artifact-jobs-mcp.ts +226 -0
  13. package/packages/cli/src/artifacts-mcp.ts +160 -0
  14. package/packages/cli/src/bootstrap/enrollment/enroll.ts +20 -8
  15. package/packages/cli/src/bootstrap/enrollment/policy.ts +125 -12
  16. package/packages/cli/src/bootstrap/enrollment/status.ts +3 -2
  17. package/packages/cli/src/cli.ts +95 -13
  18. package/packages/cli/src/format.ts +48 -10
  19. package/packages/cli/src/http.ts +1 -0
  20. package/packages/cli/src/intents.ts +23 -0
  21. package/packages/cli/src/login.ts +20 -6
  22. package/packages/cli/src/mcp.ts +380 -9
  23. package/packages/cli/src/messaging.ts +3 -1
  24. package/packages/cli/src/setup/commands.ts +3 -2
  25. package/packages/cli/src/setup/runtime.ts +3 -2
  26. package/packages/cli/src/team-intake-mcp.ts +158 -0
  27. package/packages/cli/src/teams-mcp.ts +403 -0
  28. package/packages/cli/src/watchlist.ts +135 -0
  29. package/packages/codex/package.json +1 -1
  30. package/packages/codex/src/profile.ts +163 -0
  31. package/packages/codex/src/sessions.ts +12 -0
  32. package/packages/codex/src/spoke.ts +19 -1
  33. package/packages/node/package.json +1 -1
  34. package/packages/node/src/config.ts +96 -2
  35. package/packages/node/src/opencode-warm.ts +325 -0
  36. package/packages/node/src/run.ts +27 -1
  37. package/packages/preset/package.json +1 -1
  38. package/packages/preset/src/profile.ts +34 -0
  39. package/packages/preset/src/spoke.ts +4 -0
  40. package/packages/protocol/package.json +1 -1
  41. package/packages/protocol/src/artifacts.ts +142 -0
  42. package/packages/protocol/src/concierge.ts +366 -0
  43. package/packages/protocol/src/contracts.ts +104 -0
  44. package/packages/protocol/src/graph.ts +70 -0
  45. package/packages/protocol/src/hub-workspace.ts +201 -0
  46. package/packages/protocol/src/index.ts +169 -6
  47. package/packages/protocol/src/launch.ts +22 -0
  48. package/packages/protocol/src/profiles.ts +301 -0
  49. package/packages/protocol/src/teams.ts +204 -0
  50. package/packages/runtime-openshell/package.json +1 -1
  51. package/packages/runtime-sdk/package.json +1 -1
  52. package/packages/sdk/package.json +1 -1
  53. package/packages/sdk/src/address.ts +50 -2
  54. package/packages/sdk/src/artifact-jobs.ts +194 -0
  55. package/packages/sdk/src/artifacts.ts +187 -0
  56. package/packages/sdk/src/client.ts +226 -5
  57. package/packages/sdk/src/concierge.ts +88 -0
  58. package/packages/sdk/src/errors.ts +3 -1
  59. package/packages/sdk/src/index.ts +5 -0
  60. package/packages/sdk/src/team-intake.ts +122 -0
  61. package/packages/sdk/src/teams.ts +346 -0
  62. package/packages/sdk/src/types.ts +548 -1
  63. package/packages/spoke-sdk/package.json +1 -1
  64. package/packages/spoke-sdk/src/index.ts +1 -0
  65. package/packages/spoke-sdk/src/profile-apply.ts +245 -0
  66. package/src/central/actions.ts +7 -0
  67. package/src/central/agent/acting-routes.ts +2 -1
  68. package/src/central/agent/acting-switch.ts +2 -1
  69. package/src/central/agent/acting.ts +215 -10
  70. package/src/central/agent/app-tokens.ts +3 -2
  71. package/src/central/agent/audit.ts +17 -0
  72. package/src/central/agent/body.ts +163 -0
  73. package/src/central/agent/decisions.ts +69 -2
  74. package/src/central/agent/delivery.ts +95 -15
  75. package/src/central/agent/directory-routes.ts +37 -7
  76. package/src/central/agent/directory.ts +70 -14
  77. package/src/central/agent/errors.ts +5 -0
  78. package/src/central/agent/events-routes.ts +21 -4
  79. package/src/central/agent/events.ts +11 -3
  80. package/src/central/agent/form-check.ts +85 -0
  81. package/src/central/agent/grant-requests.ts +2 -1
  82. package/src/central/agent/grant-routes.ts +4 -3
  83. package/src/central/agent/grants.ts +29 -2
  84. package/src/central/agent/idempotency.ts +133 -0
  85. package/src/central/agent/intent-routes.ts +116 -0
  86. package/src/central/agent/intents.ts +505 -0
  87. package/src/central/agent/interaction-graph-routes.ts +212 -0
  88. package/src/central/agent/interaction-graph.ts +865 -0
  89. package/src/central/agent/launch-engine.ts +16 -2
  90. package/src/central/agent/launch-routes.ts +6 -1
  91. package/src/central/agent/launch-spec.ts +14 -1
  92. package/src/central/agent/message-routes.ts +2 -1
  93. package/src/central/agent/messages.ts +27 -2
  94. package/src/central/agent/names.ts +12 -2
  95. package/src/central/agent/read-routes.ts +86 -19
  96. package/src/central/agent/routes.ts +176 -5
  97. package/src/central/agent/webhook-routes.ts +2 -1
  98. package/src/central/agent-data.ts +4 -0
  99. package/src/central/artifacts/access.ts +44 -0
  100. package/src/central/artifacts/blob-store.ts +113 -0
  101. package/src/central/artifacts/blob-support.ts +98 -0
  102. package/src/central/artifacts/events.ts +27 -0
  103. package/src/central/artifacts/host-caps.ts +162 -0
  104. package/src/central/artifacts/host.ts +315 -0
  105. package/src/central/artifacts/idempotency.ts +109 -0
  106. package/src/central/artifacts/jobs-routes.ts +773 -0
  107. package/src/central/artifacts/jobs.ts +344 -0
  108. package/src/central/artifacts/maintenance.ts +103 -0
  109. package/src/central/artifacts/routes.ts +360 -0
  110. package/src/central/artifacts/store.ts +776 -0
  111. package/src/central/artifacts/workspaces.ts +31 -0
  112. package/src/central/backend/file-blobs.ts +377 -0
  113. package/src/central/backend/index.ts +12 -3
  114. package/src/central/backend/memory-blobs.ts +116 -0
  115. package/src/central/backend/postgres-blobs.ts +513 -0
  116. package/src/central/backend/postgres-migrate.ts +33 -0
  117. package/src/central/backend/postgres-state.ts +19 -0
  118. package/src/central/backend/types.ts +8 -0
  119. package/src/central/concierge/activity.ts +142 -0
  120. package/src/central/concierge/arm.ts +63 -0
  121. package/src/central/concierge/board-render.ts +190 -0
  122. package/src/central/concierge/board-results.ts +169 -0
  123. package/src/central/concierge/board-routes.ts +339 -0
  124. package/src/central/concierge/board.ts +249 -0
  125. package/src/central/concierge/context.ts +221 -0
  126. package/src/central/concierge/entitlements.ts +13 -0
  127. package/src/central/concierge/guard.ts +120 -0
  128. package/src/central/concierge/inbound.ts +89 -0
  129. package/src/central/concierge/index.ts +84 -0
  130. package/src/central/concierge/notes.ts +41 -0
  131. package/src/central/concierge/notifier.ts +986 -0
  132. package/src/central/concierge/prompt.ts +130 -0
  133. package/src/central/concierge/proposals.ts +262 -0
  134. package/src/central/concierge/push-remap.ts +49 -0
  135. package/src/central/concierge/routes.ts +62 -0
  136. package/src/central/concierge/schedule.ts +113 -0
  137. package/src/central/concierge/service.ts +243 -0
  138. package/src/central/concierge/session-routes.ts +69 -0
  139. package/src/central/concierge/settings.ts +217 -0
  140. package/src/central/concierge/supervisor.ts +740 -0
  141. package/src/central/concierge/tiers.ts +260 -0
  142. package/src/central/concierge/watchlist-routes.ts +277 -0
  143. package/src/central/concierge/watchlist.ts +273 -0
  144. package/src/central/config.ts +7 -1
  145. package/src/central/http.ts +632 -49
  146. package/src/central/mcp-http.ts +10 -21
  147. package/src/central/profiles/apply.ts +333 -0
  148. package/src/central/profiles/content.ts +174 -0
  149. package/src/central/profiles/events.ts +40 -0
  150. package/src/central/profiles/idempotency.ts +14 -0
  151. package/src/central/profiles/import.ts +139 -0
  152. package/src/central/profiles/routes.ts +356 -0
  153. package/src/central/profiles/store.ts +597 -0
  154. package/src/central/profiles/usage-ports.ts +120 -0
  155. package/src/central/profiles/usage.ts +364 -0
  156. package/src/central/push.ts +7 -0
  157. package/src/central/registry.ts +43 -6
  158. package/src/central/store.ts +4 -0
  159. package/src/central/teams/collections.ts +34 -0
  160. package/src/central/teams/discovery.ts +346 -0
  161. package/src/central/teams/draft-edits.ts +46 -0
  162. package/src/central/teams/events.ts +46 -0
  163. package/src/central/teams/intake-routes.ts +100 -0
  164. package/src/central/teams/intake.ts +772 -0
  165. package/src/central/teams/policy.ts +231 -0
  166. package/src/central/teams/routes.ts +516 -0
  167. package/src/central/teams/seats.ts +272 -0
  168. package/src/central/teams/staffing.ts +888 -0
  169. package/src/central/teams/store.ts +667 -0
  170. package/src/central/teams/validation.ts +256 -0
  171. package/src/central/workspaces/board.ts +408 -0
  172. package/src/central/workspaces/checkpoints.ts +316 -0
  173. package/src/central/workspaces/coordinator.ts +841 -0
  174. package/src/central/workspaces/events.ts +85 -0
  175. package/src/central/workspaces/lifecycle.ts +93 -0
  176. package/src/central/workspaces/master.ts +490 -0
  177. package/src/central/workspaces/members.ts +632 -0
  178. package/src/central/workspaces/routes.ts +354 -0
  179. package/src/central/workspaces/sidechats-routes.ts +410 -0
  180. package/src/central/workspaces/sidechats.ts +482 -0
  181. package/src/central/workspaces/store.ts +571 -0
  182. package/src/central/workspaces/supervision.ts +355 -0
  183. package/src/index.ts +97 -15
  184. package/src/spoke/concierge-agent.ts +33 -0
  185. package/src/spoke/deliver.ts +22 -4
  186. package/src/spoke/fork-revert.ts +142 -0
  187. package/src/spoke/hub-rpc.ts +41 -5
  188. package/src/spoke/hub-tool.ts +383 -18
  189. package/src/spoke/listing.ts +47 -4
  190. package/src/spoke/profile-apply.ts +298 -0
  191. package/src/spoke/session-context.ts +176 -0
  192. package/src/spoke/spoke.ts +319 -20
  193. package/src/spoke/timeline.ts +6 -1
  194. package/src/spoke/tracker.ts +23 -1
  195. package/src/spoke/who-teams.ts +81 -0
  196. package/src/tui-messaging.ts +32 -18
  197. package/web/dist/assets/ArtifactPage-BS0gVkfu.js +1 -0
  198. package/web/dist/assets/ArtifactPane-PLYHNs6s.js +1 -0
  199. package/web/dist/assets/ArtifactWorkbench-9eTj1CeW.css +1 -0
  200. package/web/dist/assets/ArtifactWorkbench-LkDyJ5Dv.js +6 -0
  201. package/web/dist/assets/Library-CtxRqTjo.js +2 -0
  202. package/web/dist/assets/Library-DYsFj_5c.css +1 -0
  203. package/web/dist/assets/ProfilesDestination-BYiHKLmc.js +1 -0
  204. package/web/dist/assets/TeamsPage-BwjsAAUC.js +1 -0
  205. package/web/dist/assets/Watchtower-C4swrWVs.js +2 -0
  206. package/web/dist/assets/Watchtower-CTgqHk5g.css +1 -0
  207. package/web/dist/assets/index-DaH5GUdV.js +31 -0
  208. package/web/dist/assets/index-WWuKfWxL.css +1 -0
  209. package/web/dist/assets/live-BUofsQpz.css +1 -0
  210. package/web/dist/assets/live-YyTjE47g.js +5 -0
  211. package/web/dist/assets/pane-layout-CKC4I00k.js +1 -0
  212. package/web/dist/assets/plural-Bspvoqfq.js +1 -0
  213. package/web/dist/assets/renderer2d-CX5j3NcD.js +1 -0
  214. package/web/dist/assets/renderer3d-BXIVOvfi.js +5393 -0
  215. package/web/dist/assets/safe-area-DfPsOdWR.js +1 -0
  216. package/web/dist/assets/source-CvUHTc3O.js +1 -0
  217. package/web/dist/assets/ui-C9Igxzjm.js +1 -0
  218. package/web/dist/index.html +5 -2
  219. package/web/dist/sw.js +22 -2
  220. package/web/dist/assets/index-Dm8OmkaO.css +0 -1
  221. package/web/dist/assets/index-FUJbj_Du.js +0 -20
@@ -0,0 +1,31 @@
1
+ // The workspace facts artifact access needs (ArtifactWorkspaces, ./access.ts) from the HWA-03/04 stores: a
2
+ // workspace of the user exists, and a session is a *current* member of it (re-read on every access, so leaving or
3
+ // archiving the workspace ends a session's reads at once). Membership grants read of workspace-visible artifacts
4
+ // only; it never grants writes (v1 write policy) and role labels confer nothing.
5
+ import type { StateStore } from "../backend/types"
6
+ import { MembershipStore } from "../workspaces/members"
7
+ import { WorkspaceStore } from "../workspaces/store"
8
+ import type { ArtifactWorkspaces } from "./access"
9
+
10
+ export interface ArtifactWorkspaceSources {
11
+ readonly workspaces: Pick<WorkspaceStore, "get">
12
+ readonly members: Pick<MembershipStore, "membershipsOf">
13
+ }
14
+
15
+ export function artifactWorkspaces(sources: ArtifactWorkspaceSources): ArtifactWorkspaces {
16
+ return {
17
+ exists: async (userID, workspaceID) => (await sources.workspaces.get(userID, workspaceID)) !== undefined,
18
+ isCurrentMember: async (userID, workspaceID, sessionID) => (await sources.members.membershipsOf(userID, sessionID)).some((m) => m.workspaceID === workspaceID),
19
+ }
20
+ }
21
+
22
+ /**
23
+ * Read-only workspace facts straight from the StateStore, for a hub that has not wired the workspace routes'
24
+ * stores yet (HWA-06 passes its own). Only `get` and `membershipsOf` are ever called: machine ownership and
25
+ * session presence are not consulted for reads.
26
+ */
27
+ export function artifactWorkspacesFromState(state: StateStore): ArtifactWorkspaces {
28
+ const workspaces = new WorkspaceStore({ state, ownsMachine: () => false })
29
+ const members = new MembershipStore({ state, workspaces, presence: () => "unknown" })
30
+ return artifactWorkspaces({ workspaces, members })
31
+ }
@@ -0,0 +1,377 @@
1
+ // BlobStore in files under the hub's state directory (HWA-21, decision tctkrik8): the standalone hub's artifact
2
+ // bytes, the contract in ../artifacts/blob-store.ts (the shared suite in test/blob-store-conformance.ts).
3
+ //
4
+ // Layout (folders 0700, files 0600):
5
+ // <stateDir>/blobs/users/<user>/sha256/<ab>/<cd>/<64 hex> the bytes (mtime = touchedAt)
6
+ // <stateDir>/blobs/users/<user>/sha256/<ab>/<cd>/<64 hex>.meta {v, hash, bytes, storedAt}
7
+ // <stateDir>/blobs/users/<user>/tmp/<uuid>.tmp uploads in flight
8
+ // Blobs are per user (the tenant key): the same bytes stored by two users are two files. <user> is the user ID when
9
+ // it is a short lowercase [a-z0-9_-] name, else `~<sha256 of it>` (as the file StateStore does), so no ID becomes a
10
+ // traversal. File names come only from validated hashes.
11
+ //
12
+ // Writes: the body streams into a fresh temp file (O_EXCL, O_NOFOLLOW), hashed and bounded as it comes
13
+ // (readBlobBody); it is fsynced, then the sidecar is written (temp + rename), then the temp file is renamed into
14
+ // place: the rename is the commit, so readers never see a partial blob. The quota check, the existence check and
15
+ // the rename run in one synchronous step against the in-process index, so concurrent puts never overshoot the quota
16
+ // and the same bytes stored concurrently become one blob. delete's touchedBefore check and the unlink are one
17
+ // synchronous step too.
18
+ //
19
+ // The index (hash → bytes, storedAt, touchedAt per user) is built from disk on a user's first use: so a restart
20
+ // recovers it. That scan also sweeps crash leftovers (temp files, sidecars without bytes) and ignores anything
21
+ // that is not a regular file with a valid name (symlinks are never followed or indexed).
22
+ //
23
+ // Path defense: every folder from <stateDir>/blobs down is created and checked as a real directory (lstat, never
24
+ // through a symlink), reads open with O_NOFOLLOW after checking that the blob's folder resolves to where it should,
25
+ // and reads verify the sha256 of what they stream (bytes changed on disk fail the read instead of being served).
26
+ //
27
+ // One process owns a state directory (the standalone hub); a cluster keeps blobs in Postgres (HWA-22).
28
+ import { createHash, randomUUID } from "node:crypto"
29
+ import {
30
+ chmodSync,
31
+ closeSync,
32
+ constants,
33
+ fsyncSync,
34
+ lstatSync,
35
+ lutimesSync,
36
+ mkdirSync,
37
+ openSync,
38
+ readdirSync,
39
+ readFileSync,
40
+ realpathSync,
41
+ renameSync,
42
+ rmSync,
43
+ utimesSync,
44
+ writeFileSync,
45
+ } from "node:fs"
46
+ import { open as openFile, type FileHandle } from "node:fs/promises"
47
+ import path from "node:path"
48
+ import { isBlobHash, type BlobRef, type Page } from "../../protocol"
49
+ import {
50
+ type BlobBody,
51
+ type BlobDeleteOptions,
52
+ type BlobInfo,
53
+ type BlobListOptions,
54
+ type BlobPutOptions,
55
+ BlobQuotaError,
56
+ type BlobRead,
57
+ type BlobStore,
58
+ type BlobUsage,
59
+ } from "../artifacts/blob-store"
60
+ import { listCursor, listWindow, putLimits, readBlobBody } from "../artifacts/blob-support"
61
+ import type { Clock } from "./types"
62
+
63
+ /** The folder under the state directory. */
64
+ export const BLOBS_DIR = "blobs"
65
+ const READ_CHUNK = 64 * 1024
66
+ const HEX = /^[0-9a-f]{64}$/
67
+ const SAFE_SEGMENT = /^[a-z0-9_-]{1,64}$/
68
+
69
+ export interface FileBlobStoreOptions {
70
+ /** The hub's state directory; blobs go to <stateDir>/blobs/. */
71
+ readonly stateDir: string
72
+ readonly now?: Clock
73
+ }
74
+
75
+ interface Indexed {
76
+ readonly bytes: number
77
+ readonly storedAt: number
78
+ touchedAt: number
79
+ }
80
+
81
+ interface UserIndex {
82
+ readonly dir: string
83
+ readonly blobs: Map<string, Indexed>
84
+ total: number
85
+ }
86
+
87
+ /** A file-system-safe, case-insensitive-safe folder name for a user ID. */
88
+ const userSegment = (userID: string): string => (SAFE_SEGMENT.test(userID) ? userID : `~${createHash("sha256").update(userID).digest("hex")}`)
89
+ const errno = (cause: unknown): string | undefined => (cause as NodeJS.ErrnoException | undefined)?.code
90
+
91
+ export class FileBlobStore implements BlobStore {
92
+ private readonly now: Clock
93
+ private readonly root: string
94
+ private readonly users = new Map<string, UserIndex>()
95
+ /** Folders already restricted to the owner. */
96
+ private readonly restricted = new Set<string>()
97
+
98
+ constructor(options: FileBlobStoreOptions) {
99
+ this.now = options.now ?? Date.now
100
+ this.root = path.join(options.stateDir, BLOBS_DIR)
101
+ }
102
+
103
+ async put(userID: string, body: BlobBody, mediaType: string, options: BlobPutOptions = {}): Promise<BlobRef> {
104
+ const limits = putLimits(mediaType, options)
105
+ const user = this.user(userID)
106
+ const tmpDir = path.join(user.dir, "tmp")
107
+ this.folder(tmpDir)
108
+ const tmp = path.join(tmpDir, `${randomUUID()}.tmp`)
109
+ let handle: FileHandle | undefined = await openFile(tmp, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600)
110
+ try {
111
+ const file = handle
112
+ const { hash, bytes } = await readBlobBody(body, limits, async (chunk) => {
113
+ for (let offset = 0; offset < chunk.byteLength; ) offset += (await file.write(chunk, offset)).bytesWritten
114
+ })
115
+ await file.sync()
116
+ await file.close()
117
+ handle = undefined
118
+ // The commit: one synchronous step from here (no await), so the index and the files agree.
119
+ const at = this.now()
120
+ const final = this.blobPath(user, hash)
121
+ if (this.refresh(user, hash, at)) {
122
+ rmSync(tmp, { force: true })
123
+ } else {
124
+ if (user.total + bytes > limits.quotaBytes) throw new BlobQuotaError(limits.quotaBytes, user.total)
125
+ const dir = path.dirname(final)
126
+ this.folder(dir)
127
+ const metaTmp = `${tmp}.meta`
128
+ writeFileSync(metaTmp, JSON.stringify({ v: 1, hash, bytes, storedAt: at }), { mode: 0o600, flag: "wx" })
129
+ renameSync(metaTmp, `${final}.meta`)
130
+ utimesSync(tmp, new Date(at), new Date(at))
131
+ renameSync(tmp, final)
132
+ syncFolder(dir)
133
+ user.blobs.set(hash, { bytes, storedAt: at, touchedAt: at })
134
+ user.total += bytes
135
+ }
136
+ return { hash, bytes, mediaType }
137
+ } catch (error) {
138
+ await handle?.close().catch(() => {})
139
+ rmSync(tmp, { force: true })
140
+ rmSync(`${tmp}.meta`, { force: true })
141
+ throw error
142
+ }
143
+ }
144
+
145
+ async get(userID: string, hash: string): Promise<BlobRead | undefined> {
146
+ const found = this.find(userID, hash)
147
+ if (!found) return undefined
148
+ const { user, blob } = found
149
+ const file = this.blobPath(user, hash)
150
+ if (!this.contained(path.dirname(file))) return undefined
151
+ let handle: FileHandle
152
+ try {
153
+ handle = await openFile(file, constants.O_RDONLY | constants.O_NOFOLLOW)
154
+ } catch {
155
+ return undefined
156
+ }
157
+ const stat = await handle.stat().catch(() => undefined)
158
+ if (!stat?.isFile()) {
159
+ await handle.close().catch(() => {})
160
+ return undefined
161
+ }
162
+ return { hash, bytes: blob.bytes, storedAt: blob.storedAt, touchedAt: blob.touchedAt, body: verifiedStream(handle, hash, blob.bytes) }
163
+ }
164
+
165
+ async stat(userID: string, hash: string): Promise<BlobInfo | undefined> {
166
+ const found = this.find(userID, hash)
167
+ return found && info(hash, found.blob)
168
+ }
169
+
170
+ async has(userID: string, hash: string): Promise<boolean> {
171
+ return this.find(userID, hash) !== undefined
172
+ }
173
+
174
+ async touch(userID: string, hash: string): Promise<boolean> {
175
+ const found = this.find(userID, hash)
176
+ return found !== undefined && this.refresh(found.user, hash, this.now())
177
+ }
178
+
179
+ async delete(userID: string, hash: string, options: BlobDeleteOptions = {}): Promise<boolean> {
180
+ const found = this.find(userID, hash)
181
+ if (!found) return false
182
+ if (options.touchedBefore !== undefined && found.blob.touchedAt >= options.touchedBefore) return false
183
+ const file = this.blobPath(found.user, hash)
184
+ rmSync(file, { force: true })
185
+ rmSync(`${file}.meta`, { force: true })
186
+ this.forget(found.user, hash)
187
+ return true
188
+ }
189
+
190
+ async usage(userID: string): Promise<BlobUsage> {
191
+ const user = this.user(userID)
192
+ return { blobs: user.blobs.size, bytes: user.total }
193
+ }
194
+
195
+ async list(userID: string, options: BlobListOptions = {}): Promise<Page<BlobInfo>> {
196
+ const { limit, after } = listWindow(options)
197
+ const user = this.user(userID)
198
+ const hashes = [...user.blobs.keys()].filter((h) => after === undefined || h > after).sort()
199
+ const page = hashes.slice(0, limit)
200
+ const items = page.map((h) => info(h, user.blobs.get(h)!))
201
+ return hashes.length > limit ? { items, cursor: listCursor(page[page.length - 1]!) } : { items }
202
+ }
203
+
204
+ // ---- internals ------------------------------------------------------------------------------------------
205
+
206
+ private find(userID: string, hash: string): { user: UserIndex; blob: Indexed } | undefined {
207
+ if (!isBlobHash(hash)) return undefined
208
+ const user = this.user(userID)
209
+ const blob = user.blobs.get(hash)
210
+ return blob && { user, blob }
211
+ }
212
+
213
+ private forget(user: UserIndex, hash: string): void {
214
+ const blob = user.blobs.get(hash)
215
+ if (!blob) return
216
+ user.blobs.delete(hash)
217
+ user.total -= blob.bytes
218
+ }
219
+
220
+ /** Sets an indexed blob's touchedAt (on disk: its mtime, never through a link); false (and forgotten) when its file is gone. */
221
+ private refresh(user: UserIndex, hash: string, at: number): boolean {
222
+ const blob = user.blobs.get(hash)
223
+ if (!blob) return false
224
+ try {
225
+ lutimesSync(this.blobPath(user, hash), new Date(at), new Date(at))
226
+ } catch (cause) {
227
+ if (errno(cause) !== "ENOENT") throw cause
228
+ this.forget(user, hash)
229
+ return false
230
+ }
231
+ blob.touchedAt = at
232
+ return true
233
+ }
234
+
235
+ private blobPath(user: UserIndex, hash: string): string {
236
+ const hex = hash.slice("sha256:".length)
237
+ return path.join(user.dir, "sha256", hex.slice(0, 2), hex.slice(2, 4), hex)
238
+ }
239
+
240
+ /** The user's index, built from disk on first use (sweeping crash leftovers). */
241
+ private user(userID: string): UserIndex {
242
+ const loaded = this.users.get(userID)
243
+ if (loaded) return loaded
244
+ const dir = path.join(this.root, "users", userSegment(userID))
245
+ const user: UserIndex = { dir, blobs: new Map(), total: 0 }
246
+ this.scan(user)
247
+ this.users.set(userID, user)
248
+ return user
249
+ }
250
+
251
+ private scan(user: UserIndex): void {
252
+ if (!isRealDir(user.dir) || !this.contained(user.dir)) return
253
+ const tmpDir = path.join(user.dir, "tmp")
254
+ if (isRealDir(tmpDir)) for (const name of readdirSync(tmpDir)) rmSync(path.join(tmpDir, name), { force: true, recursive: true })
255
+ const shaDir = path.join(user.dir, "sha256")
256
+ for (const ab of subfolders(shaDir)) {
257
+ for (const cd of subfolders(path.join(shaDir, ab))) {
258
+ const dir = path.join(shaDir, ab, cd)
259
+ const entries = readdirSync(dir, { withFileTypes: true })
260
+ const data = new Set(entries.filter((e) => e.isFile() && HEX.test(e.name) && e.name.startsWith(ab + cd)).map((e) => e.name))
261
+ for (const entry of entries) {
262
+ if (!entry.isFile() || !entry.name.endsWith(".meta")) continue
263
+ if (!data.has(entry.name.slice(0, -".meta".length))) rmSync(path.join(dir, entry.name), { force: true })
264
+ }
265
+ for (const hex of data) {
266
+ const file = path.join(dir, hex)
267
+ const stat = lstatSync(file, { throwIfNoEntry: false })
268
+ if (!stat?.isFile()) continue
269
+ const touchedAt = Math.round(stat.mtimeMs)
270
+ const storedAt = readStoredAt(`${file}.meta`, `sha256:${hex}`, stat.size) ?? touchedAt
271
+ user.blobs.set(`sha256:${hex}`, { bytes: stat.size, storedAt, touchedAt })
272
+ user.total += stat.size
273
+ }
274
+ }
275
+ }
276
+ }
277
+
278
+ /**
279
+ * Creates a folder (and its parents up to the root) restricted to the owner, checking every level as a real
280
+ * directory on every call (a folder swapped for a link later is caught, never written through).
281
+ */
282
+ private folder(dir: string): void {
283
+ if (dir !== this.root) this.folder(path.dirname(dir))
284
+ try {
285
+ mkdirSync(dir, { mode: 0o700, recursive: dir === this.root })
286
+ } catch (cause) {
287
+ if (errno(cause) !== "EEXIST") throw cause
288
+ }
289
+ if (!isRealDir(dir)) throw new Error(`${dir} is not a directory (refusing to follow a link)`)
290
+ if (this.restricted.has(dir)) return
291
+ chmodSync(dir, 0o700)
292
+ this.restricted.add(dir)
293
+ }
294
+
295
+ /** Whether a folder under the root resolves to itself (no symlinked folder on the way). */
296
+ private contained(dir: string): boolean {
297
+ try {
298
+ const root = realpathSync(this.root)
299
+ return realpathSync(dir) === path.join(root, path.relative(this.root, dir))
300
+ } catch {
301
+ return false
302
+ }
303
+ }
304
+ }
305
+
306
+ const info = (hash: string, blob: Indexed): BlobInfo => ({ hash, bytes: blob.bytes, storedAt: blob.storedAt, touchedAt: blob.touchedAt })
307
+
308
+ function isRealDir(dir: string): boolean {
309
+ return lstatSync(dir, { throwIfNoEntry: false })?.isDirectory() === true
310
+ }
311
+
312
+ function subfolders(dir: string): string[] {
313
+ if (!isRealDir(dir)) return []
314
+ return readdirSync(dir, { withFileTypes: true })
315
+ .filter((e) => e.isDirectory() && /^[0-9a-f]{2}$/.test(e.name))
316
+ .map((e) => e.name)
317
+ }
318
+
319
+ function readStoredAt(meta: string, hash: string, bytes: number): number | undefined {
320
+ try {
321
+ if (!lstatSync(meta).isFile()) return undefined
322
+ const raw = JSON.parse(readFileSync(meta, "utf8")) as Record<string, unknown>
323
+ return raw.v === 1 && raw.hash === hash && raw.bytes === bytes && typeof raw.storedAt === "number" ? raw.storedAt : undefined
324
+ } catch {
325
+ return undefined
326
+ }
327
+ }
328
+
329
+ /** Makes a rename in a folder durable (best effort: some file systems refuse fsync on folders). */
330
+ function syncFolder(dir: string): void {
331
+ let fd: number | undefined
332
+ try {
333
+ fd = openSync(dir, constants.O_RDONLY)
334
+ fsyncSync(fd)
335
+ } catch {
336
+ // best effort
337
+ } finally {
338
+ if (fd !== undefined) closeSync(fd)
339
+ }
340
+ }
341
+
342
+ /** The file's bytes as a stream that fails (instead of ending) when they are not the expected ones. */
343
+ function verifiedStream(handle: FileHandle, hash: string, bytes: number): ReadableStream<Uint8Array> {
344
+ const hasher = createHash("sha256")
345
+ let read = 0
346
+ let closed = false
347
+ const close = async () => {
348
+ if (closed) return
349
+ closed = true
350
+ await handle.close().catch(() => {})
351
+ }
352
+ return new ReadableStream<Uint8Array>({
353
+ async pull(controller) {
354
+ try {
355
+ const buffer = new Uint8Array(READ_CHUNK)
356
+ const { bytesRead } = await handle.read(buffer, 0, READ_CHUNK, null)
357
+ if (bytesRead > 0) {
358
+ read += bytesRead
359
+ if (read > bytes) throw new Error("blob is larger on disk than recorded")
360
+ const chunk = buffer.subarray(0, bytesRead)
361
+ hasher.update(chunk)
362
+ controller.enqueue(chunk)
363
+ return
364
+ }
365
+ await close()
366
+ if (read !== bytes || `sha256:${hasher.digest("hex")}` !== hash) throw new Error("blob bytes on disk do not match their hash")
367
+ controller.close()
368
+ } catch (cause) {
369
+ await close()
370
+ controller.error(cause)
371
+ }
372
+ },
373
+ async cancel() {
374
+ await close()
375
+ },
376
+ })
377
+ }
@@ -2,7 +2,9 @@
2
2
  // a file StateStore under <stateDir>/agent, and the cluster bundle (Postgres + Valkey). startCentral creates the bundle its config selects (hubBackends:
3
3
  // file, or cluster when HUB_DATABASE_URL/HUB_VALKEY_URL are set) or takes one in its options, exposes it to
4
4
  // new code only, and flushes its state on stop.
5
+ import { FileBlobStore } from "./file-blobs"
5
6
  import { FileStateStore, type FileStateStoreOptions } from "./file-state"
7
+ import { MemoryBlobStore } from "./memory-blobs"
6
8
  import { MemoryBus } from "./memory-bus"
7
9
  import { MemoryCatalog } from "./memory-catalog"
8
10
  import { MemoryCounters, MemoryLocks } from "./memory-counters"
@@ -12,6 +14,7 @@ import { MemoryKv } from "./memory-kv"
12
14
  import { MemoryStateStore } from "./memory-state"
13
15
  import { randomUUID } from "node:crypto"
14
16
  import { RedisClient } from "bun"
17
+ import { PostgresBlobStore } from "./postgres-blobs"
15
18
  import { PostgresStateStore } from "./postgres-state"
16
19
  import { ValkeyBus } from "./valkey-bus"
17
20
  import { ValkeyCatalog } from "./valkey-catalog"
@@ -43,6 +46,7 @@ export function memoryBackends(options: MemoryBackendsOptions = {}): Backends {
43
46
  locks: new MemoryLocks({ now }),
44
47
  catalog: new MemoryCatalog({ now }),
45
48
  kv: new MemoryKv({ now }),
49
+ blobs: new MemoryBlobStore({ now }),
46
50
  }
47
51
  }
48
52
 
@@ -52,7 +56,7 @@ export interface FileBackendsOptions extends MemoryBackendsOptions, Pick<FileSta
52
56
  readonly stateDir: string
53
57
  }
54
58
 
55
- /** The StateStore in files under the state directory, every other backend in process memory, on one clock. */
59
+ /** The StateStore and artifact blobs in files under the state directory, every other backend in process memory, on one clock. */
56
60
  export function fileBackends(options: FileBackendsOptions): Backends {
57
61
  const { stateDir, debounceMs, warn, ...memory } = options
58
62
  const state = new FileStateStore({
@@ -62,7 +66,7 @@ export function fileBackends(options: FileBackendsOptions): Backends {
62
66
  ...(debounceMs !== undefined ? { debounceMs } : {}),
63
67
  ...(warn ? { warn } : {}),
64
68
  })
65
- return { ...memoryBackends(memory), state }
69
+ return { ...memoryBackends(memory), state, blobs: new FileBlobStore({ stateDir, ...(memory.now ? { now: memory.now } : {}) }) }
66
70
  }
67
71
 
68
72
  export interface ClusterBackendsOptions extends MemoryBackendsOptions {
@@ -91,7 +95,10 @@ export function clusterBackends(options: ClusterBackendsOptions): Backends {
91
95
  client = undefined
92
96
  },
93
97
  } as unknown as RedisClient
94
- const state = new PostgresStateStore({ url: cluster.databaseURL, ...(now ? { now } : {}) })
98
+ const timeout = cluster.statementTimeoutMs !== undefined ? { statementTimeoutMs: cluster.statementTimeoutMs } : {}
99
+ const state = new PostgresStateStore({ url: cluster.databaseURL, ...timeout, ...(now ? { now } : {}) })
100
+ // Artifact bytes in the same database (HWA-22, decision tctkrik8), on their own small pool.
101
+ const blobs = new PostgresBlobStore({ url: cluster.databaseURL, ...timeout, ...(now ? { now } : {}) })
95
102
  const bus = new ValkeyBus(cluster.valkeyURL, { instanceID: `${cluster.podID}-${randomUUID().slice(0, 8)}`, ...(options.timers ? { timers: options.timers } : {}) })
96
103
  return {
97
104
  state,
@@ -102,10 +109,12 @@ export function clusterBackends(options: ClusterBackendsOptions): Backends {
102
109
  locks: new ValkeyLocks(shared, now ? { now } : {}),
103
110
  catalog: new ValkeyCatalog(shared, now ? { now } : {}),
104
111
  kv: new ValkeyKv(shared),
112
+ blobs,
105
113
  async close() {
106
114
  bus.close()
107
115
  shared.close()
108
116
  await state.close()
117
+ await blobs.close()
109
118
  },
110
119
  }
111
120
  }
@@ -0,0 +1,116 @@
1
+ // BlobStore in process memory (tests, and the single-process hub until the file store (HWA-21) takes over): the
2
+ // contract in ../artifacts/blob-store.ts. Bytes are copied in and out; a blob appears in the map only after its
3
+ // whole body was read and verified, in one synchronous step that also checks the quota, so concurrent puts and
4
+ // readers never see a partial blob and never overshoot the quota.
5
+ import { isBlobHash, type BlobRef, type Page } from "../../protocol"
6
+ import {
7
+ type BlobBody,
8
+ type BlobDeleteOptions,
9
+ type BlobInfo,
10
+ type BlobListOptions,
11
+ type BlobPutOptions,
12
+ BlobQuotaError,
13
+ type BlobRead,
14
+ type BlobStore,
15
+ type BlobUsage,
16
+ } from "../artifacts/blob-store"
17
+ import { listCursor, listWindow, putLimits, readBlobBody } from "../artifacts/blob-support"
18
+ import type { Clock } from "./types"
19
+
20
+ interface Stored {
21
+ readonly bytes: Uint8Array
22
+ readonly storedAt: number
23
+ touchedAt: number
24
+ }
25
+
26
+ export class MemoryBlobStore implements BlobStore {
27
+ private readonly now: Clock
28
+ /** userID → hash → blob. */
29
+ private readonly users = new Map<string, Map<string, Stored>>()
30
+
31
+ constructor(options: { now?: Clock } = {}) {
32
+ this.now = options.now ?? Date.now
33
+ }
34
+
35
+ async put(userID: string, body: BlobBody, mediaType: string, options: BlobPutOptions = {}): Promise<BlobRef> {
36
+ const limits = putLimits(mediaType, options)
37
+ const chunks: Uint8Array[] = []
38
+ const { hash, bytes } = await readBlobBody(body, limits, (chunk) => void chunks.push(chunk.slice()))
39
+ let blobs = this.users.get(userID)
40
+ const existing = blobs?.get(hash)
41
+ if (existing) {
42
+ existing.touchedAt = this.now()
43
+ } else {
44
+ const used = this.usedBytes(blobs)
45
+ if (used + bytes > limits.quotaBytes) throw new BlobQuotaError(limits.quotaBytes, used)
46
+ if (!blobs) this.users.set(userID, (blobs = new Map()))
47
+ const at = this.now()
48
+ blobs.set(hash, { bytes: Buffer.concat(chunks), storedAt: at, touchedAt: at })
49
+ }
50
+ return { hash, bytes, mediaType }
51
+ }
52
+
53
+ async get(userID: string, hash: string): Promise<BlobRead | undefined> {
54
+ const stored = this.find(userID, hash)
55
+ if (!stored) return undefined
56
+ const copy = stored.bytes.slice()
57
+ const body = new ReadableStream<Uint8Array>({
58
+ start(controller) {
59
+ if (copy.byteLength > 0) controller.enqueue(copy)
60
+ controller.close()
61
+ },
62
+ })
63
+ return { ...info(hash, stored), body }
64
+ }
65
+
66
+ async stat(userID: string, hash: string): Promise<BlobInfo | undefined> {
67
+ const stored = this.find(userID, hash)
68
+ return stored && info(hash, stored)
69
+ }
70
+
71
+ async has(userID: string, hash: string): Promise<boolean> {
72
+ return this.find(userID, hash) !== undefined
73
+ }
74
+
75
+ async touch(userID: string, hash: string): Promise<boolean> {
76
+ const stored = this.find(userID, hash)
77
+ if (!stored) return false
78
+ stored.touchedAt = this.now()
79
+ return true
80
+ }
81
+
82
+ async delete(userID: string, hash: string, options: BlobDeleteOptions = {}): Promise<boolean> {
83
+ const stored = this.find(userID, hash)
84
+ if (!stored) return false
85
+ if (options.touchedBefore !== undefined && stored.touchedAt >= options.touchedBefore) return false
86
+ const blobs = this.users.get(userID)!
87
+ blobs.delete(hash)
88
+ if (blobs.size === 0) this.users.delete(userID)
89
+ return true
90
+ }
91
+
92
+ async usage(userID: string): Promise<BlobUsage> {
93
+ const blobs = this.users.get(userID)
94
+ return { blobs: blobs?.size ?? 0, bytes: this.usedBytes(blobs) }
95
+ }
96
+
97
+ async list(userID: string, options: BlobListOptions = {}): Promise<Page<BlobInfo>> {
98
+ const { limit, after } = listWindow(options)
99
+ const hashes = [...(this.users.get(userID)?.keys() ?? [])].filter((h) => after === undefined || h > after).sort()
100
+ const page = hashes.slice(0, limit)
101
+ const items = page.map((h) => info(h, this.users.get(userID)!.get(h)!))
102
+ return hashes.length > limit ? { items, cursor: listCursor(page[page.length - 1]!) } : { items }
103
+ }
104
+
105
+ private find(userID: string, hash: string): Stored | undefined {
106
+ return isBlobHash(hash) ? this.users.get(userID)?.get(hash) : undefined
107
+ }
108
+
109
+ private usedBytes(blobs: Map<string, Stored> | undefined): number {
110
+ let used = 0
111
+ for (const stored of blobs?.values() ?? []) used += stored.bytes.byteLength
112
+ return used
113
+ }
114
+ }
115
+
116
+ const info = (hash: string, stored: Stored): BlobInfo => ({ hash, bytes: stored.bytes.byteLength, storedAt: stored.storedAt, touchedAt: stored.touchedAt })