@huynle/hub 0.6.1 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +800 -0
- package/Dockerfile +30 -0
- package/README.md +870 -66
- package/package.json +45 -2
- package/packages/claude-code/package.json +12 -0
- package/packages/claude-code/src/bridge.ts +79 -0
- package/packages/claude-code/src/daemon.ts +339 -0
- package/packages/claude-code/src/hook-input.ts +44 -0
- package/packages/claude-code/src/ids.ts +27 -0
- package/packages/claude-code/src/index.ts +18 -0
- package/packages/claude-code/src/launcher.ts +265 -0
- package/packages/claude-code/src/main.ts +127 -0
- package/packages/claude-code/src/mcp.ts +109 -0
- package/packages/claude-code/src/paths.ts +34 -0
- package/packages/claude-code/src/plugin.ts +99 -0
- package/packages/claude-code/src/references.ts +51 -0
- package/packages/claude-code/src/sessions.ts +600 -0
- package/packages/claude-code/src/tools.ts +66 -0
- package/packages/claude-code/src/wire.ts +71 -0
- package/packages/claude-code/src/wrapper.ts +155 -0
- package/packages/cli/dist/hub-cli.js +10162 -0
- package/packages/codex/package.json +12 -0
- package/packages/codex/src/app-server.ts +44 -0
- package/packages/codex/src/ids.ts +28 -0
- package/packages/codex/src/index.ts +11 -0
- package/packages/codex/src/jsonrpc.ts +168 -0
- package/packages/codex/src/main.ts +122 -0
- package/packages/codex/src/paths.ts +65 -0
- package/packages/codex/src/sessions.ts +578 -0
- package/packages/codex/src/spoke.ts +249 -0
- package/packages/codex/src/stdio.ts +101 -0
- package/packages/codex/src/websocket.ts +206 -0
- package/packages/codex/src/wrapper.ts +100 -0
- package/packages/node/package.json +12 -0
- package/packages/node/src/config.ts +184 -0
- package/packages/node/src/connection.ts +283 -0
- package/packages/node/src/daemon.ts +281 -0
- package/packages/node/src/git.ts +107 -0
- package/packages/node/src/harnesses.ts +60 -0
- package/packages/node/src/host.ts +319 -0
- package/packages/node/src/index.ts +17 -0
- package/packages/node/src/main.ts +11 -0
- package/packages/node/src/openshell.ts +73 -0
- package/packages/node/src/run.ts +106 -0
- package/packages/preset/package.json +12 -0
- package/packages/preset/presets/aider.json +8 -0
- package/packages/preset/presets/goose.json +8 -0
- package/packages/preset/src/index.ts +2 -0
- package/packages/preset/src/main.ts +195 -0
- package/packages/preset/src/presets.ts +87 -0
- package/packages/preset/src/spoke.ts +251 -0
- package/packages/preset/src/tmux.ts +160 -0
- package/packages/protocol/package.json +12 -0
- package/packages/protocol/src/harness-events.ts +207 -0
- package/packages/protocol/src/index.ts +1215 -0
- package/packages/protocol/src/launch.ts +172 -0
- package/packages/protocol/src/node.ts +409 -0
- package/packages/protocol/src/preset.ts +144 -0
- package/packages/protocol/src/webhook.ts +62 -0
- package/packages/protocol/src/wire.ts +54 -0
- package/packages/runtime-openshell/image/Dockerfile +36 -0
- package/packages/runtime-openshell/image/README.md +78 -0
- package/packages/runtime-openshell/image/build.ts +203 -0
- package/packages/runtime-openshell/image/entry.sh +163 -0
- package/packages/runtime-openshell/image/image.ts +119 -0
- package/packages/runtime-openshell/image/start.sh +24 -0
- package/packages/runtime-openshell/package.json +12 -0
- package/packages/runtime-openshell/src/cli.ts +584 -0
- package/packages/runtime-openshell/src/index.ts +6 -0
- package/packages/runtime-openshell/src/policy.ts +155 -0
- package/packages/runtime-openshell/src/profiles.ts +181 -0
- package/packages/runtime-openshell/src/runtime.ts +475 -0
- package/packages/runtime-sdk/package.json +12 -0
- package/packages/runtime-sdk/src/index.ts +87 -0
- package/packages/sdk/package.json +12 -0
- package/packages/sdk/src/address.ts +25 -0
- package/packages/sdk/src/client.ts +502 -0
- package/packages/sdk/src/envelope.ts +108 -0
- package/packages/sdk/src/errors.ts +106 -0
- package/packages/sdk/src/index.ts +10 -0
- package/packages/sdk/src/sse.ts +67 -0
- package/packages/sdk/src/types.ts +458 -0
- package/packages/sdk/src/webhook.ts +102 -0
- package/packages/spoke-sdk/package.json +12 -0
- package/packages/spoke-sdk/src/activity.ts +3 -0
- package/packages/spoke-sdk/src/connection.ts +295 -0
- package/packages/spoke-sdk/src/delivery.ts +119 -0
- package/packages/spoke-sdk/src/enroll.ts +79 -0
- package/packages/spoke-sdk/src/harness-hello.ts +79 -0
- package/packages/spoke-sdk/src/harness-spoke.ts +263 -0
- package/packages/spoke-sdk/src/index.ts +12 -0
- package/packages/spoke-sdk/src/prompt-refs.ts +154 -0
- package/packages/spoke-sdk/src/prompt-source.ts +8 -0
- package/packages/spoke-sdk/src/session-ids.ts +49 -0
- package/packages/spoke-sdk/src/session-tokens.ts +78 -0
- package/scripts/hub-claude-code.service +34 -0
- package/scripts/hub-codex-spoke.service +34 -0
- package/scripts/hub-node.service +35 -0
- package/scripts/io.github.huynle.hub-claude-code.plist +53 -0
- package/scripts/io.github.huynle.hub-codex-spoke.plist +53 -0
- package/scripts/io.github.huynle.hub-node.plist +52 -0
- package/src/central/actions.ts +233 -0
- package/src/central/agent/acting-routes.ts +133 -0
- package/src/central/agent/acting-switch.ts +93 -0
- package/src/central/agent/acting.ts +599 -0
- package/src/central/agent/actor.ts +193 -0
- package/src/central/agent/app-tokens.ts +142 -0
- package/src/central/agent/audit.ts +185 -0
- package/src/central/agent/decisions.ts +530 -0
- package/src/central/agent/delivery.ts +678 -0
- package/src/central/agent/directory-routes.ts +185 -0
- package/src/central/agent/directory.ts +473 -0
- package/src/central/agent/envelope.ts +51 -0
- package/src/central/agent/errors.ts +72 -0
- package/src/central/agent/escalation.ts +219 -0
- package/src/central/agent/grant-requests.ts +221 -0
- package/src/central/agent/grant-routes.ts +70 -0
- package/src/central/agent/grants.ts +235 -0
- package/src/central/agent/launch-engine.ts +1025 -0
- package/src/central/agent/launch-routes.ts +252 -0
- package/src/central/agent/launch-spec.ts +289 -0
- package/src/central/agent/launch-store.ts +264 -0
- package/src/central/agent/launch-tokens.ts +206 -0
- package/src/central/agent/message-limits.ts +123 -0
- package/src/central/agent/message-routes.ts +385 -0
- package/src/central/agent/messages.ts +442 -0
- package/src/central/agent/names.ts +86 -0
- package/src/central/agent/placement.ts +419 -0
- package/src/central/agent/policy.ts +378 -0
- package/src/central/agent/read-routes.ts +389 -0
- package/src/central/agent/routes.ts +278 -0
- package/src/central/agent/session-tokens.ts +227 -0
- package/src/central/agent/versioned.ts +26 -0
- package/src/central/agent/watch.ts +221 -0
- package/src/central/agent/webhook-routes.ts +78 -0
- package/src/central/agent/webhooks.ts +449 -0
- package/src/central/agent-data.ts +177 -0
- package/src/central/auth.ts +224 -121
- package/src/central/backend/file-state.ts +240 -0
- package/src/central/backend/index.ts +113 -0
- package/src/central/backend/memory-bus.ts +93 -0
- package/src/central/backend/memory-catalog.ts +90 -0
- package/src/central/backend/memory-counters.ts +102 -0
- package/src/central/backend/memory-events.ts +45 -0
- package/src/central/backend/memory-kv.ts +38 -0
- package/src/central/backend/memory-state.ts +77 -0
- package/src/central/backend/postgres-migrate.ts +100 -0
- package/src/central/backend/postgres-state.ts +197 -0
- package/src/central/backend/types.ts +244 -0
- package/src/central/backend/valkey-bus.ts +425 -0
- package/src/central/backend/valkey-catalog.ts +190 -0
- package/src/central/backend/valkey-client.ts +36 -0
- package/src/central/backend/valkey-counters.ts +38 -0
- package/src/central/backend/valkey-events.ts +168 -0
- package/src/central/backend/valkey-kv.ts +54 -0
- package/src/central/backend/valkey-locks.ts +92 -0
- package/src/central/cluster/directory.ts +141 -0
- package/src/central/cluster/feed.ts +84 -0
- package/src/central/cluster/leader.ts +232 -0
- package/src/central/cluster/live.ts +151 -0
- package/src/central/cluster/recommend.ts +59 -0
- package/src/central/cluster/rpc.ts +150 -0
- package/src/central/cluster/view.ts +232 -0
- package/src/central/config.ts +197 -3
- package/src/central/done.ts +96 -0
- package/src/central/events-log.ts +100 -0
- package/src/central/http.ts +1067 -171
- package/src/central/import.ts +318 -0
- package/src/central/main.ts +211 -12
- package/src/central/nodes.ts +314 -0
- package/src/central/oidc.ts +34 -14
- package/src/central/push-target.ts +118 -18
- package/src/central/push.ts +256 -82
- package/src/central/registry.ts +902 -92
- package/src/central/say-stream.ts +128 -0
- package/src/central/session-token-push.ts +233 -0
- package/src/central/speech/openai.ts +21 -14
- package/src/central/speech/openrouter.ts +27 -10
- package/src/central/speech/provider.ts +54 -0
- package/src/central/speech/service.ts +22 -7
- package/src/central/spoke-ws.ts +522 -44
- package/src/central/store-shared.ts +518 -0
- package/src/central/store.ts +572 -189
- package/src/central/token-keys.ts +165 -0
- package/src/config.ts +49 -1
- package/src/index.ts +165 -12
- package/src/protocol.ts +3 -650
- package/src/rpc.ts +27 -2
- package/src/spoke/client.ts +3 -272
- package/src/spoke/conversation.ts +16 -12
- package/src/spoke/deliver.ts +75 -0
- package/src/spoke/enroll.ts +2 -79
- package/src/spoke/env-inject.ts +160 -0
- package/src/spoke/hub-rpc.ts +115 -0
- package/src/spoke/hub-tool.ts +409 -0
- package/src/spoke/launch-source.ts +22 -0
- package/src/spoke/launchers.ts +68 -0
- package/src/spoke/locations.ts +3 -2
- package/src/spoke/node-roots.ts +61 -0
- package/src/spoke/prompt-refs.ts +116 -0
- package/src/spoke/session-tokens.ts +2 -0
- package/src/spoke/spoke.ts +465 -28
- package/src/spoke/taskboard.ts +243 -0
- package/src/spoke/timeline.ts +184 -6
- package/src/spoke/tracker.ts +79 -4
- package/src/status.ts +203 -10
- package/src/tui-commands.ts +13 -6
- package/src/tui-messaging.ts +93 -0
- package/src/tui.ts +15 -0
- package/web/dist/assets/highlight-y4rJE6Tt.js +5 -0
- package/web/dist/assets/index-D9zMbxSg.css +1 -0
- package/web/dist/assets/index-db1z4duA.js +20 -0
- package/web/dist/assets/ort-wasm-simd-threaded-CxTQ5xH-.wasm +0 -0
- package/web/dist/assets/silero-worker-C8-8vLVS.js +11 -0
- package/web/dist/assets/silero_vad-Dt9-bTDn.onnx +0 -0
- package/web/dist/index.html +9 -2
- package/web/dist/sw.js +79 -15
- package/web/dist/assets/index-DClBvfLL.css +0 -1
- package/web/dist/assets/index-Djo_yF0a.js +0 -12
|
@@ -0,0 +1,314 @@
|
|
|
1
|
+
// NodeRegistry: the hub's view of connected `hub node` daemons (design §6.3), one entry per node connection.
|
|
2
|
+
// Kept apart from the spoke Registry on purpose: a node is never a process, never in the catalog and never makes
|
|
3
|
+
// its machine "online"; it only reports runtimes, labels, capacity and launch progress. Placement (P4-T5) reads
|
|
4
|
+
// list(ownerID); the launch engine (P4-T8) subscribes to progress and connect/disconnect events.
|
|
5
|
+
// Cluster mode (design §9): each pod is the single writer of its own nodes and exports them per owner (clusterSlice);
|
|
6
|
+
// ./cluster/view.ts applies other pods' slices (setRemote), so list/get/forMachine see every node of the cluster (a
|
|
7
|
+
// node on another pod is reached through SpokeHub.requestNode). Events and count() stay local, except for
|
|
8
|
+
// subscribers asking for remote events (the launch supervisor on the leader pod): they also get connected,
|
|
9
|
+
// capacity, progress and disconnected derived from other pods' slice changes. A local node of a machine wins over a
|
|
10
|
+
// remote one.
|
|
11
|
+
import type { NodeCapacity, NodeCapacityFrame, NodeHello, NodeLaunchPhase, NodeLaunchReport, NodeProgress, NodeRuntimeInfo } from "../protocol"
|
|
12
|
+
|
|
13
|
+
export interface NodeMachine {
|
|
14
|
+
readonly id: string
|
|
15
|
+
readonly name: string
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/** A connected node, as seen by placement and the launch engine (a copy; changing it changes nothing). */
|
|
19
|
+
export interface NodeView {
|
|
20
|
+
readonly connID: string
|
|
21
|
+
readonly machineID: string
|
|
22
|
+
readonly machineName: string
|
|
23
|
+
/** The machine's current owner, read live (ownership can move). */
|
|
24
|
+
readonly ownerID: string | undefined
|
|
25
|
+
readonly instanceID: string
|
|
26
|
+
readonly nodeVersion: string
|
|
27
|
+
readonly features: readonly string[]
|
|
28
|
+
readonly runtimes: readonly NodeRuntimeInfo[]
|
|
29
|
+
readonly labels: Readonly<Record<string, string>>
|
|
30
|
+
readonly capacity: NodeCapacity
|
|
31
|
+
readonly used: NodeCapacity
|
|
32
|
+
/** capacity − used, never below zero; memoryMB only when the node reports both. */
|
|
33
|
+
readonly free: NodeCapacity
|
|
34
|
+
readonly workspaceRoots: readonly string[]
|
|
35
|
+
readonly draining: boolean
|
|
36
|
+
/** Launches the node carries, with their latest phase (cleaned and cancelled ones are dropped). */
|
|
37
|
+
readonly launches: readonly NodeLaunchReport[]
|
|
38
|
+
readonly connectedAt: number
|
|
39
|
+
readonly lastSeen: number
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export type NodeEvent =
|
|
43
|
+
| { readonly type: "connected"; readonly node: NodeView }
|
|
44
|
+
| { readonly type: "capacity"; readonly node: NodeView }
|
|
45
|
+
| { readonly type: "progress"; readonly node: NodeView; readonly launchID: string; readonly phase: NodeLaunchPhase; readonly message?: string }
|
|
46
|
+
| { readonly type: "disconnected"; readonly node: NodeView }
|
|
47
|
+
|
|
48
|
+
export type NodeListener = (event: NodeEvent) => void
|
|
49
|
+
|
|
50
|
+
/** One owner's nodes on one pod, as other pods mirror them (JSON-compatible). */
|
|
51
|
+
export interface NodeSlice {
|
|
52
|
+
readonly nodes: NodeView[]
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** Phases after which the node no longer carries the launch. */
|
|
56
|
+
const GONE: ReadonlySet<NodeLaunchPhase> = new Set(["cleaned", "cancelled"])
|
|
57
|
+
|
|
58
|
+
interface Entry {
|
|
59
|
+
readonly connID: string
|
|
60
|
+
readonly machine: NodeMachine
|
|
61
|
+
readonly hello: NodeHello
|
|
62
|
+
labels: Record<string, string>
|
|
63
|
+
capacity: NodeCapacity
|
|
64
|
+
used: NodeCapacity
|
|
65
|
+
draining: boolean
|
|
66
|
+
readonly launches: Map<string, NodeLaunchReport>
|
|
67
|
+
readonly connectedAt: number
|
|
68
|
+
lastSeen: number
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export class NodeRegistry {
|
|
72
|
+
private readonly entries = new Map<string, Entry>()
|
|
73
|
+
private readonly listeners = new Set<NodeListener>()
|
|
74
|
+
/** Listeners that also get events derived from other pods' slices (the launch supervisor on the leader pod). */
|
|
75
|
+
private readonly remoteListeners = new Set<NodeListener>()
|
|
76
|
+
private readonly localChangeListeners = new Set<(owner: string | undefined) => void>()
|
|
77
|
+
/** Other pods' nodes: pod → owner → slice. */
|
|
78
|
+
private readonly remote = new Map<string, Map<string, NodeSlice>>()
|
|
79
|
+
|
|
80
|
+
constructor(
|
|
81
|
+
private readonly ownerOf: (machineID: string) => string | undefined,
|
|
82
|
+
private readonly now: () => number = Date.now,
|
|
83
|
+
) {}
|
|
84
|
+
|
|
85
|
+
connect(connID: string, machine: NodeMachine, hello: NodeHello): NodeView {
|
|
86
|
+
const at = this.now()
|
|
87
|
+
const entry: Entry = {
|
|
88
|
+
connID,
|
|
89
|
+
machine: { id: machine.id, name: machine.name },
|
|
90
|
+
hello,
|
|
91
|
+
labels: { ...hello.labels },
|
|
92
|
+
capacity: { ...hello.capacity },
|
|
93
|
+
used: { ...hello.used },
|
|
94
|
+
draining: hello.draining,
|
|
95
|
+
launches: new Map(hello.launches.map((l) => [l.launchID, { ...l }])),
|
|
96
|
+
connectedAt: at,
|
|
97
|
+
lastSeen: at,
|
|
98
|
+
}
|
|
99
|
+
this.entries.set(connID, entry)
|
|
100
|
+
const node = this.view(entry)
|
|
101
|
+
this.emit({ type: "connected", node })
|
|
102
|
+
return node
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
setCapacity(connID: string, frame: NodeCapacityFrame): void {
|
|
106
|
+
const entry = this.entries.get(connID)
|
|
107
|
+
if (!entry) return
|
|
108
|
+
entry.capacity = { ...frame.capacity }
|
|
109
|
+
entry.used = { ...frame.used }
|
|
110
|
+
entry.draining = frame.draining
|
|
111
|
+
if (frame.labels) entry.labels = { ...frame.labels }
|
|
112
|
+
this.emit({ type: "capacity", node: this.view(entry) })
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
progress(connID: string, frame: NodeProgress): void {
|
|
116
|
+
const entry = this.entries.get(connID)
|
|
117
|
+
if (!entry) return
|
|
118
|
+
if (GONE.has(frame.phase)) entry.launches.delete(frame.launchID)
|
|
119
|
+
else {
|
|
120
|
+
const previous = entry.launches.get(frame.launchID)
|
|
121
|
+
entry.launches.set(frame.launchID, { ...previous, launchID: frame.launchID, phase: frame.phase })
|
|
122
|
+
}
|
|
123
|
+
this.emit({
|
|
124
|
+
type: "progress",
|
|
125
|
+
node: this.view(entry),
|
|
126
|
+
launchID: frame.launchID,
|
|
127
|
+
phase: frame.phase,
|
|
128
|
+
...(frame.message !== undefined ? { message: frame.message } : {}),
|
|
129
|
+
})
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
touch(connID: string): void {
|
|
133
|
+
const entry = this.entries.get(connID)
|
|
134
|
+
if (entry) entry.lastSeen = this.now()
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
disconnect(connID: string): void {
|
|
138
|
+
const entry = this.entries.get(connID)
|
|
139
|
+
if (!entry) return
|
|
140
|
+
this.entries.delete(connID)
|
|
141
|
+
this.emit({ type: "disconnected", node: this.view(entry) })
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
get(connID: string): NodeView | undefined {
|
|
145
|
+
const entry = this.entries.get(connID)
|
|
146
|
+
return entry ? this.view(entry) : this.remoteNodes().find((node) => node.connID === connID)
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
/** The node connection of a machine (at most one: the hub turns away a second one); this pod's first. */
|
|
150
|
+
forMachine(machineID: string): NodeView | undefined {
|
|
151
|
+
return this.localForMachine(machineID) ?? this.remoteNodes().find((node) => node.machineID === machineID)
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/** The node connection of a machine held by this pod. */
|
|
155
|
+
localForMachine(machineID: string): NodeView | undefined {
|
|
156
|
+
for (const entry of this.entries.values()) if (entry.machine.id === machineID) return this.view(entry)
|
|
157
|
+
return undefined
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/** Connected nodes of `ownerID` (all owned nodes when omitted); a node whose machine has no owner is never listed. */
|
|
161
|
+
list(ownerID?: string): NodeView[] {
|
|
162
|
+
const out: NodeView[] = []
|
|
163
|
+
for (const node of [...[...this.entries.values()].map((entry) => this.view(entry)), ...this.remoteNodes()]) {
|
|
164
|
+
if (node.ownerID !== undefined && (ownerID === undefined || node.ownerID === ownerID)) out.push(node)
|
|
165
|
+
}
|
|
166
|
+
return out
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
// ---------- cluster (design §9) ----------
|
|
170
|
+
|
|
171
|
+
/** Listens for changes to this pod's nodes of an owner. */
|
|
172
|
+
onLocalChange(listener: (owner: string | undefined) => void): () => void {
|
|
173
|
+
this.localChangeListeners.add(listener)
|
|
174
|
+
return () => this.localChangeListeners.delete(listener)
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/** Owners of this pod's nodes. */
|
|
178
|
+
clusterTenants(): string[] {
|
|
179
|
+
const owners = new Set<string>()
|
|
180
|
+
for (const entry of this.entries.values()) {
|
|
181
|
+
const owner = this.ownerOf(entry.machine.id)
|
|
182
|
+
if (owner !== undefined) owners.add(owner)
|
|
183
|
+
}
|
|
184
|
+
return [...owners]
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/** This pod's nodes of an owner for other pods (undefined when none). */
|
|
188
|
+
clusterSlice(owner: string): NodeSlice | undefined {
|
|
189
|
+
const nodes = [...this.entries.values()].map((entry) => this.view(entry)).filter((node) => node.ownerID === owner)
|
|
190
|
+
return nodes.length ? { nodes } : undefined
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/** Applies another pod's nodes of an owner (undefined: it has none any more). */
|
|
194
|
+
setRemote(pod: string, owner: string, slice: NodeSlice | undefined): void {
|
|
195
|
+
let slices = this.remote.get(pod)
|
|
196
|
+
const before = slices?.get(owner)?.nodes ?? []
|
|
197
|
+
if (slice) {
|
|
198
|
+
if (!slices) this.remote.set(pod, (slices = new Map()))
|
|
199
|
+
slices.set(owner, slice)
|
|
200
|
+
} else if (slices?.delete(owner) && !slices.size) this.remote.delete(pod)
|
|
201
|
+
if (this.remoteListeners.size) this.emitRemote(before, slice?.nodes ?? [])
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
/**
|
|
205
|
+
* Events for remote listeners from a change of another pod's slice: nodes connecting and leaving, capacity, and
|
|
206
|
+
* per launch its new phase, or (gone from the slice) cancelled while preparing, else cleaned.
|
|
207
|
+
*/
|
|
208
|
+
private emitRemote(before: readonly NodeView[], after: readonly NodeView[]): void {
|
|
209
|
+
const view = (node: NodeView): NodeView => ({ ...structuredClone(node), ownerID: this.ownerOf(node.machineID) })
|
|
210
|
+
const old = new Map(before.map((node) => [node.connID, node]))
|
|
211
|
+
const events: NodeEvent[] = []
|
|
212
|
+
for (const node of after) {
|
|
213
|
+
const previous = old.get(node.connID)
|
|
214
|
+
old.delete(node.connID)
|
|
215
|
+
if (!previous) {
|
|
216
|
+
events.push({ type: "connected", node: view(node) })
|
|
217
|
+
continue
|
|
218
|
+
}
|
|
219
|
+
const capacity = (n: NodeView) => JSON.stringify([n.capacity, n.used, n.draining, n.labels])
|
|
220
|
+
if (capacity(previous) !== capacity(node)) events.push({ type: "capacity", node: view(node) })
|
|
221
|
+
const phases = new Map(previous.launches.map((l) => [l.launchID, l.phase]))
|
|
222
|
+
const current = new Set(node.launches.map((l) => l.launchID))
|
|
223
|
+
// Released launches first, then the new phases.
|
|
224
|
+
for (const [launchID, phase] of phases) {
|
|
225
|
+
if (!current.has(launchID)) events.push({ type: "progress", node: view(node), launchID, phase: phase === "preparing" ? "cancelled" : "cleaned" })
|
|
226
|
+
}
|
|
227
|
+
for (const launch of node.launches) {
|
|
228
|
+
if (phases.get(launch.launchID) !== launch.phase) events.push({ type: "progress", node: view(node), launchID: launch.launchID, phase: launch.phase })
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
for (const node of old.values()) events.push({ type: "disconnected", node: view(node) })
|
|
232
|
+
for (const event of events) this.notify(this.remoteListeners, event)
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
/** Other pods' nodes (owner read live), one per machine (the newest), none for a machine with a local node. */
|
|
236
|
+
private remoteNodes(): NodeView[] {
|
|
237
|
+
if (!this.remote.size) return []
|
|
238
|
+
const local = new Set([...this.entries.values()].map((entry) => entry.machine.id))
|
|
239
|
+
const byMachine = new Map<string, NodeView>()
|
|
240
|
+
for (const slices of this.remote.values()) {
|
|
241
|
+
for (const slice of slices.values()) {
|
|
242
|
+
for (const node of slice.nodes ?? []) {
|
|
243
|
+
if (local.has(node.machineID) || this.entries.has(node.connID)) continue
|
|
244
|
+
const known = byMachine.get(node.machineID)
|
|
245
|
+
if (!known || node.connectedAt > known.connectedAt) byMachine.set(node.machineID, node)
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
return [...byMachine.values()].map((node) => ({ ...structuredClone(node), ownerID: this.ownerOf(node.machineID) }))
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
/** Connected node connections (for /healthz). */
|
|
253
|
+
count(): number {
|
|
254
|
+
return this.entries.size
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
/** Events of this pod's nodes; with `remote`, also of other pods' nodes (derived from their slices). */
|
|
258
|
+
subscribe(listener: NodeListener, options: { readonly remote?: boolean } = {}): () => void {
|
|
259
|
+
this.listeners.add(listener)
|
|
260
|
+
if (options.remote) this.remoteListeners.add(listener)
|
|
261
|
+
return () => {
|
|
262
|
+
this.listeners.delete(listener)
|
|
263
|
+
this.remoteListeners.delete(listener)
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
private emit(event: NodeEvent): void {
|
|
268
|
+
for (const listener of [...this.localChangeListeners]) {
|
|
269
|
+
try {
|
|
270
|
+
listener(event.node.ownerID)
|
|
271
|
+
} catch (cause) {
|
|
272
|
+
console.error("[hub] node change listener failed:", cause instanceof Error ? cause.message : cause)
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
this.notify(this.listeners, event)
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
private notify(listeners: ReadonlySet<NodeListener>, event: NodeEvent): void {
|
|
279
|
+
for (const listener of [...listeners]) {
|
|
280
|
+
try {
|
|
281
|
+
listener(event)
|
|
282
|
+
} catch (cause) {
|
|
283
|
+
console.error("[hub] node listener failed:", cause instanceof Error ? cause.message : cause)
|
|
284
|
+
}
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
private view(entry: Entry): NodeView {
|
|
289
|
+
const { hello, capacity, used } = entry
|
|
290
|
+
const free: NodeCapacity = {
|
|
291
|
+
slots: Math.max(0, capacity.slots - used.slots),
|
|
292
|
+
...(capacity.memoryMB !== undefined && used.memoryMB !== undefined ? { memoryMB: Math.max(0, capacity.memoryMB - used.memoryMB) } : {}),
|
|
293
|
+
}
|
|
294
|
+
return {
|
|
295
|
+
connID: entry.connID,
|
|
296
|
+
machineID: entry.machine.id,
|
|
297
|
+
machineName: entry.machine.name,
|
|
298
|
+
ownerID: this.ownerOf(entry.machine.id),
|
|
299
|
+
instanceID: hello.instanceID,
|
|
300
|
+
nodeVersion: hello.nodeVersion,
|
|
301
|
+
features: [...(hello.features ?? [])],
|
|
302
|
+
runtimes: hello.runtimes.map((r) => structuredClone(r)),
|
|
303
|
+
labels: { ...entry.labels },
|
|
304
|
+
capacity: { ...capacity },
|
|
305
|
+
used: { ...used },
|
|
306
|
+
free,
|
|
307
|
+
workspaceRoots: [...hello.workspaceRoots],
|
|
308
|
+
draining: entry.draining,
|
|
309
|
+
launches: [...entry.launches.values()].map((l) => ({ ...l })),
|
|
310
|
+
connectedAt: entry.connectedAt,
|
|
311
|
+
lastSeen: entry.lastSeen,
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
}
|
package/src/central/oidc.ts
CHANGED
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
// OpenID Connect sign-in (authorization code flow with PKCE S256, state and nonce) for multi-user hubs.
|
|
2
2
|
// The ID token comes straight from the token endpoint over TLS (confidential client, client_secret_post),
|
|
3
|
-
// so its signature is not checked (OIDC Core 3.1.3.7, item 6); its claims are. Pending sign-ins live in
|
|
4
|
-
// memory
|
|
3
|
+
// so its signature is not checked (OIDC Core 3.1.3.7, item 6); its claims are. Pending sign-ins live in the Kv
|
|
4
|
+
// backend (process memory by default, so a restart invalidates them; Valkey in a cluster, so the callback may land
|
|
5
|
+
// on any pod) and are single use (GETDEL).
|
|
5
6
|
import { createHash, randomBytes } from "node:crypto"
|
|
7
|
+
import { MemoryCounters } from "./backend/memory-counters"
|
|
8
|
+
import { MemoryKv } from "./backend/memory-kv"
|
|
9
|
+
import type { Counters, Kv } from "./backend/types"
|
|
6
10
|
import type { OidcConfig } from "./config"
|
|
7
11
|
|
|
8
12
|
export const OIDC_COOKIE = "voice_oidc"
|
|
@@ -11,8 +15,10 @@ export const OIDC_PENDING_TTL_MS = 10 * 60_000
|
|
|
11
15
|
export const DISCOVERY_CACHE_MS = 60 * 60_000
|
|
12
16
|
/** Clock skew tolerated on `exp`. */
|
|
13
17
|
const SKEW_SECONDS = 60
|
|
14
|
-
/** Most
|
|
15
|
-
const MAX_PENDING =
|
|
18
|
+
/** Most sign-ins started per pending lifetime (every pod together); /auth/login is public. */
|
|
19
|
+
const MAX_PENDING = 10_000
|
|
20
|
+
const PENDING_KEY = (state: string) => `oidc:pending:${state}`
|
|
21
|
+
const STARTED_KEY = "oidc:started"
|
|
16
22
|
const FETCH_TIMEOUT_MS = 10_000
|
|
17
23
|
const MAX_NEXT_CHARS = 1024
|
|
18
24
|
|
|
@@ -56,6 +62,10 @@ export interface OidcClientOptions {
|
|
|
56
62
|
readonly publicURL: string
|
|
57
63
|
readonly fetch?: typeof globalThis.fetch
|
|
58
64
|
readonly now?: () => number
|
|
65
|
+
/** Pending sign-ins (default: in this process, on `now`). */
|
|
66
|
+
readonly kv?: Kv
|
|
67
|
+
/** The cap on sign-ins started (default: in this process, on `now`). */
|
|
68
|
+
readonly counters?: Counters
|
|
59
69
|
}
|
|
60
70
|
|
|
61
71
|
const random = () => randomBytes(32).toString("base64url")
|
|
@@ -129,7 +139,8 @@ export class OidcClient {
|
|
|
129
139
|
private readonly config: OidcConfig
|
|
130
140
|
private readonly fetch: typeof globalThis.fetch
|
|
131
141
|
private readonly now: () => number
|
|
132
|
-
private readonly
|
|
142
|
+
private readonly kv: Kv
|
|
143
|
+
private readonly counters: Counters
|
|
133
144
|
private discovery?: { at: number; doc: Discovery }
|
|
134
145
|
readonly redirectURI: string
|
|
135
146
|
|
|
@@ -137,18 +148,22 @@ export class OidcClient {
|
|
|
137
148
|
this.config = options.config
|
|
138
149
|
this.fetch = options.fetch ?? globalThis.fetch
|
|
139
150
|
this.now = options.now ?? Date.now
|
|
151
|
+
this.kv = options.kv ?? new MemoryKv({ now: this.now })
|
|
152
|
+
this.counters = options.counters ?? new MemoryCounters({ now: this.now })
|
|
140
153
|
this.redirectURI = `${options.publicURL}/auth/callback`
|
|
141
154
|
}
|
|
142
155
|
|
|
143
156
|
/** Starts a sign-in: the provider URL to redirect to and the state for the voice_oidc cookie. */
|
|
144
157
|
async login(next: string | null): Promise<{ location: string; state: string }> {
|
|
145
158
|
const doc = await this.discover()
|
|
146
|
-
this.
|
|
159
|
+
if ((await this.counters.incr(STARTED_KEY, OIDC_PENDING_TTL_MS)) > MAX_PENDING) {
|
|
160
|
+
throw new OidcError(429, "Too many sign-ins are in progress. Please try again in a few minutes.", "too many pending sign-ins")
|
|
161
|
+
}
|
|
147
162
|
const state = random()
|
|
148
163
|
const nonce = random()
|
|
149
164
|
const verifier = random()
|
|
150
|
-
|
|
151
|
-
this.
|
|
165
|
+
const pending: Pending = { nonce, verifier, next: safeNext(next), expiresAt: this.now() + OIDC_PENDING_TTL_MS }
|
|
166
|
+
await this.kv.set(PENDING_KEY(state), JSON.stringify(pending), { px: OIDC_PENDING_TTL_MS })
|
|
152
167
|
const url = new URL(doc.authorization_endpoint)
|
|
153
168
|
url.searchParams.set("response_type", "code")
|
|
154
169
|
url.searchParams.set("client_id", this.config.clientId)
|
|
@@ -166,13 +181,11 @@ export class OidcClient {
|
|
|
166
181
|
* pending entry is consumed whatever the outcome once the state matches the cookie.
|
|
167
182
|
*/
|
|
168
183
|
async callback(params: URLSearchParams, cookieState: string | undefined): Promise<{ identity: OidcIdentity; next: string }> {
|
|
169
|
-
this.sweep()
|
|
170
184
|
const state = params.get("state") ?? ""
|
|
171
|
-
const pending = state && cookieState === state ? this.
|
|
185
|
+
const pending = state && cookieState === state ? await this.consume(state) : undefined
|
|
172
186
|
if (!pending) {
|
|
173
187
|
throw new OidcError(400, "This sign-in has expired or was already used. Please sign in again.", "state invalid")
|
|
174
188
|
}
|
|
175
|
-
this.pending.delete(state)
|
|
176
189
|
const providerError = params.get("error")
|
|
177
190
|
if (providerError) {
|
|
178
191
|
const description = params.get("error_description")
|
|
@@ -270,9 +283,16 @@ export class OidcClient {
|
|
|
270
283
|
}
|
|
271
284
|
}
|
|
272
285
|
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
286
|
+
/** Takes a pending sign-in out of the Kv (GETDEL: one callback wins, on any pod); undefined when unknown or expired. */
|
|
287
|
+
private async consume(state: string): Promise<Pending | undefined> {
|
|
288
|
+
const raw = await this.kv.getdel(PENDING_KEY(state))
|
|
289
|
+
if (raw === undefined) return undefined
|
|
290
|
+
try {
|
|
291
|
+
const pending = JSON.parse(raw) as Pending
|
|
292
|
+
return pending.expiresAt > this.now() ? pending : undefined
|
|
293
|
+
} catch {
|
|
294
|
+
return undefined
|
|
295
|
+
}
|
|
276
296
|
}
|
|
277
297
|
}
|
|
278
298
|
|
|
@@ -1,7 +1,10 @@
|
|
|
1
|
-
//
|
|
2
|
-
// hub send requests into its own network (SSRF).
|
|
3
|
-
//
|
|
4
|
-
// unspecified or multicast
|
|
1
|
+
// URLs users give the hub (push endpoints from browsers, webhook URLs from apps) are URLs the hub POSTs to: without
|
|
2
|
+
// checks a subscription could make the hub send requests into its own network (SSRF). publicURLProblem: https on a
|
|
3
|
+
// public hostname whose every resolved address is public (not loopback, private, link-local, CGNAT, ULA,
|
|
4
|
+
// unspecified or multicast); an admin opt-in (allowPrivate, webhooks only) admits http and private targets.
|
|
5
|
+
// Push checks endpoints at subscribe time (real push services are public names on public addresses); webhooks
|
|
6
|
+
// send through publicFetch, which re-checks (and re-resolves) right before every request, never follows a
|
|
7
|
+
// redirect, times out and reads a bounded body.
|
|
5
8
|
import { lookup } from "node:dns/promises"
|
|
6
9
|
import { isIPv4, isIPv6 } from "node:net"
|
|
7
10
|
|
|
@@ -10,7 +13,7 @@ export type ResolveHost = (host: string) => Promise<readonly string[]>
|
|
|
10
13
|
/** Every address the system resolver returns for a host. */
|
|
11
14
|
export const resolveHost: ResolveHost = async (host) => (await lookup(host, { all: true, verbatim: true })).map((a) => a.address)
|
|
12
15
|
|
|
13
|
-
/** IPv4 [network, prefix length] blocks that never
|
|
16
|
+
/** IPv4 [network, prefix length] blocks that are never a public service. */
|
|
14
17
|
const V4_BLOCKED: ReadonlyArray<readonly [string, number]> = [
|
|
15
18
|
["0.0.0.0", 8], // "this network", unspecified
|
|
16
19
|
["10.0.0.0", 8], // private
|
|
@@ -47,7 +50,7 @@ function v6Groups(ip: string): number[] {
|
|
|
47
50
|
return [...left, ...fill, ...right].map((g) => parseInt(g, 16))
|
|
48
51
|
}
|
|
49
52
|
|
|
50
|
-
/** True for addresses a
|
|
53
|
+
/** True for addresses a public URL must not resolve to. Anything that is not an IP counts as private. */
|
|
51
54
|
export function isPrivateAddress(address: string): boolean {
|
|
52
55
|
const ip = address.trim().replace(/^\[(.*)\]$/, "$1")
|
|
53
56
|
if (isIPv4(ip)) return privateV4(ip)
|
|
@@ -70,28 +73,125 @@ export function isPrivateAddress(address: string): boolean {
|
|
|
70
73
|
|
|
71
74
|
const LOCAL_SUFFIXES = [".localhost", ".local", ".internal", ".localdomain", ".home.arpa", ".lan"]
|
|
72
75
|
|
|
73
|
-
|
|
74
|
-
|
|
76
|
+
export interface PublicURLOptions {
|
|
77
|
+
readonly resolve: ResolveHost
|
|
78
|
+
/** How problems name the URL (default "URL"). */
|
|
79
|
+
readonly label?: string
|
|
80
|
+
/**
|
|
81
|
+
* The admin opt-in for private targets (hub.json agents.webhooks.allowPrivate): http, loopback, private
|
|
82
|
+
* addresses and local names pass without a lookup. Other schemes and credentials still fail.
|
|
83
|
+
*/
|
|
84
|
+
readonly allowPrivate?: boolean
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** Why the hub must not send to a URL (SSRF), or undefined when it may. */
|
|
88
|
+
export async function publicURLProblem(raw: string, options: PublicURLOptions): Promise<string | undefined> {
|
|
89
|
+
const label = options.label ?? "URL"
|
|
75
90
|
let url: URL
|
|
76
91
|
try {
|
|
77
|
-
url = new URL(
|
|
92
|
+
url = new URL(raw)
|
|
78
93
|
} catch {
|
|
79
|
-
return
|
|
94
|
+
return `${label} is not a URL`
|
|
80
95
|
}
|
|
81
|
-
if (
|
|
82
|
-
|
|
96
|
+
if (options.allowPrivate) {
|
|
97
|
+
if (url.protocol !== "https:" && url.protocol !== "http:") return `${label} must be http or https`
|
|
98
|
+
} else if (url.protocol !== "https:") return `${label} must be https`
|
|
99
|
+
if (url.username || url.password) return `${label} must not carry credentials`
|
|
100
|
+
if (options.allowPrivate) return undefined
|
|
83
101
|
const host = url.hostname.toLowerCase().replace(/^\[(.*)\]$/, "$1").replace(/\.$/, "")
|
|
84
|
-
if (isIPv4(host) || isIPv6(host)) return isPrivateAddress(host) ?
|
|
102
|
+
if (isIPv4(host) || isIPv6(host)) return isPrivateAddress(host) ? `${label} is a private or local address` : undefined
|
|
85
103
|
if (host === "localhost" || !host.includes(".") || LOCAL_SUFFIXES.some((suffix) => host.endsWith(suffix))) {
|
|
86
|
-
return
|
|
104
|
+
return `${label} is a local hostname`
|
|
87
105
|
}
|
|
88
106
|
let addresses: readonly string[]
|
|
89
107
|
try {
|
|
90
|
-
addresses = await resolve(host)
|
|
108
|
+
addresses = await options.resolve(host)
|
|
91
109
|
} catch {
|
|
92
|
-
return
|
|
110
|
+
return `${label} host ${host} does not resolve`
|
|
93
111
|
}
|
|
94
|
-
if (addresses.length === 0) return
|
|
95
|
-
if (addresses.some(isPrivateAddress)) return
|
|
112
|
+
if (addresses.length === 0) return `${label} host ${host} does not resolve`
|
|
113
|
+
if (addresses.some(isPrivateAddress)) return `${label} resolves to a private or local address`
|
|
96
114
|
return undefined
|
|
97
115
|
}
|
|
116
|
+
|
|
117
|
+
/** Why an endpoint is not acceptable for Web Push, or undefined when it is. */
|
|
118
|
+
export async function pushEndpointProblem(endpoint: string, resolve: ResolveHost): Promise<string | undefined> {
|
|
119
|
+
return publicURLProblem(endpoint, { resolve, label: "Push endpoint" })
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** The URL failed publicURLProblem at send time; nothing was sent. */
|
|
123
|
+
export class PublicURLError extends Error {
|
|
124
|
+
constructor(readonly problem: string) {
|
|
125
|
+
super(problem)
|
|
126
|
+
this.name = "PublicURLError"
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
export interface PublicFetchOptions extends PublicURLOptions {
|
|
131
|
+
/** For the whole exchange, body included. */
|
|
132
|
+
readonly timeoutMs: number
|
|
133
|
+
/** Bytes of the response body read at most; the rest is discarded. */
|
|
134
|
+
readonly maxBytes: number
|
|
135
|
+
readonly fetch?: typeof globalThis.fetch
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
export interface PublicFetchResult {
|
|
139
|
+
readonly status: number
|
|
140
|
+
/** The first maxBytes of the body (UTF-8, lossy). */
|
|
141
|
+
readonly body: string
|
|
142
|
+
readonly truncated: boolean
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* One request to a user-supplied URL: the URL is checked (and its host re-resolved) right before sending, so a
|
|
147
|
+
* name that changed to a private address since registration is refused; redirects are never followed (a 3xx is
|
|
148
|
+
* returned as is); the timeout covers the body; at most maxBytes of the body are read.
|
|
149
|
+
* A rebinding between this lookup and fetch's own remains possible; private targets need the admin opt-in.
|
|
150
|
+
*/
|
|
151
|
+
export async function publicFetch(
|
|
152
|
+
url: string,
|
|
153
|
+
init: { readonly method: string; readonly headers?: Record<string, string>; readonly body: string },
|
|
154
|
+
options: PublicFetchOptions,
|
|
155
|
+
): Promise<PublicFetchResult> {
|
|
156
|
+
const problem = await publicURLProblem(url, options)
|
|
157
|
+
if (problem) throw new PublicURLError(problem)
|
|
158
|
+
const controller = new AbortController()
|
|
159
|
+
let timedOut = false
|
|
160
|
+
const timer = setTimeout(() => {
|
|
161
|
+
timedOut = true
|
|
162
|
+
controller.abort()
|
|
163
|
+
}, options.timeoutMs)
|
|
164
|
+
try {
|
|
165
|
+
const response = await (options.fetch ?? globalThis.fetch)(url, { ...init, redirect: "manual", signal: controller.signal })
|
|
166
|
+
const { body, truncated } = await readCapped(response, options.maxBytes)
|
|
167
|
+
return { status: response.status, body, truncated }
|
|
168
|
+
} catch (cause) {
|
|
169
|
+
if (timedOut) throw new Error(`timed out after ${options.timeoutMs} ms`)
|
|
170
|
+
throw cause
|
|
171
|
+
} finally {
|
|
172
|
+
clearTimeout(timer)
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
async function readCapped(response: Response, maxBytes: number): Promise<{ body: string; truncated: boolean }> {
|
|
177
|
+
if (!response.body) return { body: "", truncated: false }
|
|
178
|
+
const reader = response.body.getReader()
|
|
179
|
+
const chunks: Uint8Array[] = []
|
|
180
|
+
let size = 0
|
|
181
|
+
let truncated = false
|
|
182
|
+
while (true) {
|
|
183
|
+
const { done, value } = await reader.read()
|
|
184
|
+
if (done) break
|
|
185
|
+
const room = maxBytes - size
|
|
186
|
+
if (value.byteLength > room) {
|
|
187
|
+
chunks.push(value.subarray(0, room))
|
|
188
|
+
size = maxBytes
|
|
189
|
+
truncated = true
|
|
190
|
+
await reader.cancel().catch(() => {})
|
|
191
|
+
break
|
|
192
|
+
}
|
|
193
|
+
chunks.push(value)
|
|
194
|
+
size += value.byteLength
|
|
195
|
+
}
|
|
196
|
+
return { body: Buffer.concat(chunks).toString("utf8"), truncated }
|
|
197
|
+
}
|