@ran-sh/dsh-crew 1.0.2 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/README.zh.md +4 -0
- package/codex/AGENTS.md +101 -92
- package/docs/ui-surfaces.md +48 -21
- package/lib/client.js +235 -34
- package/official-web-bridge/lib/client.js +368 -4532
- package/official-web-bridge/package.json +1 -2
- package/package.json +130 -67
- package/scripts/build-client.mjs +30 -15
- package/scripts/remove-legacy-official-bridge.ps1 +89 -0
- package/scripts/setup.mjs +152 -140
- package/scripts/verify-npm-install.mjs +311 -310
- package/scripts/verify-official-bridge-e2e.mjs +12 -1
- package/src/client/host-readiness.mjs +62 -59
- package/src/client/index.tsx +117 -24
- package/src/client/quick-entry.tsx +10 -0
- package/src/client/quick-panel.tsx +295 -0
- package/src/client/surface-detection.mjs +43 -30
- package/src/credential-reference.mjs +38 -0
- package/src/dsh-cli-runtime.mjs +500 -40
- package/src/dsh-cohort.mjs +20 -0
- package/src/hub/index.mjs +1602 -1016
- package/src/install/npx-lifecycle.mjs +1983 -468
- package/src/install/official-web.mjs +36 -73
- package/src/jobs.mjs +20 -24
- package/src/model-catalog.mjs +8 -1
- package/src/official-web-bridge.mjs +329 -249
- package/src/provider-delete-adapters.mjs +165 -17
- package/src/provider-inventory.mjs +17 -3
- package/src/provider-layer-migration-adapters.mjs +759 -0
- package/src/provider-layer-migration.mjs +198 -0
- package/src/provider-lifecycle-state.mjs +4 -1
- package/src/provider-profile-store.mjs +193 -4
- package/src/provider-settings-store.mjs +82 -6
- package/src/provider-store-lock.mjs +67 -0
- package/src/runtime-identity.mjs +24 -1
- package/src/supervisor/restart-request.mjs +202 -0
- package/windows/start-dsh-crew.ps1 +783 -370
- package/worker.cordis.yml +32 -46
- package/zcode/AGENTS.md +35 -26
|
@@ -1,370 +1,783 @@
|
|
|
1
|
-
# DSH Crew managed Windows launcher
|
|
2
|
-
[CmdletBinding()]
|
|
3
|
-
param(
|
|
4
|
-
[ValidateSet('background', 'open', 'watch')]
|
|
5
|
-
[string] $Mode = 'open'
|
|
6
|
-
)
|
|
7
|
-
|
|
8
|
-
Set-StrictMode -Version Latest
|
|
9
|
-
$ErrorActionPreference = 'Stop'
|
|
10
|
-
|
|
11
|
-
$crewHome = Join-Path $env:USERPROFILE '.config\dsh-crew\harness'
|
|
12
|
-
$officialHome = Join-Path $env:USERPROFILE '.dsh'
|
|
13
|
-
$dshCli = Join-Path $crewHome 'runtime\node_modules\.bin\dsh.cmd'
|
|
14
|
-
$logRoot = if ($env:TEMP) { $env:TEMP } else { [System.IO.Path]::GetTempPath() }
|
|
15
|
-
$launcherLog = Join-Path $logRoot 'dsh-crew-launcher.log'
|
|
16
|
-
$startedAt = Get-Date
|
|
17
|
-
$services = @(
|
|
18
|
-
[pscustomobject]@{ Name = 'Official UI'; Profile = 'web'; Home = $officialHome; Port = 3080; Url = 'http://127.0.0.1:3080';
|
|
19
|
-
[pscustomobject]@{ Name = 'Crew backend'; Profile = 'dsh-crew'; Home = $crewHome; Port = 3210; Url = 'http://127.0.0.1:3210';
|
|
20
|
-
)
|
|
21
|
-
|
|
22
|
-
function Write-LaunchLog {
|
|
23
|
-
param([string] $Message, [ValidateSet('INFO', 'WARN', 'ERROR')] [string] $Level = 'INFO')
|
|
24
|
-
$line = '[{0}] [{1}] {2}' -f (Get-Date -Format 'yyyy-MM-ddTHH:mm:ss.fffK'), $Level, $Message
|
|
25
|
-
Add-Content -LiteralPath $launcherLog -Value $line -Encoding UTF8
|
|
26
|
-
if ($Mode -eq 'open') {
|
|
27
|
-
if ($Level -eq 'ERROR') { Write-Host $line -ForegroundColor Red }
|
|
28
|
-
elseif ($Level -eq 'WARN') { Write-Host $line -ForegroundColor Yellow }
|
|
29
|
-
else { Write-Host $line }
|
|
30
|
-
}
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
function Get-HealthState {
|
|
34
|
-
param([
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
$
|
|
122
|
-
|
|
123
|
-
try {
|
|
124
|
-
$
|
|
125
|
-
$
|
|
126
|
-
$
|
|
127
|
-
|
|
128
|
-
} catch {
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
$
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
$
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
$
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
#
|
|
280
|
-
#
|
|
281
|
-
|
|
282
|
-
$
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
}
|
|
287
|
-
$
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
}
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
}
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
}
|
|
1
|
+
# DSH Crew managed Windows launcher
|
|
2
|
+
[CmdletBinding()]
|
|
3
|
+
param(
|
|
4
|
+
[ValidateSet('background', 'open', 'watch')]
|
|
5
|
+
[string] $Mode = 'open'
|
|
6
|
+
)
|
|
7
|
+
|
|
8
|
+
Set-StrictMode -Version Latest
|
|
9
|
+
$ErrorActionPreference = 'Stop'
|
|
10
|
+
|
|
11
|
+
$crewHome = Join-Path $env:USERPROFILE '.config\dsh-crew\harness'
|
|
12
|
+
$officialHome = Join-Path $env:USERPROFILE '.dsh'
|
|
13
|
+
$dshCli = Join-Path $crewHome 'runtime\node_modules\.bin\dsh.cmd'
|
|
14
|
+
$logRoot = if ($env:TEMP) { $env:TEMP } else { [System.IO.Path]::GetTempPath() }
|
|
15
|
+
$launcherLog = Join-Path $logRoot 'dsh-crew-launcher.log'
|
|
16
|
+
$startedAt = Get-Date
|
|
17
|
+
$services = @(
|
|
18
|
+
[pscustomobject]@{ Name = 'Official UI'; Profile = 'web'; Home = $officialHome; Port = 3080; Url = 'http://127.0.0.1:3080'; CrewOwned = $false; State = 'pending'; Process = $null; RootPid = $null; RootStartedAtUtcTicks = $null; ListenerPid = $null; ListenerStartedAtUtcTicks = $null; ConsecutiveFailures = 0; LastError = $null },
|
|
19
|
+
[pscustomobject]@{ Name = 'Crew backend'; Profile = 'dsh-crew'; Home = $crewHome; Port = 3210; Url = 'http://127.0.0.1:3210'; CrewOwned = $true; State = 'pending'; Process = $null; RootPid = $null; RootStartedAtUtcTicks = $null; ListenerPid = $null; ListenerStartedAtUtcTicks = $null; ConsecutiveFailures = 0; LastError = $null }
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
function Write-LaunchLog {
|
|
23
|
+
param([string] $Message, [ValidateSet('INFO', 'WARN', 'ERROR')] [string] $Level = 'INFO')
|
|
24
|
+
$line = '[{0}] [{1}] {2}' -f (Get-Date -Format 'yyyy-MM-ddTHH:mm:ss.fffK'), $Level, $Message
|
|
25
|
+
Add-Content -LiteralPath $launcherLog -Value $line -Encoding UTF8
|
|
26
|
+
if ($Mode -eq 'open') {
|
|
27
|
+
if ($Level -eq 'ERROR') { Write-Host $line -ForegroundColor Red }
|
|
28
|
+
elseif ($Level -eq 'WARN') { Write-Host $line -ForegroundColor Yellow }
|
|
29
|
+
else { Write-Host $line }
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function Get-HealthState {
|
|
34
|
+
param([pscustomObject] $Service)
|
|
35
|
+
# Crew-owned 3210 answers the Crew extension contract. The official 3080
|
|
36
|
+
# is an external dependency: probe only its native web root, never the
|
|
37
|
+
# Crew bridge endpoint, so a missing legacy bridge cannot fail 3080 health.
|
|
38
|
+
if ($Service.CrewOwned) {
|
|
39
|
+
try {
|
|
40
|
+
$response = Invoke-RestMethod -Uri ($Service.Url + '/_dsh/dsh-crew/extension') -TimeoutSec 2
|
|
41
|
+
$runtime = $response.extension.runtime
|
|
42
|
+
$version = $runtime.runtime_version
|
|
43
|
+
# A stale 3210 process (booted before the runtime tree was swapped) can
|
|
44
|
+
# keep serving from memory while the disk tree is a different cohort.
|
|
45
|
+
# Require the reported dsh_version to equal the installed disk
|
|
46
|
+
# @deepseek-ai/dsh version so supervisor never treats a
|
|
47
|
+
# disk-rc.1/memory-alpha.5 (or vice versa) process as healthy.
|
|
48
|
+
# FAIL CLOSED: an unreadable/missing disk manifest is NOT healthy —
|
|
49
|
+
# the process cannot be proven to match the tree it will next boot.
|
|
50
|
+
$expectedDshVersion = $null
|
|
51
|
+
$diskReadable = $false
|
|
52
|
+
$runtimeManifest = Join-Path $crewHome 'runtime\node_modules\@deepseek-ai\dsh\package.json'
|
|
53
|
+
if (Test-Path -LiteralPath $runtimeManifest -PathType Leaf) {
|
|
54
|
+
try {
|
|
55
|
+
$expectedDshVersion = (Get-Content -LiteralPath $runtimeManifest -Raw | ConvertFrom-Json).version
|
|
56
|
+
if ($expectedDshVersion) { $diskReadable = $true }
|
|
57
|
+
} catch { $expectedDshVersion = $null }
|
|
58
|
+
}
|
|
59
|
+
$cohortMatches = $diskReadable -and $runtime.dsh_version -eq $expectedDshVersion
|
|
60
|
+
if ($response.ok -eq $true -and $version -and $cohortMatches) {
|
|
61
|
+
return [pscustomobject]@{ Ready = $true; Version = [string] $version; Error = $null }
|
|
62
|
+
}
|
|
63
|
+
if (-not $diskReadable) {
|
|
64
|
+
return [pscustomobject]@{ Ready = $false; Version = $null; Error = 'disk runtime manifest unreadable; cannot prove cohort identity' }
|
|
65
|
+
}
|
|
66
|
+
if ($version -and -not $cohortMatches) {
|
|
67
|
+
return [pscustomobject]@{ Ready = $false; Version = [string] $version; Error = ('runtime cohort mismatch: hub reports dsh_version={0} but disk runtime is {1}' -f $runtime.dsh_version, $expectedDshVersion) }
|
|
68
|
+
}
|
|
69
|
+
return [pscustomobject]@{ Ready = $false; Version = $null; Error = 'Response did not contain a ready runtime contract.' }
|
|
70
|
+
} catch {
|
|
71
|
+
return [pscustomobject]@{ Ready = $false; Version = $null; Error = $_.Exception.Message }
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
try {
|
|
75
|
+
# The official 3080 UI may require a token (HTTP 401/403 on the bare
|
|
76
|
+
# root) or redirect to the web entry (3xx). Accept exactly those; any
|
|
77
|
+
# other status (e.g. 404 from a foreign HTTP service squatting on 3080)
|
|
78
|
+
# is NOT healthy.
|
|
79
|
+
$response = Invoke-WebRequest -UseBasicParsing -Uri $Service.Url -TimeoutSec 2
|
|
80
|
+
if ($response.StatusCode -ge 200 -and $response.StatusCode -lt 400) {
|
|
81
|
+
return [pscustomobject]@{ Ready = $true; Version = $null; Error = $null }
|
|
82
|
+
}
|
|
83
|
+
return [pscustomobject]@{ Ready = $false; Version = $null; Error = ('Official UI returned HTTP {0}.' -f $response.StatusCode) }
|
|
84
|
+
} catch {
|
|
85
|
+
# Invoke-WebRequest surfaces 401/403 as exceptions; those prove the web
|
|
86
|
+
# service is alive and answering (token gate present).
|
|
87
|
+
$status = $null
|
|
88
|
+
if ($_.Exception.Response -and $_.Exception.Response.StatusCode) {
|
|
89
|
+
$status = [int] $_.Exception.Response.StatusCode
|
|
90
|
+
}
|
|
91
|
+
if ($status -eq 401 -or $status -eq 403) {
|
|
92
|
+
return [pscustomobject]@{ Ready = $true; Version = $null; Error = $null }
|
|
93
|
+
}
|
|
94
|
+
return [pscustomobject]@{ Ready = $false; Version = $null; Error = $_.Exception.Message }
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
# Optional legacy compatibility probe: returns true only when a 3080 that
|
|
99
|
+
# still hosts the Crew bridge answers its ping endpoint. The Crew launcher
|
|
100
|
+
# never depends on it; 3210 always boots directly first.
|
|
101
|
+
function Test-LegacyBridgeAvailable {
|
|
102
|
+
try {
|
|
103
|
+
$response = Invoke-WebRequest -UseBasicParsing -Uri 'http://127.0.0.1:3080/_dsh/dsh-crew/ping' -TimeoutSec 3
|
|
104
|
+
return ($response.StatusCode -ge 200 -and $response.StatusCode -lt 300)
|
|
105
|
+
} catch {
|
|
106
|
+
return $false
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
# ---- Crew supervisor control protocol (heartbeat + restart requests) --------
|
|
111
|
+
# The hub (3210) never spawns itself. It writes a durable restart request;
|
|
112
|
+
# this watcher is the only process authority and executes it after proving
|
|
113
|
+
# ownership (persisted identity + live runtime_id match).
|
|
114
|
+
|
|
115
|
+
$crewSupervisorRoot = Join-Path $env:USERPROFILE '.config\dsh-crew\supervisor'
|
|
116
|
+
$crewHeartbeatFile = Join-Path $crewSupervisorRoot 'heartbeat.json'
|
|
117
|
+
$crewRequestsDir = Join-Path $crewSupervisorRoot 'restart-requests'
|
|
118
|
+
$crewResultsDir = Join-Path $crewSupervisorRoot 'restart-results'
|
|
119
|
+
|
|
120
|
+
function Write-SupervisorHeartbeat {
|
|
121
|
+
$now = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds()
|
|
122
|
+
$record = @{ schema_version = 1; pid = $PID; last_seen = $now; protocol_version = 1 } | ConvertTo-Json -Compress
|
|
123
|
+
try {
|
|
124
|
+
if (-not (Test-Path -LiteralPath $crewSupervisorRoot -PathType Container)) { New-Item -ItemType Directory -Path $crewSupervisorRoot -Force | Out-Null }
|
|
125
|
+
$temp = Join-Path $crewSupervisorRoot ("heartbeat.{0}.tmp" -f $PID)
|
|
126
|
+
Set-Content -LiteralPath $temp -Value $record -Encoding UTF8 -NoNewline
|
|
127
|
+
Move-Item -LiteralPath $temp -Destination $crewHeartbeatFile -Force
|
|
128
|
+
} catch { /* heartbeat is best-effort */ }
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
function Read-RestartRequests {
|
|
132
|
+
if (-not (Test-Path -LiteralPath $crewRequestsDir -PathType Container)) { return @() }
|
|
133
|
+
$requests = @()
|
|
134
|
+
Get-ChildItem -LiteralPath $crewRequestsDir -Filter '*.json' -File -ErrorAction SilentlyContinue | ForEach-Object {
|
|
135
|
+
try {
|
|
136
|
+
$parsed = Get-Content -LiteralPath $_.FullName -Raw | ConvertFrom-Json -ErrorAction Stop
|
|
137
|
+
if ($parsed.schema_version -eq 1 -and $parsed.operation -eq 'restart' -and $parsed.request_id) {
|
|
138
|
+
$requests += $parsed
|
|
139
|
+
}
|
|
140
|
+
} catch { Remove-Item -LiteralPath $_.FullName -Force -ErrorAction SilentlyContinue }
|
|
141
|
+
}
|
|
142
|
+
return $requests
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
function Write-RestartResult {
|
|
146
|
+
param([object] $Request, [string] $State, [object] $Detail = $null)
|
|
147
|
+
try {
|
|
148
|
+
if (-not (Test-Path -LiteralPath $crewResultsDir -PathType Container)) { New-Item -ItemType Directory -Path $crewResultsDir -Force | Out-Null }
|
|
149
|
+
$result = @{
|
|
150
|
+
schema_version = 1
|
|
151
|
+
request_id = $Request.request_id
|
|
152
|
+
operation = 'restart'
|
|
153
|
+
state = $State
|
|
154
|
+
runtime_id = $Request.runtime_id
|
|
155
|
+
written_at = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds()
|
|
156
|
+
detail = $Detail
|
|
157
|
+
} | ConvertTo-Json -Depth 5
|
|
158
|
+
$file = Join-Path $crewResultsDir ("{0}.json" -f $Request.request_id)
|
|
159
|
+
$temp = Join-Path $crewResultsDir ("{0}.{1}.tmp" -f $Request.request_id, $PID)
|
|
160
|
+
Set-Content -LiteralPath $temp -Value $result -Encoding UTF8 -NoNewline
|
|
161
|
+
Move-Item -LiteralPath $temp -Destination $file -Force
|
|
162
|
+
} catch { /* best effort */ }
|
|
163
|
+
Remove-Item -LiteralPath (Join-Path $crewRequestsDir ("{0}.json" -f $Request.request_id)) -Force -ErrorAction SilentlyContinue
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
function Invoke-CrewRestartRequests {
|
|
167
|
+
# Consume durable restart requests from the hub. Called by the watch loop.
|
|
168
|
+
$now = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds()
|
|
169
|
+
foreach ($request in (Read-RestartRequests)) {
|
|
170
|
+
if (($request.expires_at -as [long]) -lt $now) {
|
|
171
|
+
Write-RestartResult $request 'RESTART_REQUEST_EXPIRED'
|
|
172
|
+
Write-LaunchLog ('Restart request {0} expired; not executed.' -f $request.request_id) 'WARN'
|
|
173
|
+
continue
|
|
174
|
+
}
|
|
175
|
+
# Ownership authority: the request must name the SAME runtime identity we
|
|
176
|
+
# own, and that identity must still be live on 3210.
|
|
177
|
+
$crew = $services | Where-Object { $_.CrewOwned } | Select-Object -First 1
|
|
178
|
+
if (-not $crew) { continue }
|
|
179
|
+
$health = Get-HealthState $crew
|
|
180
|
+
$liveIdentityOk = $false
|
|
181
|
+
try {
|
|
182
|
+
$response = Invoke-RestMethod -Uri 'http://127.0.0.1:3210/_dsh/dsh-crew/extension' -TimeoutSec 3
|
|
183
|
+
$liveRuntimeId = $response.extension.runtime.runtime_id
|
|
184
|
+
$liveIdentityOk = ($response.ok -eq $true) -and ($liveRuntimeId -eq $request.runtime_id)
|
|
185
|
+
} catch { $liveIdentityOk = $false }
|
|
186
|
+
if (-not $liveIdentityOk) {
|
|
187
|
+
Write-RestartResult $request 'SUPERVISOR_OWNERSHIP_CONFLICT'
|
|
188
|
+
Write-LaunchLog ('Restart request {0} rejected: live runtime_id does not match the request.' -f $request.request_id) 'WARN'
|
|
189
|
+
continue
|
|
190
|
+
}
|
|
191
|
+
Write-LaunchLog ('Executing restart request {0} (reason: {1}).' -f $request.request_id, $request.reason)
|
|
192
|
+
$previousRuntimeId = $request.runtime_id
|
|
193
|
+
# Re-read the listener port right before the kill so the PID identity is
|
|
194
|
+
# fresh: Stop-OwnedListener requires a PID that equals the tracked
|
|
195
|
+
# listener and belongs to the current owned process tree.
|
|
196
|
+
$port = Get-PortState $crew.Port
|
|
197
|
+
if ($port.State -ne 'occupied' -or -not $port.Pid) {
|
|
198
|
+
Write-RestartResult $request 'SUPERVISOR_STOP_FAILED'
|
|
199
|
+
Write-LaunchLog ('Restart request {0} failed: no listener on port {1}.' -f $request.request_id, $crew.Port) 'ERROR'
|
|
200
|
+
continue
|
|
201
|
+
}
|
|
202
|
+
$stopped = Stop-OwnedListener -Service $crew -ListenerPid ([int] $port.Pid)
|
|
203
|
+
if (-not $stopped) {
|
|
204
|
+
Write-RestartResult $request 'SUPERVISOR_STOP_FAILED'
|
|
205
|
+
Write-LaunchLog ('Restart request {0} failed to stop the owned runtime.' -f $request.request_id) 'ERROR'
|
|
206
|
+
continue
|
|
207
|
+
}
|
|
208
|
+
Start-CrewService $crew
|
|
209
|
+
Wait-CrewServices
|
|
210
|
+
# Verify: runtime_id must have changed and the cohort must still match.
|
|
211
|
+
$newRuntimeId = $null
|
|
212
|
+
try {
|
|
213
|
+
$response = Invoke-RestMethod -Uri 'http://127.0.0.1:3210/_dsh/dsh-crew/extension' -TimeoutSec 3
|
|
214
|
+
$newRuntimeId = $response.extension.runtime.runtime_id
|
|
215
|
+
} catch { $newRuntimeId = $null }
|
|
216
|
+
if ($newRuntimeId -and $newRuntimeId -ne $previousRuntimeId) {
|
|
217
|
+
Write-RestartResult $request 'VERIFIED' @{ previous_runtime_id = $previousRuntimeId; runtime_id = $newRuntimeId }
|
|
218
|
+
Write-LaunchLog ('Restart request {0} verified: runtime_id {1} -> {2}.' -f $request.request_id, $previousRuntimeId, $newRuntimeId)
|
|
219
|
+
} else {
|
|
220
|
+
Write-RestartResult $request 'VERIFY_FAILED' @{ previous_runtime_id = $previousRuntimeId; runtime_id = $newRuntimeId }
|
|
221
|
+
Write-LaunchLog ('Restart request {0} verification failed: runtime_id did not change.' -f $request.request_id) 'ERROR'
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
$crewMaintenanceRequestsDir = Join-Path $crewSupervisorRoot 'maintenance-requests'
|
|
227
|
+
$crewMaintenanceResultsDir = Join-Path $crewSupervisorRoot 'maintenance-results'
|
|
228
|
+
$crewMaintenanceSessionFile = Join-Path $crewSupervisorRoot 'maintenance-session.json'
|
|
229
|
+
|
|
230
|
+
function Test-MaintenanceSessionActive {
|
|
231
|
+
# Fence for ordinary Crew auto-start. Returns $true when a STOPPED session
|
|
232
|
+
# holds the launch right OR when the session file is present but MALFORMED
|
|
233
|
+
# (fail closed: a corrupt session must not let supervision auto-start
|
|
234
|
+
# 3210, because the stopped window may belong to a live npx swap).
|
|
235
|
+
# ABSENT means no fence: ordinary supervision proceeds.
|
|
236
|
+
$session = Read-MaintenanceSession
|
|
237
|
+
if ($session -eq 'ABSENT') { return $false }
|
|
238
|
+
return $true
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
function Set-MaintenanceSession {
|
|
242
|
+
param([object] $Request)
|
|
243
|
+
# Strong-failure semantics: the STOPPED result may only be published after
|
|
244
|
+
# the session is durably written AND read back with exact identity. Any
|
|
245
|
+
# failure returns $false and the caller must NOT claim the stopped window.
|
|
246
|
+
try {
|
|
247
|
+
$session = @{
|
|
248
|
+
schema_version = 1
|
|
249
|
+
state = 'STOPPED'
|
|
250
|
+
lease = $Request.lease
|
|
251
|
+
runtime_id = $Request.runtime_id
|
|
252
|
+
stopped_at = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds()
|
|
253
|
+
request_id = $Request.request_id
|
|
254
|
+
} | ConvertTo-Json -Compress
|
|
255
|
+
$temp = Join-Path $crewSupervisorRoot ("maintenance-session.{0}.tmp" -f $PID)
|
|
256
|
+
Set-Content -LiteralPath $temp -Value $session -Encoding UTF8 -NoNewline
|
|
257
|
+
Move-Item -LiteralPath $temp -Destination $crewMaintenanceSessionFile -Force
|
|
258
|
+
$back = Read-MaintenanceSession
|
|
259
|
+
if ($back -is [string]) { return $false }
|
|
260
|
+
return ($back.schema_version -eq 1 -and $back.state -eq 'STOPPED' -and $back.lease -eq $Request.lease -and $back.runtime_id -eq $Request.runtime_id -and $back.request_id -eq $Request.request_id)
|
|
261
|
+
} catch { return $false }
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
function Clear-MaintenanceSession {
|
|
265
|
+
# Verifiable one-shot consumption: returns $true only when the session
|
|
266
|
+
# file is provably ABSENT afterwards (otherwise the lease is NOT consumed
|
|
267
|
+
# and replay stays impossible because start re-validates the session).
|
|
268
|
+
try {
|
|
269
|
+
Remove-Item -LiteralPath $crewMaintenanceSessionFile -Force -ErrorAction Stop
|
|
270
|
+
} catch {
|
|
271
|
+
if (Test-Path -LiteralPath $crewMaintenanceSessionFile -PathType Leaf) { return $false }
|
|
272
|
+
return $true
|
|
273
|
+
}
|
|
274
|
+
return (-not (Test-Path -LiteralPath $crewMaintenanceSessionFile -PathType Leaf))
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
function Read-MaintenanceSession {
|
|
278
|
+
# Tri-state: 'ABSENT' (no file), a session object when valid, or the
|
|
279
|
+
# string 'MALFORMED' (present but unreadable/invalid). MALFORMED fails
|
|
280
|
+
# closed for Crew auto-start (see Test-MaintenanceSessionActive).
|
|
281
|
+
try {
|
|
282
|
+
if (-not (Test-Path -LiteralPath $crewMaintenanceSessionFile -PathType Leaf)) { return 'ABSENT' }
|
|
283
|
+
$parsed = Get-Content -LiteralPath $crewMaintenanceSessionFile -Raw | ConvertFrom-Json -ErrorAction Stop
|
|
284
|
+
if ($parsed.schema_version -ne 1 -or $parsed.state -ne 'STOPPED' -or -not $parsed.lease -or -not $parsed.runtime_id -or -not $parsed.request_id) {
|
|
285
|
+
return 'MALFORMED'
|
|
286
|
+
}
|
|
287
|
+
return $parsed
|
|
288
|
+
} catch { return 'MALFORMED' }
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
function Write-MaintenanceResult {
|
|
292
|
+
param([object] $Request, [string] $State, [object] $Detail = $null)
|
|
293
|
+
try {
|
|
294
|
+
if (-not (Test-Path -LiteralPath $crewMaintenanceResultsDir -PathType Container)) { New-Item -ItemType Directory -Path $crewMaintenanceResultsDir -Force | Out-Null }
|
|
295
|
+
$result = @{
|
|
296
|
+
schema_version = 1
|
|
297
|
+
request_id = $Request.request_id
|
|
298
|
+
operation = $Request.operation
|
|
299
|
+
state = $State
|
|
300
|
+
lease = $Request.lease
|
|
301
|
+
runtime_id = $Request.runtime_id
|
|
302
|
+
written_at = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds()
|
|
303
|
+
detail = $Detail
|
|
304
|
+
} | ConvertTo-Json -Depth 5
|
|
305
|
+
$file = Join-Path $crewMaintenanceResultsDir ("{0}.json" -f $Request.request_id)
|
|
306
|
+
$temp = Join-Path $crewMaintenanceResultsDir ("{0}.{1}.tmp" -f $Request.request_id, $PID)
|
|
307
|
+
Set-Content -LiteralPath $temp -Value $result -Encoding UTF8 -NoNewline
|
|
308
|
+
Move-Item -LiteralPath $temp -Destination $file -Force
|
|
309
|
+
} catch { /* best effort */ }
|
|
310
|
+
Remove-Item -LiteralPath (Join-Path $crewMaintenanceRequestsDir ("{0}.json" -f $Request.request_id)) -Force -ErrorAction SilentlyContinue
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
function Invoke-CrewMaintenanceRequests {
|
|
314
|
+
# Consume maintenance transactions from npx lifecycle (cohort migration).
|
|
315
|
+
# The npx process owns the runtime TREE swap; this watcher owns the
|
|
316
|
+
# PROCESS stop/start around it. Two phases, both verified:
|
|
317
|
+
# maintenance-stop -> stop owned 3210, write STOPPED (+lease)
|
|
318
|
+
# maintenance-start -> start 3210, verify new identity + cohort, VERIFIED
|
|
319
|
+
$now = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds()
|
|
320
|
+
if (-not (Test-Path -LiteralPath $crewMaintenanceRequestsDir -PathType Container)) { return }
|
|
321
|
+
$files = Get-ChildItem -LiteralPath $crewMaintenanceRequestsDir -Filter '*.json' -File -ErrorAction SilentlyContinue
|
|
322
|
+
foreach ($file in $files) {
|
|
323
|
+
try {
|
|
324
|
+
$request = Get-Content -LiteralPath $file.FullName -Raw | ConvertFrom-Json -ErrorAction Stop
|
|
325
|
+
} catch {
|
|
326
|
+
Remove-Item -LiteralPath $file.FullName -Force -ErrorAction SilentlyContinue
|
|
327
|
+
continue
|
|
328
|
+
}
|
|
329
|
+
if ($request.schema_version -ne 1 -or -not $request.request_id -or -not $request.operation) {
|
|
330
|
+
Remove-Item -LiteralPath $file.FullName -Force -ErrorAction SilentlyContinue
|
|
331
|
+
continue
|
|
332
|
+
}
|
|
333
|
+
if (($request.expires_at -as [long]) -lt $now) {
|
|
334
|
+
Write-MaintenanceResult $request 'MAINTENANCE_EXPIRED'
|
|
335
|
+
Write-LaunchLog ('Maintenance request {0} ({1}) expired; not executed.' -f $request.request_id, $request.operation) 'WARN'
|
|
336
|
+
continue
|
|
337
|
+
}
|
|
338
|
+
$crew = $services | Where-Object { $_.CrewOwned } | Select-Object -First 1
|
|
339
|
+
if (-not $crew) { continue }
|
|
340
|
+
if ($request.operation -eq 'maintenance-stop') {
|
|
341
|
+
# Authority: the request must name the SAME live runtime identity we own.
|
|
342
|
+
$liveRuntimeId = $null
|
|
343
|
+
try {
|
|
344
|
+
$resp = Invoke-RestMethod -Uri 'http://127.0.0.1:3210/_dsh/dsh-crew/extension' -TimeoutSec 3
|
|
345
|
+
if ($resp.ok -eq $true) { $liveRuntimeId = $resp.extension.runtime.runtime_id }
|
|
346
|
+
} catch { $liveRuntimeId = $null }
|
|
347
|
+
if (-not $liveRuntimeId -or $liveRuntimeId -ne $request.runtime_id) {
|
|
348
|
+
Write-MaintenanceResult $request 'SUPERVISOR_OWNERSHIP_CONFLICT'
|
|
349
|
+
Write-LaunchLog ('Maintenance-stop {0} rejected: live runtime_id mismatch.' -f $request.request_id) 'WARN'
|
|
350
|
+
continue
|
|
351
|
+
}
|
|
352
|
+
$port = Get-PortState $crew.Port
|
|
353
|
+
if ($port.State -ne 'occupied' -or -not $port.Pid) {
|
|
354
|
+
Write-MaintenanceResult $request 'SUPERVISOR_STOP_FAILED'
|
|
355
|
+
continue
|
|
356
|
+
}
|
|
357
|
+
$stopped = Stop-OwnedListener -Service $crew -ListenerPid ([int] $port.Pid)
|
|
358
|
+
if ($stopped) {
|
|
359
|
+
# The stopped window now belongs to the npx lifecycle: persist the
|
|
360
|
+
# STOPPED session (lease + proven runtime_id) so ordinary
|
|
361
|
+
# supervision will NOT auto-start 3210 until the matching start.
|
|
362
|
+
# STRONG semantics: STOPPED is published ONLY after the session is
|
|
363
|
+
# durably written AND read back with exact identity. A session write
|
|
364
|
+
# failure must never tell npx it owns a stopped window it cannot
|
|
365
|
+
# later prove (that race auto-restarts 3210 mid tree-swap).
|
|
366
|
+
$sessionDurable = Set-MaintenanceSession $request
|
|
367
|
+
if ($sessionDurable) {
|
|
368
|
+
Write-MaintenanceResult $request 'STOPPED' @{ lease = $request.lease; stopped_runtime_id = $request.runtime_id }
|
|
369
|
+
Write-LaunchLog ('Maintenance-stop {0} executed; lease issued.' -f $request.request_id)
|
|
370
|
+
} else {
|
|
371
|
+
Write-MaintenanceResult $request 'SUPERVISOR_SESSION_PERSIST_FAILED'
|
|
372
|
+
Write-LaunchLog ('Maintenance-stop {0} stopped the process but the STOPPED session could not be persisted; NOT publishing STOPPED.' -f $request.request_id) 'ERROR'
|
|
373
|
+
}
|
|
374
|
+
} else {
|
|
375
|
+
Write-MaintenanceResult $request 'SUPERVISOR_STOP_FAILED'
|
|
376
|
+
}
|
|
377
|
+
} elseif ($request.operation -eq 'maintenance-start') {
|
|
378
|
+
# Pair with the matching STOPPED session: the start request must carry
|
|
379
|
+
# the SAME lease and runtime_id the stop proved. A missing session,
|
|
380
|
+
# mismatched lease/identity, or a still-listening port all fail closed
|
|
381
|
+
# WITHOUT starting anything. The session is consumed on VERIFIED so
|
|
382
|
+
# the lease is one-shot (no replay).
|
|
383
|
+
$lease = $request.lease
|
|
384
|
+
$expectedCrew = $null
|
|
385
|
+
$expectedDsh = $null
|
|
386
|
+
if ($request.extra) {
|
|
387
|
+
$expectedCrew = $request.extra.expected_crew_version
|
|
388
|
+
$expectedDsh = $request.extra.expected_dsh_version
|
|
389
|
+
}
|
|
390
|
+
$session = Read-MaintenanceSession
|
|
391
|
+
if ($session -is [string] -or $session.lease -ne $lease -or $session.runtime_id -ne $request.runtime_id) {
|
|
392
|
+
Write-MaintenanceResult $request 'SUPERVISOR_OWNERSHIP_CONFLICT'
|
|
393
|
+
Write-LaunchLog ('Maintenance-start {0} rejected: no matching STOPPED session (lease/identity mismatch).' -f $request.request_id) 'WARN'
|
|
394
|
+
continue
|
|
395
|
+
}
|
|
396
|
+
$livePort = Get-PortState $crew.Port
|
|
397
|
+
if ($livePort.State -ne 'free') {
|
|
398
|
+
# occupied AND unknown both fail closed: the stopped window is not
|
|
399
|
+
# provably clean, so no start authority.
|
|
400
|
+
Write-MaintenanceResult $request 'SUPERVISOR_OWNERSHIP_CONFLICT'
|
|
401
|
+
Write-LaunchLog ('Maintenance-start {0} rejected: port {1} is not provably free (state={2}).' -f $request.request_id, $crew.Port, $livePort.State) 'WARN'
|
|
402
|
+
continue
|
|
403
|
+
}
|
|
404
|
+
Start-CrewService $crew
|
|
405
|
+
Wait-CrewServices
|
|
406
|
+
$newRuntime = $null
|
|
407
|
+
try {
|
|
408
|
+
$resp = Invoke-RestMethod -Uri 'http://127.0.0.1:3210/_dsh/dsh-crew/extension' -TimeoutSec 5
|
|
409
|
+
if ($resp.ok -eq $true) { $newRuntime = $resp.extension.runtime }
|
|
410
|
+
} catch { $newRuntime = $null }
|
|
411
|
+
$ok = $newRuntime -and $newRuntime.runtime_id -and ($expectedDsh -eq $null -or $newRuntime.dsh_version -eq $expectedDsh)
|
|
412
|
+
if ($ok) {
|
|
413
|
+
# Verifiable one-shot consumption: only a provably-cleared session
|
|
414
|
+
# completes the transaction; otherwise the lease stays live and any
|
|
415
|
+
# replay is still fenced (no VERIFIED without consumption proof).
|
|
416
|
+
$consumed = Clear-MaintenanceSession
|
|
417
|
+
if ($consumed) {
|
|
418
|
+
Write-MaintenanceResult $request 'VERIFIED' @{ lease = $lease; runtime_id = $newRuntime.runtime_id; runtime_version = $newRuntime.runtime_version; dsh_version = $newRuntime.dsh_version }
|
|
419
|
+
Write-LaunchLog ('Maintenance-start {0} verified: Crew {1} + DSH {2}.' -f $request.request_id, $newRuntime.runtime_version, $newRuntime.dsh_version)
|
|
420
|
+
} else {
|
|
421
|
+
Write-MaintenanceResult $request 'VERIFY_FAILED' @{ lease = $lease; runtime_id = $newRuntime.runtime_id }
|
|
422
|
+
Write-LaunchLog ('Maintenance-start {0} identity verified but session consumption unproven; NOT marking VERIFIED.' -f $request.request_id) 'ERROR'
|
|
423
|
+
}
|
|
424
|
+
} else {
|
|
425
|
+
$failedRuntimeId = $null
|
|
426
|
+
if ($newRuntime) { $failedRuntimeId = $newRuntime.runtime_id }
|
|
427
|
+
Write-MaintenanceResult $request 'VERIFY_FAILED' @{ lease = $lease; runtime_id = $failedRuntimeId }
|
|
428
|
+
Write-LaunchLog ('Maintenance-start {0} verification failed.' -f $request.request_id) 'ERROR'
|
|
429
|
+
}
|
|
430
|
+
} else {
|
|
431
|
+
# Unknown maintenance op: remove and report.
|
|
432
|
+
Write-MaintenanceResult $request 'MAINTENANCE_UNKNOWN_OP'
|
|
433
|
+
}
|
|
434
|
+
}
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
|
|
438
|
+
function Get-PortState {
|
|
439
|
+
param([int] $Port)
|
|
440
|
+
try {
|
|
441
|
+
$listeners = [System.Net.NetworkInformation.IPGlobalProperties]::GetIPGlobalProperties().GetActiveTcpListeners()
|
|
442
|
+
$occupied = @($listeners | Where-Object Port -eq $Port).Count -gt 0
|
|
443
|
+
if (-not $occupied) {
|
|
444
|
+
return [pscustomobject]@{ State = 'free'; Error = $null; Pid = $null }
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
$ownerPid = $null
|
|
448
|
+
try {
|
|
449
|
+
$ownerPid = (Get-NetTCPConnection -State Listen -LocalPort $Port -ErrorAction Stop | Select-Object -First 1).OwningProcess
|
|
450
|
+
} catch { }
|
|
451
|
+
return [pscustomobject]@{ State = 'occupied'; Error = $null; Pid = $ownerPid }
|
|
452
|
+
} catch {
|
|
453
|
+
return [pscustomobject]@{ State = 'unknown'; Error = ('Listener enumeration failed: {0}' -f $_.Exception.Message); Pid = $null }
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
function Test-TrackedProcessIdentity {
|
|
458
|
+
param([int] $ProcessId, [long] $ExpectedStartTicks, [object[]] $ProcessTable)
|
|
459
|
+
if ($ProcessId -lt 1 -or $ExpectedStartTicks -lt 1) { return $false }
|
|
460
|
+
$record = @($ProcessTable | Where-Object { [int] $_.ProcessId -eq $ProcessId } | Select-Object -First 1)
|
|
461
|
+
if ($record.Count -eq 0) { return $false }
|
|
462
|
+
$startTicks = $null
|
|
463
|
+
if ($record[0].PSObject.Properties.Name -contains 'StartTicks') {
|
|
464
|
+
$startTicks = [long] $record[0].StartTicks
|
|
465
|
+
} else {
|
|
466
|
+
try { $startTicks = (Get-Process -Id $ProcessId -ErrorAction Stop).StartTime.ToUniversalTime().Ticks } catch { return $false }
|
|
467
|
+
}
|
|
468
|
+
return $startTicks -eq $ExpectedStartTicks
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
function Get-TrackedProcessTree {
|
|
472
|
+
param([pscustomobject] $Service, [object[]] $ProcessTable = $null)
|
|
473
|
+
$processes = if ($null -ne $ProcessTable) { @($ProcessTable) } else { @(Get-CimInstance Win32_Process -ErrorAction Stop) }
|
|
474
|
+
$owned = [System.Collections.Generic.HashSet[int]]::new()
|
|
475
|
+
$rootMatches = Test-TrackedProcessIdentity -ProcessId $Service.RootPid -ExpectedStartTicks $Service.RootStartedAtUtcTicks -ProcessTable $processes
|
|
476
|
+
$hasTrackedListener = $Service.ListenerPid -and $Service.ListenerStartedAtUtcTicks
|
|
477
|
+
if ($hasTrackedListener) {
|
|
478
|
+
$listenerMatches = Test-TrackedProcessIdentity -ProcessId $Service.ListenerPid -ExpectedStartTicks $Service.ListenerStartedAtUtcTicks -ProcessTable $processes
|
|
479
|
+
if (-not $listenerMatches) { return @() }
|
|
480
|
+
[void] $owned.Add([int] $Service.ListenerPid)
|
|
481
|
+
} elseif (-not $rootMatches) {
|
|
482
|
+
return @()
|
|
483
|
+
}
|
|
484
|
+
if ($rootMatches) { [void] $owned.Add([int] $Service.RootPid) }
|
|
485
|
+
do {
|
|
486
|
+
$added = $false
|
|
487
|
+
foreach ($candidate in $processes) {
|
|
488
|
+
$candidateId = [int] $candidate.ProcessId
|
|
489
|
+
$parentId = [int] $candidate.ParentProcessId
|
|
490
|
+
if ($owned.Contains($parentId) -and $owned.Add($candidateId)) { $added = $true }
|
|
491
|
+
}
|
|
492
|
+
} while ($added)
|
|
493
|
+
return @($owned | ForEach-Object { [int] $_ })
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
function Set-TrackedListenerIdentity {
|
|
497
|
+
param([pscustomobject] $Service)
|
|
498
|
+
$port = Get-PortState $Service.Port
|
|
499
|
+
if ($port.State -ne 'occupied' -or -not $port.Pid) { return $false }
|
|
500
|
+
$tree = @(Get-TrackedProcessTree -Service $Service)
|
|
501
|
+
if ($port.Pid -notin $tree) { return $false }
|
|
502
|
+
try {
|
|
503
|
+
$listener = Get-Process -Id $port.Pid -ErrorAction Stop
|
|
504
|
+
$Service.ListenerPid = [int] $port.Pid
|
|
505
|
+
$Service.ListenerStartedAtUtcTicks = $listener.StartTime.ToUniversalTime().Ticks
|
|
506
|
+
return $true
|
|
507
|
+
} catch {
|
|
508
|
+
return $false
|
|
509
|
+
}
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
function Stop-OwnedListener {
|
|
513
|
+
param([pscustomobject] $Service, [int] $ListenerPid)
|
|
514
|
+
if ($Mode -ne 'watch' -or -not $Service.ListenerPid -or $ListenerPid -ne $Service.ListenerPid) {
|
|
515
|
+
return $false
|
|
516
|
+
}
|
|
517
|
+
try {
|
|
518
|
+
$ownedProcessIds = @(Get-TrackedProcessTree -Service $Service)
|
|
519
|
+
if ($ListenerPid -notin $ownedProcessIds) { return $false }
|
|
520
|
+
Write-LaunchLog ('Supervisor confirmed owned listener PID={0} under tracked root PID={1}; stopping that process tree after {2} consecutive failed health checks.' -f $ListenerPid, $Service.RootPid, $Service.ConsecutiveFailures) 'WARN'
|
|
521
|
+
foreach ($processId in $ownedProcessIds) {
|
|
522
|
+
Stop-Process -Id $processId -Force -ErrorAction SilentlyContinue
|
|
523
|
+
}
|
|
524
|
+
$deadline = (Get-Date).AddSeconds(5)
|
|
525
|
+
do {
|
|
526
|
+
$port = Get-PortState $Service.Port
|
|
527
|
+
if ($port.State -eq 'free') { return $true }
|
|
528
|
+
Start-Sleep -Milliseconds 250
|
|
529
|
+
} while ((Get-Date) -lt $deadline)
|
|
530
|
+
} catch {
|
|
531
|
+
Write-LaunchLog ('Supervisor could not stop its owned listener safely: {0}' -f $_.Exception.Message) 'WARN'
|
|
532
|
+
}
|
|
533
|
+
return $false
|
|
534
|
+
}
|
|
535
|
+
|
|
536
|
+
function Start-CrewService {
|
|
537
|
+
param([pscustomobject] $Service)
|
|
538
|
+
$serviceRunStamp = (Get-Date).ToString('yyyyMMdd-HHmmssfff')
|
|
539
|
+
$stdout = Join-Path $logRoot ('dsh-crew-{0}-{1}-{2}.out.log' -f $Service.Profile, $Service.Port, $serviceRunStamp)
|
|
540
|
+
$stderr = Join-Path $logRoot ('dsh-crew-{0}-{1}-{2}.err.log' -f $Service.Profile, $Service.Port, $serviceRunStamp)
|
|
541
|
+
$previousHome = $env:DSH_HOME
|
|
542
|
+
try {
|
|
543
|
+
$env:DSH_HOME = $Service.Home
|
|
544
|
+
$arguments = @('--profile', $Service.Profile, '--host', '127.0.0.1', '--port', [string] $Service.Port, '--no-open')
|
|
545
|
+
$process = Start-Process -FilePath $dshCli -ArgumentList $arguments -WindowStyle Hidden -PassThru `
|
|
546
|
+
-RedirectStandardOutput $stdout -RedirectStandardError $stderr
|
|
547
|
+
$Service.Process = $process
|
|
548
|
+
$Service.RootPid = $process.Id
|
|
549
|
+
$Service.RootStartedAtUtcTicks = $process.StartTime.ToUniversalTime().Ticks
|
|
550
|
+
$Service.ListenerPid = $null
|
|
551
|
+
$Service.ListenerStartedAtUtcTicks = $null
|
|
552
|
+
$Service.ConsecutiveFailures = 0
|
|
553
|
+
$Service.State = 'starting'
|
|
554
|
+
Write-LaunchLog ('Started {0} on port {1}; PID={2}; stdout={3}; stderr={4}' -f $Service.Profile, $Service.Port, $process.Id, $stdout, $stderr)
|
|
555
|
+
} finally {
|
|
556
|
+
$env:DSH_HOME = $previousHome
|
|
557
|
+
}
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
function Wait-CrewServices {
|
|
561
|
+
$deadline = (Get-Date).AddSeconds(90)
|
|
562
|
+
while (@($services | Where-Object State -eq 'starting').Count -gt 0 -and (Get-Date) -lt $deadline) {
|
|
563
|
+
foreach ($service in ($services | Where-Object State -eq 'starting')) {
|
|
564
|
+
$health = Get-HealthState $service
|
|
565
|
+
$service.LastError = $health.Error
|
|
566
|
+
if ($health.Ready) {
|
|
567
|
+
$service.State = 'ready'
|
|
568
|
+
$service.ConsecutiveFailures = 0
|
|
569
|
+
if ($service.CrewOwned) { [void] (Set-TrackedListenerIdentity -Service $service) }
|
|
570
|
+
Write-LaunchLog ('{0} ready on {1}; runtime={2}' -f $service.Name, $service.Port, $health.Version)
|
|
571
|
+
} elseif ($service.Process -and $service.Process.HasExited) {
|
|
572
|
+
throw ('{0} exited before becoming ready; PID={1}; exit={2}; last health error: {3}' -f $service.Name, $service.Process.Id, $service.Process.ExitCode, $health.Error)
|
|
573
|
+
}
|
|
574
|
+
}
|
|
575
|
+
if (@($services | Where-Object State -eq 'starting').Count -gt 0) { Start-Sleep -Milliseconds 500 }
|
|
576
|
+
}
|
|
577
|
+
|
|
578
|
+
$notReady = @($services | Where-Object {
|
|
579
|
+
$_.State -ne 'ready'
|
|
580
|
+
})
|
|
581
|
+
if ($notReady.Count -gt 0) {
|
|
582
|
+
$details = ($notReady | ForEach-Object { '{0}:{1} ({2})' -f $_.Profile, $_.Port, $_.LastError }) -join '; '
|
|
583
|
+
throw "Startup health deadline exceeded: $details"
|
|
584
|
+
}
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
# Non-critical diagnostic, called only after Ensure-CrewServices returns:
|
|
588
|
+
# logs legacy bridge presence for operator awareness. Never gates, delays,
|
|
589
|
+
# or claims 3210; never runs on the boot path.
|
|
590
|
+
function Write-LegacyBridgeDiagnostic {
|
|
591
|
+
if (Test-LegacyBridgeAvailable) {
|
|
592
|
+
Write-LaunchLog 'Diagnostic: legacy 3080 bridge still answers ping; Crew launcher owns 3210 directly, bridge ignored.'
|
|
593
|
+
}
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
function Ensure-CrewServices {
|
|
597
|
+
param([switch] $QuietHealthy)
|
|
598
|
+
|
|
599
|
+
foreach ($service in $services) {
|
|
600
|
+
# Maintenance fence: while a STOPPED maintenance session is active for
|
|
601
|
+
# the Crew backend, ordinary supervision must NOT auto-start 3210 —
|
|
602
|
+
# the stopped window belongs to the npx lifecycle's tree swap. Only a
|
|
603
|
+
# matching maintenance-start owns the launch right. 3080 supervision
|
|
604
|
+
# continues normally during the window.
|
|
605
|
+
if ($service.CrewOwned -and (Test-MaintenanceSessionActive)) {
|
|
606
|
+
$service.State = 'maintenance'
|
|
607
|
+
$service.LastError = $null
|
|
608
|
+
continue
|
|
609
|
+
}
|
|
610
|
+
$health = Get-HealthState $service
|
|
611
|
+
if ($health.Ready) {
|
|
612
|
+
$wasReady = $service.State -eq 'ready'
|
|
613
|
+
$service.State = 'ready'
|
|
614
|
+
$service.ConsecutiveFailures = 0
|
|
615
|
+
$service.LastError = $null
|
|
616
|
+
if ($service.CrewOwned -and -not $service.ListenerPid) { [void] (Set-TrackedListenerIdentity -Service $service) }
|
|
617
|
+
if (-not $QuietHealthy -or -not $wasReady) {
|
|
618
|
+
Write-LaunchLog ('{0} already ready on {1}; runtime={2}' -f $service.Name, $service.Port, $health.Version)
|
|
619
|
+
}
|
|
620
|
+
continue
|
|
621
|
+
}
|
|
622
|
+
|
|
623
|
+
# The Crew launcher owns 3210 directly and always boots it below.
|
|
624
|
+
# Legacy bridge presence is diagnosed only after 3210 is ready (see
|
|
625
|
+
# Wait-CrewServices tail), never on the boot path.
|
|
626
|
+
|
|
627
|
+
$service.LastError = $health.Error
|
|
628
|
+
$service.ConsecutiveFailures += 1
|
|
629
|
+
if ($service.Process -and $service.Process.HasExited) {
|
|
630
|
+
Write-LaunchLog ('{0} process exited after startup; PID={1}; exit={2}' -f $service.Name, $service.Process.Id, $service.Process.ExitCode) 'WARN'
|
|
631
|
+
$service.Process = $null
|
|
632
|
+
}
|
|
633
|
+
$service.State = 'pending'
|
|
634
|
+
|
|
635
|
+
$port = Get-PortState $service.Port
|
|
636
|
+
if ($port.State -eq 'occupied') {
|
|
637
|
+
if ($service.ConsecutiveFailures -lt 3) {
|
|
638
|
+
throw ('Health check {0}/3 failed for {1}; owned process remains untouched until failure is confirmed. Health error: {2}' -f $service.ConsecutiveFailures, $service.Name, $health.Error)
|
|
639
|
+
}
|
|
640
|
+
if (Stop-OwnedListener -Service $service -ListenerPid $port.Pid) {
|
|
641
|
+
$service.Process = $null
|
|
642
|
+
$service.RootPid = $null
|
|
643
|
+
$service.RootStartedAtUtcTicks = $null
|
|
644
|
+
$service.ListenerPid = $null
|
|
645
|
+
$service.ListenerStartedAtUtcTicks = $null
|
|
646
|
+
$service.ConsecutiveFailures = 0
|
|
647
|
+
$port = Get-PortState $service.Port
|
|
648
|
+
}
|
|
649
|
+
}
|
|
650
|
+
if ($port.State -eq 'occupied') {
|
|
651
|
+
$owner = if ($port.Pid) { "; listener PID=$($port.Pid)" } else { '' }
|
|
652
|
+
throw ('Port {0} is occupied, but {1} failed its health contract{2}. Health error: {3}' -f $service.Port, $service.Name, $owner, $health.Error)
|
|
653
|
+
}
|
|
654
|
+
if ($port.State -ne 'free') {
|
|
655
|
+
throw ('Could not determine whether port {0} is available: {1}' -f $service.Port, $port.Error)
|
|
656
|
+
}
|
|
657
|
+
if ($Mode -eq 'watch') {
|
|
658
|
+
Write-LaunchLog ('Supervisor detected {0} unavailable on {1}; restarting it.' -f $service.Name, $service.Port) 'WARN'
|
|
659
|
+
}
|
|
660
|
+
Start-CrewService $service
|
|
661
|
+
}
|
|
662
|
+
|
|
663
|
+
Wait-CrewServices
|
|
664
|
+
}
|
|
665
|
+
|
|
666
|
+
function Start-ServiceSupervisor {
|
|
667
|
+
$mutex = [System.Threading.Mutex]::new($false, 'Local\DSHCrewServiceSupervisor')
|
|
668
|
+
$ownsMutex = $false
|
|
669
|
+
try {
|
|
670
|
+
try {
|
|
671
|
+
$ownsMutex = $mutex.WaitOne(0)
|
|
672
|
+
} catch [System.Threading.AbandonedMutexException] {
|
|
673
|
+
$ownsMutex = $true
|
|
674
|
+
}
|
|
675
|
+
if (-not $ownsMutex) {
|
|
676
|
+
Write-LaunchLog 'Supervisor already active; duplicate watcher exiting.'
|
|
677
|
+
return
|
|
678
|
+
}
|
|
679
|
+
|
|
680
|
+
Write-LaunchLog 'Supervisor active; monitoring 3080 and 3210 every 10 seconds.'
|
|
681
|
+
$lastRecoveryError = $null
|
|
682
|
+
$updateLockFile = Join-Path $crewHome '..\app\update-in-progress.lock'
|
|
683
|
+
$staleLockNotified = $false
|
|
684
|
+
while ($true) {
|
|
685
|
+
try {
|
|
686
|
+
# An installer-held update lock suspends recovery restarts: the tree
|
|
687
|
+
# under Crew-owned state may be mid-migration, and a restart now
|
|
688
|
+
# would boot a half-installed runtime/profile. A lock whose owner
|
|
689
|
+
# process is dead is stale: log once and resume supervision instead
|
|
690
|
+
# of staying observe-only forever.
|
|
691
|
+
$updateHeld = $false
|
|
692
|
+
if (Test-Path -LiteralPath $updateLockFile -PathType Leaf) {
|
|
693
|
+
$lockAlive = $false
|
|
694
|
+
try {
|
|
695
|
+
$lockRecord = Get-Content -LiteralPath $updateLockFile -Raw -ErrorAction Stop | ConvertFrom-Json -ErrorAction Stop
|
|
696
|
+
$lockPid = [int] $lockRecord.pid
|
|
697
|
+
if ($lockPid -gt 0) {
|
|
698
|
+
Get-Process -Id $lockPid -ErrorAction Stop | Out-Null
|
|
699
|
+
$lockAlive = $true
|
|
700
|
+
}
|
|
701
|
+
} catch { $lockAlive = $false }
|
|
702
|
+
if ($lockAlive) {
|
|
703
|
+
$updateHeld = $true
|
|
704
|
+
$staleLockNotified = $false
|
|
705
|
+
} elseif (-not $staleLockNotified) {
|
|
706
|
+
Write-LaunchLog 'Stale update lock detected (owner process dead); resuming supervision. Re-run install/update to reconcile.' 'WARN'
|
|
707
|
+
$staleLockNotified = $true
|
|
708
|
+
}
|
|
709
|
+
} else {
|
|
710
|
+
$staleLockNotified = $false
|
|
711
|
+
}
|
|
712
|
+
if ($updateHeld) {
|
|
713
|
+
Write-LaunchLog 'Update in progress; supervisor observing only, no restarts.'
|
|
714
|
+
$lastRecoveryError = $null
|
|
715
|
+
} else {
|
|
716
|
+
# Control protocol: publish this watcher's heartbeat and consume
|
|
717
|
+
# any durable restart requests the hub wrote (3210 never spawns
|
|
718
|
+
# itself). Then consume maintenance transactions (npx cohort
|
|
719
|
+
# migration stop/start phases). Then run the ordinary health pass.
|
|
720
|
+
Write-SupervisorHeartbeat
|
|
721
|
+
Invoke-CrewRestartRequests
|
|
722
|
+
Invoke-CrewMaintenanceRequests
|
|
723
|
+
Ensure-CrewServices -QuietHealthy
|
|
724
|
+
if ($lastRecoveryError) {
|
|
725
|
+
Write-LaunchLog 'Supervisor recovery succeeded; both services are healthy.'
|
|
726
|
+
$lastRecoveryError = $null
|
|
727
|
+
}
|
|
728
|
+
}
|
|
729
|
+
} catch {
|
|
730
|
+
$recoveryError = $_.Exception.Message
|
|
731
|
+
if ($recoveryError -ne $lastRecoveryError) {
|
|
732
|
+
Write-LaunchLog ('Supervisor recovery failed; will retry: {0}' -f $recoveryError) 'WARN'
|
|
733
|
+
$lastRecoveryError = $recoveryError
|
|
734
|
+
}
|
|
735
|
+
}
|
|
736
|
+
Start-Sleep -Seconds 10
|
|
737
|
+
}
|
|
738
|
+
} finally {
|
|
739
|
+
if ($ownsMutex) { $mutex.ReleaseMutex() }
|
|
740
|
+
$mutex.Dispose()
|
|
741
|
+
}
|
|
742
|
+
}
|
|
743
|
+
|
|
744
|
+
if ($env:DSH_CREW_LAUNCHER_TEST_IMPORT -eq '1') { return }
|
|
745
|
+
|
|
746
|
+
try {
|
|
747
|
+
New-Item -ItemType Directory -Path $logRoot -Force | Out-Null
|
|
748
|
+
Write-LaunchLog ('Launcher started; mode={0}; user={1}' -f $Mode, $env:USERNAME)
|
|
749
|
+
|
|
750
|
+
if (-not (Test-Path -LiteralPath $dshCli -PathType Leaf)) {
|
|
751
|
+
throw "DSH CLI was not found at $dshCli. Run: npm install -g @ran-sh/dsh-crew@latest; dsh-crew update"
|
|
752
|
+
}
|
|
753
|
+
if (-not (Test-Path -LiteralPath (Join-Path $crewHome 'profiles\dsh-crew\package.json') -PathType Leaf)) {
|
|
754
|
+
throw "The isolated dsh-crew profile is missing under $crewHome. Run: dsh-crew update"
|
|
755
|
+
}
|
|
756
|
+
if (-not (Test-Path -LiteralPath (Join-Path $officialHome 'profiles\web\package.json') -PathType Leaf)) {
|
|
757
|
+
throw "The official web profile is missing under $officialHome. Repair or install the official DeepSeek Harness web profile outside dsh-crew; dsh-crew never mutates the official profile."
|
|
758
|
+
}
|
|
759
|
+
|
|
760
|
+
if ($Mode -eq 'watch') {
|
|
761
|
+
Start-ServiceSupervisor
|
|
762
|
+
Write-LaunchLog 'Supervisor stopped.' 'WARN'
|
|
763
|
+
exit 0
|
|
764
|
+
}
|
|
765
|
+
|
|
766
|
+
Ensure-CrewServices
|
|
767
|
+
Write-LegacyBridgeDiagnostic
|
|
768
|
+
|
|
769
|
+
if ($Mode -eq 'open') {
|
|
770
|
+
Start-Process 'http://127.0.0.1:3080/' | Out-Null
|
|
771
|
+
Write-LaunchLog 'Opened daily console at http://127.0.0.1:3080/'
|
|
772
|
+
}
|
|
773
|
+
Write-LaunchLog ('Launcher completed successfully in {0:n1}s.' -f ((Get-Date) - $startedAt).TotalSeconds)
|
|
774
|
+
exit 0
|
|
775
|
+
} catch {
|
|
776
|
+
Write-LaunchLog $_.Exception.Message 'ERROR'
|
|
777
|
+
if ($Mode -eq 'open') {
|
|
778
|
+
Write-Host ''
|
|
779
|
+
Write-Host "Diagnostic log: $launcherLog" -ForegroundColor Yellow
|
|
780
|
+
Write-Host 'Startup failed. Review the diagnostic log and the service stdout/stderr paths recorded above.' -ForegroundColor Yellow
|
|
781
|
+
}
|
|
782
|
+
exit 1
|
|
783
|
+
}
|