@trillboards/edge-sdk 0.2.220 → 0.2.222

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,538 @@
1
+ <#
2
+ test-rollout-proofs.ps1 - the connector update path, measured on a real
3
+ Windows box, with no release hold.
4
+
5
+ Installs the release's own signed code-only payload as a store PC runs it
6
+ (the TrillboardsEdge scheduled task, the shipped launcher chain, a real RTSP
7
+ camera) and points it at stand-ins (scripts/rollout-proof-rig.mjs): an API
8
+ whose sensing-profile response carries `connectorRelease` as production does,
9
+ and a local screen.trillboards.com (hosts file + a rig root CA) serving the
10
+ release channel and update payloads. Nothing is published anywhere.
11
+
12
+ PROOF 1 a newly published build is installed within -PublishDeadlineMinutes
13
+ (15): the box learns of it from the profile poll it already makes,
14
+ stages, waits for a safe point, swaps, and the new version reports
15
+ healthy and reports its version to the API.
16
+ PROOF 2 a deliberately broken build rolls itself back: it exits at
17
+ startup, install-update.ps1 sees it never report healthy, restores
18
+ the previous version, records the broken one in
19
+ failed-versions.json, and the restored connector does not take it
20
+ again (watched for -NoRetryWatchMinutes, longer than one poll).
21
+ PROOF 3 the monitoring gap of an update restart, in seconds: from the old
22
+ connector's exit to the new one's health marker (written once its
23
+ cameras are started), on PROOF 1's swap and PROOF 2's rollback.
24
+
25
+ The two published builds are the release's payload relabelled (B, C =
26
+ version + 1000, + 1001) with a catalog signed by a rig certificate whose
27
+ subject is the production publisher's CN and which only this runner trusts;
28
+ C's cli.js exits 3 whenever it is started to sense. Every .ps1/.vbs/.exe in
29
+ bin\ keeps the release's real signatures. setup-litertlm.ps1 and
30
+ setup-detector.ps1 are left out of every tree so the launcher's local-AI
31
+ repair never starts a download on the runner; the model service is a stub
32
+ that lists the pinned Gemma id.
33
+
34
+ Execute only on a fresh GitHub-hosted Windows runner. Do not run on a
35
+ customer PC: it edits the hosts file and the machine root store.
36
+ #>
37
+ [CmdletBinding()]
38
+ param(
39
+ [Parameter(Mandatory = $true)][string]$UpdateZip,
40
+ [Parameter(Mandatory = $true)][string]$PersonFrame,
41
+ [ValidateRange(1, 60)][int]$PublishDeadlineMinutes = 15,
42
+ [ValidateRange(1, 60)][int]$RollbackDeadlineMinutes = 15,
43
+ [ValidateRange(1, 30)][int]$NoRetryWatchMinutes = 6
44
+ )
45
+ $ErrorActionPreference = 'Stop'
46
+ Set-StrictMode -Version Latest
47
+ if ($env:GITHUB_ACTIONS -ne 'true' -or $env:RUNNER_OS -ne 'Windows' -or
48
+ $env:RUNNER_ENVIRONMENT -ne 'github-hosted' -or $PSVersionTable.PSVersion.Major -lt 7) {
49
+ throw 'The rollout proofs require a fresh GitHub-hosted Windows runner and PowerShell 7'
50
+ }
51
+ foreach ($p in @($UpdateZip, $PersonFrame)) { if (-not (Test-Path -LiteralPath $p -PathType Leaf)) { throw "Missing input: $p" } }
52
+
53
+ $Here = Split-Path -Parent $MyInvocation.MyCommand.Path # deploy\windows\ci
54
+ $PkgRoot = (Resolve-Path (Join-Path $Here '..\..\..')).Path # packages\edge-sdk
55
+ $RigScript = Join-Path $PkgRoot 'scripts\rollout-proof-rig.mjs'
56
+ $installDir = Join-Path $env:LOCALAPPDATA 'Trillboards\edge'
57
+ $updateDir = Join-Path $installDir '.auto-update'
58
+ $connectorLog = Join-Path $installDir 'logs\connector.log'
59
+ $hostsPath = Join-Path $env:WINDIR 'System32\drivers\etc\hosts'
60
+ if (Test-Path -LiteralPath $installDir) { throw 'The CI profile already holds a connector install' }
61
+ foreach ($name in @('TrillboardsEdge', 'TrillboardsLiteRtLm', 'TrillboardsDetector')) {
62
+ if (Get-ScheduledTask -TaskName $name -TaskPath '\' -ErrorAction SilentlyContinue) { throw ('Scheduled task already present: ' + $name) }
63
+ }
64
+
65
+ $root = [IO.Path]::GetFullPath((Join-Path $env:RUNNER_TEMP ('tb-rollout-' + [Guid]::NewGuid().ToString('N'))))
66
+ $rigOut = Join-Path $root 'rig'; $www = Join-Path $root 'www'; $trees = Join-Path $root 'trees'; $statePath = Join-Path $root 'state.json'
67
+ New-Item -ItemType Directory -Path $root, $rigOut, $www, $trees | Out-Null
68
+ $timeline = Join-Path $root 'timeline.log'
69
+ $receipt = [ordered]@{ atUtc = [DateTimeOffset]::UtcNow.ToString('o'); githubRunId = $env:GITHUB_RUN_ID; passed = $false; phase = 'start' }
70
+ $owned = New-Object System.Collections.Generic.List[System.Diagnostics.Process]
71
+ $trustedThumbprints = New-Object System.Collections.Generic.List[string]
72
+ $hostsOriginal = $null
73
+
74
+ function Note([string]$Message) {
75
+ $line = [DateTimeOffset]::UtcNow.ToString('HH:mm:ss.fff') + ' ' + $Message
76
+ Write-Host $line
77
+ Add-Content -LiteralPath $timeline -Value $line -Encoding utf8
78
+ }
79
+ function Utc { [DateTimeOffset]::UtcNow }
80
+ # PowerShell 7 turns ISO strings into local DateTime values when it parses JSON,
81
+ # so every timestamp is read as TEXT, straight out of the raw JSON.
82
+ function Get-JsonTime([string]$Raw, [string]$Key) {
83
+ $m = [regex]::Match($Raw, '"' + [regex]::Escape($Key) + '"\s*:\s*"([^"]+)"')
84
+ if (-not $m.Success) { return $null }
85
+ return [DateTimeOffset]::Parse($m.Groups[1].Value, [Globalization.CultureInfo]::InvariantCulture,
86
+ [Globalization.DateTimeStyles]::AssumeUniversal).ToUniversalTime()
87
+ }
88
+ function Get-MarkerTime([string]$Path) {
89
+ if (-not (Test-Path -LiteralPath $Path)) { return $null }
90
+ try { return Get-JsonTime ([IO.File]::ReadAllText($Path)) 'healthyAt' } catch { return $null }
91
+ }
92
+ function Seconds($From, $To) { if ($null -eq $From -or $null -eq $To) { return $null }; [Math]::Round(($To - $From).TotalSeconds, 1) }
93
+
94
+ # Children run with the native PS5.1 module path, as Task Scheduler starts them.
95
+ function Invoke-Owned {
96
+ param([string]$File, [string]$RawArguments, [string]$LogName, [int]$TimeoutSeconds)
97
+ $start = [Diagnostics.ProcessStartInfo]::new($File)
98
+ $start.UseShellExecute = $false; $start.CreateNoWindow = $true
99
+ $start.RedirectStandardOutput = $true; $start.RedirectStandardError = $true
100
+ $null = $start.Environment.Remove('PSModulePath')
101
+ $start.Arguments = $RawArguments
102
+ $process = [Diagnostics.Process]::new(); $process.StartInfo = $start
103
+ $watch = [Diagnostics.Stopwatch]::StartNew()
104
+ try {
105
+ if (-not $process.Start()) { throw ('Could not start ' + $LogName) }
106
+ $stdout = $process.StandardOutput.ReadToEndAsync(); $stderr = $process.StandardError.ReadToEndAsync()
107
+ while (-not $process.WaitForExit(5000)) {
108
+ if ($watch.Elapsed.TotalSeconds -ge $TimeoutSeconds) { $process.Kill($true); throw ('Deadline exceeded: ' + $LogName) }
109
+ }
110
+ $output = $stdout.GetAwaiter().GetResult() + "`n" + $stderr.GetAwaiter().GetResult()
111
+ [IO.File]::WriteAllText((Join-Path $root $LogName), $output, [Text.UTF8Encoding]::new($false))
112
+ if ($process.ExitCode -ne 0) { throw ($LogName + ' failed with exit ' + $process.ExitCode) }
113
+ return $output
114
+ } finally { $process.Dispose() }
115
+ }
116
+
117
+ # ---------------------------------------------------------------------------
118
+ # Certificates, minted with .NET (no certificate-store calls that can wedge).
119
+ # ---------------------------------------------------------------------------
120
+ Add-Type -AssemblyName System.Security
121
+ function New-RigCertificate {
122
+ param([string]$Subject, [switch]$Authority, $Issuer, [string]$DnsName, [string]$Eku)
123
+ $rsa = [Security.Cryptography.RSA]::Create(2048)
124
+ $request = [Security.Cryptography.X509Certificates.CertificateRequest]::new("CN=$Subject", $rsa,
125
+ [Security.Cryptography.HashAlgorithmName]::SHA256, [Security.Cryptography.RSASignaturePadding]::Pkcs1)
126
+ $request.CertificateExtensions.Add([Security.Cryptography.X509Certificates.X509BasicConstraintsExtension]::new([bool]$Authority, $false, 0, $true))
127
+ $usage = if ($Authority) { 'KeyCertSign, CrlSign, DigitalSignature' } else { 'DigitalSignature, KeyEncipherment' }
128
+ $request.CertificateExtensions.Add([Security.Cryptography.X509Certificates.X509KeyUsageExtension]::new(
129
+ [Security.Cryptography.X509Certificates.X509KeyUsageFlags]$usage, $true))
130
+ $request.CertificateExtensions.Add([Security.Cryptography.X509Certificates.X509SubjectKeyIdentifierExtension]::new($request.PublicKey, $false))
131
+ if ($Eku) {
132
+ $oids = [Security.Cryptography.OidCollection]::new(); $null = $oids.Add([Security.Cryptography.Oid]::new($Eku))
133
+ $request.CertificateExtensions.Add([Security.Cryptography.X509Certificates.X509EnhancedKeyUsageExtension]::new($oids, $false))
134
+ }
135
+ if ($DnsName) {
136
+ $san = [Security.Cryptography.X509Certificates.SubjectAlternativeNameBuilder]::new(); $san.AddDnsName($DnsName)
137
+ $request.CertificateExtensions.Add($san.Build())
138
+ }
139
+ $notBefore = [DateTimeOffset]::UtcNow.AddMinutes(-10); $notAfter = $notBefore.AddDays(2)
140
+ if ($Issuer) {
141
+ # An issued certificate may not outlive its issuer; .NET refuses one that does.
142
+ $issuerEnd = [DateTimeOffset]::new($Issuer.NotAfter.ToUniversalTime()).AddMinutes(-1)
143
+ if ($notAfter -gt $issuerEnd) { $notAfter = $issuerEnd }
144
+ $serial = [byte[]]::new(16); [Security.Cryptography.RandomNumberGenerator]::Fill($serial)
145
+ $cert = [Security.Cryptography.X509Certificates.RSACertificateExtensions]::CopyWithPrivateKey($request.Create($Issuer, $notBefore, $notAfter, $serial), $rsa)
146
+ } else {
147
+ $cert = $request.CreateSelfSigned($notBefore, $notAfter)
148
+ }
149
+ return [pscustomobject]@{ Certificate = $cert; Key = $rsa }
150
+ }
151
+ function Add-MachineRoot($Certificate) {
152
+ $public = [Security.Cryptography.X509Certificates.X509Certificate2]::new($Certificate.RawData)
153
+ $store = [Security.Cryptography.X509Certificates.X509Store]::new('Root', 'LocalMachine')
154
+ $store.Open('ReadWrite'); try { $store.Add($public) } finally { $store.Close() }
155
+ $trustedThumbprints.Add($public.Thumbprint)
156
+ }
157
+
158
+ function Get-ConnectorProcesses {
159
+ $prefix = [IO.Path]::GetFullPath($installDir).TrimEnd('\') + '\'
160
+ @(Get-CimInstance Win32_Process -Filter "Name = 'node.exe'" -ErrorAction SilentlyContinue | Where-Object {
161
+ ([string]$_.ExecutablePath).StartsWith($prefix, [StringComparison]::OrdinalIgnoreCase) -and ([string]$_.CommandLine) -match 'edge-sdk\\dist\\cli\.js'
162
+ } | ForEach-Object { [int]$_.ProcessId })
163
+ }
164
+ function Get-InstalledVersion {
165
+ $pkg = Join-Path $installDir 'lib\edge-sdk\package.json'
166
+ if (-not (Test-Path -LiteralPath $pkg)) { return $null }
167
+ try { return [string](Get-Content -LiteralPath $pkg -Raw | ConvertFrom-Json).version } catch { return $null }
168
+ }
169
+ function Read-JsonFile([string]$Path) {
170
+ if (-not (Test-Path -LiteralPath $Path)) { return $null }
171
+ try { return (Get-Content -LiteralPath $Path -Raw | ConvertFrom-Json) } catch { return $null }
172
+ }
173
+ function Get-StatusReports {
174
+ $log = Join-Path $rigOut 'api-requests.jsonl'
175
+ if (-not (Test-Path -LiteralPath $log)) { return @() }
176
+ @(Get-Content -LiteralPath $log -ErrorAction SilentlyContinue | ForEach-Object {
177
+ $raw = $_; $row = $null; try { $row = $raw | ConvertFrom-Json } catch { }
178
+ if ($row -and $row.url -like '*connector-status*' -and $row.body) {
179
+ $body = $null; try { $body = $row.body | ConvertFrom-Json } catch { }
180
+ if ($body) { [pscustomobject]@{ At = (Get-JsonTime $raw 'at'); Body = $body } }
181
+ }
182
+ })
183
+ }
184
+ function Get-ProfilePolls {
185
+ $log = Join-Path $rigOut 'api-requests.jsonl'
186
+ if (-not (Test-Path -LiteralPath $log)) { return @() }
187
+ @(Get-Content -LiteralPath $log -ErrorAction SilentlyContinue | Where-Object { $_ -like '*/v2/earner/cameras/sensing-profile*' } |
188
+ ForEach-Object { Get-JsonTime $_ 'at' })
189
+ }
190
+ $script:seenLines = 0
191
+ $script:logEvents = New-Object System.Collections.Generic.List[object]
192
+ function Update-LogEvents {
193
+ if (-not (Test-Path -LiteralPath $connectorLog)) { return }
194
+ $lines = @(Get-Content -LiteralPath $connectorLog -ErrorAction SilentlyContinue)
195
+ if ($lines.Count -lt $script:seenLines) { $script:seenLines = 0 }
196
+ for ($i = $script:seenLines; $i -lt $lines.Count; $i++) {
197
+ $line = $lines[$i]
198
+ if ($line -match '\[update\]|Fatal error|uncaught|deliberately broken') {
199
+ $script:logEvents.Add([pscustomobject]@{ At = (Utc); Line = $line })
200
+ Note ('log: ' + $line.Substring(0, [Math]::Min(240, $line.Length)))
201
+ }
202
+ }
203
+ $script:seenLines = $lines.Count
204
+ }
205
+ function First-LogEvent([string]$Pattern, $After) {
206
+ @($script:logEvents | Where-Object { $_.Line -match $Pattern -and ($null -eq $After -or $_.At -ge $After) } | Select-Object -First 1)
207
+ }
208
+ function Write-State($Manifest, $Advert) {
209
+ $json = [ordered]@{ manifest = $Manifest; advert = $Advert } | ConvertTo-Json -Depth 8
210
+ $tmp = "$statePath.tmp"
211
+ [IO.File]::WriteAllText($tmp, $json, [Text.UTF8Encoding]::new($false))
212
+ Move-Item -LiteralPath $tmp -Destination $statePath -Force
213
+ }
214
+
215
+ try {
216
+ $receipt.phase = 'certificates'
217
+ $ca = New-RigCertificate -Subject 'Trillboards rollout rig CA' -Authority
218
+ $server = New-RigCertificate -Subject 'screen.trillboards.com' -Issuer $ca.Certificate -DnsName 'screen.trillboards.com' -Eku '1.3.6.1.5.5.7.3.1'
219
+ $signing = New-RigCertificate -Subject 'Aethercorp Inc' -Eku '1.3.6.1.5.5.7.3.3'
220
+ Add-MachineRoot $ca.Certificate
221
+ Add-MachineRoot $signing.Certificate
222
+ $keyPem = Join-Path $root 'server.key.pem'; $certPem = Join-Path $root 'server.cert.pem'
223
+ [IO.File]::WriteAllText($keyPem, $server.Key.ExportPkcs8PrivateKeyPem())
224
+ [IO.File]::WriteAllText($certPem, $server.Certificate.ExportCertificatePem())
225
+ # Set-AuthenticodeSignature needs a persisted key: round-trip through PFX.
226
+ $pfxBytes = $signing.Certificate.Export([Security.Cryptography.X509Certificates.X509ContentType]::Pfx, 'rig')
227
+ $signer = [Security.Cryptography.X509Certificates.X509Certificate2]::new($pfxBytes, 'rig',
228
+ [Security.Cryptography.X509Certificates.X509KeyStorageFlags]'Exportable, PersistKeySet, UserKeySet')
229
+ Note ('rig CA and signer trusted in LocalMachine\Root (signer CN=Aethercorp Inc, ' + $signer.Thumbprint + ')')
230
+
231
+ $receipt.phase = 'payloads'
232
+ Add-Type -AssemblyName System.IO.Compression.FileSystem
233
+ $treeA = Join-Path $trees 'A'
234
+ [IO.Compression.ZipFile]::ExtractToDirectory($UpdateZip, $treeA)
235
+ foreach ($rel in @('bin\node.exe', 'bin\apply-update.ps1', 'bin\install-update.ps1', 'lib\edge-sdk\dist\cli.js', 'lib\edge-sdk\package.json')) {
236
+ if (-not (Test-Path -LiteralPath (Join-Path $treeA $rel))) { throw "The release payload is missing $rel" }
237
+ }
238
+ foreach ($rel in @('bin\setup-litertlm.ps1', 'bin\setup-detector.ps1')) { Remove-Item -LiteralPath (Join-Path $treeA $rel) -Force -ErrorAction SilentlyContinue }
239
+ $versionA = [string](Get-Content -LiteralPath (Join-Path $treeA 'lib\edge-sdk\package.json') -Raw | ConvertFrom-Json).version
240
+ if ($versionA -notmatch '^(\d+)\.(\d+)\.(\d+)$') { throw "Unreadable release version '$versionA'" }
241
+ $versionB = "$($Matches[1]).$($Matches[2]).$([int]$Matches[3] + 1000)"
242
+ $versionC = "$($Matches[1]).$($Matches[2]).$([int]$Matches[3] + 1001)"
243
+ $receipt.versions = [ordered]@{ installed = $versionA; published = $versionB; broken = $versionC }
244
+
245
+ function Set-TreeVersion([string]$Tree, [string]$Version) {
246
+ $pkgPath = Join-Path $Tree 'lib\edge-sdk\package.json'
247
+ $pkg = Get-Content -LiteralPath $pkgPath -Raw | ConvertFrom-Json -AsHashtable
248
+ $pkg['version'] = $Version
249
+ [IO.File]::WriteAllText($pkgPath, (($pkg | ConvertTo-Json -Depth 32) + "`n"), [Text.UTF8Encoding]::new($false))
250
+ }
251
+ function Publish-Tree([string]$Tree, [string]$Version) {
252
+ $zipName = "update-$Version.zip"; $catalogName = "update-$Version.catalog.ps1"
253
+ $zip = Join-Path $www $zipName
254
+ [IO.Compression.ZipFile]::CreateFromDirectory($Tree, $zip, [IO.Compression.CompressionLevel]::Fastest, $false)
255
+ $sha = (Get-FileHash -LiteralPath $zip -Algorithm SHA256).Hash.ToLowerInvariant()
256
+ $size = (Get-Item -LiteralPath $zip).Length
257
+ $catalog = Join-Path $www $catalogName
258
+ $lines = @('# Trillboards connector update catalog - rollout proof rig (trusted on this runner only).',
259
+ ("`$TrillboardsUpdateVersion = '" + $Version + "'"), ("`$TrillboardsUpdateSha256 = '" + $sha + "'"), ("`$TrillboardsUpdateSizeBytes = " + $size))
260
+ [IO.File]::WriteAllText($catalog, (($lines -join "`r`n") + "`r`n"), [Text.UTF8Encoding]::new($false))
261
+ $signed = Set-AuthenticodeSignature -FilePath $catalog -Certificate $signer -HashAlgorithm SHA256
262
+ if ($signed.Status -ne 'Valid') { throw "Could not sign the $Version catalog: $($signed.Status) $($signed.StatusMessage)" }
263
+ $publishedAt = [DateTimeOffset]::UtcNow.ToString('yyyy-MM-ddTHH:mm:ss.fffffffZ')
264
+ $url = 'https://screen.trillboards.com/connector/rig/'
265
+ return [ordered]@{ schemaVersion = 1; platform = 'windows-x64'; connectorVersion = $Version
266
+ gitSha = ((Get-FileHash -InputStream ([IO.MemoryStream]::new([Text.Encoding]::UTF8.GetBytes($Version))) -Algorithm SHA1).Hash.ToLowerInvariant())
267
+ sha256 = $sha; sizeBytes = $size; publishedAt = $publishedAt; downloadUrl = $url + $zipName
268
+ updateUrl = $url + $zipName; updateCatalogUrl = $url + $catalogName }
269
+ }
270
+ function New-Channel([object[]]$Entries) {
271
+ $sorted = @($Entries | Sort-Object { [version]$_.connectorVersion } -Descending)
272
+ $channel = [ordered]@{}
273
+ foreach ($k in $sorted[0].Keys) { $channel[$k] = $sorted[0][$k] }
274
+ $channel['releaseHistory'] = $sorted
275
+ return $channel
276
+ }
277
+
278
+ $treeB = Join-Path $trees 'B'; Copy-Item -LiteralPath $treeA -Destination $treeB -Recurse; Set-TreeVersion $treeB $versionB
279
+ $treeC = Join-Path $trees 'C'; Copy-Item -LiteralPath $treeB -Destination $treeC -Recurse; Set-TreeVersion $treeC $versionC
280
+ $cliC = Join-Path $treeC 'lib\edge-sdk\dist\cli.js'
281
+ $cliText = [IO.File]::ReadAllText($cliC)
282
+ $breaker = "if (process.argv.slice(2).some((a) => a === 'auto-sense' || a === 'local-sense')) { console.error('[rig] deliberately broken build " + $versionC + " exits at startup'); process.exit(3); }`n"
283
+ if ($cliText.StartsWith('#!')) { $nl = $cliText.IndexOf("`n"); $cliText = $cliText.Substring(0, $nl + 1) + $breaker + $cliText.Substring($nl + 1) } else { $cliText = $breaker + $cliText }
284
+ [IO.File]::WriteAllText($cliC, $cliText, [Text.UTF8Encoding]::new($false))
285
+ $entryA = Publish-Tree $treeA $versionA
286
+ Note ("payloads: installed $versionA, published $versionB, broken $versionC (catalogs signed by the rig's CN=Aethercorp Inc)")
287
+
288
+ $receipt.phase = 'rig'
289
+ $hostsOriginal = [IO.File]::ReadAllText($hostsPath)
290
+ [IO.File]::WriteAllText($hostsPath, $hostsOriginal.TrimEnd() + "`r`n127.0.0.1 screen.trillboards.com # tb-rollout-proof`r`n", [Text.Encoding]::ASCII)
291
+ & ipconfig.exe /flushdns | Out-Null
292
+ Write-State (New-Channel @($entryA)) ([ordered]@{ windows = [ordered]@{ version = $versionA; publishedAt = $entryA.publishedAt }; macos = $null; minReleaseAgeMs = 0 })
293
+ $systemNode = (Get-Command node.exe -ErrorAction Stop).Source
294
+ $owned.Add((Start-Process -FilePath $systemNode -ArgumentList @($RigScript, '--out', $rigOut, '--state', $statePath, '--www', $www,
295
+ '--api-port', '47393', '--key', $keyPem, '--cert', $certPem) -PassThru -WindowStyle Hidden `
296
+ -RedirectStandardOutput (Join-Path $root 'rig.log') -RedirectStandardError (Join-Path $root 'rig.err.log')))
297
+ $deadline = (Utc).AddSeconds(30)
298
+ while (-not (Test-Path -LiteralPath (Join-Path $rigOut 'rig-ready.json')) -and (Utc) -lt $deadline) { Start-Sleep -Milliseconds 300 }
299
+ if (-not (Test-Path -LiteralPath (Join-Path $rigOut 'rig-ready.json'))) { throw 'The rollout rig did not start (see rig.err.log)' }
300
+ # The chain the connector and the applier will see: the channel over real TLS.
301
+ $probe = Invoke-RestMethod -Uri 'https://screen.trillboards.com/connector/trillboards-connector-windows.version.json' -TimeoutSec 10
302
+ if ([string]$probe.connectorVersion -ne $versionA) { throw 'screen.trillboards.com did not answer from the rig' }
303
+ Note 'screen.trillboards.com answers from the rig over TLS the machine trusts'
304
+
305
+ $receipt.phase = 'camera'
306
+ $cam = Join-Path $root 'camera'; New-Item -ItemType Directory -Path $cam | Out-Null
307
+ $archive = Join-Path $cam 'mediamtx.zip'
308
+ Invoke-WebRequest -Uri 'https://github.com/bluenviron/mediamtx/releases/download/v1.21.2/mediamtx_v1.21.2_windows_amd64.zip' -OutFile $archive -UseBasicParsing
309
+ if ((Get-FileHash -LiteralPath $archive -Algorithm SHA256).Hash.ToLowerInvariant() -ne 'a84224f5a490855684e62ba6fb71e80ea2a5e16aed61883bb58380290e93e467') { throw 'MediaMTX checksum mismatch' }
310
+ Expand-Archive -LiteralPath $archive -DestinationPath $cam
311
+ $rtspPort = 8554; $mtxApiPort = 9997
312
+ @"
313
+ logLevel: warn
314
+ rtspAddress: 127.0.0.1:$rtspPort
315
+ rtspTransports: [tcp]
316
+ rtmp: no
317
+ hls: no
318
+ webrtc: no
319
+ srt: no
320
+ metrics: no
321
+ pprof: no
322
+ playback: no
323
+ api: yes
324
+ apiAddress: 127.0.0.1:$mtxApiPort
325
+ authInternalUsers:
326
+ - user: any
327
+ pass: ""
328
+ ips: [127.0.0.1]
329
+ permissions:
330
+ - action: publish
331
+ - action: read
332
+ - action: api
333
+ paths:
334
+ cam:
335
+ "@ | Set-Content -LiteralPath (Join-Path $cam 'mediamtx.yml') -Encoding utf8
336
+ $owned.Add((Start-Process -FilePath (Join-Path $cam 'mediamtx.exe') -ArgumentList ('"' + (Join-Path $cam 'mediamtx.yml') + '"') -WorkingDirectory $cam -PassThru -WindowStyle Hidden -RedirectStandardOutput (Join-Path $root 'mediamtx.log') -RedirectStandardError (Join-Path $root 'mediamtx.err.log')))
337
+ Start-Sleep -Seconds 2
338
+ # A COPY of the shipped ffmpeg, outside the install root, so the camera
339
+ # outlives every swap (stop-existing.ps1 stops everything under the root).
340
+ $cameraFfmpeg = Join-Path $cam 'camera-ffmpeg.exe'
341
+ Copy-Item -LiteralPath (Join-Path $treeA 'bin\ffmpeg.exe') -Destination $cameraFfmpeg
342
+ $frame = Join-Path $cam 'scene.jpg'; Copy-Item -LiteralPath $PersonFrame -Destination $frame
343
+ $publishArgs = '-hide_banner -loglevel error -re -loop 1 -framerate 4 -i "' + $frame + '" -an -c:v libx264 -preset ultrafast -tune zerolatency -pix_fmt yuv420p -r 4 -g 8 -threads 1 -f rtsp -rtsp_transport tcp rtsp://127.0.0.1:' + $rtspPort + '/cam'
344
+ $owned.Add((Start-Process -FilePath $cameraFfmpeg -ArgumentList $publishArgs -PassThru -WindowStyle Hidden -RedirectStandardOutput (Join-Path $root 'camera.log') -RedirectStandardError (Join-Path $root 'camera.err.log')))
345
+ $deadline = (Utc).AddSeconds(30); $ready = $false
346
+ do {
347
+ Start-Sleep -Seconds 1
348
+ try { $ready = @((Invoke-RestMethod -Uri ('http://127.0.0.1:' + $mtxApiPort + '/v3/paths/list') -TimeoutSec 3).items | Where-Object { $_.name -eq 'cam' -and $_.ready }).Count -eq 1 } catch { }
349
+ } while (-not $ready -and (Utc) -lt $deadline)
350
+ if (-not $ready) { throw 'The RTSP camera never became readable' }
351
+ Note ('camera live at rtsp://127.0.0.1:' + $rtspPort + '/cam')
352
+
353
+ $receipt.phase = 'install'
354
+ New-Item -ItemType Directory -Force -Path $installDir | Out-Null
355
+ foreach ($part in @('bin', 'lib')) { Copy-Item -LiteralPath (Join-Path $treeA $part) -Destination (Join-Path $installDir $part) -Recurse }
356
+ $tokens = 1..2 | ForEach-Object { -join ((1..32) | ForEach-Object { '{0:x2}' -f (Get-Random -Maximum 256) }) }
357
+ [ordered]@{
358
+ apiBaseUrl = 'http://127.0.0.1:47393'
359
+ deviceToken = $tokens[0]
360
+ cameras = @([ordered]@{ deviceToken = $tokens[1]; screenId = 'tb-e2e-rollout'; rtspUrl = ('rtsp://127.0.0.1:' + $rtspPort + '/cam'); sourceType = 'direct_url'; transport = 'tcp' })
361
+ localSensing = [ordered]@{ enabled = $true; litertLmBaseUrl = 'http://127.0.0.1:9379' }
362
+ cloudSensing = [ordered]@{ candidateBuffer = [ordered]@{ continuousReview = [ordered]@{ enabled = $false } } }
363
+ } | ConvertTo-Json -Depth 6 | Set-Content -LiteralPath (Join-Path $installDir 'trillboards.config.json') -Encoding utf8
364
+ $powershell = Join-Path $env:WINDIR 'System32\WindowsPowerShell\v1.0\powershell.exe'
365
+ $null = Invoke-Owned -File $powershell -RawArguments ('-NoProfile -NonInteractive -ExecutionPolicy Bypass -File "' + (Join-Path $installDir 'bin\register-task.ps1') + '" -InstallDir "' + $installDir + '" -Mode auto-sense') -LogName 'register-task.log' -TimeoutSeconds 300
366
+ $task = Get-ScheduledTask -TaskName 'TrillboardsEdge' -TaskPath '\' -ErrorAction Stop
367
+ if ($task.State -ne 'Running') { Enable-ScheduledTask -TaskName 'TrillboardsEdge' | Out-Null; Start-ScheduledTask -TaskName 'TrillboardsEdge' }
368
+ $installStarted = Utc
369
+ Note ("TrillboardsEdge registered and started on $versionA")
370
+
371
+ $deadline = $installStarted.AddMinutes(5); $aHealthy = $null
372
+ while ((Utc) -lt $deadline) {
373
+ Start-Sleep -Seconds 1; Update-LogEvents
374
+ $marker = Read-JsonFile (Join-Path $updateDir 'health.json')
375
+ $reported = @(Get-StatusReports | Where-Object { [string]$_.Body.connectorVersion -eq $versionA }).Count -gt 0
376
+ if ($marker -and [string]$marker.version -eq $versionA -and [int]$marker.pid -gt 0 -and (Get-Process -Id ([int]$marker.pid) -ErrorAction SilentlyContinue) -and $reported) {
377
+ $aHealthy = Get-MarkerTime (Join-Path $updateDir 'health.json'); break
378
+ }
379
+ }
380
+ if (-not $aHealthy) { throw "$versionA never reported healthy and its version within 5 minutes of its task start" }
381
+ $receipt.installedHealthySeconds = Seconds $installStarted $aHealthy
382
+ Note ("$versionA healthy $($receipt.installedHealthySeconds) s after its task started, and reporting")
383
+
384
+ # -------------------------------------------------------------------------
385
+ # PROOF 1 + 3: a newly published build reaches the box.
386
+ # -------------------------------------------------------------------------
387
+ $receipt.phase = 'proof-1-publish'
388
+ $entryB = Publish-Tree $treeB $versionB
389
+ $published1 = Utc
390
+ Write-State (New-Channel @($entryB, $entryA)) ([ordered]@{ windows = [ordered]@{ version = $versionB; publishedAt = $entryB.publishedAt }; macos = $null; minReleaseAgeMs = 0 })
391
+ Note ("PUBLISHED $versionB on the channel and the doorbell")
392
+ $before = @(Get-ConnectorProcesses); $oldExit1 = $null; $done1 = $null; $markerB = $null
393
+ $deadline = $published1.AddMinutes($PublishDeadlineMinutes + 2)
394
+ while ((Utc) -lt $deadline) {
395
+ Start-Sleep -Milliseconds 500; Update-LogEvents
396
+ $now = @(Get-ConnectorProcesses)
397
+ if (-not $oldExit1 -and $before.Count -gt 0 -and @($before | Where-Object { $now -contains $_ }).Count -eq 0) { $oldExit1 = Utc; Note "old connector ($versionA) exited" }
398
+ $marker = Read-JsonFile (Join-Path $updateDir 'health.json')
399
+ if ($marker -and [string]$marker.version -eq $versionB -and (Get-InstalledVersion) -eq $versionB -and
400
+ [int]$marker.pid -gt 0 -and (Get-Process -Id ([int]$marker.pid) -ErrorAction SilentlyContinue) -and
401
+ @(Get-StatusReports | Where-Object { [string]$_.Body.connectorVersion -eq $versionB }).Count -gt 0) {
402
+ $done1 = Utc; $markerB = $marker; $markerTimeB = Get-MarkerTime (Join-Path $updateDir 'health.json'); break
403
+ }
404
+ }
405
+ if (-not $done1) { throw "PROOF 1 FAILED: $versionB was not installed, healthy and reporting within $($PublishDeadlineMinutes + 2) min of publication (installed: $(Get-InstalledVersion))" }
406
+ $healthyB = if ($markerTimeB) { $markerTimeB } else { $done1 }
407
+ $doorbell1 = First-LogEvent ([regex]::Escape("advertises connector $versionB")) $published1
408
+ $staging1 = First-LogEvent ([regex]::Escape("staging $versionB")) $published1
409
+ $applying1 = First-LogEvent 'connector is idle|applying anyway' $published1
410
+ $polls = @(Get-ProfilePolls | Where-Object { $_ -ge $published1 })
411
+ $receipt.proof1 = [ordered]@{
412
+ publishedToHealthyMinutes = [Math]::Round(($healthyB - $published1).TotalMinutes, 2)
413
+ publishedToReportedMinutes = [Math]::Round(($done1 - $published1).TotalMinutes, 2)
414
+ firstProfilePollAfterPublishSeconds = if ($polls.Count) { Seconds $published1 $polls[0] } else { $null }
415
+ doorbellSeconds = if ($doorbell1) { Seconds $published1 $doorbell1[0].At } else { $null }
416
+ stagingStartedSeconds = if ($staging1) { Seconds $published1 $staging1[0].At } else { $null }
417
+ handoffSeconds = if ($applying1) { Seconds $published1 $applying1[0].At } else { $null }
418
+ deadlineMinutes = $PublishDeadlineMinutes
419
+ }
420
+ $receipt.proof3 = [ordered]@{
421
+ updateGapSeconds = Seconds $oldExit1 $healthyB
422
+ updateGapFromHandoffSeconds = if ($applying1) { Seconds $applying1[0].At $healthyB } else { $null }
423
+ }
424
+ Note ("PROOF 1: $versionB installed and healthy $($receipt.proof1.publishedToHealthyMinutes) min after publication (doorbell at $($receipt.proof1.doorbellSeconds) s)")
425
+ Note ("PROOF 3: monitoring gap $($receipt.proof3.updateGapSeconds) s from the old connector's exit to the new one's health marker")
426
+ if ($receipt.proof1.publishedToHealthyMinutes -gt $PublishDeadlineMinutes) { throw "PROOF 1 FAILED: $($receipt.proof1.publishedToHealthyMinutes) min exceeds $PublishDeadlineMinutes" }
427
+ if (-not $doorbell1) { throw 'PROOF 1 FAILED: the update was not started by the profile doorbell' }
428
+
429
+ # -------------------------------------------------------------------------
430
+ # PROOF 2 + 3: a deliberately broken build rolls itself back.
431
+ # -------------------------------------------------------------------------
432
+ $receipt.phase = 'proof-2-broken'
433
+ $entryC = Publish-Tree $treeC $versionC
434
+ $published2 = Utc
435
+ Write-State (New-Channel @($entryC, $entryB, $entryA)) ([ordered]@{ windows = [ordered]@{ version = $versionC; publishedAt = $entryC.publishedAt }; macos = $null; minReleaseAgeMs = 0 })
436
+ Note ("PUBLISHED broken $versionC")
437
+ $pidB = [int]$markerB.pid; $exitB = $null; $sawC = $false; $cPids = New-Object System.Collections.Generic.HashSet[int]; $rolledBack = $null; $markerBack = $null
438
+ $deadline = $published2.AddMinutes($RollbackDeadlineMinutes + 2)
439
+ while ((Utc) -lt $deadline) {
440
+ Start-Sleep -Milliseconds 500; Update-LogEvents
441
+ $now = @(Get-ConnectorProcesses)
442
+ if (-not $exitB -and -not ($now -contains $pidB)) { $exitB = Utc; Note "connector $versionB (pid $pidB) exited" }
443
+ if ((Get-InstalledVersion) -eq $versionC) { if (-not $sawC) { Note "$versionC is in place" }; $sawC = $true; foreach ($p in $now) { if ($p -ne $pidB) { [void]$cPids.Add($p) } } }
444
+ $marker = Read-JsonFile (Join-Path $updateDir 'health.json')
445
+ $outcome = Read-JsonFile (Join-Path $updateDir 'outcome.json')
446
+ if ($sawC -and $marker -and [string]$marker.version -eq $versionB -and [int]$marker.pid -gt 0 -and [int]$marker.pid -ne $pidB -and
447
+ (Get-Process -Id ([int]$marker.pid) -ErrorAction SilentlyContinue) -and (Get-InstalledVersion) -eq $versionB) {
448
+ $rolledBack = Utc; $markerBack = $marker; $markerTimeBack = Get-MarkerTime (Join-Path $updateDir 'health.json'); break
449
+ }
450
+ # outcome.json still holds PROOF 1's success until this commit finishes, so
451
+ # only a success naming the broken version is one.
452
+ if ($outcome -and [string]$outcome.result -eq 'success' -and [string]$outcome.toVersion -eq $versionC) { throw "PROOF 2 FAILED: the broken $versionC was recorded as a successful update" }
453
+ }
454
+ if (-not $rolledBack) { throw "PROOF 2 FAILED: no rollback to $versionB within $($RollbackDeadlineMinutes + 2) min of publishing $versionC (installed: $(Get-InstalledVersion))" }
455
+ # The commit writes its outcome as it exits, a moment after it starts the
456
+ # restored connector, so give it that moment before reading it.
457
+ $outcomeDeadline = (Utc).AddSeconds(60)
458
+ do {
459
+ $outcome = Read-JsonFile (Join-Path $updateDir 'outcome.json')
460
+ if ($outcome -and [string]$outcome.toVersion -eq $versionC) { break }
461
+ Start-Sleep -Milliseconds 500
462
+ } while ((Utc) -lt $outcomeDeadline)
463
+ $failedList = Read-JsonFile (Join-Path $updateDir 'failed-versions.json')
464
+ $failedVersions = if ($failedList) { @(@($failedList.versions) | ForEach-Object { [string]$_.version }) } else { @() }
465
+ if (-not $outcome -or [string]$outcome.result -ne 'failed' -or [string]$outcome.detail -notmatch [regex]::Escape($versionC)) {
466
+ throw "PROOF 2 FAILED: outcome.json does not record the failed $versionC update: $($outcome | ConvertTo-Json -Compress)"
467
+ }
468
+ if ($failedVersions -notcontains $versionC) { throw "PROOF 2 FAILED: failed-versions.json does not list $versionC" }
469
+ $healthyBack = if ($markerTimeBack) { $markerTimeBack } else { $rolledBack }
470
+ $applying2 = First-LogEvent 'connector is idle|applying anyway' $published2
471
+ Note ("rolled back to $versionB; outcome: $($outcome.detail)")
472
+
473
+ $receipt.phase = 'proof-2-no-retry'
474
+ $watchFrom = Utc; $watchUntil = $watchFrom.AddMinutes($NoRetryWatchMinutes)
475
+ while ((Utc) -lt $watchUntil) {
476
+ Start-Sleep -Seconds 2; Update-LogEvents
477
+ if ((Get-InstalledVersion) -ne $versionB) { throw "PROOF 2 FAILED: the installed version moved to $(Get-InstalledVersion) after the rollback" }
478
+ if (First-LogEvent ([regex]::Escape("staging $versionC")) $rolledBack) { throw "PROOF 2 FAILED: the restored connector staged the broken $versionC again" }
479
+ }
480
+ # From publication on, not from the marker: the restored connector reports its
481
+ # failed outcome from its launch-time check, before startup finishes.
482
+ $afterRollback = @(Get-StatusReports | Where-Object { $_.At -ge $published2 })
483
+ $heldBack = @($afterRollback | Where-Object { $_.Body.PSObject.Properties.Name -contains 'updater' -and $_.Body.updater -and
484
+ [string]$_.Body.updater.state -eq 'apply_failed' -and [string]$_.Body.updater.detail -match [regex]::Escape($versionC) })
485
+ $reportedOutcome = @($afterRollback | Where-Object { $_.Body.PSObject.Properties.Name -contains 'updateOutcome' -and $_.Body.updateOutcome -and
486
+ [string]$_.Body.updateOutcome.result -eq 'failed' })
487
+ $pollsDuringWatch = @(Get-ProfilePolls | Where-Object { $_ -ge $watchFrom }).Count
488
+ if ($heldBack.Count -eq 0) { throw "PROOF 2 FAILED: the restored connector never reported that it is holding $versionC back" }
489
+ if ($reportedOutcome.Count -eq 0) { throw 'PROOF 2 FAILED: the failed update outcome never reached the API' }
490
+ $receipt.proof2 = [ordered]@{
491
+ publishedToRolledBackMinutes = [Math]::Round(($healthyBack - $published2).TotalMinutes, 2)
492
+ handoffToRolledBackSeconds = if ($applying2) { Seconds $applying2[0].At $healthyBack } else { $null }
493
+ brokenProcessesSeen = $cPids.Count
494
+ outcome = $outcome
495
+ failedVersions = $failedVersions
496
+ noRetryWatchMinutes = $NoRetryWatchMinutes
497
+ profilePollsDuringWatch = $pollsDuringWatch
498
+ heldBackReport = [string]$heldBack[0].Body.updater.detail
499
+ }
500
+ $receipt.proof3.rollbackGapSeconds = Seconds $exitB $healthyBack
501
+ Note ("PROOF 2: $versionC rolled back to $versionB $($receipt.proof2.handoffToRolledBackSeconds) s after the hand-off; not retried in $NoRetryWatchMinutes min ($pollsDuringWatch profile polls); reported: $($receipt.proof2.heldBackReport)")
502
+ Note ("PROOF 3: monitoring gap during the failed update and rollback: $($receipt.proof3.rollbackGapSeconds) s")
503
+ $receipt.phase = 'complete'; $receipt.passed = $true
504
+ } catch {
505
+ $receipt.failure = $_.Exception.Message
506
+ Note ('FAIL in phase ' + $receipt.phase + ': ' + $_.Exception.Message)
507
+ throw
508
+ } finally {
509
+ try {
510
+ $evidence = Join-Path $root 'evidence'
511
+ New-Item -ItemType Directory -Force -Path $evidence | Out-Null
512
+ if (Test-Path -LiteralPath (Join-Path $installDir 'logs')) { Copy-Item -LiteralPath (Join-Path $installDir 'logs') -Destination (Join-Path $evidence 'connector-logs') -Recurse -Force }
513
+ if (Test-Path -LiteralPath $updateDir) { Get-ChildItem -LiteralPath $updateDir -File -Force | Where-Object { $_.Length -lt 1MB } | Copy-Item -Destination $evidence -Force }
514
+ foreach ($f in @('api-requests.jsonl', 'cdn-requests.jsonl')) { $p = Join-Path $rigOut $f; if (Test-Path -LiteralPath $p) { Copy-Item -LiteralPath $p -Destination $evidence -Force } }
515
+ } catch { Write-Warning ('Some evidence could not be collected: ' + $_.Exception.Message) }
516
+ try {
517
+ Disable-ScheduledTask -TaskName 'TrillboardsEdge' -ErrorAction SilentlyContinue | Out-Null
518
+ Stop-ScheduledTask -TaskName 'TrillboardsEdge' -ErrorAction SilentlyContinue
519
+ Unregister-ScheduledTask -TaskName 'TrillboardsEdge' -Confirm:$false -ErrorAction SilentlyContinue
520
+ foreach ($id in @(Get-ConnectorProcesses)) { Stop-Process -Id $id -Force -ErrorAction SilentlyContinue }
521
+ } catch { }
522
+ foreach ($p in $owned) { try { if (-not $p.HasExited) { $p.Kill($true) } } catch { } }
523
+ if ($null -ne $hostsOriginal) { try { [IO.File]::WriteAllText($hostsPath, $hostsOriginal, [Text.Encoding]::ASCII); & ipconfig.exe /flushdns | Out-Null } catch { } }
524
+ foreach ($thumbprint in $trustedThumbprints) {
525
+ try {
526
+ $store = [Security.Cryptography.X509Certificates.X509Store]::new('Root', 'LocalMachine'); $store.Open('ReadWrite')
527
+ try { foreach ($c in @($store.Certificates.Find('FindByThumbprint', $thumbprint, $false))) { $store.Remove($c) } } finally { $store.Close() }
528
+ } catch { }
529
+ }
530
+ $receipt.completedAtUtc = [DateTimeOffset]::UtcNow.ToString('o')
531
+ $receipt | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath (Join-Path $root 'receipt.json')
532
+ if ($env:GITHUB_OUTPUT) { ('artifact_path=' + $root) | Out-File -LiteralPath $env:GITHUB_OUTPUT -Append -Encoding utf8 }
533
+ if ($env:GITHUB_STEP_SUMMARY) {
534
+ @('### Connector update path (rollout proofs)', '', '```json', ($receipt | ConvertTo-Json -Depth 8), '```') |
535
+ Out-File -LiteralPath $env:GITHUB_STEP_SUMMARY -Append -Encoding utf8
536
+ }
537
+ $receipt | ConvertTo-Json -Depth 8
538
+ }