@trillboards/edge-sdk 0.2.220 → 0.2.221
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/deploy/macos/README.md +4 -2
- package/deploy/windows/README.md +12 -3
- package/deploy/windows/apply-update.ps1 +58 -0
- package/deploy/windows/ci/test-field-self-update.ps1 +35 -13
- package/deploy/windows/ci/test-published-install.ps1 +55 -7
- package/deploy/windows/ci/test-rollout-proofs.ps1 +538 -0
- package/deploy/windows/install-update.ps1 +147 -1
- package/dist/EdgeAgent.js +1 -1
- package/dist/SensingProfileClient.d.ts.map +1 -1
- package/dist/SensingProfileClient.js +5 -0
- package/dist/SensingProfileClient.js.map +1 -1
- package/dist/autoSense.d.ts.map +1 -1
- package/dist/autoSense.js +5 -0
- package/dist/autoSense.js.map +1 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +3 -1
- package/dist/cli.js.map +1 -1
- package/dist/connectorUpdate.d.ts +113 -28
- package/dist/connectorUpdate.d.ts.map +1 -1
- package/dist/connectorUpdate.js +289 -46
- package/dist/connectorUpdate.js.map +1 -1
- package/dist/connectorUpdateApply.d.ts +37 -0
- package/dist/connectorUpdateApply.d.ts.map +1 -1
- package/dist/connectorUpdateApply.js +70 -1
- package/dist/connectorUpdateApply.js.map +1 -1
- package/dist/diagnose.d.ts +1 -1
- package/dist/diagnose.js +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,538 @@
|
|
|
1
|
+
<#
|
|
2
|
+
test-rollout-proofs.ps1 - the connector update path, measured on a real
|
|
3
|
+
Windows box, with no release hold.
|
|
4
|
+
|
|
5
|
+
Installs the release's own signed code-only payload as a store PC runs it
|
|
6
|
+
(the TrillboardsEdge scheduled task, the shipped launcher chain, a real RTSP
|
|
7
|
+
camera) and points it at stand-ins (scripts/rollout-proof-rig.mjs): an API
|
|
8
|
+
whose sensing-profile response carries `connectorRelease` as production does,
|
|
9
|
+
and a local screen.trillboards.com (hosts file + a rig root CA) serving the
|
|
10
|
+
release channel and update payloads. Nothing is published anywhere.
|
|
11
|
+
|
|
12
|
+
PROOF 1 a newly published build is installed within -PublishDeadlineMinutes
|
|
13
|
+
(15): the box learns of it from the profile poll it already makes,
|
|
14
|
+
stages, waits for a safe point, swaps, and the new version reports
|
|
15
|
+
healthy and reports its version to the API.
|
|
16
|
+
PROOF 2 a deliberately broken build rolls itself back: it exits at
|
|
17
|
+
startup, install-update.ps1 sees it never report healthy, restores
|
|
18
|
+
the previous version, records the broken one in
|
|
19
|
+
failed-versions.json, and the restored connector does not take it
|
|
20
|
+
again (watched for -NoRetryWatchMinutes, longer than one poll).
|
|
21
|
+
PROOF 3 the monitoring gap of an update restart, in seconds: from the old
|
|
22
|
+
connector's exit to the new one's health marker (written once its
|
|
23
|
+
cameras are started), on PROOF 1's swap and PROOF 2's rollback.
|
|
24
|
+
|
|
25
|
+
The two published builds are the release's payload relabelled (B, C =
|
|
26
|
+
version + 1000, + 1001) with a catalog signed by a rig certificate whose
|
|
27
|
+
subject is the production publisher's CN and which only this runner trusts;
|
|
28
|
+
C's cli.js exits 3 whenever it is started to sense. Every .ps1/.vbs/.exe in
|
|
29
|
+
bin\ keeps the release's real signatures. setup-litertlm.ps1 and
|
|
30
|
+
setup-detector.ps1 are left out of every tree so the launcher's local-AI
|
|
31
|
+
repair never starts a download on the runner; the model service is a stub
|
|
32
|
+
that lists the pinned Gemma id.
|
|
33
|
+
|
|
34
|
+
Execute only on a fresh GitHub-hosted Windows runner. Do not run on a
|
|
35
|
+
customer PC: it edits the hosts file and the machine root store.
|
|
36
|
+
#>
|
|
37
|
+
[CmdletBinding()]
|
|
38
|
+
param(
|
|
39
|
+
[Parameter(Mandatory = $true)][string]$UpdateZip,
|
|
40
|
+
[Parameter(Mandatory = $true)][string]$PersonFrame,
|
|
41
|
+
[ValidateRange(1, 60)][int]$PublishDeadlineMinutes = 15,
|
|
42
|
+
[ValidateRange(1, 60)][int]$RollbackDeadlineMinutes = 15,
|
|
43
|
+
[ValidateRange(1, 30)][int]$NoRetryWatchMinutes = 6
|
|
44
|
+
)
|
|
45
|
+
$ErrorActionPreference = 'Stop'
|
|
46
|
+
Set-StrictMode -Version Latest
|
|
47
|
+
if ($env:GITHUB_ACTIONS -ne 'true' -or $env:RUNNER_OS -ne 'Windows' -or
|
|
48
|
+
$env:RUNNER_ENVIRONMENT -ne 'github-hosted' -or $PSVersionTable.PSVersion.Major -lt 7) {
|
|
49
|
+
throw 'The rollout proofs require a fresh GitHub-hosted Windows runner and PowerShell 7'
|
|
50
|
+
}
|
|
51
|
+
foreach ($p in @($UpdateZip, $PersonFrame)) { if (-not (Test-Path -LiteralPath $p -PathType Leaf)) { throw "Missing input: $p" } }
|
|
52
|
+
|
|
53
|
+
$Here = Split-Path -Parent $MyInvocation.MyCommand.Path # deploy\windows\ci
|
|
54
|
+
$PkgRoot = (Resolve-Path (Join-Path $Here '..\..\..')).Path # packages\edge-sdk
|
|
55
|
+
$RigScript = Join-Path $PkgRoot 'scripts\rollout-proof-rig.mjs'
|
|
56
|
+
$installDir = Join-Path $env:LOCALAPPDATA 'Trillboards\edge'
|
|
57
|
+
$updateDir = Join-Path $installDir '.auto-update'
|
|
58
|
+
$connectorLog = Join-Path $installDir 'logs\connector.log'
|
|
59
|
+
$hostsPath = Join-Path $env:WINDIR 'System32\drivers\etc\hosts'
|
|
60
|
+
if (Test-Path -LiteralPath $installDir) { throw 'The CI profile already holds a connector install' }
|
|
61
|
+
foreach ($name in @('TrillboardsEdge', 'TrillboardsLiteRtLm', 'TrillboardsDetector')) {
|
|
62
|
+
if (Get-ScheduledTask -TaskName $name -TaskPath '\' -ErrorAction SilentlyContinue) { throw ('Scheduled task already present: ' + $name) }
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
$root = [IO.Path]::GetFullPath((Join-Path $env:RUNNER_TEMP ('tb-rollout-' + [Guid]::NewGuid().ToString('N'))))
|
|
66
|
+
$rigOut = Join-Path $root 'rig'; $www = Join-Path $root 'www'; $trees = Join-Path $root 'trees'; $statePath = Join-Path $root 'state.json'
|
|
67
|
+
New-Item -ItemType Directory -Path $root, $rigOut, $www, $trees | Out-Null
|
|
68
|
+
$timeline = Join-Path $root 'timeline.log'
|
|
69
|
+
$receipt = [ordered]@{ atUtc = [DateTimeOffset]::UtcNow.ToString('o'); githubRunId = $env:GITHUB_RUN_ID; passed = $false; phase = 'start' }
|
|
70
|
+
$owned = New-Object System.Collections.Generic.List[System.Diagnostics.Process]
|
|
71
|
+
$trustedThumbprints = New-Object System.Collections.Generic.List[string]
|
|
72
|
+
$hostsOriginal = $null
|
|
73
|
+
|
|
74
|
+
function Note([string]$Message) {
|
|
75
|
+
$line = [DateTimeOffset]::UtcNow.ToString('HH:mm:ss.fff') + ' ' + $Message
|
|
76
|
+
Write-Host $line
|
|
77
|
+
Add-Content -LiteralPath $timeline -Value $line -Encoding utf8
|
|
78
|
+
}
|
|
79
|
+
function Utc { [DateTimeOffset]::UtcNow }
|
|
80
|
+
# PowerShell 7 turns ISO strings into local DateTime values when it parses JSON,
|
|
81
|
+
# so every timestamp is read as TEXT, straight out of the raw JSON.
|
|
82
|
+
function Get-JsonTime([string]$Raw, [string]$Key) {
|
|
83
|
+
$m = [regex]::Match($Raw, '"' + [regex]::Escape($Key) + '"\s*:\s*"([^"]+)"')
|
|
84
|
+
if (-not $m.Success) { return $null }
|
|
85
|
+
return [DateTimeOffset]::Parse($m.Groups[1].Value, [Globalization.CultureInfo]::InvariantCulture,
|
|
86
|
+
[Globalization.DateTimeStyles]::AssumeUniversal).ToUniversalTime()
|
|
87
|
+
}
|
|
88
|
+
function Get-MarkerTime([string]$Path) {
|
|
89
|
+
if (-not (Test-Path -LiteralPath $Path)) { return $null }
|
|
90
|
+
try { return Get-JsonTime ([IO.File]::ReadAllText($Path)) 'healthyAt' } catch { return $null }
|
|
91
|
+
}
|
|
92
|
+
function Seconds($From, $To) { if ($null -eq $From -or $null -eq $To) { return $null }; [Math]::Round(($To - $From).TotalSeconds, 1) }
|
|
93
|
+
|
|
94
|
+
# Children run with the native PS5.1 module path, as Task Scheduler starts them.
|
|
95
|
+
function Invoke-Owned {
|
|
96
|
+
param([string]$File, [string]$RawArguments, [string]$LogName, [int]$TimeoutSeconds)
|
|
97
|
+
$start = [Diagnostics.ProcessStartInfo]::new($File)
|
|
98
|
+
$start.UseShellExecute = $false; $start.CreateNoWindow = $true
|
|
99
|
+
$start.RedirectStandardOutput = $true; $start.RedirectStandardError = $true
|
|
100
|
+
$null = $start.Environment.Remove('PSModulePath')
|
|
101
|
+
$start.Arguments = $RawArguments
|
|
102
|
+
$process = [Diagnostics.Process]::new(); $process.StartInfo = $start
|
|
103
|
+
$watch = [Diagnostics.Stopwatch]::StartNew()
|
|
104
|
+
try {
|
|
105
|
+
if (-not $process.Start()) { throw ('Could not start ' + $LogName) }
|
|
106
|
+
$stdout = $process.StandardOutput.ReadToEndAsync(); $stderr = $process.StandardError.ReadToEndAsync()
|
|
107
|
+
while (-not $process.WaitForExit(5000)) {
|
|
108
|
+
if ($watch.Elapsed.TotalSeconds -ge $TimeoutSeconds) { $process.Kill($true); throw ('Deadline exceeded: ' + $LogName) }
|
|
109
|
+
}
|
|
110
|
+
$output = $stdout.GetAwaiter().GetResult() + "`n" + $stderr.GetAwaiter().GetResult()
|
|
111
|
+
[IO.File]::WriteAllText((Join-Path $root $LogName), $output, [Text.UTF8Encoding]::new($false))
|
|
112
|
+
if ($process.ExitCode -ne 0) { throw ($LogName + ' failed with exit ' + $process.ExitCode) }
|
|
113
|
+
return $output
|
|
114
|
+
} finally { $process.Dispose() }
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
# ---------------------------------------------------------------------------
|
|
118
|
+
# Certificates, minted with .NET (no certificate-store calls that can wedge).
|
|
119
|
+
# ---------------------------------------------------------------------------
|
|
120
|
+
Add-Type -AssemblyName System.Security
|
|
121
|
+
function New-RigCertificate {
|
|
122
|
+
param([string]$Subject, [switch]$Authority, $Issuer, [string]$DnsName, [string]$Eku)
|
|
123
|
+
$rsa = [Security.Cryptography.RSA]::Create(2048)
|
|
124
|
+
$request = [Security.Cryptography.X509Certificates.CertificateRequest]::new("CN=$Subject", $rsa,
|
|
125
|
+
[Security.Cryptography.HashAlgorithmName]::SHA256, [Security.Cryptography.RSASignaturePadding]::Pkcs1)
|
|
126
|
+
$request.CertificateExtensions.Add([Security.Cryptography.X509Certificates.X509BasicConstraintsExtension]::new([bool]$Authority, $false, 0, $true))
|
|
127
|
+
$usage = if ($Authority) { 'KeyCertSign, CrlSign, DigitalSignature' } else { 'DigitalSignature, KeyEncipherment' }
|
|
128
|
+
$request.CertificateExtensions.Add([Security.Cryptography.X509Certificates.X509KeyUsageExtension]::new(
|
|
129
|
+
[Security.Cryptography.X509Certificates.X509KeyUsageFlags]$usage, $true))
|
|
130
|
+
$request.CertificateExtensions.Add([Security.Cryptography.X509Certificates.X509SubjectKeyIdentifierExtension]::new($request.PublicKey, $false))
|
|
131
|
+
if ($Eku) {
|
|
132
|
+
$oids = [Security.Cryptography.OidCollection]::new(); $null = $oids.Add([Security.Cryptography.Oid]::new($Eku))
|
|
133
|
+
$request.CertificateExtensions.Add([Security.Cryptography.X509Certificates.X509EnhancedKeyUsageExtension]::new($oids, $false))
|
|
134
|
+
}
|
|
135
|
+
if ($DnsName) {
|
|
136
|
+
$san = [Security.Cryptography.X509Certificates.SubjectAlternativeNameBuilder]::new(); $san.AddDnsName($DnsName)
|
|
137
|
+
$request.CertificateExtensions.Add($san.Build())
|
|
138
|
+
}
|
|
139
|
+
$notBefore = [DateTimeOffset]::UtcNow.AddMinutes(-10); $notAfter = $notBefore.AddDays(2)
|
|
140
|
+
if ($Issuer) {
|
|
141
|
+
# An issued certificate may not outlive its issuer; .NET refuses one that does.
|
|
142
|
+
$issuerEnd = [DateTimeOffset]::new($Issuer.NotAfter.ToUniversalTime()).AddMinutes(-1)
|
|
143
|
+
if ($notAfter -gt $issuerEnd) { $notAfter = $issuerEnd }
|
|
144
|
+
$serial = [byte[]]::new(16); [Security.Cryptography.RandomNumberGenerator]::Fill($serial)
|
|
145
|
+
$cert = [Security.Cryptography.X509Certificates.RSACertificateExtensions]::CopyWithPrivateKey($request.Create($Issuer, $notBefore, $notAfter, $serial), $rsa)
|
|
146
|
+
} else {
|
|
147
|
+
$cert = $request.CreateSelfSigned($notBefore, $notAfter)
|
|
148
|
+
}
|
|
149
|
+
return [pscustomobject]@{ Certificate = $cert; Key = $rsa }
|
|
150
|
+
}
|
|
151
|
+
function Add-MachineRoot($Certificate) {
|
|
152
|
+
$public = [Security.Cryptography.X509Certificates.X509Certificate2]::new($Certificate.RawData)
|
|
153
|
+
$store = [Security.Cryptography.X509Certificates.X509Store]::new('Root', 'LocalMachine')
|
|
154
|
+
$store.Open('ReadWrite'); try { $store.Add($public) } finally { $store.Close() }
|
|
155
|
+
$trustedThumbprints.Add($public.Thumbprint)
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
function Get-ConnectorProcesses {
|
|
159
|
+
$prefix = [IO.Path]::GetFullPath($installDir).TrimEnd('\') + '\'
|
|
160
|
+
@(Get-CimInstance Win32_Process -Filter "Name = 'node.exe'" -ErrorAction SilentlyContinue | Where-Object {
|
|
161
|
+
([string]$_.ExecutablePath).StartsWith($prefix, [StringComparison]::OrdinalIgnoreCase) -and ([string]$_.CommandLine) -match 'edge-sdk\\dist\\cli\.js'
|
|
162
|
+
} | ForEach-Object { [int]$_.ProcessId })
|
|
163
|
+
}
|
|
164
|
+
function Get-InstalledVersion {
|
|
165
|
+
$pkg = Join-Path $installDir 'lib\edge-sdk\package.json'
|
|
166
|
+
if (-not (Test-Path -LiteralPath $pkg)) { return $null }
|
|
167
|
+
try { return [string](Get-Content -LiteralPath $pkg -Raw | ConvertFrom-Json).version } catch { return $null }
|
|
168
|
+
}
|
|
169
|
+
function Read-JsonFile([string]$Path) {
|
|
170
|
+
if (-not (Test-Path -LiteralPath $Path)) { return $null }
|
|
171
|
+
try { return (Get-Content -LiteralPath $Path -Raw | ConvertFrom-Json) } catch { return $null }
|
|
172
|
+
}
|
|
173
|
+
function Get-StatusReports {
|
|
174
|
+
$log = Join-Path $rigOut 'api-requests.jsonl'
|
|
175
|
+
if (-not (Test-Path -LiteralPath $log)) { return @() }
|
|
176
|
+
@(Get-Content -LiteralPath $log -ErrorAction SilentlyContinue | ForEach-Object {
|
|
177
|
+
$raw = $_; $row = $null; try { $row = $raw | ConvertFrom-Json } catch { }
|
|
178
|
+
if ($row -and $row.url -like '*connector-status*' -and $row.body) {
|
|
179
|
+
$body = $null; try { $body = $row.body | ConvertFrom-Json } catch { }
|
|
180
|
+
if ($body) { [pscustomobject]@{ At = (Get-JsonTime $raw 'at'); Body = $body } }
|
|
181
|
+
}
|
|
182
|
+
})
|
|
183
|
+
}
|
|
184
|
+
function Get-ProfilePolls {
|
|
185
|
+
$log = Join-Path $rigOut 'api-requests.jsonl'
|
|
186
|
+
if (-not (Test-Path -LiteralPath $log)) { return @() }
|
|
187
|
+
@(Get-Content -LiteralPath $log -ErrorAction SilentlyContinue | Where-Object { $_ -like '*/v2/earner/cameras/sensing-profile*' } |
|
|
188
|
+
ForEach-Object { Get-JsonTime $_ 'at' })
|
|
189
|
+
}
|
|
190
|
+
$script:seenLines = 0
|
|
191
|
+
$script:logEvents = New-Object System.Collections.Generic.List[object]
|
|
192
|
+
function Update-LogEvents {
|
|
193
|
+
if (-not (Test-Path -LiteralPath $connectorLog)) { return }
|
|
194
|
+
$lines = @(Get-Content -LiteralPath $connectorLog -ErrorAction SilentlyContinue)
|
|
195
|
+
if ($lines.Count -lt $script:seenLines) { $script:seenLines = 0 }
|
|
196
|
+
for ($i = $script:seenLines; $i -lt $lines.Count; $i++) {
|
|
197
|
+
$line = $lines[$i]
|
|
198
|
+
if ($line -match '\[update\]|Fatal error|uncaught|deliberately broken') {
|
|
199
|
+
$script:logEvents.Add([pscustomobject]@{ At = (Utc); Line = $line })
|
|
200
|
+
Note ('log: ' + $line.Substring(0, [Math]::Min(240, $line.Length)))
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
$script:seenLines = $lines.Count
|
|
204
|
+
}
|
|
205
|
+
function First-LogEvent([string]$Pattern, $After) {
|
|
206
|
+
@($script:logEvents | Where-Object { $_.Line -match $Pattern -and ($null -eq $After -or $_.At -ge $After) } | Select-Object -First 1)
|
|
207
|
+
}
|
|
208
|
+
function Write-State($Manifest, $Advert) {
|
|
209
|
+
$json = [ordered]@{ manifest = $Manifest; advert = $Advert } | ConvertTo-Json -Depth 8
|
|
210
|
+
$tmp = "$statePath.tmp"
|
|
211
|
+
[IO.File]::WriteAllText($tmp, $json, [Text.UTF8Encoding]::new($false))
|
|
212
|
+
Move-Item -LiteralPath $tmp -Destination $statePath -Force
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
try {
|
|
216
|
+
$receipt.phase = 'certificates'
|
|
217
|
+
$ca = New-RigCertificate -Subject 'Trillboards rollout rig CA' -Authority
|
|
218
|
+
$server = New-RigCertificate -Subject 'screen.trillboards.com' -Issuer $ca.Certificate -DnsName 'screen.trillboards.com' -Eku '1.3.6.1.5.5.7.3.1'
|
|
219
|
+
$signing = New-RigCertificate -Subject 'Aethercorp Inc' -Eku '1.3.6.1.5.5.7.3.3'
|
|
220
|
+
Add-MachineRoot $ca.Certificate
|
|
221
|
+
Add-MachineRoot $signing.Certificate
|
|
222
|
+
$keyPem = Join-Path $root 'server.key.pem'; $certPem = Join-Path $root 'server.cert.pem'
|
|
223
|
+
[IO.File]::WriteAllText($keyPem, $server.Key.ExportPkcs8PrivateKeyPem())
|
|
224
|
+
[IO.File]::WriteAllText($certPem, $server.Certificate.ExportCertificatePem())
|
|
225
|
+
# Set-AuthenticodeSignature needs a persisted key: round-trip through PFX.
|
|
226
|
+
$pfxBytes = $signing.Certificate.Export([Security.Cryptography.X509Certificates.X509ContentType]::Pfx, 'rig')
|
|
227
|
+
$signer = [Security.Cryptography.X509Certificates.X509Certificate2]::new($pfxBytes, 'rig',
|
|
228
|
+
[Security.Cryptography.X509Certificates.X509KeyStorageFlags]'Exportable, PersistKeySet, UserKeySet')
|
|
229
|
+
Note ('rig CA and signer trusted in LocalMachine\Root (signer CN=Aethercorp Inc, ' + $signer.Thumbprint + ')')
|
|
230
|
+
|
|
231
|
+
$receipt.phase = 'payloads'
|
|
232
|
+
Add-Type -AssemblyName System.IO.Compression.FileSystem
|
|
233
|
+
$treeA = Join-Path $trees 'A'
|
|
234
|
+
[IO.Compression.ZipFile]::ExtractToDirectory($UpdateZip, $treeA)
|
|
235
|
+
foreach ($rel in @('bin\node.exe', 'bin\apply-update.ps1', 'bin\install-update.ps1', 'lib\edge-sdk\dist\cli.js', 'lib\edge-sdk\package.json')) {
|
|
236
|
+
if (-not (Test-Path -LiteralPath (Join-Path $treeA $rel))) { throw "The release payload is missing $rel" }
|
|
237
|
+
}
|
|
238
|
+
foreach ($rel in @('bin\setup-litertlm.ps1', 'bin\setup-detector.ps1')) { Remove-Item -LiteralPath (Join-Path $treeA $rel) -Force -ErrorAction SilentlyContinue }
|
|
239
|
+
$versionA = [string](Get-Content -LiteralPath (Join-Path $treeA 'lib\edge-sdk\package.json') -Raw | ConvertFrom-Json).version
|
|
240
|
+
if ($versionA -notmatch '^(\d+)\.(\d+)\.(\d+)$') { throw "Unreadable release version '$versionA'" }
|
|
241
|
+
$versionB = "$($Matches[1]).$($Matches[2]).$([int]$Matches[3] + 1000)"
|
|
242
|
+
$versionC = "$($Matches[1]).$($Matches[2]).$([int]$Matches[3] + 1001)"
|
|
243
|
+
$receipt.versions = [ordered]@{ installed = $versionA; published = $versionB; broken = $versionC }
|
|
244
|
+
|
|
245
|
+
function Set-TreeVersion([string]$Tree, [string]$Version) {
|
|
246
|
+
$pkgPath = Join-Path $Tree 'lib\edge-sdk\package.json'
|
|
247
|
+
$pkg = Get-Content -LiteralPath $pkgPath -Raw | ConvertFrom-Json -AsHashtable
|
|
248
|
+
$pkg['version'] = $Version
|
|
249
|
+
[IO.File]::WriteAllText($pkgPath, (($pkg | ConvertTo-Json -Depth 32) + "`n"), [Text.UTF8Encoding]::new($false))
|
|
250
|
+
}
|
|
251
|
+
function Publish-Tree([string]$Tree, [string]$Version) {
|
|
252
|
+
$zipName = "update-$Version.zip"; $catalogName = "update-$Version.catalog.ps1"
|
|
253
|
+
$zip = Join-Path $www $zipName
|
|
254
|
+
[IO.Compression.ZipFile]::CreateFromDirectory($Tree, $zip, [IO.Compression.CompressionLevel]::Fastest, $false)
|
|
255
|
+
$sha = (Get-FileHash -LiteralPath $zip -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
256
|
+
$size = (Get-Item -LiteralPath $zip).Length
|
|
257
|
+
$catalog = Join-Path $www $catalogName
|
|
258
|
+
$lines = @('# Trillboards connector update catalog - rollout proof rig (trusted on this runner only).',
|
|
259
|
+
("`$TrillboardsUpdateVersion = '" + $Version + "'"), ("`$TrillboardsUpdateSha256 = '" + $sha + "'"), ("`$TrillboardsUpdateSizeBytes = " + $size))
|
|
260
|
+
[IO.File]::WriteAllText($catalog, (($lines -join "`r`n") + "`r`n"), [Text.UTF8Encoding]::new($false))
|
|
261
|
+
$signed = Set-AuthenticodeSignature -FilePath $catalog -Certificate $signer -HashAlgorithm SHA256
|
|
262
|
+
if ($signed.Status -ne 'Valid') { throw "Could not sign the $Version catalog: $($signed.Status) $($signed.StatusMessage)" }
|
|
263
|
+
$publishedAt = [DateTimeOffset]::UtcNow.ToString('yyyy-MM-ddTHH:mm:ss.fffffffZ')
|
|
264
|
+
$url = 'https://screen.trillboards.com/connector/rig/'
|
|
265
|
+
return [ordered]@{ schemaVersion = 1; platform = 'windows-x64'; connectorVersion = $Version
|
|
266
|
+
gitSha = ((Get-FileHash -InputStream ([IO.MemoryStream]::new([Text.Encoding]::UTF8.GetBytes($Version))) -Algorithm SHA1).Hash.ToLowerInvariant())
|
|
267
|
+
sha256 = $sha; sizeBytes = $size; publishedAt = $publishedAt; downloadUrl = $url + $zipName
|
|
268
|
+
updateUrl = $url + $zipName; updateCatalogUrl = $url + $catalogName }
|
|
269
|
+
}
|
|
270
|
+
function New-Channel([object[]]$Entries) {
|
|
271
|
+
$sorted = @($Entries | Sort-Object { [version]$_.connectorVersion } -Descending)
|
|
272
|
+
$channel = [ordered]@{}
|
|
273
|
+
foreach ($k in $sorted[0].Keys) { $channel[$k] = $sorted[0][$k] }
|
|
274
|
+
$channel['releaseHistory'] = $sorted
|
|
275
|
+
return $channel
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
$treeB = Join-Path $trees 'B'; Copy-Item -LiteralPath $treeA -Destination $treeB -Recurse; Set-TreeVersion $treeB $versionB
|
|
279
|
+
$treeC = Join-Path $trees 'C'; Copy-Item -LiteralPath $treeB -Destination $treeC -Recurse; Set-TreeVersion $treeC $versionC
|
|
280
|
+
$cliC = Join-Path $treeC 'lib\edge-sdk\dist\cli.js'
|
|
281
|
+
$cliText = [IO.File]::ReadAllText($cliC)
|
|
282
|
+
$breaker = "if (process.argv.slice(2).some((a) => a === 'auto-sense' || a === 'local-sense')) { console.error('[rig] deliberately broken build " + $versionC + " exits at startup'); process.exit(3); }`n"
|
|
283
|
+
if ($cliText.StartsWith('#!')) { $nl = $cliText.IndexOf("`n"); $cliText = $cliText.Substring(0, $nl + 1) + $breaker + $cliText.Substring($nl + 1) } else { $cliText = $breaker + $cliText }
|
|
284
|
+
[IO.File]::WriteAllText($cliC, $cliText, [Text.UTF8Encoding]::new($false))
|
|
285
|
+
$entryA = Publish-Tree $treeA $versionA
|
|
286
|
+
Note ("payloads: installed $versionA, published $versionB, broken $versionC (catalogs signed by the rig's CN=Aethercorp Inc)")
|
|
287
|
+
|
|
288
|
+
$receipt.phase = 'rig'
|
|
289
|
+
$hostsOriginal = [IO.File]::ReadAllText($hostsPath)
|
|
290
|
+
[IO.File]::WriteAllText($hostsPath, $hostsOriginal.TrimEnd() + "`r`n127.0.0.1 screen.trillboards.com # tb-rollout-proof`r`n", [Text.Encoding]::ASCII)
|
|
291
|
+
& ipconfig.exe /flushdns | Out-Null
|
|
292
|
+
Write-State (New-Channel @($entryA)) ([ordered]@{ windows = [ordered]@{ version = $versionA; publishedAt = $entryA.publishedAt }; macos = $null; minReleaseAgeMs = 0 })
|
|
293
|
+
$systemNode = (Get-Command node.exe -ErrorAction Stop).Source
|
|
294
|
+
$owned.Add((Start-Process -FilePath $systemNode -ArgumentList @($RigScript, '--out', $rigOut, '--state', $statePath, '--www', $www,
|
|
295
|
+
'--api-port', '47393', '--key', $keyPem, '--cert', $certPem) -PassThru -WindowStyle Hidden `
|
|
296
|
+
-RedirectStandardOutput (Join-Path $root 'rig.log') -RedirectStandardError (Join-Path $root 'rig.err.log')))
|
|
297
|
+
$deadline = (Utc).AddSeconds(30)
|
|
298
|
+
while (-not (Test-Path -LiteralPath (Join-Path $rigOut 'rig-ready.json')) -and (Utc) -lt $deadline) { Start-Sleep -Milliseconds 300 }
|
|
299
|
+
if (-not (Test-Path -LiteralPath (Join-Path $rigOut 'rig-ready.json'))) { throw 'The rollout rig did not start (see rig.err.log)' }
|
|
300
|
+
# The chain the connector and the applier will see: the channel over real TLS.
|
|
301
|
+
$probe = Invoke-RestMethod -Uri 'https://screen.trillboards.com/connector/trillboards-connector-windows.version.json' -TimeoutSec 10
|
|
302
|
+
if ([string]$probe.connectorVersion -ne $versionA) { throw 'screen.trillboards.com did not answer from the rig' }
|
|
303
|
+
Note 'screen.trillboards.com answers from the rig over TLS the machine trusts'
|
|
304
|
+
|
|
305
|
+
$receipt.phase = 'camera'
|
|
306
|
+
$cam = Join-Path $root 'camera'; New-Item -ItemType Directory -Path $cam | Out-Null
|
|
307
|
+
$archive = Join-Path $cam 'mediamtx.zip'
|
|
308
|
+
Invoke-WebRequest -Uri 'https://github.com/bluenviron/mediamtx/releases/download/v1.21.2/mediamtx_v1.21.2_windows_amd64.zip' -OutFile $archive -UseBasicParsing
|
|
309
|
+
if ((Get-FileHash -LiteralPath $archive -Algorithm SHA256).Hash.ToLowerInvariant() -ne 'a84224f5a490855684e62ba6fb71e80ea2a5e16aed61883bb58380290e93e467') { throw 'MediaMTX checksum mismatch' }
|
|
310
|
+
Expand-Archive -LiteralPath $archive -DestinationPath $cam
|
|
311
|
+
$rtspPort = 8554; $mtxApiPort = 9997
|
|
312
|
+
@"
|
|
313
|
+
logLevel: warn
|
|
314
|
+
rtspAddress: 127.0.0.1:$rtspPort
|
|
315
|
+
rtspTransports: [tcp]
|
|
316
|
+
rtmp: no
|
|
317
|
+
hls: no
|
|
318
|
+
webrtc: no
|
|
319
|
+
srt: no
|
|
320
|
+
metrics: no
|
|
321
|
+
pprof: no
|
|
322
|
+
playback: no
|
|
323
|
+
api: yes
|
|
324
|
+
apiAddress: 127.0.0.1:$mtxApiPort
|
|
325
|
+
authInternalUsers:
|
|
326
|
+
- user: any
|
|
327
|
+
pass: ""
|
|
328
|
+
ips: [127.0.0.1]
|
|
329
|
+
permissions:
|
|
330
|
+
- action: publish
|
|
331
|
+
- action: read
|
|
332
|
+
- action: api
|
|
333
|
+
paths:
|
|
334
|
+
cam:
|
|
335
|
+
"@ | Set-Content -LiteralPath (Join-Path $cam 'mediamtx.yml') -Encoding utf8
|
|
336
|
+
$owned.Add((Start-Process -FilePath (Join-Path $cam 'mediamtx.exe') -ArgumentList ('"' + (Join-Path $cam 'mediamtx.yml') + '"') -WorkingDirectory $cam -PassThru -WindowStyle Hidden -RedirectStandardOutput (Join-Path $root 'mediamtx.log') -RedirectStandardError (Join-Path $root 'mediamtx.err.log')))
|
|
337
|
+
Start-Sleep -Seconds 2
|
|
338
|
+
# A COPY of the shipped ffmpeg, outside the install root, so the camera
|
|
339
|
+
# outlives every swap (stop-existing.ps1 stops everything under the root).
|
|
340
|
+
$cameraFfmpeg = Join-Path $cam 'camera-ffmpeg.exe'
|
|
341
|
+
Copy-Item -LiteralPath (Join-Path $treeA 'bin\ffmpeg.exe') -Destination $cameraFfmpeg
|
|
342
|
+
$frame = Join-Path $cam 'scene.jpg'; Copy-Item -LiteralPath $PersonFrame -Destination $frame
|
|
343
|
+
$publishArgs = '-hide_banner -loglevel error -re -loop 1 -framerate 4 -i "' + $frame + '" -an -c:v libx264 -preset ultrafast -tune zerolatency -pix_fmt yuv420p -r 4 -g 8 -threads 1 -f rtsp -rtsp_transport tcp rtsp://127.0.0.1:' + $rtspPort + '/cam'
|
|
344
|
+
$owned.Add((Start-Process -FilePath $cameraFfmpeg -ArgumentList $publishArgs -PassThru -WindowStyle Hidden -RedirectStandardOutput (Join-Path $root 'camera.log') -RedirectStandardError (Join-Path $root 'camera.err.log')))
|
|
345
|
+
$deadline = (Utc).AddSeconds(30); $ready = $false
|
|
346
|
+
do {
|
|
347
|
+
Start-Sleep -Seconds 1
|
|
348
|
+
try { $ready = @((Invoke-RestMethod -Uri ('http://127.0.0.1:' + $mtxApiPort + '/v3/paths/list') -TimeoutSec 3).items | Where-Object { $_.name -eq 'cam' -and $_.ready }).Count -eq 1 } catch { }
|
|
349
|
+
} while (-not $ready -and (Utc) -lt $deadline)
|
|
350
|
+
if (-not $ready) { throw 'The RTSP camera never became readable' }
|
|
351
|
+
Note ('camera live at rtsp://127.0.0.1:' + $rtspPort + '/cam')
|
|
352
|
+
|
|
353
|
+
$receipt.phase = 'install'
|
|
354
|
+
New-Item -ItemType Directory -Force -Path $installDir | Out-Null
|
|
355
|
+
foreach ($part in @('bin', 'lib')) { Copy-Item -LiteralPath (Join-Path $treeA $part) -Destination (Join-Path $installDir $part) -Recurse }
|
|
356
|
+
$tokens = 1..2 | ForEach-Object { -join ((1..32) | ForEach-Object { '{0:x2}' -f (Get-Random -Maximum 256) }) }
|
|
357
|
+
[ordered]@{
|
|
358
|
+
apiBaseUrl = 'http://127.0.0.1:47393'
|
|
359
|
+
deviceToken = $tokens[0]
|
|
360
|
+
cameras = @([ordered]@{ deviceToken = $tokens[1]; screenId = 'tb-e2e-rollout'; rtspUrl = ('rtsp://127.0.0.1:' + $rtspPort + '/cam'); sourceType = 'direct_url'; transport = 'tcp' })
|
|
361
|
+
localSensing = [ordered]@{ enabled = $true; litertLmBaseUrl = 'http://127.0.0.1:9379' }
|
|
362
|
+
cloudSensing = [ordered]@{ candidateBuffer = [ordered]@{ continuousReview = [ordered]@{ enabled = $false } } }
|
|
363
|
+
} | ConvertTo-Json -Depth 6 | Set-Content -LiteralPath (Join-Path $installDir 'trillboards.config.json') -Encoding utf8
|
|
364
|
+
$powershell = Join-Path $env:WINDIR 'System32\WindowsPowerShell\v1.0\powershell.exe'
|
|
365
|
+
$null = Invoke-Owned -File $powershell -RawArguments ('-NoProfile -NonInteractive -ExecutionPolicy Bypass -File "' + (Join-Path $installDir 'bin\register-task.ps1') + '" -InstallDir "' + $installDir + '" -Mode auto-sense') -LogName 'register-task.log' -TimeoutSeconds 300
|
|
366
|
+
$task = Get-ScheduledTask -TaskName 'TrillboardsEdge' -TaskPath '\' -ErrorAction Stop
|
|
367
|
+
if ($task.State -ne 'Running') { Enable-ScheduledTask -TaskName 'TrillboardsEdge' | Out-Null; Start-ScheduledTask -TaskName 'TrillboardsEdge' }
|
|
368
|
+
$installStarted = Utc
|
|
369
|
+
Note ("TrillboardsEdge registered and started on $versionA")
|
|
370
|
+
|
|
371
|
+
$deadline = $installStarted.AddMinutes(5); $aHealthy = $null
|
|
372
|
+
while ((Utc) -lt $deadline) {
|
|
373
|
+
Start-Sleep -Seconds 1; Update-LogEvents
|
|
374
|
+
$marker = Read-JsonFile (Join-Path $updateDir 'health.json')
|
|
375
|
+
$reported = @(Get-StatusReports | Where-Object { [string]$_.Body.connectorVersion -eq $versionA }).Count -gt 0
|
|
376
|
+
if ($marker -and [string]$marker.version -eq $versionA -and [int]$marker.pid -gt 0 -and (Get-Process -Id ([int]$marker.pid) -ErrorAction SilentlyContinue) -and $reported) {
|
|
377
|
+
$aHealthy = Get-MarkerTime (Join-Path $updateDir 'health.json'); break
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
if (-not $aHealthy) { throw "$versionA never reported healthy and its version within 5 minutes of its task start" }
|
|
381
|
+
$receipt.installedHealthySeconds = Seconds $installStarted $aHealthy
|
|
382
|
+
Note ("$versionA healthy $($receipt.installedHealthySeconds) s after its task started, and reporting")
|
|
383
|
+
|
|
384
|
+
# -------------------------------------------------------------------------
|
|
385
|
+
# PROOF 1 + 3: a newly published build reaches the box.
|
|
386
|
+
# -------------------------------------------------------------------------
|
|
387
|
+
$receipt.phase = 'proof-1-publish'
|
|
388
|
+
$entryB = Publish-Tree $treeB $versionB
|
|
389
|
+
$published1 = Utc
|
|
390
|
+
Write-State (New-Channel @($entryB, $entryA)) ([ordered]@{ windows = [ordered]@{ version = $versionB; publishedAt = $entryB.publishedAt }; macos = $null; minReleaseAgeMs = 0 })
|
|
391
|
+
Note ("PUBLISHED $versionB on the channel and the doorbell")
|
|
392
|
+
$before = @(Get-ConnectorProcesses); $oldExit1 = $null; $done1 = $null; $markerB = $null
|
|
393
|
+
$deadline = $published1.AddMinutes($PublishDeadlineMinutes + 2)
|
|
394
|
+
while ((Utc) -lt $deadline) {
|
|
395
|
+
Start-Sleep -Milliseconds 500; Update-LogEvents
|
|
396
|
+
$now = @(Get-ConnectorProcesses)
|
|
397
|
+
if (-not $oldExit1 -and $before.Count -gt 0 -and @($before | Where-Object { $now -contains $_ }).Count -eq 0) { $oldExit1 = Utc; Note "old connector ($versionA) exited" }
|
|
398
|
+
$marker = Read-JsonFile (Join-Path $updateDir 'health.json')
|
|
399
|
+
if ($marker -and [string]$marker.version -eq $versionB -and (Get-InstalledVersion) -eq $versionB -and
|
|
400
|
+
[int]$marker.pid -gt 0 -and (Get-Process -Id ([int]$marker.pid) -ErrorAction SilentlyContinue) -and
|
|
401
|
+
@(Get-StatusReports | Where-Object { [string]$_.Body.connectorVersion -eq $versionB }).Count -gt 0) {
|
|
402
|
+
$done1 = Utc; $markerB = $marker; $markerTimeB = Get-MarkerTime (Join-Path $updateDir 'health.json'); break
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
if (-not $done1) { throw "PROOF 1 FAILED: $versionB was not installed, healthy and reporting within $($PublishDeadlineMinutes + 2) min of publication (installed: $(Get-InstalledVersion))" }
|
|
406
|
+
$healthyB = if ($markerTimeB) { $markerTimeB } else { $done1 }
|
|
407
|
+
$doorbell1 = First-LogEvent ([regex]::Escape("advertises connector $versionB")) $published1
|
|
408
|
+
$staging1 = First-LogEvent ([regex]::Escape("staging $versionB")) $published1
|
|
409
|
+
$applying1 = First-LogEvent 'connector is idle|applying anyway' $published1
|
|
410
|
+
$polls = @(Get-ProfilePolls | Where-Object { $_ -ge $published1 })
|
|
411
|
+
$receipt.proof1 = [ordered]@{
|
|
412
|
+
publishedToHealthyMinutes = [Math]::Round(($healthyB - $published1).TotalMinutes, 2)
|
|
413
|
+
publishedToReportedMinutes = [Math]::Round(($done1 - $published1).TotalMinutes, 2)
|
|
414
|
+
firstProfilePollAfterPublishSeconds = if ($polls.Count) { Seconds $published1 $polls[0] } else { $null }
|
|
415
|
+
doorbellSeconds = if ($doorbell1) { Seconds $published1 $doorbell1[0].At } else { $null }
|
|
416
|
+
stagingStartedSeconds = if ($staging1) { Seconds $published1 $staging1[0].At } else { $null }
|
|
417
|
+
handoffSeconds = if ($applying1) { Seconds $published1 $applying1[0].At } else { $null }
|
|
418
|
+
deadlineMinutes = $PublishDeadlineMinutes
|
|
419
|
+
}
|
|
420
|
+
$receipt.proof3 = [ordered]@{
|
|
421
|
+
updateGapSeconds = Seconds $oldExit1 $healthyB
|
|
422
|
+
updateGapFromHandoffSeconds = if ($applying1) { Seconds $applying1[0].At $healthyB } else { $null }
|
|
423
|
+
}
|
|
424
|
+
Note ("PROOF 1: $versionB installed and healthy $($receipt.proof1.publishedToHealthyMinutes) min after publication (doorbell at $($receipt.proof1.doorbellSeconds) s)")
|
|
425
|
+
Note ("PROOF 3: monitoring gap $($receipt.proof3.updateGapSeconds) s from the old connector's exit to the new one's health marker")
|
|
426
|
+
if ($receipt.proof1.publishedToHealthyMinutes -gt $PublishDeadlineMinutes) { throw "PROOF 1 FAILED: $($receipt.proof1.publishedToHealthyMinutes) min exceeds $PublishDeadlineMinutes" }
|
|
427
|
+
if (-not $doorbell1) { throw 'PROOF 1 FAILED: the update was not started by the profile doorbell' }
|
|
428
|
+
|
|
429
|
+
# -------------------------------------------------------------------------
|
|
430
|
+
# PROOF 2 + 3: a deliberately broken build rolls itself back.
|
|
431
|
+
# -------------------------------------------------------------------------
|
|
432
|
+
$receipt.phase = 'proof-2-broken'
|
|
433
|
+
$entryC = Publish-Tree $treeC $versionC
|
|
434
|
+
$published2 = Utc
|
|
435
|
+
Write-State (New-Channel @($entryC, $entryB, $entryA)) ([ordered]@{ windows = [ordered]@{ version = $versionC; publishedAt = $entryC.publishedAt }; macos = $null; minReleaseAgeMs = 0 })
|
|
436
|
+
Note ("PUBLISHED broken $versionC")
|
|
437
|
+
$pidB = [int]$markerB.pid; $exitB = $null; $sawC = $false; $cPids = New-Object System.Collections.Generic.HashSet[int]; $rolledBack = $null; $markerBack = $null
|
|
438
|
+
$deadline = $published2.AddMinutes($RollbackDeadlineMinutes + 2)
|
|
439
|
+
while ((Utc) -lt $deadline) {
|
|
440
|
+
Start-Sleep -Milliseconds 500; Update-LogEvents
|
|
441
|
+
$now = @(Get-ConnectorProcesses)
|
|
442
|
+
if (-not $exitB -and -not ($now -contains $pidB)) { $exitB = Utc; Note "connector $versionB (pid $pidB) exited" }
|
|
443
|
+
if ((Get-InstalledVersion) -eq $versionC) { if (-not $sawC) { Note "$versionC is in place" }; $sawC = $true; foreach ($p in $now) { if ($p -ne $pidB) { [void]$cPids.Add($p) } } }
|
|
444
|
+
$marker = Read-JsonFile (Join-Path $updateDir 'health.json')
|
|
445
|
+
$outcome = Read-JsonFile (Join-Path $updateDir 'outcome.json')
|
|
446
|
+
if ($sawC -and $marker -and [string]$marker.version -eq $versionB -and [int]$marker.pid -gt 0 -and [int]$marker.pid -ne $pidB -and
|
|
447
|
+
(Get-Process -Id ([int]$marker.pid) -ErrorAction SilentlyContinue) -and (Get-InstalledVersion) -eq $versionB) {
|
|
448
|
+
$rolledBack = Utc; $markerBack = $marker; $markerTimeBack = Get-MarkerTime (Join-Path $updateDir 'health.json'); break
|
|
449
|
+
}
|
|
450
|
+
# outcome.json still holds PROOF 1's success until this commit finishes, so
|
|
451
|
+
# only a success naming the broken version is one.
|
|
452
|
+
if ($outcome -and [string]$outcome.result -eq 'success' -and [string]$outcome.toVersion -eq $versionC) { throw "PROOF 2 FAILED: the broken $versionC was recorded as a successful update" }
|
|
453
|
+
}
|
|
454
|
+
if (-not $rolledBack) { throw "PROOF 2 FAILED: no rollback to $versionB within $($RollbackDeadlineMinutes + 2) min of publishing $versionC (installed: $(Get-InstalledVersion))" }
|
|
455
|
+
# The commit writes its outcome as it exits, a moment after it starts the
|
|
456
|
+
# restored connector, so give it that moment before reading it.
|
|
457
|
+
$outcomeDeadline = (Utc).AddSeconds(60)
|
|
458
|
+
do {
|
|
459
|
+
$outcome = Read-JsonFile (Join-Path $updateDir 'outcome.json')
|
|
460
|
+
if ($outcome -and [string]$outcome.toVersion -eq $versionC) { break }
|
|
461
|
+
Start-Sleep -Milliseconds 500
|
|
462
|
+
} while ((Utc) -lt $outcomeDeadline)
|
|
463
|
+
$failedList = Read-JsonFile (Join-Path $updateDir 'failed-versions.json')
|
|
464
|
+
$failedVersions = if ($failedList) { @(@($failedList.versions) | ForEach-Object { [string]$_.version }) } else { @() }
|
|
465
|
+
if (-not $outcome -or [string]$outcome.result -ne 'failed' -or [string]$outcome.detail -notmatch [regex]::Escape($versionC)) {
|
|
466
|
+
throw "PROOF 2 FAILED: outcome.json does not record the failed $versionC update: $($outcome | ConvertTo-Json -Compress)"
|
|
467
|
+
}
|
|
468
|
+
if ($failedVersions -notcontains $versionC) { throw "PROOF 2 FAILED: failed-versions.json does not list $versionC" }
|
|
469
|
+
$healthyBack = if ($markerTimeBack) { $markerTimeBack } else { $rolledBack }
|
|
470
|
+
$applying2 = First-LogEvent 'connector is idle|applying anyway' $published2
|
|
471
|
+
Note ("rolled back to $versionB; outcome: $($outcome.detail)")
|
|
472
|
+
|
|
473
|
+
$receipt.phase = 'proof-2-no-retry'
|
|
474
|
+
$watchFrom = Utc; $watchUntil = $watchFrom.AddMinutes($NoRetryWatchMinutes)
|
|
475
|
+
while ((Utc) -lt $watchUntil) {
|
|
476
|
+
Start-Sleep -Seconds 2; Update-LogEvents
|
|
477
|
+
if ((Get-InstalledVersion) -ne $versionB) { throw "PROOF 2 FAILED: the installed version moved to $(Get-InstalledVersion) after the rollback" }
|
|
478
|
+
if (First-LogEvent ([regex]::Escape("staging $versionC")) $rolledBack) { throw "PROOF 2 FAILED: the restored connector staged the broken $versionC again" }
|
|
479
|
+
}
|
|
480
|
+
# From publication on, not from the marker: the restored connector reports its
|
|
481
|
+
# failed outcome from its launch-time check, before startup finishes.
|
|
482
|
+
$afterRollback = @(Get-StatusReports | Where-Object { $_.At -ge $published2 })
|
|
483
|
+
$heldBack = @($afterRollback | Where-Object { $_.Body.PSObject.Properties.Name -contains 'updater' -and $_.Body.updater -and
|
|
484
|
+
[string]$_.Body.updater.state -eq 'apply_failed' -and [string]$_.Body.updater.detail -match [regex]::Escape($versionC) })
|
|
485
|
+
$reportedOutcome = @($afterRollback | Where-Object { $_.Body.PSObject.Properties.Name -contains 'updateOutcome' -and $_.Body.updateOutcome -and
|
|
486
|
+
[string]$_.Body.updateOutcome.result -eq 'failed' })
|
|
487
|
+
$pollsDuringWatch = @(Get-ProfilePolls | Where-Object { $_ -ge $watchFrom }).Count
|
|
488
|
+
if ($heldBack.Count -eq 0) { throw "PROOF 2 FAILED: the restored connector never reported that it is holding $versionC back" }
|
|
489
|
+
if ($reportedOutcome.Count -eq 0) { throw 'PROOF 2 FAILED: the failed update outcome never reached the API' }
|
|
490
|
+
$receipt.proof2 = [ordered]@{
|
|
491
|
+
publishedToRolledBackMinutes = [Math]::Round(($healthyBack - $published2).TotalMinutes, 2)
|
|
492
|
+
handoffToRolledBackSeconds = if ($applying2) { Seconds $applying2[0].At $healthyBack } else { $null }
|
|
493
|
+
brokenProcessesSeen = $cPids.Count
|
|
494
|
+
outcome = $outcome
|
|
495
|
+
failedVersions = $failedVersions
|
|
496
|
+
noRetryWatchMinutes = $NoRetryWatchMinutes
|
|
497
|
+
profilePollsDuringWatch = $pollsDuringWatch
|
|
498
|
+
heldBackReport = [string]$heldBack[0].Body.updater.detail
|
|
499
|
+
}
|
|
500
|
+
$receipt.proof3.rollbackGapSeconds = Seconds $exitB $healthyBack
|
|
501
|
+
Note ("PROOF 2: $versionC rolled back to $versionB $($receipt.proof2.handoffToRolledBackSeconds) s after the hand-off; not retried in $NoRetryWatchMinutes min ($pollsDuringWatch profile polls); reported: $($receipt.proof2.heldBackReport)")
|
|
502
|
+
Note ("PROOF 3: monitoring gap during the failed update and rollback: $($receipt.proof3.rollbackGapSeconds) s")
|
|
503
|
+
$receipt.phase = 'complete'; $receipt.passed = $true
|
|
504
|
+
} catch {
|
|
505
|
+
$receipt.failure = $_.Exception.Message
|
|
506
|
+
Note ('FAIL in phase ' + $receipt.phase + ': ' + $_.Exception.Message)
|
|
507
|
+
throw
|
|
508
|
+
} finally {
|
|
509
|
+
try {
|
|
510
|
+
$evidence = Join-Path $root 'evidence'
|
|
511
|
+
New-Item -ItemType Directory -Force -Path $evidence | Out-Null
|
|
512
|
+
if (Test-Path -LiteralPath (Join-Path $installDir 'logs')) { Copy-Item -LiteralPath (Join-Path $installDir 'logs') -Destination (Join-Path $evidence 'connector-logs') -Recurse -Force }
|
|
513
|
+
if (Test-Path -LiteralPath $updateDir) { Get-ChildItem -LiteralPath $updateDir -File -Force | Where-Object { $_.Length -lt 1MB } | Copy-Item -Destination $evidence -Force }
|
|
514
|
+
foreach ($f in @('api-requests.jsonl', 'cdn-requests.jsonl')) { $p = Join-Path $rigOut $f; if (Test-Path -LiteralPath $p) { Copy-Item -LiteralPath $p -Destination $evidence -Force } }
|
|
515
|
+
} catch { Write-Warning ('Some evidence could not be collected: ' + $_.Exception.Message) }
|
|
516
|
+
try {
|
|
517
|
+
Disable-ScheduledTask -TaskName 'TrillboardsEdge' -ErrorAction SilentlyContinue | Out-Null
|
|
518
|
+
Stop-ScheduledTask -TaskName 'TrillboardsEdge' -ErrorAction SilentlyContinue
|
|
519
|
+
Unregister-ScheduledTask -TaskName 'TrillboardsEdge' -Confirm:$false -ErrorAction SilentlyContinue
|
|
520
|
+
foreach ($id in @(Get-ConnectorProcesses)) { Stop-Process -Id $id -Force -ErrorAction SilentlyContinue }
|
|
521
|
+
} catch { }
|
|
522
|
+
foreach ($p in $owned) { try { if (-not $p.HasExited) { $p.Kill($true) } } catch { } }
|
|
523
|
+
if ($null -ne $hostsOriginal) { try { [IO.File]::WriteAllText($hostsPath, $hostsOriginal, [Text.Encoding]::ASCII); & ipconfig.exe /flushdns | Out-Null } catch { } }
|
|
524
|
+
foreach ($thumbprint in $trustedThumbprints) {
|
|
525
|
+
try {
|
|
526
|
+
$store = [Security.Cryptography.X509Certificates.X509Store]::new('Root', 'LocalMachine'); $store.Open('ReadWrite')
|
|
527
|
+
try { foreach ($c in @($store.Certificates.Find('FindByThumbprint', $thumbprint, $false))) { $store.Remove($c) } } finally { $store.Close() }
|
|
528
|
+
} catch { }
|
|
529
|
+
}
|
|
530
|
+
$receipt.completedAtUtc = [DateTimeOffset]::UtcNow.ToString('o')
|
|
531
|
+
$receipt | ConvertTo-Json -Depth 8 | Set-Content -LiteralPath (Join-Path $root 'receipt.json')
|
|
532
|
+
if ($env:GITHUB_OUTPUT) { ('artifact_path=' + $root) | Out-File -LiteralPath $env:GITHUB_OUTPUT -Append -Encoding utf8 }
|
|
533
|
+
if ($env:GITHUB_STEP_SUMMARY) {
|
|
534
|
+
@('### Connector update path (rollout proofs)', '', '```json', ($receipt | ConvertTo-Json -Depth 8), '```') |
|
|
535
|
+
Out-File -LiteralPath $env:GITHUB_STEP_SUMMARY -Append -Encoding utf8
|
|
536
|
+
}
|
|
537
|
+
$receipt | ConvertTo-Json -Depth 8
|
|
538
|
+
}
|