@trillboards/edge-sdk 0.2.246 → 0.2.253
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/deploy/detector/pyproject.toml +1 -1
- package/deploy/detector/tests/test_actor_tracking.py +136 -0
- package/deploy/detector/tests/test_actor_tracking_wire.py +11 -0
- package/deploy/detector/tests/test_loader.py +25 -0
- package/deploy/detector/tests/test_registry.py +53 -0
- package/deploy/detector/trillboards_detector/_version.py +2 -2
- package/deploy/detector/trillboards_detector/actor_tracking.py +95 -9
- package/deploy/detector/trillboards_detector/loader.py +28 -0
- package/deploy/detector/trillboards_detector/registry.py +27 -1
- package/deploy/detector/trillboards_detector/server.py +6 -5
- package/deploy/windows/build-windows.ps1 +1 -0
- package/deploy/windows/ci/test-detector-looks-field.ps1 +15 -3
- package/deploy/windows/ci/test-detector-reinstall.ps1 +173 -2
- package/deploy/windows/ci/test-rollout-proofs.ps1 +3 -2
- package/deploy/windows/ensure-detector.ps1 +255 -0
- package/deploy/windows/install-update.ps1 +4 -3
- package/deploy/windows/stop-existing.ps1 +1 -0
- package/deploy/windows/trillboards-edge.cmd +10 -22
- package/dist/CloudSensingManager.d.ts +9 -0
- package/dist/CloudSensingManager.d.ts.map +1 -1
- package/dist/CloudSensingManager.js +46 -3
- package/dist/CloudSensingManager.js.map +1 -1
- package/dist/ConnectorStatusClient.d.ts +8 -0
- package/dist/ConnectorStatusClient.d.ts.map +1 -1
- package/dist/ConnectorStatusClient.js +3 -0
- package/dist/ConnectorStatusClient.js.map +1 -1
- package/dist/EdgeAgent.js +1 -1
- package/dist/LocalDetectorClient.d.ts +19 -0
- package/dist/LocalDetectorClient.d.ts.map +1 -1
- package/dist/LocalDetectorClient.js +28 -4
- package/dist/LocalDetectorClient.js.map +1 -1
- package/dist/NativeReadiness.d.ts +21 -2
- package/dist/NativeReadiness.d.ts.map +1 -1
- package/dist/NativeReadiness.js +25 -6
- package/dist/NativeReadiness.js.map +1 -1
- package/dist/OrderedReviewClient.d.ts +3 -0
- package/dist/OrderedReviewClient.d.ts.map +1 -1
- package/dist/OrderedReviewClient.js +3 -0
- package/dist/OrderedReviewClient.js.map +1 -1
- package/dist/TemporalReviewClient.d.ts +3 -0
- package/dist/TemporalReviewClient.d.ts.map +1 -1
- package/dist/TemporalReviewClient.js +3 -0
- package/dist/TemporalReviewClient.js.map +1 -1
- package/dist/autoSense.d.ts.map +1 -1
- package/dist/autoSense.js +3 -0
- package/dist/autoSense.js.map +1 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +3 -0
- package/dist/cli.js.map +1 -1
- package/dist/connectorUpdate.d.ts +18 -9
- package/dist/connectorUpdate.d.ts.map +1 -1
- package/dist/connectorUpdate.js +66 -3
- package/dist/connectorUpdate.js.map +1 -1
- package/dist/connectorUpdateApply.d.ts +20 -1
- package/dist/connectorUpdateApply.d.ts.map +1 -1
- package/dist/connectorUpdateApply.js +55 -1
- package/dist/connectorUpdateApply.js.map +1 -1
- package/dist/connectorUpdateApplyMac.d.ts +3 -1
- package/dist/connectorUpdateApplyMac.d.ts.map +1 -1
- package/dist/connectorUpdateApplyMac.js +6 -0
- package/dist/connectorUpdateApplyMac.js.map +1 -1
- package/dist/detectorEndpointState.d.ts +30 -0
- package/dist/detectorEndpointState.d.ts.map +1 -0
- package/dist/detectorEndpointState.js +125 -0
- package/dist/detectorEndpointState.js.map +1 -0
- package/dist/detectorLauncher.d.ts +23 -0
- package/dist/detectorLauncher.d.ts.map +1 -0
- package/dist/detectorLauncher.js +94 -0
- package/dist/detectorLauncher.js.map +1 -0
- package/dist/diagnose.d.ts +1 -1
- package/dist/diagnose.js +1 -1
- package/dist/localSensing.d.ts +69 -0
- package/dist/localSensing.d.ts.map +1 -1
- package/dist/localSensing.js +211 -28
- package/dist/localSensing.js.map +1 -1
- package/package.json +2 -2
- package/retained/ProfileAuthorityClient.cjs +4 -0
- package/retained/RetainedCaptureClient.cjs +4 -0
|
@@ -10,7 +10,7 @@ name = "trillboards-detector"
|
|
|
10
10
|
# constant, a `labels.py`-only change survives `uv tool install` AND
|
|
11
11
|
# `uv tool install --force`; move the version and a bare `uv tool install`
|
|
12
12
|
# reinstalls. See `stamp_version.py`.
|
|
13
|
-
version = "0.2.0+src.
|
|
13
|
+
version = "0.2.0+src.74a44d1fb817"
|
|
14
14
|
description = "On-device object-detector service for the Trillboards BYO-CCTV connector (loopback HTTP, ONNX Runtime)."
|
|
15
15
|
requires-python = ">=3.10"
|
|
16
16
|
|
|
@@ -218,6 +218,142 @@ class ActorTrackingTest(unittest.TestCase):
|
|
|
218
218
|
with self.assertRaisesRegex(TrackingError, "Corrupt"):
|
|
219
219
|
self.track(1400)
|
|
220
220
|
|
|
221
|
+
# ── each frame's own records (server.py:669 Codex P2) ────────────────────
|
|
222
|
+
|
|
223
|
+
def track_and_records(self, at, detections=None, **kwargs):
|
|
224
|
+
return self.journal.track_and_records(
|
|
225
|
+
request(at, **kwargs), detections if detections is not None else [person()],
|
|
226
|
+
image_sha256=SHA, model_hashes=MODEL, frame_size=(640, 480))
|
|
227
|
+
|
|
228
|
+
def test_two_interleaved_requests_on_one_camera_each_get_their_own_records(self):
|
|
229
|
+
"""The bug: the server read the records with a second lock acquisition after
|
|
230
|
+
track(), so a request for the camera's next frame could commit in between and this
|
|
231
|
+
frame was answered with the next frame's records. Here the next frame's request is
|
|
232
|
+
started from INSIDE this frame's transaction; it must wait, and each answer must be
|
|
233
|
+
its own frame's."""
|
|
234
|
+
import threading
|
|
235
|
+
self.track(1000)
|
|
236
|
+
original = self.journal._write_track_records
|
|
237
|
+
other = {}
|
|
238
|
+
def write_then_start_the_next_frame(*args):
|
|
239
|
+
original(*args)
|
|
240
|
+
if "thread" not in other:
|
|
241
|
+
other["thread"] = threading.Thread(
|
|
242
|
+
target=lambda: other.setdefault("answer", self.track_and_records(1400, [person(14, .9)])))
|
|
243
|
+
other["thread"].start()
|
|
244
|
+
other["thread"].join(.3) # it cannot get in: this frame still holds the journal
|
|
245
|
+
other["waited"] = other["thread"].is_alive()
|
|
246
|
+
self.journal._write_track_records = write_then_start_the_next_frame
|
|
247
|
+
result, records = self.track_and_records(1200)
|
|
248
|
+
other["thread"].join(10)
|
|
249
|
+
self.journal._write_track_records = original
|
|
250
|
+
self.assertTrue(other["waited"])
|
|
251
|
+
record, = records
|
|
252
|
+
self.assertEqual((record["lastSeenAtMs"], record["frames"], record["box"]["x"]), (1200, 1, 10))
|
|
253
|
+
self.assertEqual(record["actorId"], result["actors"][0]["actor"]["id"])
|
|
254
|
+
_, next_records = other["answer"]
|
|
255
|
+
self.assertEqual((next_records[0]["lastSeenAtMs"], next_records[0]["frames"],
|
|
256
|
+
next_records[0]["box"]["x"]), (1400, 2, 14))
|
|
257
|
+
# The live records moved on; the first answer did not move with them.
|
|
258
|
+
live, = self.journal.track_records("cam_a")
|
|
259
|
+
self.assertEqual((live.last_seen_ms, live.frames), (1400, 2))
|
|
260
|
+
|
|
261
|
+
def test_a_replay_serves_the_frames_own_saved_records_not_the_live_ones(self):
|
|
262
|
+
self.track(1000)
|
|
263
|
+
result, records = self.track_and_records(1200)
|
|
264
|
+
self.track(1400, [person(14, .9)])
|
|
265
|
+
self.assertEqual(self.track_and_records(1200), (result, records))
|
|
266
|
+
self.journal.close()
|
|
267
|
+
self.journal = ActorTrackerJournal(self.path)
|
|
268
|
+
self.assertEqual(self.track_and_records(1200), (result, records))
|
|
269
|
+
self.assertEqual(self.track(1200), result)
|
|
270
|
+
|
|
271
|
+
def test_a_frame_with_no_records_kept_replays_no_records(self):
|
|
272
|
+
"""A frame journalled before the snapshot existed, or by an older build."""
|
|
273
|
+
import sqlite3
|
|
274
|
+
self.track(1000)
|
|
275
|
+
result, records = self.track_and_records(1200)
|
|
276
|
+
self.assertEqual(len(records), 1)
|
|
277
|
+
with sqlite3.connect(self.path) as db:
|
|
278
|
+
db.execute("DELETE FROM actor_frame_records WHERE frame_id='frame_1200'")
|
|
279
|
+
self.assertEqual(self.track_and_records(1200), (result, []))
|
|
280
|
+
|
|
281
|
+
def test_records_kept_for_different_bytes_under_the_same_frame_id_are_not_served(self):
|
|
282
|
+
import sqlite3
|
|
283
|
+
self.track(1000)
|
|
284
|
+
result, _ = self.track_and_records(1200)
|
|
285
|
+
with sqlite3.connect(self.path) as db:
|
|
286
|
+
db.execute("UPDATE actor_frame_records SET fingerprint=? WHERE frame_id='frame_1200'", ("d" * 64,))
|
|
287
|
+
self.assertEqual(self.track_and_records(1200), (result, []))
|
|
288
|
+
|
|
289
|
+
def test_a_journal_from_before_the_snapshot_gains_its_table_and_keeps_its_frames(self):
|
|
290
|
+
import sqlite3
|
|
291
|
+
self.track(1000)
|
|
292
|
+
result, _ = self.track_and_records(1200)
|
|
293
|
+
self.journal.close()
|
|
294
|
+
with sqlite3.connect(self.path) as db:
|
|
295
|
+
db.execute("DROP TABLE actor_frame_records")
|
|
296
|
+
self.journal = ActorTrackerJournal(self.path)
|
|
297
|
+
self.assertEqual(self.track_and_records(1200), (result, []))
|
|
298
|
+
_, records = self.track_and_records(1400)
|
|
299
|
+
self.assertEqual(records[0]["frames"], 2)
|
|
300
|
+
self.assertEqual(self.track_and_records(1400)[1], records)
|
|
301
|
+
|
|
302
|
+
def test_an_older_build_can_still_write_this_journal_after_a_rollback(self):
|
|
303
|
+
"""Older builds insert frames by position (7 values): actor_frame keeps its columns."""
|
|
304
|
+
import sqlite3
|
|
305
|
+
self.track(1000)
|
|
306
|
+
with sqlite3.connect(self.path) as db:
|
|
307
|
+
self.assertEqual([r[1] for r in db.execute("PRAGMA table_info(actor_frame)")],
|
|
308
|
+
["camera_id", "frame_id", "seq", "epoch", "fingerprint", "input_json", "result_json"])
|
|
309
|
+
|
|
310
|
+
def test_records_kept_beside_frames_are_pruned_with_them(self):
|
|
311
|
+
import sqlite3
|
|
312
|
+
for at in range(1000, 1000 + 300 * 200, 200):
|
|
313
|
+
self.track(at)
|
|
314
|
+
with sqlite3.connect(self.path) as db:
|
|
315
|
+
frames = db.execute("SELECT count(*) FROM actor_frame WHERE camera_id='cam_a'").fetchone()[0]
|
|
316
|
+
kept = db.execute("SELECT count(*) FROM actor_frame_records WHERE camera_id='cam_a'").fetchone()[0]
|
|
317
|
+
orphans = db.execute("SELECT count(*) FROM actor_frame_records r WHERE NOT EXISTS (SELECT 1 FROM "
|
|
318
|
+
"actor_frame f WHERE f.camera_id=r.camera_id AND f.frame_id=r.frame_id)").fetchone()[0]
|
|
319
|
+
self.assertLessEqual(frames, 256)
|
|
320
|
+
self.assertEqual((kept, orphans), (frames, 0))
|
|
321
|
+
|
|
322
|
+
def test_a_replay_serves_a_face_only_while_its_actor_still_holds_that_slot(self):
|
|
323
|
+
import sqlite3
|
|
324
|
+
from trillboards_detector.track_records import FaceSlot
|
|
325
|
+
who = self.confirmed()
|
|
326
|
+
face = FaceSlot("face_model", "c" * 64, (0.6, 0.8), "frame_1200", 1200, .7)
|
|
327
|
+
self.assertTrue(self.journal.record_face("cam_a", who["id"], face))
|
|
328
|
+
_, records = self.track_and_records(1400)
|
|
329
|
+
self.assertEqual(records[0]["face"], face.to_wire())
|
|
330
|
+
self.assertEqual(self.track_and_records(1400)[1], records)
|
|
331
|
+
# The journal row keeps the slot's digest, never the vector: a face never
|
|
332
|
+
# outlives its track (track_records.py).
|
|
333
|
+
with sqlite3.connect(self.path) as db:
|
|
334
|
+
stored = db.execute("SELECT records_json FROM actor_frame_records WHERE frame_id='frame_1400'").fetchone()[0]
|
|
335
|
+
self.assertNotIn("0.6", stored)
|
|
336
|
+
self.assertNotIn("face_model", stored)
|
|
337
|
+
# A newer slot on the live record is not the one this frame had.
|
|
338
|
+
newer = FaceSlot("face_model", "c" * 64, (0.8, 0.6), "frame_1400", 1400, .9)
|
|
339
|
+
self.assertTrue(self.journal.record_face("cam_a", who["id"], newer))
|
|
340
|
+
replayed, = self.track_and_records(1400)[1]
|
|
341
|
+
self.assertIsNone(replayed["face"])
|
|
342
|
+
self.assertEqual({k: v for k, v in replayed.items() if k != "face"},
|
|
343
|
+
{k: v for k, v in records[0].items() if k != "face"})
|
|
344
|
+
# And once the actor ends, no replay brings its face back.
|
|
345
|
+
self.assertTrue(self.journal.record_face("cam_a", who["id"], face))
|
|
346
|
+
self.track(10000)
|
|
347
|
+
self.assertIsNone(self.track_and_records(1400)[1][0]["face"])
|
|
348
|
+
|
|
349
|
+
def test_a_corrupt_records_snapshot_is_not_an_idempotent_success(self):
|
|
350
|
+
import sqlite3
|
|
351
|
+
self.confirmed()
|
|
352
|
+
with sqlite3.connect(self.path) as db:
|
|
353
|
+
db.execute("UPDATE actor_frame_records SET records_json='{\"not\":\"a list\"}' WHERE frame_id='frame_1200'")
|
|
354
|
+
with self.assertRaisesRegex(TrackingError, "Corrupt"):
|
|
355
|
+
self.track_and_records(1200)
|
|
356
|
+
|
|
221
357
|
|
|
222
358
|
if __name__ == "__main__":
|
|
223
359
|
unittest.main()
|
|
@@ -92,6 +92,17 @@ class ActorTrackingWireTest(unittest.TestCase):
|
|
|
92
92
|
self.assertEqual(self.post(1400)[1]["tracks"][0]["frames"],2)
|
|
93
93
|
self.assertNotIn("tracks",self.post(1600,tracking=None)[1])
|
|
94
94
|
|
|
95
|
+
def test_a_replayed_frame_serves_its_own_tracks_not_the_live_ones(self):
|
|
96
|
+
self.post(1000)
|
|
97
|
+
_,second=self.post(1200)
|
|
98
|
+
_,third=self.post(1400)
|
|
99
|
+
self.assertEqual(third["tracks"][0]["frames"],2)
|
|
100
|
+
status,replayed=self.post(1200)
|
|
101
|
+
self.assertEqual(status,200)
|
|
102
|
+
self.assertEqual(replayed["tracking"],second["tracking"])
|
|
103
|
+
self.assertEqual(replayed["tracks"],second["tracks"])
|
|
104
|
+
self.assertEqual(replayed["tracks"][0]["frames"],1)
|
|
105
|
+
|
|
95
106
|
def test_invalid_or_unavailable_tracking_never_becomes_an_empty_scene(self):
|
|
96
107
|
for change in [{"tracking":{}},{"min_confidence":.5},{"expected_model":None},
|
|
97
108
|
{"classes":["cell phone"]}]:
|
|
@@ -13,6 +13,7 @@ property of `/health` — both of which are about `DetectorLoader` specifically.
|
|
|
13
13
|
|
|
14
14
|
from __future__ import annotations
|
|
15
15
|
|
|
16
|
+
import importlib.util
|
|
16
17
|
import unittest
|
|
17
18
|
|
|
18
19
|
try:
|
|
@@ -42,6 +43,30 @@ if HAVE_RUNTIME:
|
|
|
42
43
|
)
|
|
43
44
|
|
|
44
45
|
|
|
46
|
+
# The loader imports on a runner without Pillow or NumPy (guards-fast's detector step); the
|
|
47
|
+
# warm-up frame needs both, so this case skips there on its own, with its reason.
|
|
48
|
+
@unittest.skipUnless(HAVE_RUNTIME and importlib.util.find_spec("numpy") and importlib.util.find_spec("PIL"),
|
|
49
|
+
"onnxruntime/numpy/Pillow are not installed")
|
|
50
|
+
class TestFullSizeWarmup(unittest.TestCase):
|
|
51
|
+
def test_the_warmup_frame_is_a_textured_live_sized_jpeg_at_the_probes_threshold(self):
|
|
52
|
+
import io
|
|
53
|
+
from PIL import Image
|
|
54
|
+
calls = []
|
|
55
|
+
|
|
56
|
+
class Session:
|
|
57
|
+
def detect(self, jpeg, min_confidence, wanted):
|
|
58
|
+
calls.append((jpeg, min_confidence, wanted))
|
|
59
|
+
|
|
60
|
+
DetectorLoader._warm_full_size(Session())
|
|
61
|
+
(jpeg, min_confidence, wanted), = calls
|
|
62
|
+
image = Image.open(io.BytesIO(jpeg))
|
|
63
|
+
self.assertEqual((image.format, image.size), ("JPEG", (640, 480)))
|
|
64
|
+
self.assertEqual((min_confidence, wanted), (0.99, None))
|
|
65
|
+
# Texture, not a flat fill: a flat frame decodes in no time and would warm
|
|
66
|
+
# the forward pass but not the decode a live frame costs.
|
|
67
|
+
self.assertGreater(len(jpeg), 20_000)
|
|
68
|
+
|
|
69
|
+
|
|
45
70
|
@unittest.skipUnless(HAVE_RUNTIME, "onnxruntime/numpy/Pillow are not installed")
|
|
46
71
|
class TestBuildCatalogue(unittest.TestCase):
|
|
47
72
|
def test_non_box_paths_do_not_inherit_the_detector_only_override(self):
|
|
@@ -162,6 +162,7 @@ class RegistryCase(unittest.TestCase):
|
|
|
162
162
|
vocabularies,
|
|
163
163
|
pinned: str,
|
|
164
164
|
budget_mb: "int | None" = 1536,
|
|
165
|
+
warm=None,
|
|
165
166
|
) -> ModelRegistry:
|
|
166
167
|
"""Every case INJECTS a ceiling, because the service owns none.
|
|
167
168
|
|
|
@@ -179,9 +180,61 @@ class RegistryCase(unittest.TestCase):
|
|
|
179
180
|
ram_budget_bytes=None if budget_mb is None else budget_mb * MB,
|
|
180
181
|
log=self.logs.append,
|
|
181
182
|
clock=lambda: self.now,
|
|
183
|
+
warm=warm,
|
|
182
184
|
)
|
|
183
185
|
|
|
184
186
|
|
|
187
|
+
class TestThePinnedModelWarmsAtFullSize(RegistryCase):
|
|
188
|
+
"""The probe proves a forward pass on a 64x48 frame; the pinned model then warms
|
|
189
|
+
once more at full size before READY, so the first live look is not the slow one
|
|
190
|
+
(Texas, 0.2.246 swap: first live call 3.2 s, the next 18 averaged 1.4 s)."""
|
|
191
|
+
|
|
192
|
+
def test_the_pinned_model_warms_once_before_ready_and_a_specialist_never_does(self):
|
|
193
|
+
self.write_artifact("general")
|
|
194
|
+
self.write_artifact("ppe")
|
|
195
|
+
warmed = []
|
|
196
|
+
reg = None
|
|
197
|
+
def warm(session):
|
|
198
|
+
warmed.append((session.model_name, reg.snapshot()["models"][session.model_name]["state"]))
|
|
199
|
+
reg = self.registry([GENERAL, PPE], pinned="general", warm=warm)
|
|
200
|
+
reg.refresh()
|
|
201
|
+
self.assertEqual(warmed, [("general", "loading")])
|
|
202
|
+
self.assertEqual(reg.snapshot()["models"]["general"]["state"], STATE_READY)
|
|
203
|
+
self.assertTrue(any("full-size warm-up" in line and "general" in line for line in self.logs))
|
|
204
|
+
reg.detect(b"jpeg", 0.25, reg.resolve_classes(["glove"]))
|
|
205
|
+
self.assertEqual(self.probed, ["general", "ppe"])
|
|
206
|
+
self.assertEqual(warmed, [("general", "loading")])
|
|
207
|
+
|
|
208
|
+
def test_a_failed_full_size_warmup_leaves_the_model_ready_on_its_probe(self):
|
|
209
|
+
self.write_artifact("general")
|
|
210
|
+
def warm(session):
|
|
211
|
+
raise RuntimeError("no full-size frame today")
|
|
212
|
+
reg = self.registry([GENERAL], pinned="general", warm=warm)
|
|
213
|
+
reg.refresh()
|
|
214
|
+
row = reg.snapshot()["models"]["general"]
|
|
215
|
+
self.assertEqual(row["state"], STATE_READY)
|
|
216
|
+
self.assertIn("warmup_inference_ms", row)
|
|
217
|
+
self.assertTrue(any("full-size warm-up failed; ready on its probe: no full-size frame today" in line
|
|
218
|
+
for line in self.logs))
|
|
219
|
+
self.assertEqual(reg.detect(b"jpeg", 0.25, reg.resolve_classes(["person"])).models, ["general"])
|
|
220
|
+
|
|
221
|
+
def test_the_warmup_is_not_timed_into_the_probes_inference_time(self):
|
|
222
|
+
import time
|
|
223
|
+
self.write_artifact("general")
|
|
224
|
+
reg = self.registry([GENERAL], pinned="general", warm=lambda session: time.sleep(0.25))
|
|
225
|
+
reg.refresh()
|
|
226
|
+
self.assertLess(reg.snapshot()["models"]["general"]["warmup_inference_ms"], 200)
|
|
227
|
+
|
|
228
|
+
def test_a_reload_of_a_new_pinned_artifact_warms_again(self):
|
|
229
|
+
self.write_artifact("general")
|
|
230
|
+
warmed = []
|
|
231
|
+
reg = self.registry([GENERAL], pinned="general", warm=lambda session: warmed.append(session.model_name))
|
|
232
|
+
reg.refresh()
|
|
233
|
+
self.write_artifact("general", b"onnx-v2-different-size")
|
|
234
|
+
reg.refresh()
|
|
235
|
+
self.assertEqual(warmed, ["general", "general"])
|
|
236
|
+
|
|
237
|
+
|
|
185
238
|
class TestTwoModelsAnswerOneRequest(RegistryCase):
|
|
186
239
|
"""§3: "A request spanning two models runs both and merges." """
|
|
187
240
|
|
|
@@ -4,6 +4,16 @@ SQLite commits frame inputs and identities together. In-memory Kalman state is a
|
|
|
4
4
|
cache: restart recovery replays the bounded detection journal through the same
|
|
5
5
|
tracker, restoring external actor IDs from exact saved detection associations.
|
|
6
6
|
Predicted boxes are never returned as new visual observations.
|
|
7
|
+
|
|
8
|
+
A frame's per-track records (track_records.py) are read inside the same lock and
|
|
9
|
+
transaction that commits the frame, and kept beside it (actor_frame_records), so
|
|
10
|
+
`track_and_records` answers a frame with ITS records: never the records of a
|
|
11
|
+
request on the same camera that committed in between, and on a replay the ones
|
|
12
|
+
the frame had. A face vector is never kept there (a face never outlives its
|
|
13
|
+
track): the row keeps the face slot's digest, and a replay serves the face only
|
|
14
|
+
while the actor's live record still holds that same slot. A side table, not a
|
|
15
|
+
column on actor_frame: an older build inserts frames by position and must still
|
|
16
|
+
be able to write this journal after a rollback.
|
|
7
17
|
"""
|
|
8
18
|
from __future__ import annotations
|
|
9
19
|
|
|
@@ -105,6 +115,9 @@ class ActorTrackerJournal:
|
|
|
105
115
|
CREATE TABLE IF NOT EXISTS actor_motion_frame (
|
|
106
116
|
camera_id TEXT NOT NULL, frame_id TEXT NOT NULL, result_json TEXT NOT NULL,
|
|
107
117
|
PRIMARY KEY(camera_id,frame_id));
|
|
118
|
+
CREATE TABLE IF NOT EXISTS actor_frame_records (
|
|
119
|
+
camera_id TEXT NOT NULL, frame_id TEXT NOT NULL, fingerprint TEXT NOT NULL,
|
|
120
|
+
records_json TEXT NOT NULL, PRIMARY KEY(camera_id,frame_id));
|
|
108
121
|
CREATE TABLE IF NOT EXISTS actor_identity_checkpoint (
|
|
109
122
|
camera_id TEXT PRIMARY KEY, seq INTEGER NOT NULL, payload_json TEXT NOT NULL);
|
|
110
123
|
CREATE TABLE IF NOT EXISTS actor_checkpoint (
|
|
@@ -146,15 +159,63 @@ class ActorTrackerJournal:
|
|
|
146
159
|
"actor_track_record WHERE camera_id=? ORDER BY last_seen_ms DESC, actor_id LIMIT ?)",
|
|
147
160
|
(camera, camera, MAX_RECORDS_PER_CAMERA))
|
|
148
161
|
|
|
149
|
-
def
|
|
150
|
-
"""The camera's live actors' records, oldest first
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
"FROM actor_track_record WHERE camera_id=? ORDER BY first_seen_ms, actor_id", (camera_id,)).fetchall()
|
|
162
|
+
def _read_track_records(self, camera_id):
|
|
163
|
+
"""The camera's live actors' records, oldest first. The caller holds the lock."""
|
|
164
|
+
rows = self._db.execute(
|
|
165
|
+
"SELECT actor_id,kind,epoch,first_seen_ms,last_seen_ms,frames,box_json,confidence,face_json "
|
|
166
|
+
"FROM actor_track_record WHERE camera_id=? ORDER BY first_seen_ms, actor_id", (camera_id,)).fetchall()
|
|
155
167
|
return [TrackRecord(r[0], r[1], r[2], r[3], r[4], r[5], tuple(json.loads(r[6])), r[7],
|
|
156
168
|
None if r[8] is None else FaceSlot.from_wire(json.loads(r[8]))) for r in rows]
|
|
157
169
|
|
|
170
|
+
def track_records(self, camera_id):
|
|
171
|
+
"""The camera's live actors' records, oldest first (list of TrackRecord).
|
|
172
|
+
|
|
173
|
+
The LIVE state: a /detect response serves the records of its own frame from
|
|
174
|
+
track_and_records instead, which another frame cannot move.
|
|
175
|
+
"""
|
|
176
|
+
with self._lock:
|
|
177
|
+
return self._read_track_records(camera_id)
|
|
178
|
+
|
|
179
|
+
@staticmethod
|
|
180
|
+
def _stored_records(records):
|
|
181
|
+
"""A frame's records as its journal row keeps them: the wire form, each face replaced
|
|
182
|
+
by its slot's digest. The vector stays only on the live record, which ends with its
|
|
183
|
+
actor (track_records.py)."""
|
|
184
|
+
stored = []
|
|
185
|
+
for record in records:
|
|
186
|
+
wire = record.to_wire()
|
|
187
|
+
if record.face is not None:
|
|
188
|
+
wire["face"] = {"sha256": _hash(record.face.to_wire())}
|
|
189
|
+
stored.append(wire)
|
|
190
|
+
return _json(stored)
|
|
191
|
+
|
|
192
|
+
def _replayed_records(self, camera, frame_id, fingerprint):
|
|
193
|
+
"""A replayed frame's own records, as they were kept beside it. Each face comes back
|
|
194
|
+
only while the actor's live record still holds that same slot. A frame with none kept
|
|
195
|
+
(journalled before the snapshot existed, or by an older build) has none to serve: [].
|
|
196
|
+
The caller holds the lock."""
|
|
197
|
+
row = self._db.execute(
|
|
198
|
+
"SELECT fingerprint,records_json FROM actor_frame_records WHERE camera_id=? AND frame_id=?",
|
|
199
|
+
(camera, frame_id)).fetchone()
|
|
200
|
+
if row is None or row[0] != fingerprint:
|
|
201
|
+
return []
|
|
202
|
+
try:
|
|
203
|
+
stored = json.loads(row[1])
|
|
204
|
+
if not isinstance(stored, list) or any(not isinstance(r, dict) for r in stored):
|
|
205
|
+
raise ValueError("records snapshot is not a list of records")
|
|
206
|
+
live = {r.actor_id: r.face for r in self._read_track_records(camera)}
|
|
207
|
+
replayed = []
|
|
208
|
+
for record in stored:
|
|
209
|
+
saved, face = record.get("face"), None
|
|
210
|
+
if saved is not None:
|
|
211
|
+
current = live.get(record.get("actorId"))
|
|
212
|
+
if current is not None and saved.get("sha256") == _hash(current.to_wire()):
|
|
213
|
+
face = current.to_wire()
|
|
214
|
+
replayed.append({**record, "face": face})
|
|
215
|
+
return replayed
|
|
216
|
+
except (ValueError, TypeError, AttributeError) as error:
|
|
217
|
+
raise TrackingError("Corrupt actor frame journal") from error
|
|
218
|
+
|
|
158
219
|
def record_face(self, camera_id, actor_id, face):
|
|
159
220
|
"""Store one FaceSlot on a LIVE actor's record; False when the actor is not live.
|
|
160
221
|
|
|
@@ -359,6 +420,21 @@ class ActorTrackerJournal:
|
|
|
359
420
|
|
|
360
421
|
def track(self, request, detections, *, image_sha256, model_hashes, frame_size,
|
|
361
422
|
appearance=None):
|
|
423
|
+
"""The signed tracking result for one frame (track_and_records without its records)."""
|
|
424
|
+
return self.track_and_records(request, detections, image_sha256=image_sha256,
|
|
425
|
+
model_hashes=model_hashes, frame_size=frame_size,
|
|
426
|
+
appearance=appearance)[0]
|
|
427
|
+
|
|
428
|
+
def track_and_records(self, request, detections, *, image_sha256, model_hashes, frame_size,
|
|
429
|
+
appearance=None):
|
|
430
|
+
"""Track one frame: (signed tracking result, the camera's per-track records AS OF
|
|
431
|
+
THIS FRAME in wire form, oldest first).
|
|
432
|
+
|
|
433
|
+
Both are read inside one lock and one transaction, so a request on the same camera
|
|
434
|
+
that commits in between cannot hand this frame its records, and a replayed frame
|
|
435
|
+
(the same frame again, e.g. a retry after a lost response) gets the records it had
|
|
436
|
+
then, from its journal row, not the camera's live ones.
|
|
437
|
+
"""
|
|
362
438
|
request = validate_tracking_request(request)
|
|
363
439
|
if (not isinstance(image_sha256, str) or not _SHA.fullmatch(image_sha256)
|
|
364
440
|
or not isinstance(model_hashes, dict) or not 1 <= len(model_hashes) <= 16
|
|
@@ -410,8 +486,9 @@ class ActorTrackerJournal:
|
|
|
410
486
|
if own[0] != fingerprint:
|
|
411
487
|
raise TrackingError("Retained actor frame identity changed")
|
|
412
488
|
_, saved = self._checked_frame(camera, request["frameId"], own[3], own[0], own[1], own[2])
|
|
489
|
+
records = self._replayed_records(camera, request["frameId"], own[0])
|
|
413
490
|
self._db.execute("COMMIT")
|
|
414
|
-
return saved
|
|
491
|
+
return saved, records
|
|
415
492
|
head = self._db.execute(
|
|
416
493
|
"SELECT context_hash,epoch,captured_at,seq,active_json FROM actor_camera WHERE camera_id=?",
|
|
417
494
|
(camera,)).fetchone()
|
|
@@ -517,13 +594,19 @@ class ActorTrackerJournal:
|
|
|
517
594
|
(camera, request["frameId"], motion_json))
|
|
518
595
|
self._db.execute("INSERT INTO actor_frame VALUES (?,?,?,?,?,?,?)",
|
|
519
596
|
(camera, request["frameId"], seq, epoch, fingerprint, _json(inputs), _json(result)))
|
|
597
|
+
# This frame's records, read in its own transaction and kept beside it.
|
|
598
|
+
self._write_track_records(camera, result, values, captured_at)
|
|
599
|
+
records = self._read_track_records(camera)
|
|
600
|
+
self._db.execute(
|
|
601
|
+
"INSERT INTO actor_frame_records VALUES (?,?,?,?) ON CONFLICT(camera_id,frame_id) DO UPDATE SET "
|
|
602
|
+
"fingerprint=excluded.fingerprint,records_json=excluded.records_json",
|
|
603
|
+
(camera, request["frameId"], fingerprint, self._stored_records(records)))
|
|
520
604
|
self._db.execute(
|
|
521
605
|
"INSERT INTO actor_camera VALUES (?,?,?,?,?,?) ON CONFLICT(camera_id) DO UPDATE SET "
|
|
522
606
|
"context_hash=excluded.context_hash,epoch=excluded.epoch,captured_at=excluded.captured_at,"
|
|
523
607
|
"seq=excluded.seq,active_json=excluded.active_json",
|
|
524
608
|
(camera, context, epoch, captured_at, seq, _json(active)))
|
|
525
609
|
self._save_checkpoint(camera,context,epoch,captured_at,seq,fingerprint,tracker,identities)
|
|
526
|
-
self._write_track_records(camera, result, values, captured_at)
|
|
527
610
|
if appearance is not None:
|
|
528
611
|
self._save_identity_checkpoint(
|
|
529
612
|
camera, request, context, epoch, seq, fingerprint,
|
|
@@ -535,13 +618,16 @@ class ActorTrackerJournal:
|
|
|
535
618
|
self._db.execute(
|
|
536
619
|
"DELETE FROM actor_motion_frame WHERE camera_id=? AND frame_id NOT IN "
|
|
537
620
|
"(SELECT frame_id FROM actor_frame WHERE camera_id=?)", (camera,camera))
|
|
621
|
+
self._db.execute(
|
|
622
|
+
"DELETE FROM actor_frame_records WHERE camera_id=? AND frame_id NOT IN "
|
|
623
|
+
"(SELECT frame_id FROM actor_frame WHERE camera_id=?)", (camera,camera))
|
|
538
624
|
self._db.execute("COMMIT")
|
|
539
625
|
self._cache[camera] = (seq, tracker, identities)
|
|
540
626
|
if identity is not None:
|
|
541
627
|
self._identity_cache[camera] = (seq, identity)
|
|
542
628
|
else:
|
|
543
629
|
self._identity_cache.pop(camera, None)
|
|
544
|
-
return json.loads(_json(result))
|
|
630
|
+
return json.loads(_json(result)), [record.to_wire() for record in records]
|
|
545
631
|
except BaseException:
|
|
546
632
|
if self._db.in_transaction:
|
|
547
633
|
self._db.execute("ROLLBACK")
|
|
@@ -79,6 +79,24 @@ def _warmup_jpeg() -> bytes:
|
|
|
79
79
|
return buf.getvalue()
|
|
80
80
|
|
|
81
81
|
|
|
82
|
+
#: The frame size a live look sends (the connector's 640x480 analysis frame).
|
|
83
|
+
FULL_SIZE_WARMUP = (640, 480)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _full_size_warmup_jpeg() -> bytes:
|
|
87
|
+
"""A 640x480 JPEG with real texture, for the pinned model's full-size warm-up.
|
|
88
|
+
|
|
89
|
+
Noise, not a flat fill: a flat frame compresses to almost nothing and decodes
|
|
90
|
+
in no time, so it would warm the forward pass but not the decode a live frame
|
|
91
|
+
costs. See ModelRegistry._ensure_loaded for why this pass exists.
|
|
92
|
+
"""
|
|
93
|
+
from PIL import Image
|
|
94
|
+
|
|
95
|
+
buf = io.BytesIO()
|
|
96
|
+
Image.effect_noise(FULL_SIZE_WARMUP, 48).convert("RGB").save(buf, format="JPEG", quality=80)
|
|
97
|
+
return buf.getvalue()
|
|
98
|
+
|
|
99
|
+
|
|
82
100
|
def build_catalogue(
|
|
83
101
|
default_model_name: str,
|
|
84
102
|
default_model_path: "Path | None" = None,
|
|
@@ -161,6 +179,7 @@ class DetectorLoader:
|
|
|
161
179
|
probe=self._probe,
|
|
162
180
|
ram_budget_bytes=ram_budget_bytes,
|
|
163
181
|
log=log,
|
|
182
|
+
warm=self._warm_full_size,
|
|
164
183
|
)
|
|
165
184
|
|
|
166
185
|
self._stop = threading.Event()
|
|
@@ -198,6 +217,15 @@ class DetectorLoader:
|
|
|
198
217
|
else:
|
|
199
218
|
session.detect(_warmup_jpeg(), 0.99, None)
|
|
200
219
|
|
|
220
|
+
@staticmethod
|
|
221
|
+
def _warm_full_size(session) -> None:
|
|
222
|
+
"""One more forward pass, on a live-sized frame, for the pinned cascade gate.
|
|
223
|
+
|
|
224
|
+
The registry runs this after `_probe` and never as a gate (see
|
|
225
|
+
ModelRegistry._ensure_loaded). `0.99` for the same reason as the probe.
|
|
226
|
+
"""
|
|
227
|
+
session.detect(_full_size_warmup_jpeg(), 0.99, None)
|
|
228
|
+
|
|
201
229
|
def start(self) -> None:
|
|
202
230
|
self._thread.start()
|
|
203
231
|
|
|
@@ -214,6 +214,7 @@ class ModelRegistry:
|
|
|
214
214
|
ram_budget_bytes: "int | None",
|
|
215
215
|
log=print,
|
|
216
216
|
clock=time.monotonic,
|
|
217
|
+
warm=None,
|
|
217
218
|
):
|
|
218
219
|
if not entries:
|
|
219
220
|
raise ValueError("a registry with no models can answer nothing")
|
|
@@ -242,6 +243,8 @@ class ModelRegistry:
|
|
|
242
243
|
self._entries = list(entries)
|
|
243
244
|
self._session_factory = session_factory
|
|
244
245
|
self._probe = probe
|
|
246
|
+
#: The pinned model's full-size warm-up (see `_ensure_loaded`), or None.
|
|
247
|
+
self._warm = warm
|
|
245
248
|
#: `None` means NO CEILING WAS INJECTED — not "unlimited".
|
|
246
249
|
self._budget = None if ram_budget_bytes is None else int(ram_budget_bytes)
|
|
247
250
|
self._log = log
|
|
@@ -818,6 +821,26 @@ class ModelRegistry:
|
|
|
818
821
|
rss_delta = max(0, after[0] - before[0])
|
|
819
822
|
rss_source = after[1]
|
|
820
823
|
|
|
824
|
+
# THE PINNED MODEL WARMS AT FULL SIZE BEFORE IT IS READY (plan v7 W8 follow-up).
|
|
825
|
+
# The probe above proves a forward pass completes, on a 64x48 frame, and that
|
|
826
|
+
# frame is too small to warm the path a live look takes: after the 0.2.246
|
|
827
|
+
# detector swap at Texas the first live call took 3.2 s and the next 18 averaged
|
|
828
|
+
# 1.4 s. So the cascade gate runs once more on a full-size frame before READY.
|
|
829
|
+
# Only the pinned model: it loads at start or on a new artifact, ahead of the
|
|
830
|
+
# looks it gates, while a specialist loads inside the first request that needs
|
|
831
|
+
# it, where another pass would only lengthen that request. Never a gate: a
|
|
832
|
+
# failure here is logged and the model is ready on its proof. Not timed into
|
|
833
|
+
# warmup_ms (`warmup_inference_ms`), which stays the proof pass's own time, and
|
|
834
|
+
# after the RSS reading, so the admission number is unchanged.
|
|
835
|
+
warm_ms = None
|
|
836
|
+
if entry.pinned and self._warm is not None:
|
|
837
|
+
try:
|
|
838
|
+
warm_started = time.perf_counter()
|
|
839
|
+
self._warm(session)
|
|
840
|
+
warm_ms = (time.perf_counter() - warm_started) * 1000.0
|
|
841
|
+
except Exception as err:
|
|
842
|
+
self._log(f"[detector] {entry.name}: full-size warm-up failed; ready on its probe: {err}")
|
|
843
|
+
|
|
821
844
|
with self._lock:
|
|
822
845
|
state.session = session
|
|
823
846
|
state.stamp = stamp
|
|
@@ -830,9 +853,10 @@ class ModelRegistry:
|
|
|
830
853
|
state.rss_source = rss_source
|
|
831
854
|
self._index = None
|
|
832
855
|
measured = f", rss delta {rss_delta} B via {rss_source}" if rss_delta is not None else ""
|
|
856
|
+
warmed = f", full-size warm-up {warm_ms:.0f} ms" if warm_ms is not None else ""
|
|
833
857
|
self._log(
|
|
834
858
|
f"[detector] ready — {entry.name} at {entry.path} "
|
|
835
|
-
f"(probe {warmup_ms:.0f} ms, task {getattr(entry.vocabulary, 'task', 'object_detection')}{measured})"
|
|
859
|
+
f"(probe {warmup_ms:.0f} ms{warmed}, task {getattr(entry.vocabulary, 'task', 'object_detection')}{measured})"
|
|
836
860
|
)
|
|
837
861
|
|
|
838
862
|
def _fail(self, state: _ModelState, stamp: "tuple[int, int]", reason: str) -> None:
|
|
@@ -892,6 +916,8 @@ class ModelRegistry:
|
|
|
892
916
|
if s.reason:
|
|
893
917
|
row["reason"] = s.reason
|
|
894
918
|
if s.warmup_ms is not None:
|
|
919
|
+
# The proof pass (64x48) only; the pinned model's full-size warm-up
|
|
920
|
+
# is in the ready log line, not here (see `_ensure_loaded`).
|
|
895
921
|
row["warmup_inference_ms"] = round(s.warmup_ms, 1)
|
|
896
922
|
if s.loaded_at is not None:
|
|
897
923
|
row["loaded_at"] = s.loaded_at
|
|
@@ -660,13 +660,14 @@ class DetectorHandler(BaseHTTPRequestHandler):
|
|
|
660
660
|
result = loader.estimate_appearance(jpeg, boxes, appearance_model)
|
|
661
661
|
appearance = build_appearance_evidence(
|
|
662
662
|
result, hashlib.sha256(jpeg).hexdigest(), frame_size, batch.detections)
|
|
663
|
-
|
|
663
|
+
# The camera's per-track records (track_records.py) as of THIS frame, beside the
|
|
664
|
+
# signed tracking proof and outside its digest. They come from the same lock and
|
|
665
|
+
# transaction as the proof: a second request on the same camera can no longer
|
|
666
|
+
# commit in between and hand this frame its records, and a replayed frame gets
|
|
667
|
+
# the records it had then, not the live ones.
|
|
668
|
+
body["tracking"], body["tracks"] = self.server.actor_tracker.track_and_records(
|
|
664
669
|
tracking, batch.detections, image_sha256=hashlib.sha256(jpeg).hexdigest(),
|
|
665
670
|
model_hashes=body.get("model_hashes"), frame_size=frame_size, appearance=appearance)
|
|
666
|
-
# The camera's live per-track records (track_records.py), beside the
|
|
667
|
-
# signed tracking proof and outside its digest.
|
|
668
|
-
body["tracks"] = [record.to_wire() for record in
|
|
669
|
-
self.server.actor_tracker.track_records(tracking["cameraId"])]
|
|
670
671
|
if detection_pose is not None:
|
|
671
672
|
indices = detection_pose_indices(batch.detections, detection_pose)
|
|
672
673
|
boxes = [[d.x, d.y, d.x + d.width, d.y + d.height]
|
|
@@ -274,6 +274,7 @@ Copy-Item -Force "$Here\register-litertlm-task.ps1" "$Payload\bin\register-liter
|
|
|
274
274
|
# regex over the whole file, so a comment naming it enrols a file that does not
|
|
275
275
|
# exist. This paragraph is deliberately phrased so it cannot match.)
|
|
276
276
|
Copy-Item -Force "$Here\setup-detector.ps1" "$Payload\bin\setup-detector.ps1"
|
|
277
|
+
Copy-Item -Force "$Here\ensure-detector.ps1" "$Payload\bin\ensure-detector.ps1"
|
|
277
278
|
Copy-Item -Force "$Here\register-detector-task.ps1" "$Payload\bin\register-detector-task.ps1"
|
|
278
279
|
# The Python package SOURCE that `uv tool install` builds into its own venv.
|
|
279
280
|
# Tens of KB; the 124 MB fp32 ONNX it runs is OTA'd, never bundled. Not matched
|
|
@@ -26,6 +26,14 @@
|
|
|
26
26
|
The connector's process tree is sampled every 10 s for its ffmpeg sessions
|
|
27
27
|
and CPU (receipt.json).
|
|
28
28
|
|
|
29
|
+
Then two detector-restart cases (plan v7 W8 item 2), judged apart from the
|
|
30
|
+
window above and still with no Gemma call: the connector process frozen for
|
|
31
|
+
-StallMs (16.5 s) across a readiness proof's renewal window, so the proof
|
|
32
|
+
lapses between two looks; and a detector swap (connections refused for
|
|
33
|
+
-SwapSeconds, then 503 while it loads) during which the alert cameras pause
|
|
34
|
+
with their sessions kept, retry every 2 s, and look again within 5 s of the
|
|
35
|
+
detector's first answer. -StallMs 0 -SwapSeconds 0 turns them off.
|
|
36
|
+
|
|
29
37
|
The install tree is assembled from the built workspace (bin\node.exe is the
|
|
30
38
|
runner's Node 22, bin\ffmpeg.exe the pinned static build, lib\edge-sdk a
|
|
31
39
|
junction to the built package) because a pull request has no published
|
|
@@ -36,7 +44,9 @@
|
|
|
36
44
|
param(
|
|
37
45
|
[ValidateRange(3, 30)][double]$ObserveMinutes = 6,
|
|
38
46
|
[ValidateRange(1, 12)][int]$IdleCameras = 6,
|
|
39
|
-
[ValidateRange(0, 2)][int]$MainCameras = 1
|
|
47
|
+
[ValidateRange(0, 2)][int]$MainCameras = 1,
|
|
48
|
+
[ValidateRange(0, 60000)][int]$StallMs = 16500,
|
|
49
|
+
[ValidateRange(0, 300)][int]$SwapSeconds = 100
|
|
40
50
|
)
|
|
41
51
|
$ErrorActionPreference = 'Stop'
|
|
42
52
|
Set-StrictMode -Version Latest
|
|
@@ -206,7 +216,7 @@ $paths
|
|
|
206
216
|
'--rtsp-base', ('rtsp://127.0.0.1:' + $rtspPort + '/'), '--mediamtx-api', ('http://127.0.0.1:' + $apiPort),
|
|
207
217
|
'--idle-cameras', $IdleCameras, '--main-cameras', $MainCameras, '--data-dir', $dataDir,
|
|
208
218
|
'--write-config', $configPath, '--ready-file', $readyFile,
|
|
209
|
-
'--minutes', $ObserveMinutes) | ForEach-Object { '"' + $_ + '"' }
|
|
219
|
+
'--minutes', $ObserveMinutes, '--stall-ms', $StallMs, '--swap-sec', $SwapSeconds) | ForEach-Object { '"' + $_ + '"' }
|
|
210
220
|
$proof = Start-Process -FilePath $systemNode -ArgumentList ($proofArgs -join ' ') -PassThru -WindowStyle Hidden `
|
|
211
221
|
-RedirectStandardOutput (Join-Path $root 'proof.log') -RedirectStandardError (Join-Path $root 'proof.err.log')
|
|
212
222
|
$owned.Add($proof)
|
|
@@ -229,7 +239,9 @@ $paths
|
|
|
229
239
|
$seen = 0
|
|
230
240
|
$lastTask = ''
|
|
231
241
|
$samples = New-Object System.Collections.Generic.List[object]
|
|
232
|
-
|
|
242
|
+
# The restart cases add up to three freezes (~1 min each) and the swap (~2.5 min).
|
|
243
|
+
$restartMinutes = if ($StallMs -gt 0 -or $SwapSeconds -gt 0) { 3 + [Math]::Ceiling($SwapSeconds / 60) + 4 } else { 0 }
|
|
244
|
+
$limit = [DateTime]::UtcNow.AddMinutes(12 + $ObserveMinutes + 6 + $restartMinutes)
|
|
233
245
|
while (-not $proof.HasExited -and [DateTime]::UtcNow -lt $limit) {
|
|
234
246
|
Start-Sleep -Seconds 10
|
|
235
247
|
$samples.Add((Get-ConnectorSample -Ffmpeg ([IO.Path]::GetFullPath($ffmpeg))))
|