@trillboards/edge-sdk 0.2.209 → 0.2.213

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. package/deploy/windows/ci/test-alert-window-field.ps1 +252 -0
  2. package/deploy/windows/ci/test-published-install.ps1 +6 -2
  3. package/dist/AlertClips.d.ts +131 -0
  4. package/dist/AlertClips.d.ts.map +1 -0
  5. package/dist/AlertClips.js +476 -0
  6. package/dist/AlertClips.js.map +1 -0
  7. package/dist/CloudSensingManager.d.ts +27 -0
  8. package/dist/CloudSensingManager.d.ts.map +1 -1
  9. package/dist/CloudSensingManager.js +138 -16
  10. package/dist/CloudSensingManager.js.map +1 -1
  11. package/dist/ConnectorStatusClient.d.ts +10 -0
  12. package/dist/ConnectorStatusClient.d.ts.map +1 -1
  13. package/dist/ConnectorStatusClient.js +11 -1
  14. package/dist/ConnectorStatusClient.js.map +1 -1
  15. package/dist/DurableVlmCandidateSpool.d.ts +4 -0
  16. package/dist/DurableVlmCandidateSpool.d.ts.map +1 -1
  17. package/dist/DurableVlmCandidateSpool.js +13 -0
  18. package/dist/DurableVlmCandidateSpool.js.map +1 -1
  19. package/dist/EdgeAgent.js +1 -1
  20. package/dist/LocalVideoBuffer.d.ts +59 -0
  21. package/dist/LocalVideoBuffer.d.ts.map +1 -0
  22. package/dist/LocalVideoBuffer.js +214 -0
  23. package/dist/LocalVideoBuffer.js.map +1 -0
  24. package/dist/NativeReadiness.d.ts +9 -1
  25. package/dist/NativeReadiness.d.ts.map +1 -1
  26. package/dist/NativeReadiness.js +21 -5
  27. package/dist/NativeReadiness.js.map +1 -1
  28. package/dist/SensingProfileClient.d.ts +62 -5
  29. package/dist/SensingProfileClient.d.ts.map +1 -1
  30. package/dist/SensingProfileClient.js +76 -1
  31. package/dist/SensingProfileClient.js.map +1 -1
  32. package/dist/VideoSegmentUploader.d.ts +169 -5
  33. package/dist/VideoSegmentUploader.d.ts.map +1 -1
  34. package/dist/VideoSegmentUploader.js +543 -32
  35. package/dist/VideoSegmentUploader.js.map +1 -1
  36. package/dist/autoSense.d.ts.map +1 -1
  37. package/dist/autoSense.js +4 -0
  38. package/dist/autoSense.js.map +1 -1
  39. package/dist/cli.d.ts.map +1 -1
  40. package/dist/cli.js +3 -0
  41. package/dist/cli.js.map +1 -1
  42. package/dist/connectorUpdate.d.ts +22 -0
  43. package/dist/connectorUpdate.d.ts.map +1 -1
  44. package/dist/connectorUpdate.js +72 -8
  45. package/dist/connectorUpdate.js.map +1 -1
  46. package/dist/diagnose.d.ts +1 -1
  47. package/dist/diagnose.js +1 -1
  48. package/dist/localSenseE2eHarness.d.ts +1 -1
  49. package/dist/localSenseE2eHarness.d.ts.map +1 -1
  50. package/dist/localSenseE2eHarness.js +7 -4
  51. package/dist/localSenseE2eHarness.js.map +1 -1
  52. package/dist/localSensing.d.ts +58 -0
  53. package/dist/localSensing.d.ts.map +1 -1
  54. package/dist/localSensing.js +126 -16
  55. package/dist/localSensing.js.map +1 -1
  56. package/package.json +5 -5
  57. package/retained/ArchiveLifecycle.cjs +11 -5
  58. package/retained/RetainedCaptureClient.cjs +206 -49
  59. package/retained/WebsiteAdmissionCaptureClient.cjs +40 -8
  60. package/retained/__tests__/evidence-uploads.test.cjs +130 -0
  61. package/retained/__tests__/realtime-schedule.test.cjs +71 -11
  62. package/retained/__tests__/upgrade-backlog.test.cjs +239 -0
  63. package/retained/__tests__/website-admission-realtime.test.cjs +48 -9
  64. package/retained/manager.cjs +2 -0
  65. package/retained/website-api-capability.cjs +7 -1
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@trillboards/edge-sdk",
3
- "version": "0.2.209",
3
+ "version": "0.2.213",
4
4
  "description": "Composable cross-platform Edge AI SDK for audience sensing on DOOH/CTV devices",
5
5
  "main": "dist/index.js",
6
6
  "types": "dist/index.d.ts",
@@ -11,15 +11,15 @@
11
11
  "scripts": {
12
12
  "build": "tsc",
13
13
  "postbuild": "chmod +x dist/cli.js dist/mcp-server.js",
14
- "test": "jest --passWithNoTests --forceExit && node --test retained/__tests__/runtime.test.cjs retained/__tests__/archive-lifecycle.test.cjs retained/__tests__/actor-perception.test.cjs retained/__tests__/actor-window.test.cjs retained/__tests__/website-admission.test.cjs retained/__tests__/realtime-schedule.test.cjs retained/__tests__/website-admission-realtime.test.cjs scripts/__tests__/local-sense-wiring-detector.test.cjs",
15
- "test:retained": "node --test retained/__tests__/runtime.test.cjs retained/__tests__/archive-lifecycle.test.cjs retained/__tests__/actor-perception.test.cjs retained/__tests__/actor-window.test.cjs retained/__tests__/website-admission.test.cjs retained/__tests__/realtime-schedule.test.cjs retained/__tests__/website-admission-realtime.test.cjs scripts/__tests__/local-sense-wiring-detector.test.cjs",
14
+ "test": "jest --passWithNoTests --forceExit && node --test retained/__tests__/runtime.test.cjs retained/__tests__/archive-lifecycle.test.cjs retained/__tests__/actor-perception.test.cjs retained/__tests__/actor-window.test.cjs retained/__tests__/website-admission.test.cjs retained/__tests__/realtime-schedule.test.cjs retained/__tests__/upgrade-backlog.test.cjs retained/__tests__/website-admission-realtime.test.cjs retained/__tests__/evidence-uploads.test.cjs scripts/__tests__/local-sense-wiring-detector.test.cjs",
15
+ "test:retained": "node --test retained/__tests__/runtime.test.cjs retained/__tests__/archive-lifecycle.test.cjs retained/__tests__/actor-perception.test.cjs retained/__tests__/actor-window.test.cjs retained/__tests__/website-admission.test.cjs retained/__tests__/realtime-schedule.test.cjs retained/__tests__/upgrade-backlog.test.cjs retained/__tests__/website-admission-realtime.test.cjs retained/__tests__/evidence-uploads.test.cjs scripts/__tests__/local-sense-wiring-detector.test.cjs",
16
16
  "clean": "rm -rf dist",
17
17
  "prepublishOnly": "npm run build"
18
18
  },
19
19
  "dependencies": {
20
20
  "@trillboards/edge-ads": "^0.2.0",
21
21
  "@trillboards/edge-cloud": "^0.2.0",
22
- "@trillboards/edge-core": "^0.4.16",
22
+ "@trillboards/edge-core": "^0.4.18",
23
23
  "@trillboards/edge-federated": "^0.2.0 || ^1.0.0",
24
24
  "@trillboards/edge-sensing": "^0.2.4",
25
25
  "jpeg-js": "^0.4.4",
@@ -29,7 +29,7 @@
29
29
  },
30
30
  "optionalDependencies": {
31
31
  "@trillboards/edge-platform-linux": "^0.2.0",
32
- "@trillboards/edge-platform-rtsp": "^0.2.14",
32
+ "@trillboards/edge-platform-rtsp": "^0.2.16",
33
33
  "@trillboards/edge-platform-windows": "^0.2.0"
34
34
  },
35
35
  "files": [
@@ -42,7 +42,7 @@ class ArchiveLifecycle {
42
42
  assert(fs.lstatSync(absolute).isFile(), 'Archive cleanup target is not a regular file');
43
43
  return true;
44
44
  }
45
- completePending() {
45
+ completePending(liveImage = null) {
46
46
  const pending = this.state.pending;
47
47
  if (!pending) return;
48
48
  assert(Array.isArray(pending.records) && pending.records.length > 0 && pending.records.length <= 64
@@ -56,7 +56,11 @@ class ArchiveLifecycle {
56
56
  }
57
57
  }
58
58
  // A shared JPEG is owned by every remaining record, not just the one retired.
59
- const live = new Set(fs.readdirSync(this.directory).filter(name => /^[a-f0-9]{32}\.json$/.test(name))
59
+ // The running archive counts those owners in memory; only a plan replayed
60
+ // after a crash re-reads every record, because then nothing else knows.
61
+ // (Re-reading on every batch cost a full archive read per capture.)
62
+ const live = liveImage ? { has: liveImage } : new Set(fs.readdirSync(this.directory)
63
+ .filter(name => /^[a-f0-9]{32}\.json$/.test(name))
60
64
  .map(name => JSON.parse(fs.readFileSync(path.join(this.directory, name), 'utf8')).imageSha256));
61
65
  for (const sha of pending.images) {
62
66
  assert(/^[a-f0-9]{64}$/.test(sha), 'Invalid cleanup image identity');
@@ -70,17 +74,19 @@ class ArchiveLifecycle {
70
74
  this.state = { ...this.state, pending: null };
71
75
  atomic(this.file, this.state);
72
76
  }
73
- retire(rows, remaining) {
77
+ /** `liveImage(sha)`, when given, says whether a remaining record still owns
78
+ * that JPEG; otherwise ownership is read from `remaining`. */
79
+ retire(rows, remaining, liveImage = null) {
74
80
  assert(!this.state.pending, 'Archive reclamation must finish before another batch');
75
81
  if (!rows.length) return;
76
82
  const pruned = { ...this.state.pruned };
77
83
  for (const row of rows) for (const [key, count] of Object.entries(recordCounts(row))) pruned[key] += count;
78
- const live = new Set([...remaining.values()].map(row => row.imageSha256));
84
+ const live = liveImage ? { has: liveImage } : new Set([...remaining.values()].map(row => row.imageSha256));
79
85
  const records = rows.map(row => ({ id: row.id, sha256: digest(fs.readFileSync(path.join(this.directory, row.id + '.json'))) }));
80
86
  const images = [...new Set(rows.map(row => row.imageSha256))].filter(sha => !live.has(sha));
81
87
  this.state = { version: 1, pruned, pending: { records, images } };
82
88
  atomic(this.file, this.state);
83
- this.completePending();
89
+ this.completePending(liveImage);
84
90
  }
85
91
  }
86
92
  module.exports = { ArchiveLifecycle };
@@ -13,6 +13,16 @@ const UNREVIEWED=new Set(['shed_superseded','expired_unreviewed']);
13
13
  const unreviewed=r=>UNREVIEWED.has(r?.disposition);
14
14
  // Timed rules and ordered sequences need every observation in capture order.
15
15
  const realtimeContext=c=>!c?.temporalReviewContract&&!c?.orderedSequence&&!c?.orderedCaptureContract;
16
+ // Realtime retirements, one JSON line each, appended once per pump. A retired
17
+ // frame is bookkeeping, not evidence: a line lost to a crash leaves that frame
18
+ // pending and the schedule decides it again. Rewriting the frame's own record
19
+ // (write + fsync + rename, up to 16 a pump per camera) is what starved 0.2.206.
20
+ const DECISION_LOG='SCHEDULE-DECISIONS.jsonl';
21
+ // A historical (backlog) review may hold the one model slot this long. A live
22
+ // frame that arrives meanwhile waits at most this; a cut review is retried.
23
+ const BACKLOG_REVIEW_BUDGET_MS=60000;
24
+ // Started this many times without an answer, a backlog frame waits for restart.
25
+ const BACKLOG_MAX_STARTS=2;
16
26
  function commit(file,value) {
17
27
  const bytes=Buffer.isBuffer(value)?value:Buffer.from(JSON.stringify(value));
18
28
  const tmp=file+'.tmp';const fd=fs.openSync(tmp,'w');try{fs.writeFileSync(fd,bytes);fs.fsyncSync(fd);}finally{fs.closeSync(fd);}fs.renameSync(tmp,file);
@@ -41,6 +51,19 @@ class RetainedCaptureClient {
41
51
  this.groupReferences=new Map();this.previousCaptureRecord=null;this.previousSelectedRecord=null;
42
52
  this.entries=new Map();this.queued=new Set();this.anchor=null;this.nextDue=null;this.lastAt=-1;this.bytes=0;
43
53
  this.counts={captures:0,selected:0,reviewed:0,received:0,inactive:0,grouped:0,shed:0,expired:0};
54
+ // Realtime originals already past the live bound when this archive loaded (an
55
+ // older release's capture-order queue): kept pending and unrewritten, never
56
+ // live work, reviewed only while the review lane is idle. `backlogUndecided`
57
+ // holds such originals no rule had decided yet. Both stay in capture order.
58
+ this.backlog=new Map();this.backlogUndecided=new Map();this.backlogPins=new Set();
59
+ this.backlogSkipped=new Set();this.backlogStarts=new Map();this.backlogPerception=null;this.backlogSwept=false;
60
+ // Every record except the backlog, in capture order: what cleanup walks.
61
+ this.retireOrder=new Map();
62
+ // Records still owed their original actor perception, in capture order.
63
+ this.perceptionPending=new Map();
64
+ // JPEG digest -> records that own it; cleanup deletes a JPEG at zero.
65
+ this.imageRefs=new Map();
66
+ this.decisions=[];this.decisionLogLines=0;this.loggedDecisions=new Set();this.lastRetiredIds=[];
44
67
  fs.mkdirSync(directory,{recursive:true});fs.mkdirSync(path.join(directory,'images'),{recursive:true});
45
68
  this.records=new Map();
46
69
  const policy={version:1,cameraId,intervalMs,maxCaptures,maxBytes,...(configuredApiBaseUrl?{configuredApiBaseUrl}:{}),...(archiveRetentionMs===undefined?{}:{archiveRetentionMs,archiveLifecycle:'resolved_and_unneeded_only_v1'}),algorithm:'first_capture_at_or_after_fixed_slot; no synthetic timestamps',...(selectionMode==='all'?{}:{selectionMode,groupingPolicy,temporalPolicy:'explicit_grouped_continuity_opt_in_else_preserve_all'})};
@@ -50,7 +73,32 @@ class RetainedCaptureClient {
50
73
  if(this.lifecycle)this.counts={...this.counts,...this.lifecycle.state.pruned};
51
74
  const files=fs.readdirSync(directory).filter(f=>/^[a-f0-9]{32}\.json$/.test(f));
52
75
  const records=files.map(f=>JSON.parse(fs.readFileSync(path.join(directory,f)))).sort((a,b)=>a.capturedAt-b.capturedAt);
53
- for(const r of records){assert.equal(r.cameraId,cameraId);assert.equal(sha(fs.readFileSync(this.image(r))),r.imageSha256);this.records.set(r.id,r);if(r.actorPerceptionPolicy){assert(this.actorPerceiver,'Archived actor perception requires its runtime');if(r.actorPerception)this.actorPerceiver.validate(r.actorPerception,this.perceptionInput(r),r.actorPerceptionPolicy);}if(r.selected&&!r.acknowledged&&!unreviewed(r))this.entries.set(r.id,r);this.counts.captures++;if(r.selected)this.counts.selected++;if(r.result)this.counts.reviewed++;if(r.acknowledged)this.counts.received++;if(r.reviewStage===null)this.counts.inactive++;if(r.disposition==='archived_grouped_unchanged')this.counts.grouped++;if(r.disposition==='shed_superseded')this.counts.shed++;if(r.disposition==='expired_unreviewed')this.counts.expired++;if(r.result)this.noteReviewed(r);this.bytes+=r.imageBytes;this.advance(r);}
76
+ const decided=this.readDecisions(),loadedAt=this.realtime?this.realtime.now():null;
77
+ for(const r of records){
78
+ assert.equal(r.cameraId,cameraId);
79
+ const decision=decided.get(r.id);
80
+ if(decision&&r.selected&&!r.acknowledged&&!r.result&&!unreviewed(r)){Object.assign(r,decision);this.loggedDecisions.add(r.id);}
81
+ const held=this.heldAtLoad(r,loadedAt);
82
+ // A held original is verified when it is offered (and again by prepare);
83
+ // hashing every backlog JPEG here held startup for minutes on a large archive.
84
+ if(!held)assert.equal(sha(fs.readFileSync(this.image(r))),r.imageSha256);
85
+ this.records.set(r.id,r);
86
+ if(r.actorPerceptionPolicy){assert(this.actorPerceiver,'Archived actor perception requires its runtime');if(r.actorPerception&&!held)this.actorPerceiver.validate(r.actorPerception,this.perceptionInput(r),r.actorPerceptionPolicy);}
87
+ if(held==='selected')this.backlog.set(r.id,r);
88
+ else if(held==='undecided')this.backlogUndecided.set(r.id,r);
89
+ else{this.retireOrder.set(r.id,true);if(r.selected&&!r.acknowledged&&!unreviewed(r))this.entries.set(r.id,r);if(r.actorPerceptionPolicy&&!r.actorPerception)this.perceptionPending.set(r.id,r);}
90
+ this.ownImage(r.imageSha256,1);
91
+ this.counts.captures++;if(r.selected)this.counts.selected++;if(r.result)this.counts.reviewed++;if(r.acknowledged)this.counts.received++;if(r.reviewStage===null)this.counts.inactive++;if(r.disposition==='archived_grouped_unchanged')this.counts.grouped++;if(r.disposition==='shed_superseded')this.counts.shed++;if(r.disposition==='expired_unreviewed')this.counts.expired++;if(r.result)this.noteReviewed(r);this.bytes+=r.imageBytes;this.advance(r,false);}
92
+ // Only the newest 256 selections are grouping references; hashing every
93
+ // selected context to keep 256 cost seconds per camera on a large archive.
94
+ const selections=[];for(let i=records.length-1;i>=0&&selections.length<256;i--)if(records[i].selected)selections.push(records[i]);
95
+ for(const r of selections.reverse())this.groupReferences.set(this.referenceKey(r.capturedAt,r.imageSha256,r.candidate.reviewContext),r.id);
96
+ // A backlog frame with an actor window still needs its history at review time.
97
+ for(const r of this.backlog.values()){
98
+ if(!r.candidate?.reviewContext?.modelSelection?.tasks?.some?.(task=>task?.task==='actor_actions'))continue;
99
+ try{pinActorWindowHistory(this,r,this.backlogPins,new Map());}catch{/* reviewed without a window, or cut */}
100
+ }
101
+ this.compactDecisions();
54
102
  }
55
103
  image(r){return path.join(this.directory,'images',r.imageSha256+'.jpg');}
56
104
  noteReviewed(r){
@@ -58,7 +106,50 @@ class RetainedCaptureClient {
58
106
  if(!this.lastReviewed||r.capturedAt>=this.lastReviewed.capturedAt)this.lastReviewed={id:r.id,capturedAt:r.capturedAt};
59
107
  }
60
108
  file(id){return path.join(this.directory,id+'.json');}
61
- advance(r){this.previousCaptureRecord=r;if(r.selected){this.previousSelectedRecord=r;const key=this.referenceKey(r.capturedAt,r.imageSha256,r.candidate.reviewContext);this.groupReferences.set(key,r.id);if(this.groupReferences.size>256)this.groupReferences.delete(this.groupReferences.keys().next().value);}if(this.anchor===null){this.anchor=r.capturedAt;this.nextDue=r.capturedAt;}if(r.selected)this.nextDue=this.anchor+(Math.floor((r.capturedAt-this.anchor)/this.intervalMs)+1)*this.intervalMs;this.lastAt=r.capturedAt;}
109
+ ownImage(digest,delta){const n=(this.imageRefs.get(digest)??0)+delta;if(n>0)this.imageRefs.set(digest,n);else this.imageRefs.delete(digest);}
110
+ /** Realtime originals past the live bound at load, never answered: held apart
111
+ * from live work. 'selected' was chosen for review; 'undecided' awaits a rule. */
112
+ heldAtLoad(r,now){
113
+ if(!this.realtime||!(this.realtime.freshnessMs>0)||now<r.capturedAt+this.realtime.freshnessMs
114
+ ||r.result||r.acknowledged||unreviewed(r)||!this.realtimeReviewable(r))return null;
115
+ return r.selected?'selected':r.disposition==='awaiting_rule_decision'?'undecided':null;
116
+ }
117
+ /** The realtime schedule's work: not a timed rule or ordered sequence. */
118
+ realtimeReviewable(r){return realtimeContext(r.candidate?.reviewContext);}
119
+ decisionFile(){return path.join(this.directory,DECISION_LOG);}
120
+ decisionLine(r){return JSON.stringify({version:1,id:r.id,disposition:r.disposition,scheduleOutcome:r.scheduleOutcome});}
121
+ readDecisions(){
122
+ const out=new Map();let text;
123
+ try{text=fs.readFileSync(this.decisionFile(),'utf8');}catch(error){if(error.code==='ENOENT')return out;throw error;}
124
+ for(const line of text.split('\n')){
125
+ if(!line)continue;let d;
126
+ try{d=JSON.parse(line);}catch{continue;} // a torn last line: that frame is decided again
127
+ if(d?.version!==1||!/^[a-f0-9]{32}$/.test(d.id??'')||!UNREVIEWED.has(d.disposition)||!d.scheduleOutcome||typeof d.scheduleOutcome!=='object')continue;
128
+ out.set(d.id,{disposition:d.disposition,scheduleOutcome:d.scheduleOutcome});
129
+ }
130
+ return out;
131
+ }
132
+ /** Rewrite the log down to decisions whose records still exist. */
133
+ compactDecisions(){
134
+ for(const id of this.loggedDecisions)if(!this.records.has(id))this.loggedDecisions.delete(id);
135
+ const lines=[...this.loggedDecisions].map(id=>this.decisionLine(this.records.get(id)));
136
+ if(lines.length)commit(this.decisionFile(),Buffer.from(lines.join('\n')+'\n'));else fs.rmSync(this.decisionFile(),{force:true});
137
+ this.decisionLogLines=lines.length;
138
+ }
139
+ /** This pump's retirements, as one append. */
140
+ flushDecisions(){
141
+ if(!this.decisions.length)return;
142
+ const lines=this.decisions.splice(0);
143
+ fs.appendFileSync(this.decisionFile(),lines.join('\n')+'\n');
144
+ this.decisionLogLines+=lines.length;
145
+ if(this.decisionLogLines>Math.max(4096,4*this.loggedDecisions.size))this.compactDecisions();
146
+ }
147
+ /** A record exactly as a restart would load it: its file plus any logged retirement. */
148
+ durableRecord(id){
149
+ const r=JSON.parse(fs.readFileSync(this.file(id))),d=this.readDecisions().get(id);
150
+ return d&&r.selected&&!r.acknowledged&&!r.result&&!unreviewed(r)?{...r,...d}:r;
151
+ }
152
+ advance(r,reference=true){this.previousCaptureRecord=r;if(r.selected){this.previousSelectedRecord=r;if(reference){const key=this.referenceKey(r.capturedAt,r.imageSha256,r.candidate.reviewContext);this.groupReferences.set(key,r.id);if(this.groupReferences.size>256)this.groupReferences.delete(this.groupReferences.keys().next().value);}}if(this.anchor===null){this.anchor=r.capturedAt;this.nextDue=r.capturedAt;}if(r.selected)this.nextDue=this.anchor+(Math.floor((r.capturedAt-this.anchor)/this.intervalMs)+1)*this.intervalMs;this.lastAt=r.capturedAt;}
62
153
  referenceKey(at,digest,context){return JSON.stringify([at,digest,sha(Buffer.from(JSON.stringify(context??null)))]);}
63
154
  chooseReview(c,digest,due,reviewStage){
64
155
  if(reviewStage===null)return{selected:false,disposition:'archived_profile_inactive'};
@@ -110,11 +201,14 @@ class RetainedCaptureClient {
110
201
  async reportCameraHealth(health){return this.client.reportCameraHealth(health);}
111
202
  baseContext(c){if(!c)return c;const x={...c};if(x.retainedCapture?.kind==='retained_single_capture')delete x.retainedCapture;return x;}
112
203
  archivePins(){
113
- const pins=new Set([...this.entries.keys(),...this.queued,...[...this.records.values()].filter(r=>r.actorPerceptionPolicy&&!r.actorPerception).map(r=>r.id),this.previousCaptureRecord?.id,this.previousSelectedRecord?.id].filter(Boolean));
204
+ // Live work only: backlog frames are pending (cleanup never takes them) and
205
+ // their actor history was pinned once, at load.
206
+ const pins=new Set([...this.entries.keys(),...this.queued,...this.perceptionPending.keys(),this.previousCaptureRecord?.id,this.previousSelectedRecord?.id].filter(Boolean));
114
207
  const visited=new Map();for(const id of [...pins])pinActorWindowHistory(this,this.records.get(id),pins,visited);
115
208
  return pins;
116
209
  }
117
210
  reclaim(at){
211
+ this.lastRetiredIds=[];
118
212
  if(!this.lifecycle)return;
119
213
  const pins=this.archivePins(),todo=[...pins];
120
214
  const pin=id=>{if(id&&!pins.has(id)){assert(this.records.has(id),'Archive reference missing before cleanup');pins.add(id);todo.push(id);}};
@@ -131,18 +225,22 @@ class RetainedCaptureClient {
131
225
  if(scope.referenceCapturedAtMs!==undefined){const id=this.groupReferences.get(this.referenceKey(scope.referenceCapturedAtMs,scope.referenceSha256,r.candidate.reviewContext));if(id)pin(id);}
132
226
  }
133
227
  }
228
+ for(const id of this.backlogPins)pins.add(id);
134
229
  const rows=[];
135
- for(const r of this.records.values()){
230
+ for(const id of this.retireOrder.keys()){
136
231
  if(rows.length>=64)break;
232
+ const r=this.records.get(id);
233
+ if(!r){this.retireOrder.delete(id);continue;}
137
234
  if(pins.has(r.id)||r.selected&&!r.acknowledged&&!unreviewed(r)||r.disposition==='awaiting_rule_decision'
138
235
  ||Math.max(r.capturedAt,r.acknowledgedAtCaptureMs??r.capturedAt)>at-this.archiveRetentionMs)continue;
139
236
  rows.push(r);
140
237
  }
141
238
  if(!rows.length)return;
142
- const remaining=new Map(this.records);for(const row of rows)remaining.delete(row.id);
143
- this.lifecycle.retire(rows,remaining);
144
- this.records=remaining;for(const row of rows)this.bytes-=row.imageBytes;
145
- for(const[key,id]of this.groupReferences)if(!remaining.has(id))this.groupReferences.delete(key);
239
+ const dropping=new Map();for(const row of rows)dropping.set(row.imageSha256,(dropping.get(row.imageSha256)??0)+1);
240
+ this.lifecycle.retire(rows,null,digest=>(this.imageRefs.get(digest)??0)>(dropping.get(digest)??0));
241
+ for(const row of rows){this.records.delete(row.id);this.retireOrder.delete(row.id);this.loggedDecisions.delete(row.id);this.ownImage(row.imageSha256,-1);this.bytes-=row.imageBytes;}
242
+ this.lastRetiredIds=rows.map(row=>row.id);
243
+ for(const[key,id]of this.groupReferences)if(!this.records.has(id))this.groupReferences.delete(key);
146
244
  }
147
245
  retainOrderedCapture(c) {
148
246
  assert.equal(c.name,this.cameraId);assert(Number.isSafeInteger(c.enqueuedAtMs));
@@ -151,17 +249,20 @@ class RetainedCaptureClient {
151
249
  if(fs.existsSync(this.file(id)))return;
152
250
  assert(c.enqueuedAtMs>=this.lastAt,'Capture time moved backwards');
153
251
  this.reclaim(c.enqueuedAtMs);
154
- if((this.lifecycle?this.records.size:this.counts.captures)>=this.maxCaptures || this.bytes+c.jpeg.length>this.maxBytes)throw Error('CAPTURE_STORAGE_EXHAUSTED: unresolved evidence retained; capture must pause visibly');
252
+ // The backlog is a fixed legacy set (it only shrinks); counting it would pause
253
+ // capture on a camera an older release filled, for as long as it stays.
254
+ if((this.lifecycle?this.records.size-this.backlog.size-this.backlogUndecided.size:this.counts.captures)>=this.maxCaptures || this.bytes+c.jpeg.length>this.maxBytes)throw Error('CAPTURE_STORAGE_EXHAUSTED: unresolved evidence retained; capture must pause visibly');
155
255
  const reviewStage=this.reviewStage(c.enqueuedAtMs);assert(reviewStage===null||typeof reviewStage==='string');
156
256
  const choice=this.chooseReview(c,digest,this.nextDue===null || c.enqueuedAtMs>=this.nextDue,reviewStage);const selected=choice.selected;
157
257
  const actorPolicy=this.actorPerceiver?.policyForContext?this.actorPerceiver.policyForContext(c.reviewContext):this.actorPerceiver?.policy;
158
258
  const r={version:1,id,cameraId:c.name,capturedAt:c.enqueuedAtMs,imageSha256:digest,imageBytes:c.jpeg.length,selected,reviewStage,
159
259
  ...choice,previousCaptureId:this.previousCaptureRecord?.id??null,previousSelectedId:this.previousSelectedRecord?.id??null,candidate:{...c,jpeg:undefined},result:null,acknowledged:false,...(actorPolicy?{actorPerceptionPolicy:structuredClone(actorPolicy)}:{})};
160
260
  if(!fs.existsSync(this.image(r)))commit(this.image(r),c.jpeg);
161
- commit(this.file(id),r);this.records.set(id,r);if(selected)this.entries.set(id,r);this.counts.captures++;if(selected)this.counts.selected++;if(reviewStage===null)this.counts.inactive++;if(r.disposition==='archived_grouped_unchanged')this.counts.grouped++;this.bytes+=c.jpeg.length;this.advance(r);
261
+ commit(this.file(id),r);this.records.set(id,r);this.retireOrder.set(id,true);this.ownImage(digest,1);if(actorPolicy)this.perceptionPending.set(id,r);if(selected)this.entries.set(id,r);this.counts.captures++;if(selected)this.counts.selected++;if(reviewStage===null)this.counts.inactive++;if(r.disposition==='archived_grouped_unchanged')this.counts.grouped++;this.bytes+=c.jpeg.length;this.advance(r);
162
262
  }
163
263
  pendingOrderedCandidates(){
164
- return this.realtime?this.realtimeCandidates():this.captureOrderCandidates(this.entries.values());
264
+ try{return this.realtime?this.realtimeCandidates():this.captureOrderCandidates(this.entries.values());}
265
+ finally{this.flushDecisions();}
165
266
  }
166
267
  offerFor(r,extra={}){
167
268
  return {...r.candidate,jpeg:fs.readFileSync(this.image(r)),retainedId:r.id,...extra,
@@ -187,6 +288,11 @@ class RetainedCaptureClient {
187
288
  * retention and are counted; none is reviewed late. Answered frames awaiting
188
289
  * receipt are never touched.
189
290
  *
291
+ * Only live work is scanned: frames already past the bound when the archive
292
+ * loaded wait in the backlog (pendingBacklogCandidate), so one pass over a
293
+ * handful of entries replaces sorting the whole archive twice a call, and this
294
+ * pump's retirements land as one append to the decision log.
295
+ *
190
296
  * Across cameras, the spool's least-recently-served rotation serves these
191
297
  * slots round robin. Earliest-capture-first was rejected: every slot holds a
192
298
  * frame at most one selection old, so it would favor whichever camera selects
@@ -195,35 +301,28 @@ class RetainedCaptureClient {
195
301
  */
196
302
  realtimeCandidates(){
197
303
  assert(this.reviewQueue,'Realtime review scheduling requires its review queue');
304
+ this.sweepBacklogCopies();
198
305
  const now=this.realtime.now(),freshnessMs=this.realtime.freshnessMs;
199
306
  const stale=r=>freshnessMs>0&&now>=r.capturedAt+freshnessMs;
200
- const unanswered=[],answered=[],ordered=[];
307
+ const byCapture=(a,b)=>a.capturedAt-b.capturedAt||a.id.localeCompare(b.id);
308
+ // The newest view this camera has: reviewable now, or already answered. A
309
+ // frame still waiting for its original actor perception cannot be offered yet.
310
+ const unanswered=[],answered=[],ordered=[];let newest=this.lastReviewed;
201
311
  for(const r of this.entries.values()){
202
312
  if(r.acknowledged)continue;
203
- if(!realtimeContext(r.candidate.reviewContext))ordered.push(r);
204
- else (r.result?answered:unanswered).push(r);
313
+ if(!realtimeContext(r.candidate.reviewContext)){ordered.push(r);continue;}
314
+ if(r.result)answered.push(r);else unanswered.push(r);
315
+ if((r.result||!(r.actorPerceptionPolicy&&!r.actorPerception))&&(!newest||byCapture(newest,r)<0))newest=r;
205
316
  }
206
- const byCapture=(a,b)=>a.capturedAt-b.capturedAt||a.id.localeCompare(b.id);
207
317
  unanswered.sort(byCapture);
208
- // The newest view this camera has: reviewable now, or already answered. A
209
- // frame still waiting for its original actor perception cannot be offered yet.
210
- const ready=unanswered.filter(r=>!(r.actorPerceptionPolicy&&!r.actorPerception));
211
- const newest=[...ready,...answered,...(this.lastReviewed?[this.lastReviewed]:[])].sort(byCapture).pop()??null;
212
318
  const reserved=new Map();
213
- for(const r of ready){const f=this.followUpOf(r);if(f&&!reserved.has(f.routeId))reserved.set(f.routeId,r);}
319
+ for(const r of unanswered){if(r.actorPerceptionPolicy&&!r.actorPerception)continue;const f=this.followUpOf(r);if(f&&!reserved.has(f.routeId))reserved.set(f.routeId,r);}
214
320
  const keep=new Set([...reserved.values()].map(r=>r.id));
215
321
  // Expired first: a frame past the bound (including one whose review was cut
216
322
  // at its deadline) could not have become a live alert, whatever replaced it.
323
+ // A queued copy can win the next take, so it leaves now.
217
324
  const decide=r=>stale(r)?'expired':newest&&r.id!==newest.id&&!keep.has(r.id)&&byCapture(r,newest)<0?'shed':null;
218
- // A queued copy can win the next take, so it leaves now. The rest is
219
- // bookkeeping, bounded per pump so capture stays responsive while a backlog drains.
220
- let budget=16;
221
- for(const r of [...unanswered.filter(r=>this.queued.has(r.id)),...unanswered.filter(r=>!this.queued.has(r.id))]){
222
- const outcome=decide(r);if(!outcome)continue;
223
- const queued=this.queued.has(r.id);
224
- if(!queued&&budget<=0)break;
225
- if(this.retireUnreviewed(r,outcome,now,newest)&&!queued)budget--;
226
- }
325
+ for(const r of unanswered){const outcome=decide(r);if(outcome)this.retireUnreviewed(r,outcome,now,newest);}
227
326
  const current=newest&&!keep.has(newest.id)?this.entries.get(newest.id):null;
228
327
  const slots=[...reserved.values(),...(current&&!current.result?[current]:[])];
229
328
  const out=slots.filter(r=>this.entries.has(r.id)&&!this.queued.has(r.id)&&!stale(r))
@@ -232,6 +331,44 @@ class RetainedCaptureClient {
232
331
  out.push(...this.captureOrderCandidates(answered,this.offerWindow));
233
332
  return [...out,...this.captureOrderCandidates(ordered)];
234
333
  }
334
+ /** An older release queued backlog frames in capture order; a recovered copy
335
+ * would be taken ahead of live work. The originals stay pending here. */
336
+ sweepBacklogCopies(){
337
+ if(this.backlogSwept||!(this.backlog.size||this.backlogUndecided.size)||typeof this.reviewQueue?.queuedIds!=='function')return;
338
+ this.backlogSwept=true;
339
+ for(const id of this.reviewQueue.queuedIds(this.cameraId))
340
+ if((this.backlog.has(id)||this.backlogUndecided.has(id))&&!this.queued.has(id))this.reviewQueue.withdraw(id);
341
+ }
342
+ /** A backlog frame of this camera is queued, or waits for its perception. */
343
+ backlogBusy(){
344
+ for(const id of this.queued)if(this.backlog.has(id)){if(this.reviewQueue?.holds?.(id))return true;this.queued.delete(id);}
345
+ return !!this.backlogPerception&&!this.backlogPerception.actorPerception&&this.backlog.has(this.backlogPerception.id);
346
+ }
347
+ /** The offer for one backlog original, or null when its JPEG no longer
348
+ * matches (it stays on disk, unoffered until restart). */
349
+ backlogOffer(r){
350
+ if(r.actorPerceptionPolicy&&!r.actorPerception){this.backlogPerception=r;return null;}
351
+ let jpeg=null;try{jpeg=fs.readFileSync(this.image(r));}catch{/* missing: skipped below */}
352
+ if(!jpeg||sha(jpeg)!==r.imageSha256){this.backlogSkipped.add(r.id);return undefined;}
353
+ // Never live news: its deadline is a budget from now, not capture + bound,
354
+ // and the server's stale_event rule keeps its answer from alerting.
355
+ return {...r.candidate,jpeg,retainedId:r.id,reviewDeadlineAtMs:this.realtime.now()+BACKLOG_REVIEW_BUDGET_MS,
356
+ reviewContext:{...r.candidate.reviewContext,retainedCapture:{version:1,id:r.id,kind:'retained_single_capture'}}};
357
+ }
358
+ /**
359
+ * One historical original, for a manager whose review lane is idle with no
360
+ * live frame waiting. Oldest first; one queued per camera at a time.
361
+ */
362
+ pendingBacklogCandidate(){
363
+ if(!this.realtime||!this.reviewQueue||this.backlogBusy())return null;
364
+ for(const r of this.backlog.values()){
365
+ if(this.backlogSkipped.has(r.id)||(this.backlogStarts.get(r.id)??0)>=BACKLOG_MAX_STARTS)continue;
366
+ const offer=this.backlogOffer(r);
367
+ if(offer===undefined)continue;
368
+ return offer;
369
+ }
370
+ return null;
371
+ }
235
372
  /** Admission's mark for a wait/recheck endpoint, reserved while the server
236
373
  * would still pair it with its anchor: captured inside
237
374
  * [anchor + wait, anchor + wait + RECHECK_ANCHOR_GRACE_MS]. */
@@ -242,15 +379,16 @@ class RetainedCaptureClient {
242
379
  const dueAtMs=f.anchorCapturedAtMs+f.waitMs;
243
380
  return r.capturedAt>=dueAtMs&&r.capturedAt<=dueAtMs+RECHECK_ANCHOR_GRACE_MS?f:null;
244
381
  }
245
- /** The queue copy leaves first. Work the model already took is never retired. */
382
+ /** The queue copy leaves first. Work the model already took is never retired.
383
+ * The decision is applied now and logged with this pump's batch. */
246
384
  retireUnreviewed(r,outcome,now,by){
247
385
  const queued=this.reviewQueue.withdraw(r.id);
248
386
  if(queued==='in_flight'||queued==='reviewed')return false;
249
387
  const f=this.followUpOf(r);
250
- const next={...r,disposition:outcome==='shed'?'shed_superseded':'expired_unreviewed',
388
+ Object.assign(r,{disposition:outcome==='shed'?'shed_superseded':'expired_unreviewed',
251
389
  scheduleOutcome:{version:1,outcome,decidedAtMs:now,slot:f?'follow_up':'latest',...(f?{routeId:f.routeId}:{}),
252
- ...(outcome==='shed'?{supersededBy:by.id,supersededByCapturedAtMs:by.capturedAt}:{freshnessMs:this.realtime.freshnessMs})}};
253
- commit(this.file(r.id),next);Object.assign(r,next);this.records.set(r.id,r);
390
+ ...(outcome==='shed'?{supersededBy:by.id,supersededByCapturedAtMs:by.capturedAt}:{freshnessMs:this.realtime.freshnessMs})}});
391
+ this.decisions.push(this.decisionLine(r));this.loggedDecisions.add(r.id);
254
392
  this.entries.delete(r.id);this.queued.delete(r.id);this.counts[outcome]++;
255
393
  return true;
256
394
  }
@@ -262,9 +400,11 @@ class RetainedCaptureClient {
262
400
  getReviewScheduleStats(){
263
401
  if(!this.realtime)return null;
264
402
  const {selected,reviewed,shed,expired}=this.counts;
265
- // selected = reviewed + shed + expired + pending: an answer awaiting receipt is reviewed.
266
- const pending=[...this.entries.values()].filter(r=>!r.result).length;
267
- return {freshnessMs:this.realtime.freshnessMs,pending,selected,reviewed,shed,expired};
403
+ // selected = reviewed + shed + expired + pending: an answer awaiting receipt
404
+ // is reviewed; a backlog frame is pending.
405
+ let pending=this.backlog.size;for(const r of this.entries.values())if(!r.result)pending++;
406
+ return {freshnessMs:this.realtime.freshnessMs,pending,selected,reviewed,shed,expired,
407
+ ...(this.backlog.size?{backlog:this.backlog.size}:{}),...(this.backlogUndecided.size?{undecidedBacklog:this.backlogUndecided.size}:{})};
268
408
  }
269
409
  offered(c,result){if(result==='buffered'||result==='coalesced')this.queued.add(c.retainedId);}
270
410
  flushOrderedCapture(){}
@@ -274,12 +414,16 @@ class RetainedCaptureClient {
274
414
  contextSha256:sha(Buffer.from(JSON.stringify(r.candidate.reviewContext??null)))};
275
415
  }
276
416
  pendingPerception(at=Date.now()){
277
- for(const r of this.records.values()){
278
- if(!r.actorPerceptionPolicy||r.actorPerception)continue;
279
- // Preserve capture order within a camera, including retries. Other cameras
280
- // remain independently schedulable through the manager's bounded CPU lane.
417
+ // Preserve capture order within a camera, including retries. Other cameras
418
+ // remain independently schedulable through the manager's bounded CPU lane.
419
+ for(const r of this.perceptionPending.values()){
420
+ if(r.actorPerception){this.perceptionPending.delete(r.id);continue;}
281
421
  return !r.actorPerceptionFailure||r.actorPerceptionFailure.retryAtMs<=at?r:null;
282
- }return null;
422
+ }
423
+ // Live work first; a backlog frame only once an idle manager asked for it.
424
+ const r=this.backlogPerception;
425
+ if(r&&!r.actorPerception&&this.backlog.has(r.id))return !r.actorPerceptionFailure||r.actorPerceptionFailure.retryAtMs<=at?r:null;
426
+ this.backlogPerception=null;return null;
283
427
  }
284
428
  hasPendingOrderedPerception(at=Date.now()){
285
429
  return !this.perceptionBusy&&!!this.actorPerceiver&&!!this.pendingPerception(at);
@@ -294,6 +438,7 @@ class RetainedCaptureClient {
294
438
  delete updated.actorPerceptionFailure;commit(this.file(r.id),updated);
295
439
  // Advance the in-memory stage only after its original proof is durable.
296
440
  Object.assign(r,updated);delete r.actorPerceptionFailure;
441
+ this.perceptionPending.delete(r.id);if(this.backlogPerception===r)this.backlogPerception=null;
297
442
  return true;
298
443
  }catch(error){
299
444
  const attempts=(r.actorPerceptionFailure?.attempts??0)+1,at=Date.now();
@@ -303,25 +448,37 @@ class RetainedCaptureClient {
303
448
  commit(this.file(r.id),r);throw error;
304
449
  }finally{this.perceptionBusy=false;}
305
450
  }
451
+ /** The in-memory record (with logged decisions applied), else the file. */
452
+ recordFor(id){
453
+ return this.entries.get(id)??this.backlog.get(id)??this.records.get(id)
454
+ ??(/^[a-f0-9]{32}$/.test(id)&&fs.existsSync(this.file(id))?JSON.parse(fs.readFileSync(this.file(id))):null);
455
+ }
456
+ /** A backlog frame with a saved answer is ordinary delivery work from here on. */
457
+ leaveBacklog(r){
458
+ if(!this.backlog.delete(r.id))return;
459
+ this.backlogStarts.delete(r.id);this.retireOrder.set(r.id,true);
460
+ if(!r.acknowledged)this.entries.set(r.id,r);
461
+ }
306
462
  async prepare(request){
307
- const r=this.entries.get(request.evidenceId)??(/^[a-f0-9]{32}$/.test(request.evidenceId)&&fs.existsSync(this.file(request.evidenceId))?JSON.parse(fs.readFileSync(this.file(request.evidenceId))):null);assert(r&&r.selected,'Unknown selected evidence');
463
+ const r=this.recordFor(request.evidenceId);assert(r&&r.selected,'Unknown selected evidence');
308
464
  // A queue copy that outlived its retirement (a failed delete, then restart):
309
465
  // coded so the manager drops the copy instead of retrying it forever.
310
466
  if(unreviewed(r))throw Object.assign(Error('Evidence was retired unreviewed ('+r.disposition+'); this queue copy is stale'),
311
467
  {code:require('../dist/continuousReview').RETAINED_EVIDENCE_RETIRED});
312
468
  assert.equal(request.capturedAt,r.capturedAt);assert.equal(sha(request.jpeg),r.imageSha256);
313
469
  assert.deepEqual(this.baseContext(request.reviewContext),r.candidate.reviewContext,'Capture context changed');
314
- const coverageCache=new Map();
470
+ const coverageCache=new Map(),historical=this.backlog.has(r.id);
315
471
  if(r.actorPerceptionPolicy)assert(r.actorPerception,'Original actor perception must complete before review');
472
+ if(historical&&!r.result)this.backlogStarts.set(r.id,(this.backlogStarts.get(r.id)??0)+1);
316
473
  // Frozen once, when the review starts: every upload retry reports the same facts.
317
474
  if(this.realtime&&!r.result&&!r.reviewSchedule&&realtimeContext(r.candidate.reviewContext)){
318
- const f=this.followUpOf(r),{selected,reviewed,shed,expired}=this.counts;
319
- r.reviewSchedule={version:1,slot:f?'follow_up':'latest',...(f?{routeId:f.routeId}:{}),freshnessMs:this.realtime.freshnessMs,
475
+ const f=historical?null:this.followUpOf(r),{selected,reviewed,shed,expired}=this.counts;
476
+ r.reviewSchedule={version:1,slot:historical?'backlog':f?'follow_up':'latest',...(f?{routeId:f.routeId}:{}),freshnessMs:this.realtime.freshnessMs,
320
477
  startedAtMs:this.realtime.now(),selected,reviewed,shed,expired};
321
478
  commit(this.file(r.id),r);
322
479
  }
323
480
  const req={...request,...(r.reviewSchedule?{reviewSchedule:r.reviewSchedule}:{}),...(r.actorPerception?{actorPerception:this.actorPerceiver.validate(r.actorPerception,this.perceptionInput(r),r.actorPerceptionPolicy)}:{}),reviewContext:r.candidate.reviewContext,resolveActorWindow:(actor,modelId)=>resolveActorWindow(this,r,actor,modelId),temporalContinuityBreak:r.temporalContinuityBroken===true,resolveTemporalCoverage:contract=>{const key=JSON.stringify(contract);if(!coverageCache.has(key))coverageCache.set(key,resolveTemporalCoverage(this,r,contract));return coverageCache.get(key);},cachedAnalysis:request.cachedAnalysis??r.result,
324
- checkpointAnalysis:result=>{const first=!r.result;r.result=result;commit(this.file(r.id),r);if(first)this.counts.reviewed++;this.noteReviewed(r);request.checkpointAnalysis(result);}};
481
+ checkpointAnalysis:result=>{const first=!r.result;r.result=result;commit(this.file(r.id),r);if(first)this.counts.reviewed++;this.noteReviewed(r);this.leaveBacklog(r);request.checkpointAnalysis(result);}};
325
482
  await reviewExecution.run({evidenceId:r.id,cameraId:r.cameraId},()=>this.client.reviewCandidate(req));assert(r.result,'No durable result');return {...req,cachedAnalysis:r.result};
326
483
  }
327
484
  async reviewCandidate(request){await this.prepare(request);}
@@ -330,9 +487,9 @@ class RetainedCaptureClient {
330
487
  // Match the continuous worker's receipt contract before releasing archive state.
331
488
  if(response.ok&&response.body?.accepted===true&&response.body.evidenceId===request.evidenceId&&response.body.evidenceReview?.version===1&&response.body.evidenceReview.capturedAtMs===request.capturedAt){
332
489
  await this.client.acknowledgeAcceptedReview?.(req);
333
- const r=this.entries.get(request.evidenceId)??JSON.parse(fs.readFileSync(this.file(request.evidenceId)));const first=!r.acknowledged;r.acknowledged=true;r.acknowledgedAtCaptureMs=this.lastAt;r.disposition='reviewed_and_received';commit(this.file(r.id),r);this.records.set(r.id,r);if(first)this.counts.received++;this.queued.delete(r.id);this.entries.delete(r.id);
490
+ const r=this.recordFor(request.evidenceId);const first=!r.acknowledged;r.acknowledged=true;r.acknowledgedAtCaptureMs=this.lastAt;r.disposition='reviewed_and_received';commit(this.file(r.id),r);this.records.set(r.id,r);this.leaveBacklog(r);if(first)this.counts.received++;this.queued.delete(r.id);this.entries.delete(r.id);
334
491
  }return response;
335
492
  }
336
- stats(){const actorRows=[...this.records.values()].filter(r=>r.actorPerceptionPolicy),actorPending=actorRows.filter(r=>!r.actorPerception);const {captures,selected,reviewed,received,inactive,grouped,shed,expired}=this.counts;return{...(this.actorPerceiver?{actorPerception:{completed:actorRows.length-actorPending.length,pending:actorPending.length,oldestPendingCapturedAt:actorPending[0]?.capturedAt??null,unavailable:actorPending.some(r=>r.actorPerceptionFailure)}}:{}),captures,selected,archivedProfileInactive:inactive,archivedBetweenReviews:captures-selected-inactive-grouped,archivedGroupedUnreviewed:grouped,reviewed,received,shedUnreviewed:shed,expiredUnreviewed:expired,pending:this.entries.size,oldestPendingCapturedAt:this.entries.values().next().value?.capturedAt??null,logicalImageBytes:this.bytes,retainedCaptures:this.records.size,prunedCaptures:this.lifecycle?.state.pruned.captures??0};}
493
+ stats(){const actorRows=[...this.records.values()].filter(r=>r.actorPerceptionPolicy),actorPending=actorRows.filter(r=>!r.actorPerception);const {captures,selected,reviewed,received,inactive,grouped,shed,expired}=this.counts;const first=[this.entries.values().next().value?.capturedAt,this.backlog.values().next().value?.capturedAt].filter(Number.isSafeInteger);return{...(this.actorPerceiver?{actorPerception:{completed:actorRows.length-actorPending.length,pending:actorPending.length,oldestPendingCapturedAt:actorPending[0]?.capturedAt??null,unavailable:actorPending.some(r=>r.actorPerceptionFailure)}}:{}),captures,selected,archivedProfileInactive:inactive,archivedBetweenReviews:captures-selected-inactive-grouped,archivedGroupedUnreviewed:grouped,reviewed,received,shedUnreviewed:shed,expiredUnreviewed:expired,pending:this.entries.size+this.backlog.size,oldestPendingCapturedAt:first.length?Math.min(...first):null,logicalImageBytes:this.bytes,retainedCaptures:this.records.size,prunedCaptures:this.lifecycle?.state.pruned.captures??0};}
337
494
  }
338
- module.exports={RetainedCaptureClient,reviewExecution,RECHECK_ANCHOR_GRACE_MS};
495
+ module.exports={RetainedCaptureClient,reviewExecution,RECHECK_ANCHOR_GRACE_MS,BACKLOG_REVIEW_BUDGET_MS,BACKLOG_MAX_STARTS,DECISION_LOG};
@@ -86,9 +86,11 @@ class WebsiteAdmissionCaptureClient extends RetainedCaptureClient {
86
86
  this.phoneProofChecks = new Map();
87
87
  this.admissionNowMs = options.admissionNowMs ?? Date.now;
88
88
  assert(typeof this.admissionNowMs === 'function', 'Admission decision clock required');
89
+ // Metadata is computed when a row is first consulted, not for every
90
+ // archived original at startup. Undecided originals already past the live
91
+ // bound wait in backlogUndecided (see pendingBacklogCandidate), not here.
89
92
  for (const row of this.records.values()) {
90
- if (row.disposition === 'awaiting_rule_decision') this.deferred.set(row.id, row);
91
- this.metadata(row);
93
+ if (row.disposition === 'awaiting_rule_decision' && !this.backlogUndecided.has(row.id)) this.deferred.set(row.id, row);
92
94
  if (row.result) this.remember(row);
93
95
  }
94
96
  for (const row of this.records.values()) {
@@ -150,6 +152,9 @@ class WebsiteAdmissionCaptureClient extends RetainedCaptureClient {
150
152
  return context && context.profileState === 'active' && context.schema && typeof context.schema === 'object' && !Array.isArray(context.schema)
151
153
  && !context.temporalReviewContract && !context.orderedSequence && !context.orderedCaptureContract;
152
154
  }
155
+ // Unknown and ordered contexts stay capture-order barriers; only known
156
+ // realtime originals past the bound are held as backlog.
157
+ realtimeReviewable(row) { return !!this.knownRealtime(row); }
153
158
  captureOrderAdmissionCandidates(onlyAnswered = false) {
154
159
  const now = this.realtime.now(), freshnessMs = this.realtime.freshnessMs;
155
160
  const stale = row => !row.result && this.knownRealtime(row)
@@ -192,7 +197,8 @@ class WebsiteAdmissionCaptureClient extends RetainedCaptureClient {
192
197
  if (!this.knownRealtime(row) || meta.admission && meta.phoneSearch) break;
193
198
  prefix.push(row);
194
199
  }
195
- if (!prefix.length || [...this.entries.values()].some(row => !prefix.includes(row))) return null;
200
+ const inPrefix = new Set(prefix.map(row => row.id));
201
+ if (!prefix.length || [...this.entries.keys()].some(id => !inPrefix.has(id))) return null;
196
202
  const choices = new Map(prefix.map(row => [row.id, this.admissionSelection(row)]));
197
203
  const eligible = prefix.filter(row => row.selected || choices.get(row.id).choice.selected);
198
204
  const target = eligible.find(row => this.followUpOf(row) || choices.get(row.id).followUp) ?? eligible.at(-1);
@@ -229,13 +235,35 @@ class WebsiteAdmissionCaptureClient extends RetainedCaptureClient {
229
235
  }
230
236
  return super.chooseReview(c, digest, due, stage);
231
237
  }
238
+ // Records leave only through cleanup, so only the rows it just pruned can
239
+ // leave metadata behind. (Walking every row's metadata on each pump and each
240
+ // capture cost a whole-archive scan.)
232
241
  releaseAdmissionMetadata() {
233
- for (const id of this.admissionMetadata?.keys() ?? []) if (!this.records.has(id)) this.admissionMetadata.delete(id);
234
- for (const id of this.phoneReceipts?.keys() ?? []) if (!this.records.has(id)) this.phoneReceipts.delete(id);
242
+ for (const id of this.lastRetiredIds ?? []) { this.admissionMetadata?.delete(id); this.phoneReceipts?.delete(id); }
235
243
  }
236
244
  reclaim(at) {
237
245
  try { super.reclaim(at); } finally { this.releaseAdmissionMetadata(); }
238
246
  }
247
+ /**
248
+ * A selected backlog original first; otherwise the oldest undecided one. An
249
+ * undecided original past the live bound gets no live wait decision (those
250
+ * belong to the moment of capture): it is recorded as selected for
251
+ * historical review and reviewed like any backlog frame. Its answer never
252
+ * moves live admission state, which only advances on a newer capture.
253
+ */
254
+ pendingBacklogCandidate() {
255
+ const selected = super.pendingBacklogCandidate();
256
+ if (selected || !this.realtime || !this.reviewQueue || this.backlogBusy()) return selected;
257
+ for (const row of this.backlogUndecided.values()) {
258
+ if (this.backlogSkipped.has(row.id)) continue;
259
+ const next = { ...row, selected: true, disposition: 'pending_review', selectionReason: 'historical_backlog' };
260
+ save(this.file(row.id), next); // disk first; a failed write leaves it undecided
261
+ Object.assign(row, next); this.backlogUndecided.delete(row.id); this.backlog.set(row.id, row); this.counts.selected++;
262
+ const offer = this.backlogOffer(row);
263
+ if (offer !== undefined) return offer;
264
+ }
265
+ return null;
266
+ }
239
267
  retainOrderedCapture(c) {
240
268
  const before = this.counts.captures;
241
269
  super.retainOrderedCapture(c);
@@ -436,10 +464,13 @@ class WebsiteAdmissionCaptureClient extends RetainedCaptureClient {
436
464
  return prepared;
437
465
  }
438
466
  pendingOrderedCandidates() {
467
+ try { return this.admissionCandidates(); } finally { this.flushDecisions(); }
468
+ }
469
+ admissionCandidates() {
439
470
  this.phoneProofChecks.clear();
440
- this.releaseAdmissionMetadata();
441
471
  this.settlePhoneOmissions();
442
472
  if (this.realtime) {
473
+ this.sweepBacklogCopies();
443
474
  const offered = this.realtimeAdmissionCandidates();
444
475
  if (offered !== null) return offered;
445
476
  }
@@ -464,8 +495,9 @@ class WebsiteAdmissionCaptureClient extends RetainedCaptureClient {
464
495
  ? { ...c, reviewPriority: this.entries.get(c.retainedId).scheduledReviewPriority } : c);
465
496
  }
466
497
  stats() {
467
- const s = super.stats(), pendingAdmission = this.deferred?.size ?? 0;
468
- const first = this.deferred?.values().next().value?.capturedAt;
498
+ const s = super.stats(), pendingAdmission = (this.deferred?.size ?? 0) + (this.backlogUndecided?.size ?? 0);
499
+ const first = [this.deferred?.values().next().value?.capturedAt, this.backlogUndecided?.values().next().value?.capturedAt]
500
+ .filter(Number.isSafeInteger).reduce((a, b) => Math.min(a, b), Infinity);
469
501
  return { ...s, selectedPending: s.pending, pendingAdmission, pending: s.pending + pendingAdmission,
470
502
  archivedBetweenReviews: s.archivedBetweenReviews - pendingAdmission,
471
503
  retainedWaitWindowCaptures: [...this.records.values()].filter(r => r.disposition === 'archived_wait_window').length,