@memberjunction/server 6.2.0-edge.6 → 6.2.0-edge.8

package/src/__tests__/RSUPostRestartProgress.test.ts

@@ -0,0 +1,343 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { mkdtempSync, rmSync, readFileSync, existsSync } from 'node:fs';
+import { join } from 'node:path';
+import { tmpdir } from 'node:os';
+import { IntegrationProgressEmitter, IntegrationProgressReader, SweepInterruptedRuns } from '@memberjunction/integration-progress-artifacts';
+import type { IntegrationProgressEvent, IntegrationRunManifest, IntegrationRunResult } from '@memberjunction/integration-progress-artifacts';
+import { RSUPostRestartProgressSession } from '../integration/RSUPostRestartProgress.js';
+
+/**
+ * The half of an RSU run that happens after the pipeline restarts the API on itself.
+ *
+ * What must hold, in order of how badly it hurts when it doesn't:
+ *  1. the post-restart events land in the SAME run, above a tailing client's cursor — otherwise the
+ *     client polls forever and never learns the connector went live;
+ *  2. the run reaches a terminal state — otherwise it stays flagged in-flight forever;
+ *  3. a failure names the objects that were never mapped — otherwise the operator's only signal that
+ *     something needs re-applying is a log line nobody reads.
+ */
+describe('RSUPostRestartProgress', () => {
+    let rootDir: string;
+
+    beforeEach(() => {
+        rootDir = mkdtempSync(join(tmpdir(), 'rsu-post-'));
+    });
+
+    afterEach(() => {
+        rmSync(rootDir, { recursive: true, force: true });
+    });
+
+    /**
+     * Leaves behind exactly what a killed pre-restart process leaves: a manifest, a journal ending at
+     * the restart boundary, and NO result. Returns the client's cursor at the moment of the kill.
+     */
+    async function killedRunAt(runID: string, companyIntegrationIDs: string[], opts: { checkpoint?: boolean } = {}): Promise<number> {
+        const manifest: IntegrationRunManifest = {
+            runID,
+            runKind: 'RSU',
+            triggerType: 'Pipeline',
+            startedAt: new Date().toISOString(),
+            companyIntegrationIDs,
+            companyIntegrationID: companyIntegrationIDs.length === 1 ? companyIntegrationIDs[0] : undefined,
+        };
+        const e = new IntegrationProgressEmitter(manifest, { rootDir });
+        e.runStart('RSU pipeline started');
+        e.stageStart('RunCodeGen', 'step 7 of 12');
+        e.stageComplete('RunCodeGen');
+        e.stageStart('RestartMJAPI', 'step 10 of 12');
+        e.stageComplete('RestartMJAPI');
+        if (opts.checkpoint !== false) {
+            e.checkpoint('RestartMJAPI', {
+                runID,
+                remainingSteps: ['CreateEntityMaps', 'StartSync'],
+                stepIndex: 10,
+                stepTotal: 12,
+                companyIntegrationIDs,
+                pendingWorkIDs: ['PW-1'],
+            });
+        }
+        await e.flush();
+        return e.LatestSeq;
+    }
+
+    function events(runID: string): IntegrationProgressEvent[] {
+        const out: IntegrationProgressEvent[] = [];
+        for (const line of readFileSync(join(rootDir, runID, 'progress.jsonl'), 'utf-8').split('\n')) {
+            if (!line.trim()) continue;
+            try { out.push(JSON.parse(line) as IntegrationProgressEvent); } catch { /* torn */ }
+        }
+        return out;
+    }
+
+    function result(runID: string): IntegrationRunResult {
+        return JSON.parse(readFileSync(join(rootDir, runID, 'result.json'), 'utf-8')) as IntegrationRunResult;
+    }
+
+    it('the startup sweep leaves a restart-killed run in flight when told it is a continuation, so For() re-attaches', async () => {
+        // #4469's SweepInterruptedRuns runs at boot, BEFORE processRSUPendingWork. Without an
+        // exemption it closes the very run this process is meant to continue.
+        await killedRunAt('rsu-s1', ['CI-S1']);
+
+        const sweep = await SweepInterruptedRuns(rootDir, undefined, RSUPostRestartProgressSession.WasKilledAtRestart);
+
+        expect(sweep.Skipped).toEqual(['rsu-s1']);
+        expect(sweep.Swept).toEqual([]);
+        expect(existsSync(join(rootDir, 'rsu-s1', 'result.json'))).toBe(false);
+        const session = new RSUPostRestartProgressSession({ rootDir });
+        expect(await session.For('CI-S1')).not.toBeNull();
+        await session.FinishAll();
+        expect(existsSync(join(rootDir, 'rsu-s1', 'result.json'))).toBe(true);
+    });
+
+    it('without that exemption the sweep closes the run first and For() finds nothing', async () => {
+        await killedRunAt('rsu-s2', ['CI-S2']);
+
+        await SweepInterruptedRuns(rootDir);
+
+        expect(result('rsu-s2').exitReason).toBe('killed');
+        expect(await new RSUPostRestartProgressSession({ rootDir }).For('CI-S2')).toBeNull();
+    });
+
+    it('after the resuming pass, a restart-killed run nothing re-attached to is swept as killed; an attached one is left to its session', async () => {
+        await killedRunAt('rsu-a', ['CI-A']);
+        await killedRunAt('rsu-b', ['CI-B']);
+        const boot = await SweepInterruptedRuns(rootDir, undefined, RSUPostRestartProgressSession.WasKilledAtRestart);
+        expect([...boot.Skipped].sort()).toEqual(['rsu-a', 'rsu-b']);
+        const session = new RSUPostRestartProgressSession({ rootDir });
+        expect(await session.For('CI-A')).not.toBeNull(); // only A has pending work to resume it
+
+        const after = await RSUPostRestartProgressSession.SweepUnattached(boot.Skipped, rootDir);
+
+        expect(after.Swept).toEqual(['rsu-b']);
+        expect(result('rsu-b').exitReason).toBe('killed');
+        expect(existsSync(join(rootDir, 'rsu-a', 'result.json'))).toBe(false);
+        await session.FinishAll();
+        expect(existsSync(join(rootDir, 'rsu-a', 'result.json'))).toBe(true);
+    });
+
+    it('SweepUnattached judges only the runs the startup sweep left in flight — never a run this process started', async () => {
+        await killedRunAt('rsu-c', ['CI-C']);
+
+        const after = await RSUPostRestartProgressSession.SweepUnattached(['some-other-run'], rootDir);
+
+        expect(after.Swept).toEqual([]);
+        expect(existsSync(join(rootDir, 'rsu-c', 'result.json'))).toBe(false);
+        expect(await RSUPostRestartProgressSession.SweepUnattached([], rootDir)).toEqual({ Swept: [], Failed: [], Skipped: [] });
+    });
+
+    it('emits both missing steps into the original run, visible to a client tailing past the restart', async () => {
+        const cursor = await killedRunAt('rsu-1', ['CI-1']);
+        const session = new RSUPostRestartProgressSession({ rootDir });
+
+        const progress = await session.For('CI-1');
+        expect(progress).not.toBeNull();
+        progress!.BeginEntityMaps(3);
+        progress!.ObjectMapped('Contacts', 1, 3);
+        progress!.ObjectMapped('Accounts', 2, 3);
+        progress!.ObjectMapped('Orders', 3, 3);
+        progress!.CompleteEntityMaps(3, 3, 0);
+        progress!.BeginStartSync();
+        progress!.CompleteStartSync('SYNC-RUN-42');
+        await session.FinishAll();
+
+        const reader = new IntegrationProgressReader(rootDir);
+        const tail = await reader.Tail('rsu-1', cursor);
+
+        // Everything the second process wrote is ABOVE the cursor the first process left the client on.
+        expect(tail.length).toBeGreaterThan(0);
+        for (const ev of tail) expect(ev.seq).toBeGreaterThan(cursor);
+
+        const kinds = tail.map(e => `${e.eventType}:${e.stage ?? ''}`);
+        expect(kinds).toContain('run.resumed:');
+        expect(kinds).toContain('stage.start:CreateEntityMaps');
+        expect(kinds).toContain('stage.complete:CreateEntityMaps');
+        expect(kinds).toContain('stage.start:StartSync');
+        expect(kinds).toContain('stage.complete:StartSync');
+        expect(kinds).toContain('run.complete:');
+    });
+
+    it('reports per-object heartbeats while mapping, without double-counting the stage rollup', async () => {
+        await killedRunAt('rsu-1', ['CI-1']);
+        const session = new RSUPostRestartProgressSession({ rootDir });
+        const progress = (await session.For('CI-1'))!;
+        progress.BeginEntityMaps(2);
+        progress.ObjectMapped('Contacts', 1, 2);
+        progress.ObjectMapped('Accounts', 2, 2);
+        progress.CompleteEntityMaps(2, 2, 0);
+        progress.BeginStartSync();
+        progress.CompleteStartSync(null);
+        await session.FinishAll();
+
+        const beats = events('rsu-1').filter(e => e.eventType === 'progress.heartbeat' && e.stage === 'CreateEntityMaps');
+        expect(beats.length).toBe(3); // one on open + one per object
+        expect(beats.at(-1)?.counts).toMatchObject({ processed: 2, totalKnown: 2 });
+
+        // Heartbeat counts must NOT feed the aggregate; only the stage rollup does. 2, not 5.
+        expect(result('rsu-1').aggregateCounts).toMatchObject({ processed: 2, succeeded: 2, failed: 0 });
+    });
+
+    it('carries the sync run id on StartSync so a client can hop to the sync stream', async () => {
+        await killedRunAt('rsu-1', ['CI-1']);
+        const session = new RSUPostRestartProgressSession({ rootDir });
+        const progress = (await session.For('CI-1'))!;
+        progress.BeginEntityMaps(1);
+        progress.CompleteEntityMaps(1, 1, 0);
+        progress.BeginStartSync();
+        progress.CompleteStartSync('SYNC-RUN-42');
+        await session.FinishAll();
+
+        const done = events('rsu-1').find(e => e.eventType === 'stage.complete' && e.stage === 'StartSync');
+        expect(done?.data).toMatchObject({ syncRunID: 'SYNC-RUN-42' });
+        // No counts on this stage — the records belong to the sync's own run.
+        expect(done?.counts).toBeUndefined();
+    });
+
+    it('continues the determinate stepper from the checkpoint the killed process left', async () => {
+        await killedRunAt('rsu-1', ['CI-1']);
+        const session = new RSUPostRestartProgressSession({ rootDir });
+        const progress = (await session.For('CI-1'))!;
+        progress.BeginEntityMaps(1);
+        progress.CompleteEntityMaps(1, 1, 0);
+        progress.BeginStartSync();
+        progress.CompleteStartSync(null);
+        await session.FinishAll();
+
+        const all = events('rsu-1');
+        expect(all.find(e => e.eventType === 'stage.start' && e.stage === 'CreateEntityMaps')?.message).toBe('step 11 of 12');
+        expect(all.find(e => e.eventType === 'stage.start' && e.stage === 'StartSync')?.message).toBe('step 12 of 12');
+    });
+
+    it('still re-attaches when the checkpoint did NOT survive the kill — it just loses the step numbers', async () => {
+        // Re-attachment keys off the manifest, not the checkpoint, precisely so a checkpoint racing
+        // the kill signal cannot cost the whole continuation.
+        const cursor = await killedRunAt('rsu-1', ['CI-1'], { checkpoint: false });
+        const session = new RSUPostRestartProgressSession({ rootDir });
+        const progress = await session.For('CI-1');
+        expect(progress).not.toBeNull();
+        progress!.BeginEntityMaps(1);
+        progress!.CompleteEntityMaps(1, 1, 0);
+        progress!.BeginStartSync();
+        progress!.CompleteStartSync(null);
+        await session.FinishAll();
+
+        const reader = new IntegrationProgressReader(rootDir);
+        const tail = await reader.Tail('rsu-1', cursor);
+        expect(tail.some(e => e.eventType === 'stage.start' && e.stage === 'CreateEntityMaps')).toBe(true);
+        expect(tail.find(e => e.eventType === 'stage.start' && e.stage === 'CreateEntityMaps')?.message).toBeUndefined();
+        expect(result('rsu-1').success).toBe(true);
+    });
+
+    it('names the objects that were never mapped when the work fails', async () => {
+        await killedRunAt('rsu-1', ['CI-1']);
+        const session = new RSUPostRestartProgressSession({ rootDir });
+        const progress = (await session.For('CI-1'))!;
+        progress.BeginEntityMaps(3);
+        progress.ObjectMapped('Contacts', 1, 3);
+        progress.FailEntityMaps('provider timed out', ['Accounts', 'Orders']);
+        await session.FinishAll();
+
+        const err = events('rsu-1').find(e => e.eventType === 'stage.error' && e.stage === 'CreateEntityMaps');
+        expect(err?.message).toContain('provider timed out');
+        expect(err?.message).toContain('Accounts');
+        expect(err?.message).toContain('Orders');
+        expect(err?.data).toMatchObject({ code: 'RSU_POST_RESTART_ENTITY_MAPS_FAILED', unmappedObjects: ['Accounts', 'Orders'] });
+
+        const res = result('rsu-1');
+        expect(res.success).toBe(false);
+        expect(res.exitReason).toBe('failed');
+    });
+
+    it('never leaves a stage open when the run terminates', async () => {
+        // A stage left open pins a client's stepper on it forever.
+        await killedRunAt('rsu-1', ['CI-1']);
+        const session = new RSUPostRestartProgressSession({ rootDir });
+        const progress = (await session.For('CI-1'))!;
+        progress.BeginEntityMaps(2); // and then the process gives up mid-stage
+        await session.FinishAll();
+
+        const all = events('rsu-1');
+        const opened = all.filter(e => e.eventType === 'stage.start' && e.stage === 'CreateEntityMaps').length;
+        const closed = all.filter(e => (e.eventType === 'stage.complete' || e.eventType === 'stage.error') && e.stage === 'CreateEntityMaps').length;
+        expect(opened).toBe(closed);
+        expect(result('rsu-1').success).toBe(false);
+    });
+
+    it('terminates the run EXACTLY once even when several pending-work items share it', async () => {
+        // One RSU batch registers one pending-work row per input; attaching per row would try to
+        // resume and terminate the same run repeatedly, and the second attempt would report nothing.
+        const cursor = await killedRunAt('rsu-batch', ['CI-1', 'CI-2']);
+        const session = new RSUPostRestartProgressSession({ rootDir });
+
+        const a = await session.For('CI-1');
+        const b = await session.For('CI-2');
+        expect(a).not.toBeNull();
+        expect(b).toBe(a); // the SAME continuation, not a second one
+
+        a!.BeginEntityMaps(1);
+        a!.CompleteEntityMaps(1, 1, 0);
+        a!.BeginStartSync();
+        a!.CompleteStartSync('SYNC-A');
+        await session.FinishAll();
+        await session.FinishAll(); // idempotent
+
+        const terminals = events('rsu-batch').filter(e => e.eventType === 'run.complete' || e.eventType === 'run.fail');
+        expect(terminals).toHaveLength(1);
+        expect(result('rsu-batch').success).toBe(true);
+
+        const reader = new IntegrationProgressReader(rootDir);
+        expect((await reader.Tail('rsu-batch', cursor)).length).toBeGreaterThan(0);
+    });
+
+    it('returns null — and never throws — when there is no run to continue', async () => {
+        const session = new RSUPostRestartProgressSession({ rootDir });
+        expect(await session.For('CI-UNKNOWN')).toBeNull();
+        await expect(session.FinishAll()).resolves.toBeUndefined();
+    });
+
+    it('does not adopt a run that was not killed at the restart', async () => {
+        // An RSU run still legitimately in flight at an earlier step belongs to another process; the
+        // last-stage check is what keeps this one from hijacking and terminating it.
+        const e = new IntegrationProgressEmitter(
+            { runID: 'rsu-early', runKind: 'RSU', startedAt: new Date().toISOString(), companyIntegrationIDs: ['CI-1'] },
+            { rootDir }
+        );
+        e.runStart();
+        e.stageStart('RunCodeGen');
+        await e.flush();
+
+        const session = new RSUPostRestartProgressSession({ rootDir });
+        expect(await session.For('CI-1')).toBeNull();
+        expect(existsSync(join(rootDir, 'rsu-early', 'result.json'))).toBe(false);
+    });
+
+    it('does not adopt an already-finished run', async () => {
+        await killedRunAt('rsu-1', ['CI-1']);
+        const first = new RSUPostRestartProgressSession({ rootDir });
+        const progress = (await first.For('CI-1'))!;
+        progress.BeginEntityMaps(1);
+        progress.CompleteEntityMaps(1, 1, 0);
+        progress.BeginStartSync();
+        progress.CompleteStartSync(null);
+        await first.FinishAll();
+
+        // A later boot re-reading the queue must not reopen the closed run.
+        const second = new RSUPostRestartProgressSession({ rootDir });
+        expect(await second.For('CI-1')).toBeNull();
+        expect(result('rsu-1').success).toBe(true);
+    });
+
+    it('records a skipped sync as a completed stage, not a silent gap', async () => {
+        await killedRunAt('rsu-1', ['CI-1']);
+        const session = new RSUPostRestartProgressSession({ rootDir });
+        const progress = (await session.For('CI-1'))!;
+        progress.BeginEntityMaps(1);
+        progress.CompleteEntityMaps(1, 1, 0);
+        progress.BeginStartSync();
+        progress.SkipStartSync('the caller asked for no initial sync (StartSync=false)');
+        await session.FinishAll();
+
+        const done = events('rsu-1').find(e => e.eventType === 'stage.complete' && e.stage === 'StartSync');
+        expect(done?.data).toMatchObject({ skipped: true, syncRunID: null });
+        expect(result('rsu-1').success).toBe(true);
+    });
+});

package/src/__tests__/RSUProgressBridge.test.ts

@@ -1,5 +1,5 @@
 import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
-import { mkdtempSync, rmSync, readFileSync, existsSync } from 'node:fs';
+import { mkdtempSync, rmSync, readFileSync, existsSync, statSync } from 'node:fs';
 import { join } from 'node:path';
 import { tmpdir } from 'node:os';
 import { RSUProgressBridge } from '../integration/RSUProgressBridge.js';
@@ -44,12 +44,39 @@ describe('RSUProgressBridge', () => {
         ItemCount: 2,
         Descriptions: ['add Orders', 'add OrderItems'],
         AffectedTables: ['Orders', 'OrderItems'],
-        StepTotal: 12,
+        StepTotal: 14,
+        CompanyIntegrationIDs: ['CI-1'],
     };
 
-    /** The emitter's writes are chained promises; let them drain before reading from disk. */
+    /**
+     * Waits for the emitter's append chain to quiesce.
+     *
+     * This was a flat 50 ms sleep, which is plenty on an idle machine and NOT always enough with
+     * the whole suite running in parallel — the emitter appends through a promise chain the test
+     * cannot await, so a slow write made the assertion read "the event was never emitted" when it
+     * simply had not landed yet. Observed as an intermittent failure of the stage.error test.
+     * Poll for the journal to stop growing instead of guessing how long that takes.
+     */
     async function settle(): Promise<void> {
-        await new Promise(resolve => setTimeout(resolve, 50));
+        const path = join(rootDir, bridge.CurrentRunID ?? '', 'progress.jsonl');
+        const deadline = Date.now() + 5000;
+        let last = -1;
+        let stableReads = 0;
+        while (Date.now() < deadline) {
+            await new Promise(resolve => setTimeout(resolve, 10));
+            let size = -1;
+            try { size = statSync(path).size; } catch { /* not written yet */ }
+            if (size === last) {
+                stableReads++;
+                // A journal that exists and has stopped growing is settled. One that does not
+                // exist at all gets a short grace period and then counts as settled too, so a
+                // test asserting that NOTHING was written does not pay the full deadline.
+                if (stableReads >= 3 && (size >= 0 || stableReads >= 20)) return;
+            } else {
+                stableReads = 0;
+                last = size;
+            }
+        }
     }
 
     function runDir(): string {
@@ -100,7 +127,7 @@ describe('RSUProgressBridge', () => {
         expect(manifest.context).toMatchObject({
             itemCount: 2,
             affectedTables: ['Orders', 'OrderItems'],
-            stepTotal: 12,
+            stepTotal: 14,
         });
 
         const events = readEvents(dir);
@@ -237,6 +264,162 @@ describe('RSUProgressBridge', () => {
         expect(start?.message).toBe('step 0 of 12');
     });
 
+    // ── Connection identity: what makes an RSU run readable over the API at all ──────────────
+
+    it('stamps the batch connections onto the manifest, which is what the read check tests', async () => {
+        // Before this, an RSU manifest carried no connection, `userCanReadRunArtifact` returned false
+        // on the missing field, and EVERY RSU run answered "not authorized" to IntegrationGetRun and
+        // IntegrationTailRunEvents. The manifest field is the fix.
+        bridge.Observe({ ...RUN_START, CompanyIntegrationIDs: ['CI-1'] });
+        const dir = runDir();
+        await settle();
+
+        const manifest = JSON.parse(readFileSync(join(dir, 'manifest.json'), 'utf-8')) as IntegrationRunManifest;
+        expect(manifest.companyIntegrationIDs).toEqual(['CI-1']);
+        // Singular is set too, so the per-connection list filter and the GQL summary keep working.
+        expect(manifest.companyIntegrationID).toBe('CI-1');
+    });
+
+    it('leaves the SINGULAR connection empty for a batch spanning several, keeping the full set', async () => {
+        // Picking one member of the set to report as "the" connection would be a lie a client could
+        // act on; the AND authorization rule reads the set instead.
+        bridge.Observe({ ...RUN_START, CompanyIntegrationIDs: ['CI-1', 'CI-2'] });
+        const dir = runDir();
+        await settle();
+
+        const manifest = JSON.parse(readFileSync(join(dir, 'manifest.json'), 'utf-8')) as IntegrationRunManifest;
+        expect(manifest.companyIntegrationIDs).toEqual(['CI-1', 'CI-2']);
+        expect(manifest.companyIntegrationID).toBeUndefined();
+    });
+
+    it('records no connection when the batch supplied none, rather than inventing one', async () => {
+        bridge.Observe({ ...RUN_START, CompanyIntegrationIDs: [] });
+        const dir = runDir();
+        await settle();
+
+        const manifest = JSON.parse(readFileSync(join(dir, 'manifest.json'), 'utf-8')) as IntegrationRunManifest;
+        expect(manifest.companyIntegrationIDs).toEqual([]);
+        expect(manifest.companyIntegrationID).toBeUndefined();
+    });
+
+    // ── The restart boundary ─────────────────────────────────────────────────────────────────
+
+    it('closes the RestartMJAPI stage and checkpoints BEFORE the process is killed', async () => {
+        // `step.end` for RestartMJAPI is unreachable — pm2 kills the process inside the step — so
+        // without this the stream ends on an open stage and the run is in-flight forever.
+        bridge.Observe(RUN_START);
+        const dir = runDir();
+        bridge.Observe({ Kind: 'step.start', Name: 'RestartMJAPI', StepIndex: 10, StepTotal: 14 });
+
+        await bridge.Observe({
+            Kind: 'restart.pending',
+            RemainingSteps: ['CreateEntityMaps', 'StartSync'],
+            StepIndex: 10,
+            StepTotal: 14,
+            CompanyIntegrationIDs: ['CI-1'],
+            PendingWorkIDs: ['PW-1'],
+            TotalCount: 2,
+            SuccessCount: 2,
+        });
+
+        const events = readEvents(dir);
+        const done = events.find(e => e.eventType === 'stage.complete' && e.stage === 'RestartMJAPI');
+        const checkpoint = events.find(e => e.eventType === 'checkpoint' && e.stage === 'RestartMJAPI');
+        expect(done).toBeDefined();
+        expect(checkpoint?.resumableState).toMatchObject({
+            remainingSteps: ['CreateEntityMaps', 'StartSync'],
+            stepIndex: 10,
+            stepTotal: 14,
+            companyIntegrationIDs: ['CI-1'],
+            pendingWorkIDs: ['PW-1'],
+        });
+        // Still in flight: the run is finished by the process on the other side of the restart.
+        expect(existsSync(join(dir, 'result.json'))).toBe(false);
+    });
+
+    it('AWAITS the restart checkpoint to disk — an unflushed write would be lost to the kill', async () => {
+        // Deliberately does NOT call settle(): the returned promise must have flushed already.
+        bridge.Observe(RUN_START);
+        const dir = runDir();
+        await bridge.Observe({
+            Kind: 'restart.pending',
+            RemainingSteps: ['CreateEntityMaps', 'StartSync'],
+            CompanyIntegrationIDs: ['CI-1'],
+            PendingWorkIDs: [],
+            TotalCount: 1,
+            SuccessCount: 1,
+        });
+
+        const events = readEvents(dir);
+        expect(events.some(e => e.eventType === 'checkpoint' && e.stage === 'RestartMJAPI')).toBe(true);
+    });
+
+    it('ignores a restart.pending that arrives with no run open', async () => {
+        await expect(bridge.Observe({
+            Kind: 'restart.pending',
+            RemainingSteps: [],
+            CompanyIntegrationIDs: [],
+            PendingWorkIDs: [],
+            TotalCount: 0,
+            SuccessCount: 0,
+        })).resolves.toBeUndefined();
+    });
+
+    // ── Heartbeat staleness ──────────────────────────────────────────────────────────────────
+
+    it('never lets one run heartbeat stamp its stage name onto the NEXT run', async () => {
+        // The timer captures its stage; if it outlives the run it was armed for, it writes that
+        // stale name into whatever emitter the bridge points at next.
+        vi.useFakeTimers();
+        try {
+            bridge.Observe(RUN_START);
+            bridge.Observe({ Kind: 'step.start', Name: 'CompileTypeScript', StepIndex: 8, StepTotal: 14 });
+
+            // A retry starts a brand-new run while the previous step's heartbeat is still armed.
+            bridge.Observe(RUN_START);
+            const secondDir = runDir();
+            vi.advanceTimersByTime(300_000); // ten ticks' worth
+
+            vi.useRealTimers();
+            await settle();
+
+            const beats = readEvents(secondDir).filter(e => e.eventType === 'progress.heartbeat');
+            expect(beats.filter(b => b.stage === 'CompileTypeScript')).toHaveLength(0);
+            expect(beats).toHaveLength(0);
+        } finally {
+            vi.useRealTimers();
+        }
+    });
+
+    it('stops heart-beating once the restart boundary is published', async () => {
+        vi.useFakeTimers();
+        try {
+            bridge.Observe(RUN_START);
+            const dir = runDir();
+            bridge.Observe({ Kind: 'step.start', Name: 'RestartMJAPI', StepIndex: 10, StepTotal: 14 });
+            const pending = bridge.Observe({
+                Kind: 'restart.pending',
+                RemainingSteps: ['CreateEntityMaps', 'StartSync'],
+                CompanyIntegrationIDs: ['CI-1'],
+                PendingWorkIDs: [],
+                TotalCount: 1,
+                SuccessCount: 1,
+            });
+            vi.useRealTimers();
+            await pending;
+
+            vi.useFakeTimers();
+            vi.advanceTimersByTime(300_000);
+            vi.useRealTimers();
+            await settle();
+
+            const beats = readEvents(dir).filter(e => e.eventType === 'progress.heartbeat');
+            expect(beats).toHaveLength(0);
+        } finally {
+            vi.useRealTimers();
+        }
+    });
+
     it('ignores step and run.end events that arrive with no run open', async () => {
         // Defensive: an observer registered mid-run would see a trailing run.end for a run it never
         // saw start. That must be a no-op, not a crash.

package/src/__tests__/RSUStartupSweepWiring.test.ts

@@ -0,0 +1,30 @@
+/**
+ * Startup order in index.ts: the boot sweep must exempt restart-killed RSU runs on a replica that
+ * runs background workers (processRSUPendingWork is their continuation), and whichever of those
+ * nothing re-attached to must be swept once the resuming pass is over. The behaviour of each half
+ * is tested in RSUPostRestartProgress.test.ts; this pins the wiring between them, which lives in
+ * a bootstrap function no unit test can run.
+ */
+import { describe, it, expect } from 'vitest';
+import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
+
+const src = readFileSync(join(__dirname, '..', 'index.ts'), 'utf-8');
+
+describe('RSU startup sweep wiring', () => {
+    it('the boot sweep exempts restart-killed RSU runs only where background workers will resume them', () => {
+        expect(src).toMatch(
+            /const bootSweep = await SweepInterruptedRuns\(\s*undefined,\s*msg => console\.warn\(msg\),\s*runBackgroundWorkers \? RSUPostRestartProgressSession\.WasKilledAtRestart : undefined\s*\);/
+        );
+    });
+
+    it('the leftover sweep runs after the resuming pass, over exactly the runs the boot sweep left in flight', () => {
+        const pending = src.indexOf('processRSUPendingWork().catch(');
+        const leftover = src.indexOf('RSUPostRestartProgressSession.SweepUnattached(bootSweep.Skipped');
+        const boot = src.indexOf('const bootSweep = await SweepInterruptedRuns(');
+        expect(boot).toBeGreaterThan(-1);
+        expect(pending).toBeGreaterThan(boot);
+        expect(leftover).toBeGreaterThan(pending);
+        expect(src.slice(pending, leftover)).toMatch(/\.finally\(/);
+    });
+});

package/src/__tests__/ReactivateRefreshMessage.test.ts

@@ -19,6 +19,8 @@ function summary(over: Partial<SchemaRefreshSummaryLike> = {}): SchemaRefreshSum
         Succeeded: true,
         ObjectsCreated: 12,
         ObjectsUpdated: 3,
+        FieldsCreated: 69,
+        FieldsUpdated: 487,
         UnresolvedObjects: ['Widgets'],
         ...over,
     };
@@ -28,7 +30,7 @@ describe('BuildReactivateMessage', () => {
     it('leads with the reactivation on every path', () => {
         const paths = [
             BuildReactivateMessage(undefined),
-            BuildReactivateMessage(summary({ InProgress: true, ObjectsCreated: 0, ObjectsUpdated: 0, UnresolvedObjects: [] })),
+            BuildReactivateMessage(summary({ InProgress: true, ObjectsCreated: 0, ObjectsUpdated: 0, FieldsCreated: 0, FieldsUpdated: 0, UnresolvedObjects: [] })),
             BuildReactivateMessage(summary()),
             BuildReactivateMessage(summary({ Succeeded: false, FailureMessage: 'boom' })),
         ];
@@ -40,6 +42,8 @@ describe('BuildReactivateMessage', () => {
             InProgress: true,
             ObjectsCreated: 0,
             ObjectsUpdated: 0,
+            FieldsCreated: 0,
+            FieldsUpdated: 0,
             UnresolvedObjects: [],
         }));
 
@@ -50,10 +54,11 @@ describe('BuildReactivateMessage', () => {
         expect(msg).not.toMatch(/PK-unresolved/);
     });
 
-    it('reports real counts when the caller asked to block', () => {
+    it('reports real counts when the caller asked to block — fields included', () => {
         const msg = BuildReactivateMessage(summary());
-        expect(msg).toContain('12 created');
+        expect(msg).toContain('12 objects created');
         expect(msg).toContain('3 updated');
+        expect(msg).toContain('69 fields created');
         expect(msg).toContain('1 PK-unresolved');
         expect(msg).not.toMatch(/running/i);
     });
@@ -67,6 +72,8 @@ describe('BuildReactivateMessage', () => {
             FailureMessage: 'ConnectionTest failed: No Totara credentials found',
             ObjectsCreated: 0,
             ObjectsUpdated: 0,
+            FieldsCreated: 0,
+            FieldsUpdated: 0,
             UnresolvedObjects: [],
         }));
 

package/src/__tests__/RealtimeClientSessionResolver.test.ts

@@ -2695,6 +2695,28 @@ describe('RealtimeClientSessionResolver.RelayRealtimeUsage', () => {
         expect(promptRun.Save).not.toHaveBeenCalled();
     });
 
+    it('logs a missing co-agent promptRunID once per session, not once per relay', async () => {
+        errSpy = vi.spyOn(console, 'error').mockImplementation(() => undefined);
+        const missingRunLines = (): string[] =>
+            (errSpy?.mock.calls ?? []).map((c) => c.map(String).join(' ')).filter((m) => m.includes('has no co-agent promptRunID'));
+        const resolver = makeResolver();
+
+        for (const sessionID of ['session-no-run-a', 'session-no-run-b']) {
+            const { provider } = makeUsageProvider({
+                session: makeSessionEntity({ ID: sessionID, Config_: JSON.stringify({ targetAgentID: 'target-1' }) }),
+            });
+            currentProvider = provider;
+            for (let i = 0; i < 3; i++) {
+                await expect(resolver.RelayRealtimeUsage(sessionID, 10, 5, makeCtx())).resolves.toBe(false);
+            }
+        }
+
+        const lines = missingRunLines();
+        expect(lines).toHaveLength(2);
+        expect(lines[0]).toContain('session-no-run-a');
+        expect(lines[1]).toContain('session-no-run-b');
+    });
+
     it('returns false (tolerant, no throw) when the session config is missing/malformed', async () => {
         const { provider } = makeUsageProvider({ session: makeSessionEntity({ Config_: '{broken json' }) });
         currentProvider = provider;