@remnic/core 10.0.98 → 10.0.100
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/access-admin-ops-surface.js +2 -2
- package/dist/access-cli.js +2 -2
- package/dist/access-extraction-force-flush.js +2 -2
- package/dist/access-http-briefing.js +2 -2
- package/dist/access-http-query.js +2 -2
- package/dist/access-http.js +2 -2
- package/dist/access-identity-continuity-surface.js +2 -2
- package/dist/access-lcm-surface.js +2 -2
- package/dist/access-mcp.js +2 -2
- package/dist/access-observe-write-surface.js +2 -2
- package/dist/access-operations-batch.js +2 -2
- package/dist/access-operations.js +2 -2
- package/dist/access-recall-concurrency.js +2 -2
- package/dist/access-recall-response.js +2 -2
- package/dist/access-recall-surface.js +2 -2
- package/dist/access-service.js +2 -2
- package/dist/{chunk-5DGK7E4W.js → chunk-MMOXCCHF.js} +27 -4
- package/dist/chunk-MMOXCCHF.js.map +1 -0
- package/dist/{chunk-4PXDXCNE.js → chunk-PL36IQPL.js} +47 -15
- package/dist/chunk-PL36IQPL.js.map +1 -0
- package/dist/{chunk-TZKK72BY.js → chunk-QIPDTAWZ.js} +2 -2
- package/dist/cli.js +2 -2
- package/dist/explicit-capture.js +2 -2
- package/dist/harmonic-retrieval.d.ts +5 -4
- package/dist/harmonic-retrieval.d.ts.map +1 -1
- package/dist/harmonic-retrieval.js +1 -1
- package/dist/index.js +2 -2
- package/dist/orchestration/backlog-scheduler.d.ts +6 -1
- package/dist/orchestration/backlog-scheduler.d.ts.map +1 -1
- package/dist/orchestration/backlog-scheduler.js +21 -5
- package/dist/orchestration/backlog-scheduler.js.map +1 -1
- package/dist/orchestration/buffer-drain-runner.js +3 -3
- package/dist/orchestration/task-exit.d.ts +15 -0
- package/dist/orchestration/task-exit.d.ts.map +1 -0
- package/dist/orchestration/turn-ingestion.d.ts +1 -7
- package/dist/orchestration/turn-ingestion.d.ts.map +1 -1
- package/dist/orchestrator.js +2 -2
- package/dist/recall-navigation.js +2 -2
- package/package.json +2 -2
- package/src/backlog-scheduler.test.ts +48 -0
- package/src/harmonic-graph-cache.test.ts +187 -2
- package/src/harmonic-retrieval.ts +77 -19
- package/src/orchestration/backlog-scheduler.ts +23 -2
- package/src/orchestration/extraction-queue-bounded-retention.test.ts +20 -0
- package/src/orchestration/task-exit.ts +27 -0
- package/src/orchestration/turn-ingestion.ts +7 -9
- package/dist/chunk-4PXDXCNE.js.map +0 -1
- package/dist/chunk-5DGK7E4W.js.map +0 -1
- /package/dist/{chunk-TZKK72BY.js.map → chunk-QIPDTAWZ.js.map} +0 -0
|
@@ -219,13 +219,7 @@ export declare class TurnIngestionCoordinator {
|
|
|
219
219
|
abortSignal?: AbortSignal;
|
|
220
220
|
/** Automatic buffered work only. Replay and import must not set this. */
|
|
221
221
|
preemptable?: boolean;
|
|
222
|
-
/**
|
|
223
|
-
* Invoked when the queued closure has FULLY exited (after settlement).
|
|
224
|
-
* Settlement can fire before the closure exits — e.g. an abort listener
|
|
225
|
-
* settles while persistence is still in flight — so callers that
|
|
226
|
-
* release resources on settlement (claim leases) can use this to wait
|
|
227
|
-
* for the actual task exit first.
|
|
228
|
-
*/
|
|
222
|
+
/** Fires once when the task is done: closure exited, or settled before it started. */
|
|
229
223
|
onTaskExit?: () => void;
|
|
230
224
|
/** Fail-closed force-flush only. Not implied by failOnExtractionFailure. */
|
|
231
225
|
preemptRunning?: boolean;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"turn-ingestion.d.ts","sourceRoot":"","sources":["../../src/orchestration/turn-ingestion.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAIH,OAAO,EAAE,WAAW,EAAuB,MAAM,cAAc,CAAC;AAChE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAE1D,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,qCAAqC,CAAC;AAI7E,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C,OAAO,EAAE,0BAA0B,EAAE,MAAM,mCAAmC,CAAC;AAC/E,OAAO,EAEL,wBAAwB,EACxB,KAAK,mBAAmB,EACzB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,KAAK,UAAU,EAA6B,MAAM,oBAAoB,CAAC;AAChF,OAAO,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC;AAEpE,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,yBAAyB,EAAE,MAAM,aAAa,CAAC;AAKvF,OAAO,EAAuE,KAAK,8BAA8B,EAAE,MAAM,+BAA+B,CAAC;AACzJ,OAAO,EAAoC,iCAAiC,EAAwC,MAAM,wCAAwC,CAAC;AACnK,OAAO,EAA2B,KAAK,wBAAwB,EAA8B,MAAM,iCAAiC,CAAC;AAIrI,OAAO,EAIL,KAAK,2BAA2B,EACjC,MAAM,oBAAoB,CAAC;AAI5B,OAAO,EAAE,2BAA2B,EAAE,0BAA0B,EAAE,MAAM,wBAAwB,CAAC;
|
|
1
|
+
{"version":3,"file":"turn-ingestion.d.ts","sourceRoot":"","sources":["../../src/orchestration/turn-ingestion.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAIH,OAAO,EAAE,WAAW,EAAuB,MAAM,cAAc,CAAC;AAChE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAE1D,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,qCAAqC,CAAC;AAI7E,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAC7C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C,OAAO,EAAE,0BAA0B,EAAE,MAAM,mCAAmC,CAAC;AAC/E,OAAO,EAEL,wBAAwB,EACxB,KAAK,mBAAmB,EACzB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAE,KAAK,UAAU,EAA6B,MAAM,oBAAoB,CAAC;AAChF,OAAO,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC;AAEpE,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,KAAK,EAAE,UAAU,EAAE,YAAY,EAAE,yBAAyB,EAAE,MAAM,aAAa,CAAC;AAKvF,OAAO,EAAuE,KAAK,8BAA8B,EAAE,MAAM,+BAA+B,CAAC;AACzJ,OAAO,EAAoC,iCAAiC,EAAwC,MAAM,wCAAwC,CAAC;AACnK,OAAO,EAA2B,KAAK,wBAAwB,EAA8B,MAAM,iCAAiC,CAAC;AAIrI,OAAO,EAIL,KAAK,2BAA2B,EACjC,MAAM,oBAAoB,CAAC;AAI5B,OAAO,EAAE,2BAA2B,EAAE,0BAA0B,EAAE,MAAM,wBAAwB,CAAC;AAEjG,OAAO,EAAE,2BAA2B,EAAE,0BAA0B,EAAE,CAAC;AAEnE,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B,wBAAwB,IAAI,MAAM,CAAC;IACnC,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC;IAC9B,QAAQ,CAAC,0BAA0B,EAAE,0BAA0B,CAAC;IAChE,UAAU,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;IACxD,QAAQ,CAAC,uBAAuB,EAAE,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7D,QAAQ,CAAC,SAAS,EAAE,SAAS,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,sBAAsB,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;IAC7C,wBAAwB,IAAI,iBAAiB,CAAC;IAC9C,QAAQ,CAAC,0BAA0B,EAAE;QACnC,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,WAAW,EAAE,MAAM,CAAC;QACpB,sBAAsB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAChD,CAAC;IACF,uBAAuB,CACrB,cAAc,EAAE,UAAU,EAAE,EAC5B,MAAM,EAAE,cAAc,GAAG,oBAAoB,EAC7C,OAAO,CAAC,EAAE;QACR,eAAe,CAAC,EAAE,OAAO,CAAC;QAC1B,0BAA0B,CAAC,EAAE,OAAO,CAAC;QACrC,kBAAkB,CAAC,EAAE,OAAO,CAAC;QAC7B,iBAAiB,CAAC,EAAE,OAAO,CAAC;QAC5B,qBAAqB,CAAC,EAAE,OAAO,CAAC;QAChC,oBAAoB,CAAC,EAAE,MAAM,CAAC;QAC9B,uBAAuB,CAAC,EAAE,OAAO,CAAC;QAClC,eAAe,CAAC,EAAE,MAAM,IAAI,CAAC;QAC7B,sBAAsB,CAAC,EAAE,OAAO,CAAC;QACjC,aAAa,CAAC,EAAE,CACd,KAAK,CAAC,EAAE,OAAO,EACf,MAAM,CAAC,EAAE,mBAAmB,KACzB,IAAI,CAAC;QACV,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,WAAW,CAAC,EAAE,WAAW,CAAC;QAC1B,WAAW,CAAC,EAAE,OAAO,CAAC;QACtB,cAAc,CAAC,EAAE,OAAO,CAAC;QACzB,sBAAsB,CAAC,EAAE,MAAM,CAAC;QAChC;;;;WAIG;QACH,qBAAqB,CAAC,EAAE,wBAAwB,GAAG,IAAI,CAAC;QACxD,iBAAiB,CAAC,EAAE,MAAM,CAAC;KAC5B,GACA,OAAO,CAAC,IAAI,CAAC,CAAC;IACjB,uBAAuB,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAClE,yGAAyG;IACzG,2BAA2B,CAAC,IAAI,MAAM,CAAC;IACvC,aAAa,CACX,GAAG,IAAI,EAAE,UAAU,CAAC,wBAAwB,CAAC,eAAe,CAAC,CAAC,GAC7D,OAAO,CAAC,mBAAmB,CAAC,CAAC;IAChC,QAAQ,CAAC,eAAe,EAAE,oBAAoB,CAAC;IAC/C,qBAAqB,CACnB,KAAK,EAAE,UAAU,EAAE,EACnB,OAAO,CAAC,EAAE;QAAE,MAAM,CAAC,EAAE,OAAO,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAA;KAAE,GACjD,OAAO,CAAC;IACX,QAAQ,CAAC,UAAU,EAAE,iBAAiB,CAAC;CACxC;AAED;;;GAGG;AACH,MAAM,WAAW,oBAAqB,SAAQ,yBAAyB;IACrE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAqBD,qBAAa,wBAAwB;IAIvB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAHjC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAkB;IACxC,wFAAwF;IACxF,QAAQ,CAAC,eAAe,oCAA2C;gBACtC,IAAI,EAAE,iBAAiB;IAI9C,WAAW,CACf,IAAI,EAAE,MAAM,GAAG,WAAW,EAC1B,OAAO,EAAE,MAAM,EACf,UAAU,CAAC,EAAE,MAAM,EACnB,OAAO,GAAE,oBAAyB,GACjC,OAAO,CAAC,IAAI,CAAC;IAgEV,iBAAiB,CACrB,KAAK,EAAE,UAAU,EAAE,EACnB,OAAO,GAAE;QACP,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,UAAU,CAAC,EAAE,OAAO,CAAC;QACrB,WAAW,CAAC,EAAE,WAAW,CAAC;QAC1B;;;;;;;;;WASG;QACH,sBAAsB,CAAC,EAAE,MAAM,CAAC;QAChC;;;;;;;;;WASG;QACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;QAC3B;;;;WAIG;QACH,qBAAqB,CAAC,EAAE,wBAAwB,GAAG,IAAI,CAAC;QACxD;;;;WAIG;QACH,qBAAqB,CAAC,EAAE,MAAM,CAAC;QAC/B;;;;;;WAMG;QACH,cAAc,CAAC,EAAE,WAAW,GAAG,UAAU,CAAC;QAC1C,4EAA4E;QAC5E,kBAAkB,CAAC,EAAE,CAAC,UAAU,EAAE,OAAO,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC;QACzD;;;;WAIG;QACH,aAAa,CAAC,EAAE,OAAO,aAAa,EAAE,aAAa,CAAC;KAChD,GACL,OAAO,CAAC,IAAI,CAAC;IAoHhB,2FAA2F;YAC7E,yBAAyB;IA4EvC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAsCG;IACG,qBAAqB,CACzB,KAAK,EAAE,UAAU,EAAE,EACnB,OAAO,GAAE;QACP,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,uBAAuB,CAAC,EAAE,OAAO,CAAC;QAClC,2BAA2B,CAAC,EAAE,OAAO,CAAC;KAClC,GACL,OAAO,CAAC,2BAA2B,CAAC;IA8KjC,uBAAuB,CAC3B,UAAU,EAAE,MAAM,EAClB,OAAO,GAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAA;KAAO,GACnC,OAAO,CAAC,IAAI,CAAC;IA0EV,uBAAuB,CAC3B,cAAc,EAAE,UAAU,EAAE,EAC5B,MAAM,EAAE,cAAc,GAAG,oBAAoB,EAC7C,OAAO,GAAE;QACP,eAAe,CAAC,EAAE,OAAO,CAAC;QAC1B,0BAA0B,CAAC,EAAE,OAAO,CAAC;QACrC,kBAAkB,CAAC,EAAE,OAAO,CAAC;QAC7B,iBAAiB,CAAC,EAAE,OAAO,CAAC;QAC5B,qBAAqB,CAAC,EAAE,OAAO,CAAC;QAChC,oBAAoB,CAAC,EAAE,MAAM,CAAC;QAC9B,uBAAuB,CAAC,EAAE,OAAO,CAAC;QAClC,eAAe,CAAC,EAAE,MAAM,IAAI,CAAC;QAC7B,sBAAsB,CAAC,EAAE,OAAO,CAAC;QACjC,aAAa,CAAC,EAAE,CACd,KAAK,CAAC,EAAE,OAAO,EACf,MAAM,CAAC,EAAE,mBAAmB,KACzB,IAAI,CAAC;QACV,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,WAAW,CAAC,EAAE,WAAW,CAAC;QAC1B,yEAAyE;QACzE,WAAW,CAAC,EAAE,OAAO,CAAC;QACtB,sFAAsF;QACtF,UAAU,CAAC,EAAE,MAAM,IAAI,CAAC;QACxB,4EAA4E;QAC5E,cAAc,CAAC,EAAE,OAAO,CAAC;QACzB,sBAAsB,CAAC,EAAE,MAAM,CAAC;QAChC;;;;WAIG;QACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;QAC3B,qBAAqB,CAAC,EAAE,wBAAwB,GAAG,IAAI,CAAC;QACxD,wEAAwE;QACxE,SAAS,CAAC,EAAE,8BAA8B,CAAC;KACvC,GACL,OAAO,CAAC,IAAI,CAAC;IAmShB;;;;;;;;;OASG;IACG,8BAA8B,CAClC,KAAK,EAAE,SAAS,UAAU,EAAE,EAC5B,IAAI,EAAE;QACJ,UAAU,EAAE,MAAM,CAAC;QACnB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,SAAS,EAAE,MAAM,CAAC;QAClB,SAAS,EAAE,MAAM,CAAC;QAClB,aAAa,EAAE,OAAO,CAAC;QACvB,WAAW,CAAC,EAAE,WAAW,CAAC;KAC3B,GACA,OAAO,CAAC,IAAI,CAAC;CAkEjB;AAED,sFAAsF;AACtF,MAAM,MAAM,wBAAwB,GAAG,UAAU,CAAC,wBAAwB,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC"}
|
package/dist/orchestrator.js
CHANGED
|
@@ -6,7 +6,7 @@ import {
|
|
|
6
6
|
mergeGraphExpandedResults,
|
|
7
7
|
qmdStartupCollectionChecksWithTimeout,
|
|
8
8
|
resolveEffectiveIdentityInjectionMode
|
|
9
|
-
} from "./chunk-
|
|
9
|
+
} from "./chunk-MMOXCCHF.js";
|
|
10
10
|
import "./chunk-6P7U3HWG.js";
|
|
11
11
|
import "./chunk-PQM3B54K.js";
|
|
12
12
|
import "./chunk-B3DR2JBZ.js";
|
|
@@ -183,7 +183,7 @@ import "./chunk-MDPT6QFK.js";
|
|
|
183
183
|
import "./chunk-UOBPK2OS.js";
|
|
184
184
|
import "./chunk-2YA5N2BY.js";
|
|
185
185
|
import "./chunk-3HSRWJTC.js";
|
|
186
|
-
import "./chunk-
|
|
186
|
+
import "./chunk-PL36IQPL.js";
|
|
187
187
|
import "./chunk-HUDPXTXW.js";
|
|
188
188
|
import "./chunk-AZEZSQ5R.js";
|
|
189
189
|
import "./chunk-YAZNBMNF.js";
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
runRecallNavigation
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-MMOXCCHF.js";
|
|
4
4
|
import "./chunk-6P7U3HWG.js";
|
|
5
5
|
import "./chunk-PQM3B54K.js";
|
|
6
6
|
import "./chunk-B3DR2JBZ.js";
|
|
@@ -173,7 +173,7 @@ import "./chunk-MDPT6QFK.js";
|
|
|
173
173
|
import "./chunk-UOBPK2OS.js";
|
|
174
174
|
import "./chunk-2YA5N2BY.js";
|
|
175
175
|
import "./chunk-3HSRWJTC.js";
|
|
176
|
-
import "./chunk-
|
|
176
|
+
import "./chunk-PL36IQPL.js";
|
|
177
177
|
import "./chunk-HUDPXTXW.js";
|
|
178
178
|
import "./chunk-AZEZSQ5R.js";
|
|
179
179
|
import "./chunk-YAZNBMNF.js";
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@remnic/core",
|
|
3
|
-
"version": "10.0.
|
|
3
|
+
"version": "10.0.100",
|
|
4
4
|
"description": "Framework-agnostic Remnic memory engine — orchestrator, storage, extraction, search, trust zones",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
@@ -3386,7 +3386,7 @@
|
|
|
3386
3386
|
"core"
|
|
3387
3387
|
],
|
|
3388
3388
|
"peerDependencies": {
|
|
3389
|
-
"@remnic/coding-graph": "^10.0.
|
|
3389
|
+
"@remnic/coding-graph": "^10.0.100"
|
|
3390
3390
|
},
|
|
3391
3391
|
"peerDependenciesMeta": {
|
|
3392
3392
|
"@remnic/coding-graph": {
|
|
@@ -12,6 +12,7 @@ import assert from "node:assert/strict";
|
|
|
12
12
|
import test from "node:test";
|
|
13
13
|
|
|
14
14
|
import { hostname } from "node:os";
|
|
15
|
+
import { log } from "./logger.js";
|
|
15
16
|
import {
|
|
16
17
|
BacklogScheduler,
|
|
17
18
|
type BacklogSchedulerDeps,
|
|
@@ -390,6 +391,53 @@ test("the in-flight bound is the queue's own maxConcurrency", async () => {
|
|
|
390
391
|
}
|
|
391
392
|
});
|
|
392
393
|
|
|
394
|
+
test("stop returns within its settle bound and names a drain that ignores abort", async () => {
|
|
395
|
+
const queue = new ExtractionQueueCoordinator(1);
|
|
396
|
+
const state: StubState = {
|
|
397
|
+
counts: new Map([["stuck-key", { turns: 3, retainedTurns: 0 }]]),
|
|
398
|
+
claimed: new Set(),
|
|
399
|
+
claimTurns: new Map(),
|
|
400
|
+
claimedThisRun: new Set(),
|
|
401
|
+
outcomes: [],
|
|
402
|
+
drained: [],
|
|
403
|
+
};
|
|
404
|
+
const entered = Promise.withResolvers<void>();
|
|
405
|
+
const forever = new Promise<DrainSessionOutcome>(() => {});
|
|
406
|
+
const scheduler = new BacklogScheduler(
|
|
407
|
+
{
|
|
408
|
+
...makeDeps(state, queue),
|
|
409
|
+
// Ignores options.signal entirely: the shape that held a daemon stop
|
|
410
|
+
// until the 700 s systemd kill.
|
|
411
|
+
drainKey: async () => {
|
|
412
|
+
entered.resolve();
|
|
413
|
+
return forever;
|
|
414
|
+
},
|
|
415
|
+
},
|
|
416
|
+
{ pollMs: 3_600_000, stopSettleTimeoutMs: 50 }
|
|
417
|
+
);
|
|
418
|
+
const warnings: string[] = [];
|
|
419
|
+
const originalWarn = log.warn;
|
|
420
|
+
log.warn = (msg: string) => {
|
|
421
|
+
warnings.push(msg);
|
|
422
|
+
};
|
|
423
|
+
try {
|
|
424
|
+
scheduler.start();
|
|
425
|
+
await scheduler.tickForTest();
|
|
426
|
+
await entered.promise;
|
|
427
|
+
const outcome = await Promise.race([
|
|
428
|
+
scheduler.stop().then(() => "returned"),
|
|
429
|
+
new Promise<string>((resolve) => setTimeout(() => resolve("hung"), 2_000)),
|
|
430
|
+
]);
|
|
431
|
+
assert.equal(outcome, "returned", "stop must not wait forever on a drain that ignores abort");
|
|
432
|
+
assert.ok(
|
|
433
|
+
warnings.some((w) => w.includes("stuck-key")),
|
|
434
|
+
`the stuck key is named in a warning: ${JSON.stringify(warnings)}`
|
|
435
|
+
);
|
|
436
|
+
} finally {
|
|
437
|
+
log.warn = originalWarn;
|
|
438
|
+
}
|
|
439
|
+
});
|
|
440
|
+
|
|
393
441
|
test("stop aborts in-flight drains; a fresh start resumes the still-claimed backlog", async () => {
|
|
394
442
|
const queue = new ExtractionQueueCoordinator(1);
|
|
395
443
|
const state: StubState = {
|
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
import assert from "node:assert/strict";
|
|
2
|
-
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
|
2
|
+
import { mkdtemp, readFile, rename, rm, writeFile } from "node:fs/promises";
|
|
3
3
|
import { tmpdir } from "node:os";
|
|
4
4
|
import path from "node:path";
|
|
5
|
-
import test from "node:test";
|
|
5
|
+
import test, { type TestContext } from "node:test";
|
|
6
6
|
import { recordAbstractionNode, upsertAbstractionNodes, type AbstractionNode } from "./abstraction-nodes.js";
|
|
7
7
|
import { pruneOrphanCueAnchors, recordCueAnchor, upsertCueAnchors, type CueAnchor } from "./cue-anchors.js";
|
|
8
8
|
import {
|
|
9
9
|
clearHarmonicGraphCache,
|
|
10
10
|
harmonicGraphCacheStats,
|
|
11
|
+
type ProjectedHarmonicGraph,
|
|
11
12
|
readProjectedHarmonicGraph,
|
|
12
13
|
searchHarmonicRetrieval,
|
|
13
14
|
} from "./harmonic-retrieval.js";
|
|
@@ -228,6 +229,61 @@ test("pruneOrphanCueAnchors alone invalidates the cached graph", async () => {
|
|
|
228
229
|
});
|
|
229
230
|
});
|
|
230
231
|
|
|
232
|
+
// Recall waits for the graph build, so a cache miss costs every recall that lands on it. Production: a
|
|
233
|
+
// 2106 node, 9174 anchor graph rebuilt in 5 s standalone, 7 s inside the daemon, and 17 to 32 s when two
|
|
234
|
+
// recalls overlapped and each ran its own build.
|
|
235
|
+
test("concurrent reads on a cold cache share one build", async () => {
|
|
236
|
+
await withDir(async (memoryDir, storage) => {
|
|
237
|
+
const memory = await storage.writeMemory("fact", "theta shared fact", { source: "test" });
|
|
238
|
+
await recordAbstractionNode({ memoryDir, node: node("n1", "theta shared fact", [memory.id]) });
|
|
239
|
+
const before = { ...harmonicGraphCacheStats };
|
|
240
|
+
const [a, b, c] = await Promise.all([read(memoryDir), read(memoryDir), read(memoryDir)]);
|
|
241
|
+
assert.equal(harmonicGraphCacheStats.misses, before.misses + 1, "one build, not one per caller");
|
|
242
|
+
assert.equal(a, b);
|
|
243
|
+
assert.equal(b, c);
|
|
244
|
+
});
|
|
245
|
+
});
|
|
246
|
+
|
|
247
|
+
test("a caller that aborts leaves the shared build for the others", async () => {
|
|
248
|
+
await withDir(async (memoryDir, storage) => {
|
|
249
|
+
const memory = await storage.writeMemory("fact", "iota abort fact", { source: "test" });
|
|
250
|
+
await recordAbstractionNode({ memoryDir, node: node("n1", "iota abort fact", [memory.id]) });
|
|
251
|
+
const controller = new AbortController();
|
|
252
|
+
const aborted = readProjectedHarmonicGraph({ memoryDir, anchorsEnabled: true, useCache: true, abortSignal: controller.signal });
|
|
253
|
+
const other = read(memoryDir);
|
|
254
|
+
controller.abort();
|
|
255
|
+
await assert.rejects(aborted);
|
|
256
|
+
assert.equal((await other).nodes.length, 1);
|
|
257
|
+
});
|
|
258
|
+
});
|
|
259
|
+
|
|
260
|
+
// Unreadable node store: a file where the nodes directory belongs is not ENOENT, so the strict listing throws
|
|
261
|
+
// (a corrupt node file would not: readAbstractionNodes skips those on purpose).
|
|
262
|
+
async function breakNodeStore(memoryDir: string) {
|
|
263
|
+
const nodesDir = path.join(memoryDir, "state", "abstraction-nodes", "nodes");
|
|
264
|
+
const parked = `${nodesDir}.parked`;
|
|
265
|
+
await rename(nodesDir, parked);
|
|
266
|
+
await writeFile(nodesDir, "not a directory");
|
|
267
|
+
return async () => {
|
|
268
|
+
await rm(nodesDir, { force: true });
|
|
269
|
+
await rename(parked, nodesDir);
|
|
270
|
+
};
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
test("callers that joined a failing build all reject, and the next read retries and succeeds", async () => {
|
|
274
|
+
await withDir(async (memoryDir, storage) => {
|
|
275
|
+
const memory = await storage.writeMemory("fact", "mu retry fact", { source: "test" });
|
|
276
|
+
await recordAbstractionNode({ memoryDir, node: node("n1", "mu retry fact", [memory.id]) });
|
|
277
|
+
const restore = await breakNodeStore(memoryDir);
|
|
278
|
+
const missesBefore = harmonicGraphCacheStats.misses;
|
|
279
|
+
const outcomes = await Promise.allSettled([read(memoryDir), read(memoryDir), read(memoryDir)]);
|
|
280
|
+
assert.deepEqual(outcomes.map((outcome) => outcome.status), ["rejected", "rejected", "rejected"]);
|
|
281
|
+
assert.equal(harmonicGraphCacheStats.misses, missesBefore + 1, "the three callers shared one failing build");
|
|
282
|
+
await restore();
|
|
283
|
+
assert.equal((await read(memoryDir)).nodes.length, 1);
|
|
284
|
+
});
|
|
285
|
+
});
|
|
286
|
+
|
|
231
287
|
test("an entry older than the TTL is rebuilt", async (t) => {
|
|
232
288
|
await withDir(async (memoryDir, storage) => {
|
|
233
289
|
const memory = await storage.writeMemory("fact", "zeta ttl fact", { source: "test" });
|
|
@@ -242,3 +298,132 @@ test("an entry older than the TTL is rebuilt", async (t) => {
|
|
|
242
298
|
assert.equal((await read(memoryDir)).nodes.length, 0, "rebuilt past the TTL");
|
|
243
299
|
});
|
|
244
300
|
});
|
|
301
|
+
|
|
302
|
+
interface BuildHold {
|
|
303
|
+
release: () => void;
|
|
304
|
+
reached: Promise<void>;
|
|
305
|
+
calls: number;
|
|
306
|
+
signal?: AbortSignal;
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
/**
|
|
310
|
+
* Hold the first build inside its source-memory read so a second read can land while it runs. The held call
|
|
311
|
+
* records the build's own abort signal; builds started after it are not held.
|
|
312
|
+
*/
|
|
313
|
+
function holdFirstBuild(t: TestContext): BuildHold {
|
|
314
|
+
const original = StorageManager.prototype.readAllMemories;
|
|
315
|
+
let release = () => {};
|
|
316
|
+
const opened = new Promise<void>((resolve) => {
|
|
317
|
+
release = resolve;
|
|
318
|
+
});
|
|
319
|
+
let reachedHold = () => {};
|
|
320
|
+
const reached = new Promise<void>((resolve) => {
|
|
321
|
+
reachedHold = resolve;
|
|
322
|
+
});
|
|
323
|
+
const hold: BuildHold = { release, reached, calls: 0 };
|
|
324
|
+
t.mock.method(StorageManager.prototype, "readAllMemories", async function (this: StorageManager, ...args: Parameters<typeof original>) {
|
|
325
|
+
hold.calls++;
|
|
326
|
+
if (hold.calls === 1) {
|
|
327
|
+
hold.signal = args[0]?.abortSignal;
|
|
328
|
+
reachedHold();
|
|
329
|
+
await opened;
|
|
330
|
+
}
|
|
331
|
+
return original.apply(this, args);
|
|
332
|
+
});
|
|
333
|
+
return hold;
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
/** Fail the test instead of hanging when a promise that must settle does not. */
|
|
337
|
+
function within<T>(promise: Promise<T>, ms: number, what: string): Promise<T> {
|
|
338
|
+
return Promise.race([
|
|
339
|
+
promise,
|
|
340
|
+
new Promise<never>((_resolve, reject) => setTimeout(() => reject(new Error(`${what} did not settle in ${ms} ms`)), ms).unref()),
|
|
341
|
+
]);
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
test("the shared build aborts once every joined caller has aborted, and the next read starts a fresh one", async (t) => {
|
|
345
|
+
await withDir(async (memoryDir, storage) => {
|
|
346
|
+
const memory = await storage.writeMemory("fact", "nu abort fact", { source: "test" });
|
|
347
|
+
await recordAbstractionNode({ memoryDir, node: node("n1", "nu abort fact", [memory.id]) });
|
|
348
|
+
const hold = holdFirstBuild(t);
|
|
349
|
+
const first = new AbortController();
|
|
350
|
+
const second = new AbortController();
|
|
351
|
+
const options = { memoryDir, anchorsEnabled: true, useCache: true };
|
|
352
|
+
const a = readProjectedHarmonicGraph({ ...options, abortSignal: first.signal });
|
|
353
|
+
const b = readProjectedHarmonicGraph({ ...options, abortSignal: second.signal });
|
|
354
|
+
await within(hold.reached, 2000, "build start");
|
|
355
|
+
assert.equal(hold.calls, 1, "one build for both callers");
|
|
356
|
+
first.abort();
|
|
357
|
+
await assert.rejects(within(a, 1000, "first caller"), { name: "AbortError" });
|
|
358
|
+
assert.equal(hold.signal?.aborted, false, "one caller is still waiting, so the build keeps running");
|
|
359
|
+
second.abort();
|
|
360
|
+
await assert.rejects(within(b, 1000, "second caller"), { name: "AbortError" });
|
|
361
|
+
assert.equal(hold.signal?.aborted, true, "no caller is left, so the build is cancelled");
|
|
362
|
+
const missesBefore = harmonicGraphCacheStats.misses;
|
|
363
|
+
const next = read(memoryDir);
|
|
364
|
+
assert.equal(harmonicGraphCacheStats.misses, missesBefore + 1, "the next read does not join the cancelled build");
|
|
365
|
+
assert.equal((await within(next, 5000, "next read")).nodes.length, 1);
|
|
366
|
+
hold.release();
|
|
367
|
+
});
|
|
368
|
+
});
|
|
369
|
+
|
|
370
|
+
test("a read after a stamp change does not join the build that started on the old stamp", async (t) => {
|
|
371
|
+
await withDir(async (memoryDir, storage) => {
|
|
372
|
+
const first = await storage.writeMemory("fact", "xi stamp fact one", { source: "test" });
|
|
373
|
+
await recordAbstractionNode({ memoryDir, node: node("n1", "xi stamp fact one", [first.id]) });
|
|
374
|
+
const hold = holdFirstBuild(t);
|
|
375
|
+
const missesBefore = harmonicGraphCacheStats.misses;
|
|
376
|
+
const stale = read(memoryDir);
|
|
377
|
+
await within(hold.reached, 2000, "old build start");
|
|
378
|
+
const second = await storage.writeMemory("fact", "xi stamp fact two", { source: "test" });
|
|
379
|
+
await recordAbstractionNode({ memoryDir, node: node("n2", "xi stamp fact two", [second.id]) });
|
|
380
|
+
try {
|
|
381
|
+
const fresh = read(memoryDir);
|
|
382
|
+
assert.equal(harmonicGraphCacheStats.misses, missesBefore + 2, "the second read started its own build");
|
|
383
|
+
assert.equal((await within(fresh, 5000, "fresh read")).nodes.length, 2, "and it sees the write");
|
|
384
|
+
} finally {
|
|
385
|
+
hold.release();
|
|
386
|
+
}
|
|
387
|
+
await stale;
|
|
388
|
+
});
|
|
389
|
+
});
|
|
390
|
+
|
|
391
|
+
test("a joined caller that aborts rejects at once, while the build it joined is still running", async (t) => {
|
|
392
|
+
await withDir(async (memoryDir, storage) => {
|
|
393
|
+
const memory = await storage.writeMemory("fact", "omicron join fact", { source: "test" });
|
|
394
|
+
await recordAbstractionNode({ memoryDir, node: node("n1", "omicron join fact", [memory.id]) });
|
|
395
|
+
const hold = holdFirstBuild(t);
|
|
396
|
+
const owner = read(memoryDir);
|
|
397
|
+
const controller = new AbortController();
|
|
398
|
+
const joiner = readProjectedHarmonicGraph({ memoryDir, anchorsEnabled: true, useCache: true, abortSignal: controller.signal });
|
|
399
|
+
controller.abort();
|
|
400
|
+
try {
|
|
401
|
+
await assert.rejects(within(joiner, 1000, "aborted joiner"), { name: "AbortError" });
|
|
402
|
+
} finally {
|
|
403
|
+
hold.release();
|
|
404
|
+
}
|
|
405
|
+
assert.equal((await owner).nodes.length, 1, "the owner is not affected");
|
|
406
|
+
});
|
|
407
|
+
});
|
|
408
|
+
|
|
409
|
+
test("clearing the cache drops a running build: the next read starts its own and the old build stores nothing", async (t) => {
|
|
410
|
+
await withDir(async (memoryDir, storage) => {
|
|
411
|
+
const memory = await storage.writeMemory("fact", "pi clear fact", { source: "test" });
|
|
412
|
+
await recordAbstractionNode({ memoryDir, node: node("n1", "pi clear fact", [memory.id]) });
|
|
413
|
+
const hold = holdFirstBuild(t);
|
|
414
|
+
const missesBefore = harmonicGraphCacheStats.misses;
|
|
415
|
+
const old = read(memoryDir);
|
|
416
|
+
await within(hold.reached, 2000, "old build start");
|
|
417
|
+
clearHarmonicGraphCache();
|
|
418
|
+
let kept: ProjectedHarmonicGraph;
|
|
419
|
+
try {
|
|
420
|
+
const fresh = read(memoryDir);
|
|
421
|
+
assert.equal(harmonicGraphCacheStats.misses, missesBefore + 2, "the read after the clear did not join the old build");
|
|
422
|
+
kept = await within(fresh, 5000, "fresh read");
|
|
423
|
+
} finally {
|
|
424
|
+
hold.release();
|
|
425
|
+
}
|
|
426
|
+
await old;
|
|
427
|
+
assert.equal(await read(memoryDir), kept, "the finished old build did not overwrite the newer cache entry");
|
|
428
|
+
});
|
|
429
|
+
});
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import path from "node:path";
|
|
2
2
|
import { realpath } from "node:fs/promises";
|
|
3
3
|
import { listJsonFilesStrict, readJsonFile } from "./json-store.js";
|
|
4
|
-
import { throwIfAborted } from "./abort-error.js";
|
|
4
|
+
import { raceAbort, throwIfAborted } from "./abort-error.js";
|
|
5
5
|
import { readHarmonicGraphStamp } from "./harmonic-graph-version.js";
|
|
6
6
|
import { inferMemoryStatus, toMemoryPathRel } from "./memory-lifecycle-ledger-utils.js";
|
|
7
7
|
import {
|
|
@@ -471,8 +471,20 @@ interface GraphCacheEntry {
|
|
|
471
471
|
}
|
|
472
472
|
|
|
473
473
|
const graphCache = new Map<string, GraphCacheEntry>();
|
|
474
|
-
|
|
475
|
-
|
|
474
|
+
|
|
475
|
+
interface GraphBuild {
|
|
476
|
+
stamp: string;
|
|
477
|
+
promise: Promise<ProjectedHarmonicGraph>;
|
|
478
|
+
/** Cancels the build's source scan; fired when no caller is waiting for it. */
|
|
479
|
+
controller: AbortController;
|
|
480
|
+
/** Callers currently waiting for this build. */
|
|
481
|
+
waiting: number;
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
/** One build per cache key at a time: every recall that lands on a miss waits for the same build. */
|
|
485
|
+
const graphBuilds = new Map<string, GraphBuild>();
|
|
486
|
+
/** Counters for tests and diagnostics. `shared` counts reads that joined a build already running. */
|
|
487
|
+
export const harmonicGraphCacheStats = { hits: 0, misses: 0, shared: 0 };
|
|
476
488
|
|
|
477
489
|
/** Freeze records and their nested arrays/objects so a caller cannot corrupt the shared cache. */
|
|
478
490
|
function deepFreeze<T extends object>(value: T): T {
|
|
@@ -485,6 +497,60 @@ function deepFreeze<T extends object>(value: T): T {
|
|
|
485
497
|
|
|
486
498
|
export function clearHarmonicGraphCache(): void {
|
|
487
499
|
graphCache.clear();
|
|
500
|
+
graphBuilds.clear();
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
type GraphReadOptions = Parameters<typeof buildProjectedHarmonicGraph>[0];
|
|
504
|
+
|
|
505
|
+
/**
|
|
506
|
+
* Start the one build for `key`. The build owns an abort controller instead of using one caller's signal,
|
|
507
|
+
* so a caller that leaves does not cancel it for the others; `joinGraphBuild` cancels it when the last caller leaves.
|
|
508
|
+
*/
|
|
509
|
+
function startGraphBuild(key: string, stamp: string, options: GraphReadOptions): GraphBuild {
|
|
510
|
+
harmonicGraphCacheStats.misses++;
|
|
511
|
+
const startedAtMs = Date.now();
|
|
512
|
+
const controller = new AbortController();
|
|
513
|
+
const promise: Promise<ProjectedHarmonicGraph> = buildProjectedHarmonicGraph({
|
|
514
|
+
memoryDir: options.memoryDir,
|
|
515
|
+
abstractionNodeStoreDir: options.abstractionNodeStoreDir,
|
|
516
|
+
anchorsEnabled: options.anchorsEnabled,
|
|
517
|
+
temporalExpiredInInjection: options.temporalExpiredInInjection,
|
|
518
|
+
abortSignal: controller.signal,
|
|
519
|
+
})
|
|
520
|
+
.then((built) => {
|
|
521
|
+
const graph = { nodes: deepFreeze(built.nodes), anchors: deepFreeze(built.anchors) };
|
|
522
|
+
// A cleared cache or a newer build for this key has replaced this one: do not store a stale graph.
|
|
523
|
+
if (graphBuilds.get(key)?.promise !== promise) return graph;
|
|
524
|
+
graphCache.delete(key);
|
|
525
|
+
graphCache.set(key, { stamp, startedAtMs, graph });
|
|
526
|
+
while (graphCache.size > GRAPH_CACHE_MAX_ENTRIES) {
|
|
527
|
+
const oldest = graphCache.keys().next().value;
|
|
528
|
+
if (oldest === undefined) break;
|
|
529
|
+
graphCache.delete(oldest);
|
|
530
|
+
}
|
|
531
|
+
return graph;
|
|
532
|
+
})
|
|
533
|
+
.finally(() => {
|
|
534
|
+
if (graphBuilds.get(key)?.promise === promise) graphBuilds.delete(key);
|
|
535
|
+
});
|
|
536
|
+
const build: GraphBuild = { stamp, promise, controller, waiting: 0 };
|
|
537
|
+
graphBuilds.set(key, build);
|
|
538
|
+
return build;
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
/**
|
|
542
|
+
* Wait for `build` on behalf of one caller. The caller's own signal ends only its own wait. When the last
|
|
543
|
+
* waiting caller has left, the build is cancelled and dropped, so abandoned recalls do not leave full scans running
|
|
544
|
+
* (cancelling a build that already finished does nothing).
|
|
545
|
+
*/
|
|
546
|
+
function joinGraphBuild(key: string, build: GraphBuild, signal: AbortSignal | undefined): Promise<ProjectedHarmonicGraph> {
|
|
547
|
+
build.waiting++;
|
|
548
|
+
return raceAbort(build.promise, signal, "harmonic retrieval aborted").finally(() => {
|
|
549
|
+
build.waiting--;
|
|
550
|
+
if (build.waiting > 0) return;
|
|
551
|
+
if (graphBuilds.get(key) === build) graphBuilds.delete(key);
|
|
552
|
+
build.controller.abort();
|
|
553
|
+
});
|
|
488
554
|
}
|
|
489
555
|
|
|
490
556
|
/**
|
|
@@ -495,9 +561,9 @@ export function clearHarmonicGraphCache(): void {
|
|
|
495
561
|
* cold-write version sentinels (bumped by every memory write, in any process) and
|
|
496
562
|
* the graph version sentinel (bumped by node and anchor writes). A stamp read is
|
|
497
563
|
* four stats, not a scan of the memories. The TTL bounds the one input a sentinel
|
|
498
|
-
* cannot see, which is wall-clock validity expiry.
|
|
499
|
-
* cache so tests stay deterministic. The
|
|
500
|
-
* mutate them.
|
|
564
|
+
* cannot see, which is wall-clock validity expiry. Concurrent misses share one
|
|
565
|
+
* build. An explicit `nowMs` bypasses the cache so tests stay deterministic. The
|
|
566
|
+
* cached arrays are frozen: callers must not mutate them.
|
|
501
567
|
*/
|
|
502
568
|
export async function readProjectedHarmonicGraph(options: {
|
|
503
569
|
memoryDir: string;
|
|
@@ -529,20 +595,12 @@ export async function readProjectedHarmonicGraph(options: {
|
|
|
529
595
|
graphCache.set(key, hit);
|
|
530
596
|
return hit.graph;
|
|
531
597
|
}
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
anchors: deepFreeze(built.anchors),
|
|
537
|
-
};
|
|
538
|
-
graphCache.delete(key);
|
|
539
|
-
graphCache.set(key, { stamp, startedAtMs, graph });
|
|
540
|
-
while (graphCache.size > GRAPH_CACHE_MAX_ENTRIES) {
|
|
541
|
-
const oldest = graphCache.keys().next().value;
|
|
542
|
-
if (oldest === undefined) break;
|
|
543
|
-
graphCache.delete(oldest);
|
|
598
|
+
const running = graphBuilds.get(key);
|
|
599
|
+
if (running?.stamp === stamp) {
|
|
600
|
+
harmonicGraphCacheStats.shared++;
|
|
601
|
+
return joinGraphBuild(key, running, options.abortSignal);
|
|
544
602
|
}
|
|
545
|
-
return
|
|
603
|
+
return joinGraphBuild(key, startGraphBuild(key, stamp, options), options.abortSignal);
|
|
546
604
|
}
|
|
547
605
|
|
|
548
606
|
export async function searchHarmonicRetrieval(options: {
|
|
@@ -70,6 +70,8 @@ export interface BacklogSchedulerOptions {
|
|
|
70
70
|
pollMs?: number;
|
|
71
71
|
retryBaseMs?: number;
|
|
72
72
|
retryMaxMs?: number;
|
|
73
|
+
/** Bound on `stop()`'s wait for aborted drains to settle. */
|
|
74
|
+
stopSettleTimeoutMs?: number;
|
|
73
75
|
now?: () => number;
|
|
74
76
|
}
|
|
75
77
|
|
|
@@ -114,6 +116,9 @@ interface KeyState {
|
|
|
114
116
|
const DEFAULT_POLL_MS = 15_000;
|
|
115
117
|
const DEFAULT_RETRY_BASE_MS = 30_000;
|
|
116
118
|
const DEFAULT_RETRY_MAX_MS = 15 * 60_000;
|
|
119
|
+
// Aborted drains normally settle within seconds (a chunk boundary); this only
|
|
120
|
+
// bounds the pathological case, well inside the daemon's stop budget.
|
|
121
|
+
const DEFAULT_STOP_SETTLE_TIMEOUT_MS = 30_000;
|
|
117
122
|
|
|
118
123
|
export class BacklogScheduler {
|
|
119
124
|
private readonly deps: BacklogSchedulerDeps;
|
|
@@ -141,6 +146,7 @@ export class BacklogScheduler {
|
|
|
141
146
|
pollMs: options.pollMs ?? DEFAULT_POLL_MS,
|
|
142
147
|
retryBaseMs: options.retryBaseMs ?? DEFAULT_RETRY_BASE_MS,
|
|
143
148
|
retryMaxMs: options.retryMaxMs ?? DEFAULT_RETRY_MAX_MS,
|
|
149
|
+
stopSettleTimeoutMs: options.stopSettleTimeoutMs ?? DEFAULT_STOP_SETTLE_TIMEOUT_MS,
|
|
144
150
|
now: options.now,
|
|
145
151
|
};
|
|
146
152
|
}
|
|
@@ -155,7 +161,10 @@ export class BacklogScheduler {
|
|
|
155
161
|
}
|
|
156
162
|
|
|
157
163
|
/** Idempotent. Stops the timer, aborts in-flight drains (their claims stay
|
|
158
|
-
* retryable) and waits for them to settle.
|
|
164
|
+
* retryable) and waits up to `stopSettleTimeoutMs` for them to settle. A
|
|
165
|
+
* drain that ignores abort is named in a warning and left behind: the
|
|
166
|
+
* caller's later shutdown phases (orchestrator flush and dispose) must
|
|
167
|
+
* still run before the service manager's kill. */
|
|
159
168
|
async stop(): Promise<void> {
|
|
160
169
|
if (this.stopped) return;
|
|
161
170
|
this.stopped = true;
|
|
@@ -168,7 +177,19 @@ export class BacklogScheduler {
|
|
|
168
177
|
entry.controller.abort(new Error("backlog scheduler stopping"));
|
|
169
178
|
aborts.push(entry.settled);
|
|
170
179
|
}
|
|
171
|
-
|
|
180
|
+
const bound = Promise.withResolvers<"timeout">();
|
|
181
|
+
const boundTimer = setTimeout(() => bound.resolve("timeout"), this.options.stopSettleTimeoutMs);
|
|
182
|
+
try {
|
|
183
|
+
const result = await Promise.race([Promise.allSettled(aborts), bound.promise]);
|
|
184
|
+
if (result === "timeout") {
|
|
185
|
+
log.warn(
|
|
186
|
+
`backlog scheduler stop: ${this.inFlight.size} drain(s) ignored abort for ${this.options.stopSettleTimeoutMs}ms; ` +
|
|
187
|
+
`continuing shutdown (stuck keys: ${[...this.inFlight.keys()].join(", ")})`,
|
|
188
|
+
);
|
|
189
|
+
}
|
|
190
|
+
} finally {
|
|
191
|
+
clearTimeout(boundTimer);
|
|
192
|
+
}
|
|
172
193
|
}
|
|
173
194
|
|
|
174
195
|
/** Bounded snapshot for ops/watchers: counters + per-key states + live sets. */
|
|
@@ -299,6 +299,26 @@ test("abort while queued: settled task starts, skips extraction, queue drains",
|
|
|
299
299
|
assert.equal(settled.length, 1, "queued task settles exactly once from the abort");
|
|
300
300
|
});
|
|
301
301
|
|
|
302
|
+
test("abort while queued: onTaskExit fires at once, not after the queue head finishes", async () => {
|
|
303
|
+
const h = makeHarness();
|
|
304
|
+
const controller = new AbortController();
|
|
305
|
+
// Occupies the single worker; the drain stays blocked until release().
|
|
306
|
+
await h.coordinator.processTurn("user", "running", "k");
|
|
307
|
+
let exited = 0;
|
|
308
|
+
await h.coordinator.queueBufferedExtraction([makeTurn("queued then aborted")], "trigger_mode", {
|
|
309
|
+
bufferKey: "k",
|
|
310
|
+
preemptable: true,
|
|
311
|
+
abortSignal: controller.signal,
|
|
312
|
+
onTaskExit: () => { exited += 1; },
|
|
313
|
+
});
|
|
314
|
+
controller.abort();
|
|
315
|
+
// A claim-lease owner waiting on this exit must not be held by an unrelated
|
|
316
|
+
// running extraction: nothing of this task ever ran.
|
|
317
|
+
assert.equal(exited, 1, "queued task that never started reports exit on abort");
|
|
318
|
+
await h.release();
|
|
319
|
+
assert.equal(exited, 1, "the later no-op closure run does not report a second exit");
|
|
320
|
+
});
|
|
321
|
+
|
|
302
322
|
test("sharing bound: distinct reference slots equal the union, not the snapshot sum", async () => {
|
|
303
323
|
const h = makeHarness();
|
|
304
324
|
const turns = 30;
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Exit-once bookkeeping for a queued extraction task (`onTaskExit`).
|
|
3
|
+
*
|
|
4
|
+
* A task settled (abort, queue deadline) before its closure ever ran has
|
|
5
|
+
* nothing to exit, so `settled()` reports exit at once; a claim-lease owner
|
|
6
|
+
* waiting on the exit is then not held behind an unrelated running extraction.
|
|
7
|
+
* A closure that did start reports exit when it finishes. Either way the
|
|
8
|
+
* callback fires exactly once.
|
|
9
|
+
*/
|
|
10
|
+
export function createTaskExit(onExit?: () => void) {
|
|
11
|
+
let started = false;
|
|
12
|
+
let reported = false;
|
|
13
|
+
const report = (): void => {
|
|
14
|
+
if (reported) return;
|
|
15
|
+
reported = true;
|
|
16
|
+
onExit?.();
|
|
17
|
+
};
|
|
18
|
+
return {
|
|
19
|
+
start: () => {
|
|
20
|
+
started = true;
|
|
21
|
+
},
|
|
22
|
+
settled: () => {
|
|
23
|
+
if (!started) report();
|
|
24
|
+
},
|
|
25
|
+
report,
|
|
26
|
+
};
|
|
27
|
+
}
|