@bongos/core 1.19.362 → 1.19.363
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.bongos-core.json +24 -14
- package/migrations/core_225_search_chunks_updated_at_idx.sql +24 -0
- package/modules/memory/search.js +45 -12
- package/package-lock.json +2 -2
- package/package.json +1 -1
- package/scripts/gds/recall.js +11 -1
- package/scripts/gds/run-unit-tests.js +8 -1
- package/src/module-api.js +3 -1
- package/tests/recall_corpus_health.mjs +38 -3
- package/tests/recall_freshness_plan_db.mjs +171 -0
package/.bongos-core.json
CHANGED
|
@@ -2,22 +2,22 @@
|
|
|
2
2
|
"artifact": "bongos-core",
|
|
3
3
|
"manifest_schema": 1,
|
|
4
4
|
"generator": "scripts/gds/package-core.js",
|
|
5
|
-
"core_version": "1.19.
|
|
6
|
-
"core_contract": "1.19.
|
|
7
|
-
"source_commit": "
|
|
5
|
+
"core_version": "1.19.363",
|
|
6
|
+
"core_contract": "1.19.363",
|
|
7
|
+
"source_commit": "dfe3198cbb7ad4104276759b209d6064e4075753",
|
|
8
8
|
"source_ref": "HEAD",
|
|
9
|
-
"built_at": "2026-08-
|
|
9
|
+
"built_at": "2026-08-25T19:24:12.550Z",
|
|
10
10
|
"redaction": {
|
|
11
11
|
"model": "docs-redacted+functional-verbatim",
|
|
12
12
|
"docs_redacted": 330,
|
|
13
13
|
"agent_docs_stubbed": 22,
|
|
14
|
-
"functional_verbatim":
|
|
14
|
+
"functional_verbatim": 1527,
|
|
15
15
|
"rules": 3,
|
|
16
16
|
"gate_literals": 3,
|
|
17
17
|
"gate": "passed"
|
|
18
18
|
},
|
|
19
|
-
"file_count":
|
|
20
|
-
"tree_sha256": "
|
|
19
|
+
"file_count": 1879,
|
|
20
|
+
"tree_sha256": "11a13769c884c5b71eefc6c2d9d62e17d1096f782583e2b0cdd46ae4a92cf52d",
|
|
21
21
|
"files": [
|
|
22
22
|
{
|
|
23
23
|
"path": ".claude/skills/blocker-review/SKILL.md",
|
|
@@ -2499,6 +2499,11 @@
|
|
|
2499
2499
|
"mode": "0000644",
|
|
2500
2500
|
"sha256": "a379e9f926274559d8c57c9127024463e38b99e3f73962022b9f5d0f4e156a57"
|
|
2501
2501
|
},
|
|
2502
|
+
{
|
|
2503
|
+
"path": "migrations/core_225_search_chunks_updated_at_idx.sql",
|
|
2504
|
+
"mode": "0000644",
|
|
2505
|
+
"sha256": "1913405bdb9de48ac3ca7ad6705cf44244d20c77d3a6f73700f88e717fb84043"
|
|
2506
|
+
},
|
|
2502
2507
|
{
|
|
2503
2508
|
"path": "modules/agents/migrations/agents_001_definitions.sql",
|
|
2504
2509
|
"mode": "0000644",
|
|
@@ -4287,7 +4292,7 @@
|
|
|
4287
4292
|
{
|
|
4288
4293
|
"path": "modules/memory/search.js",
|
|
4289
4294
|
"mode": "0000644",
|
|
4290
|
-
"sha256": "
|
|
4295
|
+
"sha256": "43ede2e90d0124c42f6f9dfc3695898b83c8e876c01f70b0ae6e9d2f61d41085"
|
|
4291
4296
|
},
|
|
4292
4297
|
{
|
|
4293
4298
|
"path": "modules/onboarding/CLAUDE.md",
|
|
@@ -4847,12 +4852,12 @@
|
|
|
4847
4852
|
{
|
|
4848
4853
|
"path": "package-lock.json",
|
|
4849
4854
|
"mode": "0000644",
|
|
4850
|
-
"sha256": "
|
|
4855
|
+
"sha256": "44644726a2b97729a15e7220d9bc5aa7b7a394580ae95da58b452e8824688ff6"
|
|
4851
4856
|
},
|
|
4852
4857
|
{
|
|
4853
4858
|
"path": "package.json",
|
|
4854
4859
|
"mode": "0000644",
|
|
4855
|
-
"sha256": "
|
|
4860
|
+
"sha256": "2dc2136427cf6df206810b410527d3cc09cb3ac72c663f49d0c0ad3bb8a3951c"
|
|
4856
4861
|
},
|
|
4857
4862
|
{
|
|
4858
4863
|
"path": "public-docs/index.html",
|
|
@@ -5912,7 +5917,7 @@
|
|
|
5912
5917
|
{
|
|
5913
5918
|
"path": "scripts/gds/recall.js",
|
|
5914
5919
|
"mode": "0000644",
|
|
5915
|
-
"sha256": "
|
|
5920
|
+
"sha256": "8bb682eebeec8738082a2988f59697da6982d7c727e82352c169abc72a623b78"
|
|
5916
5921
|
},
|
|
5917
5922
|
{
|
|
5918
5923
|
"path": "scripts/gds/redteam-patrol.js",
|
|
@@ -5962,7 +5967,7 @@
|
|
|
5962
5967
|
{
|
|
5963
5968
|
"path": "scripts/gds/run-unit-tests.js",
|
|
5964
5969
|
"mode": "0000644",
|
|
5965
|
-
"sha256": "
|
|
5970
|
+
"sha256": "21726e53ad33bef62c5a2a0e0d59c380927e06cfa00994d5bbfd9669b83ce167"
|
|
5966
5971
|
},
|
|
5967
5972
|
{
|
|
5968
5973
|
"path": "scripts/gds/sandbox-stage.js",
|
|
@@ -6717,7 +6722,7 @@
|
|
|
6717
6722
|
{
|
|
6718
6723
|
"path": "src/module-api.js",
|
|
6719
6724
|
"mode": "0000644",
|
|
6720
|
-
"sha256": "
|
|
6725
|
+
"sha256": "df48c1828cda7dc37f655dcb890871fd2fd5867f223823eff07a8ae74dd1febe"
|
|
6721
6726
|
},
|
|
6722
6727
|
{
|
|
6723
6728
|
"path": "src/module-loader/catalog.js",
|
|
@@ -8762,7 +8767,12 @@
|
|
|
8762
8767
|
{
|
|
8763
8768
|
"path": "tests/recall_corpus_health.mjs",
|
|
8764
8769
|
"mode": "0000644",
|
|
8765
|
-
"sha256": "
|
|
8770
|
+
"sha256": "3b379d1fe9c938dac2ad596c5a74022c5476164a57541481f427f22ab17af35e"
|
|
8771
|
+
},
|
|
8772
|
+
{
|
|
8773
|
+
"path": "tests/recall_freshness_plan_db.mjs",
|
|
8774
|
+
"mode": "0000644",
|
|
8775
|
+
"sha256": "94fd5052df816ffad4eaeaff3ec1dfe580a66e7b588756f3b9ca84b68e01081d"
|
|
8766
8776
|
},
|
|
8767
8777
|
{
|
|
8768
8778
|
"path": "tests/recall_hints.mjs",
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
-- core_225_search_chunks_updated_at_idx.sql — task 1003145 (panel finding on 1003097).
|
|
2
|
+
--
|
|
3
|
+
-- corpusHealth() runs a freshness probe on every zero-result search: the newest
|
|
4
|
+
-- updated_at among the chunks VISIBLE to the caller. Migration 103 indexed tsv,
|
|
5
|
+
-- body trigrams, (source_kind, source_ref) and the owner partial — nothing on
|
|
6
|
+
-- updated_at — so that probe was an aggregate over the whole visible corpus. For a
|
|
7
|
+
-- Metic+ caller the scope predicate matches nearly every row, and during an ingest
|
|
8
|
+
-- outage EVERY search is a zero-result search, so it ran on every request from
|
|
9
|
+
-- every builder for the duration. That is the same cost shape that got the
|
|
10
|
+
-- COUNT(*) removed from this path in the first place; the third subquery kept it.
|
|
11
|
+
--
|
|
12
|
+
-- LEADING COLUMN IS updated_at, NOT THE SCOPE COLUMNS. The obvious index here
|
|
13
|
+
-- looks like (min_rank, owner_builder_id, updated_at), but it cannot serve this
|
|
14
|
+
-- query: the scope predicate is `min_rank = ANY($1)` plus an OR on
|
|
15
|
+
-- owner_builder_id, and neither shape lets a btree deliver rows ordered by a
|
|
16
|
+
-- TRAILING column. Ordering on updated_at instead lets the probe walk the index
|
|
17
|
+
-- newest-first and stop at the first row that passes the scope filter — which, for
|
|
18
|
+
-- the privileged caller that was the worst case, is the first row it reads. The
|
|
19
|
+
-- filter costs a heap fetch per candidate, so the win is bounded by how quickly a
|
|
20
|
+
-- visible row turns up, not by the corpus size. search.js asks for that plan
|
|
21
|
+
-- explicitly (ORDER BY updated_at DESC LIMIT 1) rather than leaving it to the
|
|
22
|
+
-- planner's MIN/MAX transform.
|
|
23
|
+
CREATE INDEX IF NOT EXISTS idx_search_chunks_updated_at
|
|
24
|
+
ON search_chunks (updated_at DESC);
|
package/modules/memory/search.js
CHANGED
|
@@ -331,7 +331,8 @@ async function markChunkUsefulness(
|
|
|
331
331
|
// NOTE for anyone reading task 1003097's brief: it points at "migration 127
|
|
332
332
|
// search_ingest_runs" as an ingest ledger to consult. No such table exists in
|
|
333
333
|
// this repo (the search schema is migration 103, search_chunks alone), so
|
|
334
|
-
// freshness here is derived from
|
|
334
|
+
// freshness here is derived from updated_at on the chunks themselves — which is
|
|
335
|
+
// why it reports the last CONTENT CHANGE and not the last ingest run (1003145).
|
|
335
336
|
// Who may be told about content they cannot read. Metic+ is the boundary this
|
|
336
337
|
// repo already uses for the permission/methodology core (ADR 0043), and
|
|
337
338
|
// distinguishing "the index is empty" from "everything is above your rank"
|
|
@@ -350,7 +351,7 @@ const PRIVILEGED_DIAGNOSTIC_RANKS = ['metic', 'archon'];
|
|
|
350
351
|
// keeping in view:
|
|
351
352
|
//
|
|
352
353
|
// SCOPING (ADR 0016). The first cut returned unfiltered total/doc/db counts and
|
|
353
|
-
// a global
|
|
354
|
+
// a global freshness stamp to every caller — an oracle for the size,
|
|
354
355
|
// composition and freshness of content the caller may not read. This route
|
|
355
356
|
// family deliberately avoids that (fetchChunkBody answers one 404 for both
|
|
356
357
|
// "missing" and "not yours"). Now: the caller's own view is scoped with the
|
|
@@ -360,10 +361,23 @@ const PRIVILEGED_DIAGNOSTIC_RANKS = ['metic', 'archon'];
|
|
|
360
361
|
// COST. It also ran a full-table COUNT(*) — and during an actual outage EVERY
|
|
361
362
|
// search is a zero-result search, so that scan would run on every request from
|
|
362
363
|
// every builder for the duration. But nothing here needs a COUNT: the states
|
|
363
|
-
// turn on whether ANY row exists, not how many.
|
|
364
|
-
// first matching row, so this is two index-assisted probes rather than a scan,
|
|
365
|
-
// and it stays flat as the corpus grows. Dropping the exact numbers also
|
|
364
|
+
// turn on whether ANY row exists, not how many. Dropping the exact numbers also
|
|
366
365
|
// removes the last of the oracle.
|
|
366
|
+
//
|
|
367
|
+
// WHAT THIS QUERY COSTS (task 1003145). The comment here used to say "two
|
|
368
|
+
// index-assisted probes rather than a scan", counting only the EXISTS pair and
|
|
369
|
+
// forgetting the third subquery — which was a filtered MAX(updated_at), an
|
|
370
|
+
// AGGREGATE over the whole visible corpus, on a column migration 103 never
|
|
371
|
+
// indexed. Wrong, and load-bearing wrong: a future reader sizing this path would
|
|
372
|
+
// have trusted it. What runs now, all three bounded in the common case:
|
|
373
|
+
// * the two EXISTS probes stop at the first matching row;
|
|
374
|
+
// * the freshness probe is ORDER BY updated_at DESC LIMIT 1 over
|
|
375
|
+
// idx_search_chunks_updated_at (migration 225), so it walks the index
|
|
376
|
+
// newest-first and stops at the first row that passes the scope filter.
|
|
377
|
+
// The honest remaining worst case is a caller who can see NOTHING: no row passes
|
|
378
|
+
// the filter, so the visible-probe and the freshness probe each read to the end
|
|
379
|
+
// before they can answer. That is the `nothing_visible` state — one of the
|
|
380
|
+
// outages these three states exist to NAME, not a steady-state cost.
|
|
367
381
|
async function corpusHealth({ builderId, builderRank } = {}, { pool: p = pool, client = null } = {}) {
|
|
368
382
|
const allowedRanks = ranksAtOrBelow(builderRank);
|
|
369
383
|
const ownerId = Number.isInteger(builderId) ? builderId : (Number(builderId) || null);
|
|
@@ -379,10 +393,27 @@ async function corpusHealth({ builderId, builderRank } = {}, { pool: p = pool, c
|
|
|
379
393
|
-- for everyone else), and cheap either way because EXISTS stops at
|
|
380
394
|
-- the first row.
|
|
381
395
|
EXISTS (SELECT 1 FROM search_chunks) AS any_at_all,
|
|
382
|
-
|
|
396
|
+
-- FRESHNESS — and NOT "when the ingest last ran". Both ingesters
|
|
397
|
+
-- delta-gate their upsert (updated_at = now() fires only when a
|
|
398
|
+
-- chunk's content actually CHANGED — search-ingest-md.js,
|
|
399
|
+
-- search-ingest-db.js), so this is the newest CONTENT CHANGE the
|
|
400
|
+
-- caller can see, and a healthy nightly ingest over an unchanged
|
|
401
|
+
-- corpus leaves it perfectly still. Named for what it measures after
|
|
402
|
+
-- task 1003145: the old name, last_indexed_at, promised a liveness
|
|
403
|
+
-- signal it cannot deliver. Real ingest liveness would need an
|
|
404
|
+
-- ingest-run ledger, not a proxy on chunk mtimes.
|
|
405
|
+
--
|
|
406
|
+
-- ORDER BY … LIMIT 1 rather than MAX(): the same answer on a NOT NULL
|
|
407
|
+
-- column, but it ASKS for the bounded plan instead of depending on the
|
|
408
|
+
-- planner's MIN/MAX transform still firing. Scoped with the SAME
|
|
409
|
+
-- predicate as the search — an unscoped stamp was the freshness half
|
|
410
|
+
-- of the oracle described above.
|
|
411
|
+
(SELECT updated_at FROM search_chunks
|
|
383
412
|
WHERE min_rank = ANY($1::text[])
|
|
384
413
|
AND (owner_builder_id IS NULL OR owner_builder_id = $2::bigint)
|
|
385
|
-
|
|
414
|
+
ORDER BY updated_at DESC
|
|
415
|
+
LIMIT 1
|
|
416
|
+
) AS last_content_change_at
|
|
386
417
|
`;
|
|
387
418
|
const runner = client || p;
|
|
388
419
|
let rows;
|
|
@@ -392,13 +423,13 @@ async function corpusHealth({ builderId, builderRank } = {}, { pool: p = pool, c
|
|
|
392
423
|
// ENRICHMENT ONLY — it runs to EXPLAIN a zero result, so it must never turn
|
|
393
424
|
// a working search into an error. Nulls fail closed through
|
|
394
425
|
// classifyEmptyResult, which never reports health it cannot prove.
|
|
395
|
-
return { any_visible: null, any_at_all: null,
|
|
426
|
+
return { any_visible: null, any_at_all: null, last_content_change_at: null, privileged };
|
|
396
427
|
}
|
|
397
428
|
const r = rows[0] || {};
|
|
398
429
|
return {
|
|
399
430
|
any_visible: r.any_visible === true || r.any_visible === false ? r.any_visible : null,
|
|
400
431
|
any_at_all: r.any_at_all === true || r.any_at_all === false ? r.any_at_all : null,
|
|
401
|
-
|
|
432
|
+
last_content_change_at: r.last_content_change_at || null,
|
|
402
433
|
privileged,
|
|
403
434
|
};
|
|
404
435
|
}
|
|
@@ -409,9 +440,11 @@ async function corpusHealth({ builderId, builderRank } = {}, { pool: p = pool, c
|
|
|
409
440
|
function classifyEmptyResult(health) {
|
|
410
441
|
const anyVisible = health && typeof health.any_visible === 'boolean' ? health.any_visible : null;
|
|
411
442
|
const privileged = !!(health && health.privileged);
|
|
412
|
-
// Never echo the unscoped probe.
|
|
413
|
-
// caller's own view by the query
|
|
414
|
-
|
|
443
|
+
// Never echo the unscoped probe. last_content_change_at is already scoped to
|
|
444
|
+
// the caller's own view by the query, and is non-null ONLY when any_visible is
|
|
445
|
+
// true (the same predicate matches nothing in every other state), so in
|
|
446
|
+
// practice it rides along with no_match and is null on all three bad ones.
|
|
447
|
+
const safe = { last_content_change_at: (health && health.last_content_change_at) || null };
|
|
415
448
|
|
|
416
449
|
if (anyVisible === null) {
|
|
417
450
|
return {
|
package/package-lock.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bongos/core",
|
|
3
|
-
"version": "1.19.
|
|
3
|
+
"version": "1.19.363",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "@bongos/core",
|
|
9
|
-
"version": "1.19.
|
|
9
|
+
"version": "1.19.363",
|
|
10
10
|
"license": "AGPL-3.0-or-later",
|
|
11
11
|
"dependencies": {
|
|
12
12
|
"express": "^4.21.2",
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@bongos/core",
|
|
3
|
-
"version": "1.19.
|
|
3
|
+
"version": "1.19.363",
|
|
4
4
|
"description": "Cloud Bongos — the AI-first build platform core (GDS + platform surfaces + module system), installed as a versioned dependency (ADR 0108).",
|
|
5
5
|
"license": "AGPL-3.0-or-later",
|
|
6
6
|
"main": "src/platform-server.js",
|
package/scripts/gds/recall.js
CHANGED
|
@@ -93,7 +93,17 @@ function renderNoMatch({ q, corpus }) {
|
|
|
93
93
|
lines: [`No matches for "${q}". The index covers repo docs + (nightly) DB prose; try fewer / different terms, or it may not be indexed yet.`],
|
|
94
94
|
};
|
|
95
95
|
}
|
|
96
|
-
|
|
96
|
+
// "Corpus last changed", NOT "last indexed" — the server derives this from
|
|
97
|
+
// updated_at on the chunks, and both ingesters delta-gate their upsert, so it
|
|
98
|
+
// moves when CONTENT changes and stands still through a healthy ingest over an
|
|
99
|
+
// unchanged corpus. It was labelled "Last indexed:" until task 1003145, which
|
|
100
|
+
// read as an ingest-liveness signal it was never measuring.
|
|
101
|
+
//
|
|
102
|
+
// Both keys are read for the version-skew window: this CLI ships before the
|
|
103
|
+
// server pin moves, so an OLD server still answers with last_indexed_at. Same
|
|
104
|
+
// value either way — only the name was wrong.
|
|
105
|
+
const changedAt = corpus.last_content_change_at || corpus.last_indexed_at;
|
|
106
|
+
const stamp = changedAt ? ` Corpus last changed: ${changedAt}.` : '';
|
|
97
107
|
if (corpus.healthy) {
|
|
98
108
|
return { stream: 'out', exitCode: 0, lines: [`No matches for "${q}". ${corpus.message}${stamp}`] };
|
|
99
109
|
}
|
|
@@ -107,7 +107,14 @@ const INTEGRATION = new Set([
|
|
|
107
107
|
// never governs under the widened predicate.
|
|
108
108
|
'government_board_membership_db',
|
|
109
109
|
'government_fold_bench', 'government_live_proof', 'linkify_refs', 'memory_isolation',
|
|
110
|
-
'memory_pull', 'needs_rebase', 'parallel-load', 'profile_rollup_consent',
|
|
110
|
+
'memory_pull', 'needs_rebase', 'parallel-load', 'profile_rollup_consent',
|
|
111
|
+
// task 1003145: whether the PLANNER uses the migration 225 index for the recall
|
|
112
|
+
// freshness probe. The DB-free sibling (recall_corpus_health) pins the query
|
|
113
|
+
// shape and that the migration declares the index; only a real Postgres can
|
|
114
|
+
// answer whether the probe stops at the first visible row or reads the corpus —
|
|
115
|
+
// and an unverified claim about exactly that is the comment this task fixed.
|
|
116
|
+
'recall_freshness_plan_db',
|
|
117
|
+
'reward_mode_cost_plus_only',
|
|
111
118
|
'scouting_rollup_fanout', 'session_records',
|
|
112
119
|
// task 1003217: the cost-row RAISE against real SQL. Every claim in it IS the
|
|
113
120
|
// SQL — an ON CONFLICT DO UPDATE guarded to move a row only upward, and a
|
package/src/module-api.js
CHANGED
|
@@ -1154,8 +1154,10 @@ const { validateOrRespond, LIMITS, parseId, asyncHandler, corsPublicGet, parsePa
|
|
|
1154
1154
|
// landed since 1.19.360 with no explicit bump. run 32883230235. (task 1002620)
|
|
1155
1155
|
// 1.19.362 — CI auto-patch (publish-on-merge, ADR 0161): carrier for merges
|
|
1156
1156
|
// landed since 1.19.361 with no explicit bump. run 32886106098. (task 1002620)
|
|
1157
|
+
// 1.19.363 — CI auto-patch (publish-on-merge, ADR 0161): carrier for merges
|
|
1158
|
+
// landed since 1.19.362 with no explicit bump. run 32889324497. (task 1002620)
|
|
1157
1159
|
// ---------------------------------------------------------------------------
|
|
1158
|
-
const CORE_VERSION = '1.19.
|
|
1160
|
+
const CORE_VERSION = '1.19.363'; // CI auto-patch carrier (ADR 0161); see changelog above
|
|
1159
1161
|
|
|
1160
1162
|
// A namespaced logger so a module's log lines are attributable + consistent.
|
|
1161
1163
|
// Usage: const log = api.logger('dev-box'); log.info('mounted');
|
|
@@ -122,13 +122,27 @@ test('an outage goes to STDERR with a non-zero exit', () => {
|
|
|
122
122
|
});
|
|
123
123
|
|
|
124
124
|
test('a genuine miss stays ordinary stdout with exit 0', () => {
|
|
125
|
-
const corpus = { ...classifyEmptyResult({ any_visible: true, any_at_all: true, privileged: false }),
|
|
125
|
+
const corpus = { ...classifyEmptyResult({ any_visible: true, any_at_all: true, privileged: false }), last_content_change_at: '2026-08-20T00:00:00.000Z' };
|
|
126
126
|
const out = renderNoMatch({ q: 'nonexistent gibberish', corpus });
|
|
127
127
|
assert.equal(out.stream, 'out');
|
|
128
128
|
assert.equal(out.exitCode, 0);
|
|
129
129
|
const text = out.lines.join('\n');
|
|
130
130
|
assert.doesNotMatch(text, /RECALL IS NOT ANSWERING/, 'never cry outage on an honest miss');
|
|
131
|
-
assert.match(text, /
|
|
131
|
+
assert.match(text, /Corpus last changed: 2026-08-20/, 'freshness is still surfaced');
|
|
132
|
+
// task 1003145: the stamp comes from chunk updated_at, and both ingesters
|
|
133
|
+
// delta-gate their upsert — so it moves on CONTENT CHANGE and stands perfectly
|
|
134
|
+
// still through a healthy ingest of an unchanged corpus. Labelling it "Last
|
|
135
|
+
// indexed" sold it as ingest liveness, the one thing it cannot report.
|
|
136
|
+
assert.doesNotMatch(text, /Last indexed/, 'the stamp must not claim to be an ingest-liveness signal');
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
test('the stamp still renders for an OLD server sending the former key', () => {
|
|
140
|
+
// Same version-skew reason the no-diagnostic case below is covered: the CLI
|
|
141
|
+
// ships before the server pin moves, so for a while it talks to a server that
|
|
142
|
+
// still answers last_indexed_at. Same value, older name — read both.
|
|
143
|
+
const corpus = { ...classifyEmptyResult({ any_visible: true, any_at_all: true, privileged: false }), last_indexed_at: '2026-08-19T00:00:00.000Z' };
|
|
144
|
+
const text = renderNoMatch({ q: 'anything', corpus }).lines.join('\n');
|
|
145
|
+
assert.match(text, /Corpus last changed: 2026-08-19/, 'the older key still renders, under the true label');
|
|
132
146
|
});
|
|
133
147
|
|
|
134
148
|
test('a server that sends no diagnostic keeps the previous behaviour', () => {
|
|
@@ -178,7 +192,28 @@ test('corpusHealth probes EXISTENCE, scoped with the same predicate as the searc
|
|
|
178
192
|
assert.match(body, /EXISTS \(/, 'existence probes instead');
|
|
179
193
|
assert.match(body, /min_rank = ANY/, 'the visible probe is rank-scoped');
|
|
180
194
|
assert.match(body, /owner_builder_id IS NULL OR owner_builder_id/, 'with the search\'s own ownership predicate');
|
|
181
|
-
|
|
195
|
+
// task 1003145: the freshness probe used to be MAX(updated_at) — an AGGREGATE
|
|
196
|
+
// over the whole visible corpus, on an unindexed column, while the comment
|
|
197
|
+
// beside it claimed the path held no scan. ORDER BY … LIMIT 1 over
|
|
198
|
+
// idx_search_chunks_updated_at (migration 225) stops at the first row that
|
|
199
|
+
// passes the scope filter instead.
|
|
200
|
+
assert.doesNotMatch(body, /MAX\(updated_at\)/, 'freshness must not be an aggregate scan');
|
|
201
|
+
assert.match(body, /ORDER BY updated_at DESC\s+LIMIT 1/, 'it asks for the bounded plan explicitly');
|
|
202
|
+
const freshness = body.slice(body.indexOf('SELECT updated_at FROM search_chunks'));
|
|
203
|
+
assert.match(freshness.slice(0, freshness.indexOf('LIMIT 1')), /min_rank = ANY/,
|
|
204
|
+
'and the freshness probe carries the search’s scope predicate too');
|
|
205
|
+
});
|
|
206
|
+
|
|
207
|
+
test('the freshness probe has an index to walk (migration 225)', () => {
|
|
208
|
+
// The LIMIT 1 only bounds anything if updated_at is ORDERED by an index —
|
|
209
|
+
// otherwise the planner sorts the visible corpus to answer it, which is the
|
|
210
|
+
// same scan wearing a new name. Migration 103 indexed tsv / body / source /
|
|
211
|
+
// owner and NOT updated_at, which is how the aggregate went unnoticed.
|
|
212
|
+
const dir = path.join(ROOT, 'migrations');
|
|
213
|
+
const sql = fs.readdirSync(dir).filter((f) => /search_chunks/.test(f))
|
|
214
|
+
.map((f) => fs.readFileSync(path.join(dir, f), 'utf8')).join('\n');
|
|
215
|
+
assert.match(sql, /CREATE INDEX IF NOT EXISTS idx_search_chunks_updated_at[\s\S]{0,80}\(updated_at DESC\)/,
|
|
216
|
+
'search_chunks must carry a descending index on updated_at');
|
|
182
217
|
});
|
|
183
218
|
|
|
184
219
|
test('the route hands the caller ONLY the classifier output', () => {
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
// tests/recall_freshness_plan_db.mjs — task 1003145, against a REAL Postgres.
|
|
2
|
+
//
|
|
3
|
+
// WHY THIS EXISTS BESIDE tests/recall_corpus_health.mjs. That file is DB-free: it
|
|
4
|
+
// pins the SHAPE of the freshness probe (ORDER BY ... LIMIT 1, not an aggregate)
|
|
5
|
+
// and that migration 225 declares an index on updated_at. Neither of those is the
|
|
6
|
+
// claim that actually matters, which is a claim about the DATABASE: that the
|
|
7
|
+
// planner USES that index, so the probe stops at the first visible row instead of
|
|
8
|
+
// reading the visible corpus.
|
|
9
|
+
//
|
|
10
|
+
// That distinction IS the bug being fixed here. The old code said MAX(updated_at)
|
|
11
|
+
// under a comment promising "two index-assisted probes rather than a scan" — a
|
|
12
|
+
// statement about the plan that nothing checked, and that was false. Answering it
|
|
13
|
+
// with only a text tripwire on the SQL would repeat the mistake one level up: the
|
|
14
|
+
// shape can be right while the plan is still a scan (no index, a lost ANALYZE, an
|
|
15
|
+
// ORDER BY the index cannot serve). So this asks the planner directly.
|
|
16
|
+
//
|
|
17
|
+
// It also proves the REWRITE IS SEMANTICALLY IDENTICAL — ORDER BY updated_at DESC
|
|
18
|
+
// LIMIT 1 returns exactly what MAX(updated_at) returned under the same predicate —
|
|
19
|
+
// because the point was to make the probe cheaper, not to change the answer.
|
|
20
|
+
//
|
|
21
|
+
// The SQL under test is SLICED OUT OF modules/memory/search.js rather than copied,
|
|
22
|
+
// so this cannot keep passing against a query the module no longer runs.
|
|
23
|
+
//
|
|
24
|
+
// It SELF-SKIPS (exit 0) with no reachable Postgres, and is listed in
|
|
25
|
+
// run-unit-tests.js INTEGRATION set so the DB-free lane never runs it.
|
|
26
|
+
//
|
|
27
|
+
// Run: DATABASE_URL=postgres://... node tests/recall_freshness_plan_db.mjs
|
|
28
|
+
//
|
|
29
|
+
// It CREATES its own throwaway search_chunks and drops it again, so point it at a
|
|
30
|
+
// scratch database, never a real one.
|
|
31
|
+
|
|
32
|
+
import pg from 'pg';
|
|
33
|
+
import fs from 'node:fs';
|
|
34
|
+
import path from 'node:path';
|
|
35
|
+
import url from 'node:url';
|
|
36
|
+
import { strict as assert } from 'node:assert';
|
|
37
|
+
|
|
38
|
+
const ROOT = path.resolve(path.dirname(url.fileURLToPath(import.meta.url)), '..');
|
|
39
|
+
const pool = new pg.Pool({ connectionString: process.env.DATABASE_URL || undefined });
|
|
40
|
+
|
|
41
|
+
try {
|
|
42
|
+
await pool.query('SELECT 1');
|
|
43
|
+
} catch (e) {
|
|
44
|
+
console.log(`recall_freshness_plan_db.mjs SKIPPED — no reachable Postgres (${e.code || e.message}). Set DATABASE_URL to run.`);
|
|
45
|
+
await pool.end().catch(() => {});
|
|
46
|
+
process.exit(0);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// Same posture as tests/claims_requested_worktree_db.mjs: this DROPs and CREATEs
|
|
50
|
+
// search_chunks, so refuse anything that looks like a real GDS database.
|
|
51
|
+
const { rows: realRows } = await pool.query(
|
|
52
|
+
`SELECT to_regclass('public.builders') IS NOT NULL AS has_builders`);
|
|
53
|
+
if (realRows[0].has_builders) {
|
|
54
|
+
console.log('recall_freshness_plan_db.mjs SKIPPED — this database has a `builders` table, so it is not a scratch DB. It DROPs and CREATEs search_chunks; point it at a throwaway Postgres.');
|
|
55
|
+
await pool.end().catch(() => {});
|
|
56
|
+
process.exit(0);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
let passed = 0;
|
|
60
|
+
let failed = 0;
|
|
61
|
+
async function test(name, fn) {
|
|
62
|
+
try { await fn(); passed++; console.log(` ok ${name}`); }
|
|
63
|
+
catch (err) { failed++; console.log(` FAIL ${name}`); console.log(` ${err.message}`); }
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
// ---- the query under test, taken from the module itself ----
|
|
67
|
+
|
|
68
|
+
const searchSrc = fs.readFileSync(path.join(ROOT, 'modules/memory/search.js'), 'utf8');
|
|
69
|
+
const OPEN = '(SELECT updated_at FROM search_chunks';
|
|
70
|
+
const start = searchSrc.indexOf(OPEN);
|
|
71
|
+
assert.ok(start > 0, 'the freshness subquery must be findable in modules/memory/search.js');
|
|
72
|
+
const end = searchSrc.indexOf(') AS last_content_change_at', start);
|
|
73
|
+
assert.ok(end > start, 'the freshness subquery must still be aliased last_content_change_at');
|
|
74
|
+
// Drop the wrapping parens so it stands alone as a statement.
|
|
75
|
+
const FRESHNESS_SQL = searchSrc.slice(start + 1, end);
|
|
76
|
+
assert.match(FRESHNESS_SQL, /ORDER BY updated_at DESC/, 'sanity: sliced the ordered form');
|
|
77
|
+
|
|
78
|
+
// The privileged caller — the case the task named, where the scope predicate
|
|
79
|
+
// matches nearly every row and the aggregate therefore scanned nearly everything.
|
|
80
|
+
const ALL_RANKS = ['xenos', 'thetes', 'metic', 'archon'];
|
|
81
|
+
const OWNER = 1;
|
|
82
|
+
|
|
83
|
+
async function explain(sql) {
|
|
84
|
+
const { rows } = await pool.query(`EXPLAIN (FORMAT JSON) ${sql}`, [ALL_RANKS, OWNER]);
|
|
85
|
+
return JSON.stringify(rows[0]['QUERY PLAN']);
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// ---- fixture ----
|
|
89
|
+
|
|
90
|
+
await pool.query('DROP TABLE IF EXISTS search_chunks');
|
|
91
|
+
// Only the columns this probe touches. The real table (migration 103) carries the
|
|
92
|
+
// tsvector, the trigram body and the rest; none of them participate in this plan.
|
|
93
|
+
await pool.query(`
|
|
94
|
+
CREATE TABLE search_chunks (
|
|
95
|
+
id bigserial PRIMARY KEY,
|
|
96
|
+
min_rank text NOT NULL DEFAULT 'xenos',
|
|
97
|
+
owner_builder_id bigint,
|
|
98
|
+
updated_at timestamptz NOT NULL DEFAULT now()
|
|
99
|
+
)`);
|
|
100
|
+
// Enough rows that a sequential scan plus a top-N sort is visibly the worse plan.
|
|
101
|
+
await pool.query(`
|
|
102
|
+
INSERT INTO search_chunks (min_rank, owner_builder_id, updated_at)
|
|
103
|
+
SELECT (ARRAY['xenos','thetes','metic','archon'])[1 + (i % 4)],
|
|
104
|
+
CASE WHEN i % 50 = 0 THEN 1 + (i % 7) ELSE NULL END,
|
|
105
|
+
now() - (i || ' minutes')::interval
|
|
106
|
+
FROM generate_series(1, 20000) AS i`);
|
|
107
|
+
await pool.query('ANALYZE search_chunks');
|
|
108
|
+
|
|
109
|
+
await test('WITHOUT the index the probe is the scan the old comment denied', async () => {
|
|
110
|
+
const plan = await explain(FRESHNESS_SQL);
|
|
111
|
+
assert.match(plan, /Seq Scan/, 'baseline: with no index on updated_at this reads the table');
|
|
112
|
+
assert.doesNotMatch(plan, /idx_search_chunks_updated_at/, 'and cannot be using an index that does not exist yet');
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
await test('migration 225 applies cleanly', async () => {
|
|
116
|
+
const sql = fs.readFileSync(path.join(ROOT, 'migrations/core_225_search_chunks_updated_at_idx.sql'), 'utf8');
|
|
117
|
+
await pool.query(sql);
|
|
118
|
+
await pool.query('ANALYZE search_chunks');
|
|
119
|
+
const { rows } = await pool.query(
|
|
120
|
+
`SELECT 1 FROM pg_indexes WHERE tablename = 'search_chunks' AND indexname = 'idx_search_chunks_updated_at'`);
|
|
121
|
+
assert.equal(rows.length, 1, 'the migration must create idx_search_chunks_updated_at');
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
await test('WITH the index the planner walks it and stops — no scan', async () => {
|
|
125
|
+
// THE CLAIM THE COMMENT MAKES. Everything else in this file is scaffolding for
|
|
126
|
+
// this assertion: the probe must reach its answer through the index, not by
|
|
127
|
+
// reading the visible corpus.
|
|
128
|
+
const plan = await explain(FRESHNESS_SQL);
|
|
129
|
+
assert.match(plan, /idx_search_chunks_updated_at/, 'the freshness probe must use the updated_at index');
|
|
130
|
+
assert.match(plan, /Index Scan/, 'and reach it by an index scan');
|
|
131
|
+
assert.doesNotMatch(plan, /Seq Scan/, 'no sequential scan may remain on this path');
|
|
132
|
+
assert.doesNotMatch(plan, /"Node Type":"Sort"/, 'and no sort — the index already supplies the order');
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
await test('the probe reads a bounded number of rows, not the corpus', async () => {
|
|
136
|
+
const { rows } = await pool.query(
|
|
137
|
+
`EXPLAIN (ANALYZE, FORMAT JSON) ${FRESHNESS_SQL}`, [ALL_RANKS, OWNER]);
|
|
138
|
+
const plan = rows[0]['QUERY PLAN'][0].Plan;
|
|
139
|
+
// A privileged caller matches nearly every row, so the first row the index hands
|
|
140
|
+
// back already passes the filter. That is the inversion this index buys: the
|
|
141
|
+
// worst case for the old aggregate is the best case for this plan.
|
|
142
|
+
assert.ok(plan['Actual Rows'] <= 1, `expected at most 1 row out, got ${plan['Actual Rows']}`);
|
|
143
|
+
const read = JSON.stringify(plan).match(/"Actual Rows": ?(\d+)/g).map((m) => Number(m.split(':')[1]));
|
|
144
|
+
assert.ok(Math.max(...read) < 100, `plan should touch a handful of rows, not 20000 (saw ${Math.max(...read)})`);
|
|
145
|
+
});
|
|
146
|
+
|
|
147
|
+
await test('the rewrite returns exactly what MAX(updated_at) returned', async () => {
|
|
148
|
+
// The point was to make the probe cheaper, NOT to change the answer. Equivalent
|
|
149
|
+
// on a NOT NULL column, and this pins it rather than trusting the reasoning.
|
|
150
|
+
const { rows: viaOrder } = await pool.query(FRESHNESS_SQL, [ALL_RANKS, OWNER]);
|
|
151
|
+
const { rows: viaMax } = await pool.query(
|
|
152
|
+
`SELECT MAX(updated_at) AS updated_at FROM search_chunks
|
|
153
|
+
WHERE min_rank = ANY($1::text[])
|
|
154
|
+
AND (owner_builder_id IS NULL OR owner_builder_id = $2::bigint)`, [ALL_RANKS, OWNER]);
|
|
155
|
+
assert.equal(String(viaOrder[0].updated_at), String(viaMax[0].updated_at));
|
|
156
|
+
});
|
|
157
|
+
|
|
158
|
+
await test('a caller who can see NOTHING still gets null, not a wrong stamp', async () => {
|
|
159
|
+
// The scope predicate is load-bearing (ADR 0016): a freshness stamp that outran
|
|
160
|
+
// the caller scope would be the freshness half of the oracle this diagnostic was
|
|
161
|
+
// rewritten to remove. No visible row must mean no stamp.
|
|
162
|
+
const { rows } = await pool.query(FRESHNESS_SQL, [['nobody'], 999999]);
|
|
163
|
+
assert.equal(rows.length, 0, 'nothing visible must yield no row, hence a null stamp');
|
|
164
|
+
});
|
|
165
|
+
|
|
166
|
+
await pool.query('DROP TABLE IF EXISTS search_chunks');
|
|
167
|
+
await pool.end();
|
|
168
|
+
|
|
169
|
+
console.log('');
|
|
170
|
+
console.log(`recall_freshness_plan_db: ${passed} passed, ${failed} failed`);
|
|
171
|
+
process.exit(failed === 0 ? 0 : 1);
|