@inetafrica/open-claudia 3.0.35 → 3.0.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/bin/cli.js +24 -0
- package/bin/pack.js +19 -2
- package/bin/schedule.js +71 -2
- package/core/actions.js +2 -2
- package/core/auth-flow.js +5 -3
- package/core/credential-detect.js +55 -0
- package/core/handlers.js +73 -0
- package/core/loopback.js +52 -0
- package/core/packs.js +51 -1
- package/core/recall/discoverer.js +123 -20
- package/core/recall/kpi.js +23 -2
- package/core/recall/tuning.js +18 -0
- package/core/router.js +53 -9
- package/core/scheduler.js +105 -5
- package/core/system-prompt.js +71 -27
- package/package.json +9 -3
- package/test-cokept-warming.js +117 -0
- package/test-credential-capture.js +109 -0
- package/test-extractive-injection.js +70 -0
- package/test-pack-search.js +79 -0
- package/test-provider-language.js +28 -9
- package/test-provider-scheduler.js +90 -0
- package/test-recall-evolution.js +9 -1
- package/test-score-gate.js +104 -0
- package/test-walker-score.js +69 -0
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
// WS1: extractive per-line search (packs.searchLines). Unlike matchPacks,
|
|
2
|
+
// which routes whole packs, searchLines returns the individual lines that
|
|
3
|
+
// best match a query, tagged by section — the substrate the extractive
|
|
4
|
+
// injector uses to surface a verbatim fact from anywhere in a pack, not just
|
|
5
|
+
// Stance. Covers: cross-section retrieval, section tagging, verbatim text,
|
|
6
|
+
// distinct-term scoring, dir scoping, k cap, stopword/empty guards, archived
|
|
7
|
+
// exclusion.
|
|
8
|
+
const assert = require("assert");
|
|
9
|
+
const fs = require("fs");
|
|
10
|
+
const os = require("os");
|
|
11
|
+
const path = require("path");
|
|
12
|
+
|
|
13
|
+
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "pack-search-"));
|
|
14
|
+
process.env.PACKS_DIR = path.join(tmp, "packs");
|
|
15
|
+
|
|
16
|
+
const packs = require("./core/packs");
|
|
17
|
+
|
|
18
|
+
packs.createPack({
|
|
19
|
+
dir: "deploy-notes",
|
|
20
|
+
name: "Deploy Notes",
|
|
21
|
+
description: "how we ship",
|
|
22
|
+
stance: "Prefer GitOps for what runs in the cluster.",
|
|
23
|
+
procedure: "Run the smoke test before every deploy.\nTag the release then push via git-release.",
|
|
24
|
+
state: "The ArgoCD sync for payments is currently paused.",
|
|
25
|
+
journal: "2026-07-01 deployed v3 to the cluster without incident.",
|
|
26
|
+
});
|
|
27
|
+
packs.createPack({
|
|
28
|
+
dir: "billing",
|
|
29
|
+
name: "Billing",
|
|
30
|
+
description: "invoices and refunds",
|
|
31
|
+
stance: "Refunds always require a manager approval.",
|
|
32
|
+
procedure: "Generate the invoice, then email it to the customer.",
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
// ── cross-section: a fact in Procedure/State/Journal is retrievable, proving
|
|
36
|
+
// searchLines reaches beyond Stance (the felt-regression fix). ──
|
|
37
|
+
const argo = packs.searchLines("argocd sync payments");
|
|
38
|
+
assert.ok(argo.length > 0, "finds a matching line");
|
|
39
|
+
assert.strictEqual(argo[0].dir, "deploy-notes", "matched the right pack");
|
|
40
|
+
assert.strictEqual(argo[0].section, "State", "line is tagged with its section (State, not Stance)");
|
|
41
|
+
assert.ok(/ArgoCD sync for payments/.test(argo[0].text), "returns the verbatim source line");
|
|
42
|
+
assert.ok(argo[0].lineno >= 1, "reports a 1-based line number");
|
|
43
|
+
assert.ok(argo[0].score > 0, "reports a positive score");
|
|
44
|
+
|
|
45
|
+
// ── verbatim: returned text is the source line trimmed, never paraphrased ──
|
|
46
|
+
const smoke = packs.searchLines("smoke test deploy", { dir: "deploy-notes" });
|
|
47
|
+
assert.ok(smoke.some((h) => h.text === "Run the smoke test before every deploy." && h.section === "Procedure"),
|
|
48
|
+
"verbatim line preserved exactly, correct section");
|
|
49
|
+
|
|
50
|
+
// ── scoring: a line covering more distinct query terms outranks one covering
|
|
51
|
+
// fewer, so the injector's top hit is the densest match. ──
|
|
52
|
+
const rank = packs.searchLines("release git-release push tag");
|
|
53
|
+
assert.strictEqual(rank[0].section, "Procedure", "the line with the most distinct query terms ranks first");
|
|
54
|
+
assert.ok(/git-release/.test(rank[0].text), "top hit is the multi-term line");
|
|
55
|
+
|
|
56
|
+
// ── dir scoping: --pack restricts the search to one pack ──
|
|
57
|
+
const scoped = packs.searchLines("approval invoice refund", { dir: "billing" });
|
|
58
|
+
assert.ok(scoped.length > 0, "dir-scoped search returns billing lines");
|
|
59
|
+
assert.ok(scoped.every((h) => h.dir === "billing"), "dir scoping excludes other packs");
|
|
60
|
+
|
|
61
|
+
// ── k cap: never returns more than k hits ──
|
|
62
|
+
const capped = packs.searchLines("the", { k: 2 }); // 'the' is a stopword → no terms
|
|
63
|
+
assert.strictEqual(capped.length, 0, "a stopword-only query yields no terms and no hits");
|
|
64
|
+
const capped2 = packs.searchLines("deploy invoice refund approval release", { k: 2 });
|
|
65
|
+
assert.ok(capped2.length <= 2, "k caps the number of hits");
|
|
66
|
+
|
|
67
|
+
// ── guards: empty query → [] ──
|
|
68
|
+
assert.deepStrictEqual(packs.searchLines(""), [], "empty query returns nothing");
|
|
69
|
+
assert.deepStrictEqual(packs.searchLines(" "), [], "whitespace query returns nothing");
|
|
70
|
+
|
|
71
|
+
// ── no match → [] ──
|
|
72
|
+
assert.deepStrictEqual(packs.searchLines("kubernetes helm quantum"), [], "no matching lines returns []");
|
|
73
|
+
|
|
74
|
+
// ── archived packs are excluded from a corpus-wide search ──
|
|
75
|
+
packs.archivePack("billing");
|
|
76
|
+
const afterArchive = packs.searchLines("approval invoice refund");
|
|
77
|
+
assert.ok(afterArchive.every((h) => h.dir !== "billing"), "archived pack lines are not surfaced");
|
|
78
|
+
|
|
79
|
+
console.log("pack search OK");
|
|
@@ -32,15 +32,34 @@ assert.doesNotMatch(setup, /Your Claude Code bot is connected|Description=Claude
|
|
|
32
32
|
|
|
33
33
|
const pkg = JSON.parse(read("package.json"));
|
|
34
34
|
assert.match(pkg.description, /provider-agnostic coding-agent harness/i);
|
|
35
|
-
// 3.0.
|
|
36
|
-
//
|
|
37
|
-
//
|
|
38
|
-
//
|
|
39
|
-
//
|
|
40
|
-
//
|
|
41
|
-
//
|
|
42
|
-
//
|
|
43
|
-
|
|
35
|
+
// 3.0.36 approved by Sumeet 2026-07-17 ("One bug is when you change model to a
|
|
36
|
+
// Claude it asks you to sign in again" → "Go ahead"): picking a Claude model no
|
|
37
|
+
// longer forces a re-login. The model-button handler and the /login picker
|
|
38
|
+
// judged "connected?" from a shallow token probe (getClaudeOAuthToken) that
|
|
39
|
+
// only sees .env/process-env/vault tokens, so a Claude CLI-store login read as
|
|
40
|
+
// disconnected and every tap restarted login — a loop, since that login stores
|
|
41
|
+
// no bot token. Both now use claudeProviderAuthStatus() (honours a bot token
|
|
42
|
+
// AND a `claude auth status` CLI-store login), matching the Codex branch.
|
|
43
|
+
// 3.0.37 approved by Sumeet 2026-07-17 ("when done please push out the new
|
|
44
|
+
// version"): adds silent background watches (a cron carrying a shell `check`
|
|
45
|
+
// that only wakes the agent on exit 0, owner-gated via /watch) and
|
|
46
|
+
// provider-agnostic loose-credential capture (a pasted secret is stored
|
|
47
|
+
// out-of-band and the turn stopped before the model — Codex refuses secrets in
|
|
48
|
+
// its prompt, so the harness intercepts the paste itself).
|
|
49
|
+
// 3.0.40 approved by Sumeet 2026-07-18 ("please go ahead and work through end
|
|
50
|
+
// to end till it's fully built and deploy out a new version for me ... I want
|
|
51
|
+
// you to fully complete the plan then let me know to upgrade and test"): the
|
|
52
|
+
// Pass-4 recall overhaul. Third cut — v3.0.38 bumped the version but forgot to
|
|
53
|
+
// move THIS pin (pretest caught it); v3.0.39 fixed the pin but a new test
|
|
54
|
+
// (test-cokept-warming.js) asserted a sqlite-graph write that no-ops on CI's
|
|
55
|
+
// node:20 (no node:sqlite), so it now skips that path like test-recall-graph.js.
|
|
56
|
+
// Payload unchanged: extractive on-open injection (Stance verbatim + fuzzy-
|
|
57
|
+
// matched, section-tagged lines) surfaces the actual matched facts instead of a
|
|
58
|
+
// headline; a walker score-gate drops weakly-relevant nodes below the relevance
|
|
59
|
+
// floor and tightens the cascade so sticky auto-keeps route back to the judge; a
|
|
60
|
+
// KPI ruler stamps per-version relevance/gate/warm columns. Co-kept Hebbian
|
|
61
|
+
// warming ships dark (default OFF).
|
|
62
|
+
assert.strictEqual(pkg.version, "3.0.40", "release version must remain unchanged without explicit approval");
|
|
44
63
|
for (const keyword of ["claude", "codex", "coding-agent", "provider-agnostic"]) {
|
|
45
64
|
assert.ok(pkg.keywords.includes(keyword), `package keyword missing: ${keyword}`);
|
|
46
65
|
}
|
|
@@ -10,6 +10,14 @@ const path = require("path");
|
|
|
10
10
|
const root = __dirname;
|
|
11
11
|
const fixturesRoot = path.join(root, "test-fixtures", "migrations");
|
|
12
12
|
|
|
13
|
+
// Keep watch state off the real ~/.open-claudia during the run. scheduler.js
|
|
14
|
+
// reads these paths from env at require time, so they must be set before main()
|
|
15
|
+
// pulls in ./core/scheduler.
|
|
16
|
+
const watchTmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "oc-watch-test-"));
|
|
17
|
+
process.env.OPEN_CLAUDIA_WATCH_LOG_FILE = path.join(watchTmpDir, "watch.log");
|
|
18
|
+
process.env.OPEN_CLAUDIA_WATCHES_ENABLED_FILE = path.join(watchTmpDir, "watches-enabled");
|
|
19
|
+
const watchesEnabledFile = process.env.OPEN_CLAUDIA_WATCHES_ENABLED_FILE;
|
|
20
|
+
|
|
13
21
|
function readFixture(name, file = "jobs.json") {
|
|
14
22
|
return JSON.parse(fs.readFileSync(path.join(fixturesRoot, name, file), "utf8"));
|
|
15
23
|
}
|
|
@@ -423,6 +431,7 @@ function schedulerHarness(createSchedulerService, seed, options = {}) {
|
|
|
423
431
|
retryDelayMs: 1000,
|
|
424
432
|
busyDelayMs: 250,
|
|
425
433
|
missedGraceMs: 60 * 60 * 1000,
|
|
434
|
+
...(options.runWatchCheck ? { runWatchCheck: options.runWatchCheck } : {}),
|
|
426
435
|
});
|
|
427
436
|
return { service, jobsStore, scheduled, admissions, runCalls, sends, state };
|
|
428
437
|
}
|
|
@@ -727,6 +736,78 @@ function loopbackMetadataProbe(runner, loopbackClient) {
|
|
|
727
736
|
}
|
|
728
737
|
}
|
|
729
738
|
|
|
739
|
+
async function watchGateProbe(createSchedulerService) {
|
|
740
|
+
const successOutcome = () => ({
|
|
741
|
+
ok: true,
|
|
742
|
+
status: "succeeded",
|
|
743
|
+
persistence: { transcript: { ok: true }, usage: { ok: true }, session: { ok: true } },
|
|
744
|
+
delivery: { ok: true },
|
|
745
|
+
});
|
|
746
|
+
const watchJob = (overrides = {}) => baseJob({
|
|
747
|
+
id: "watch-1",
|
|
748
|
+
kind: "cron",
|
|
749
|
+
schedule: "* * * * *",
|
|
750
|
+
fireAt: undefined,
|
|
751
|
+
occurrenceId: "watch-occurrence",
|
|
752
|
+
check: "true",
|
|
753
|
+
repeat: false,
|
|
754
|
+
label: "disk watch",
|
|
755
|
+
...overrides,
|
|
756
|
+
});
|
|
757
|
+
|
|
758
|
+
// Failing check → idle skip: agent never wakes, nothing hits chat, job stays pending.
|
|
759
|
+
const idle = schedulerHarness(createSchedulerService, [watchJob({ id: "watch-idle" })], {
|
|
760
|
+
runWatchCheck: async () => ({ code: 1, timedOut: false }),
|
|
761
|
+
outcomes: [successOutcome()],
|
|
762
|
+
});
|
|
763
|
+
const idleResult = await idle.service.fireJob("watch-idle");
|
|
764
|
+
assert.strictEqual(idleResult.status, "watch-idle");
|
|
765
|
+
assert.strictEqual(idleResult.skipped, true);
|
|
766
|
+
assert.strictEqual(idle.runCalls.length, 0, "failed check must not wake the agent");
|
|
767
|
+
assert.strictEqual(idle.sends.length, 0, "idle poll never touches chat");
|
|
768
|
+
const idleJob = idle.jobsStore.get("watch-idle");
|
|
769
|
+
assert.ok(idleJob, "idle watch is left in the store");
|
|
770
|
+
assert.strictEqual(idleJob.status, "pending", "idle watch stays pending for the next tick");
|
|
771
|
+
|
|
772
|
+
// Passing check → wakes agent once with the Watch header; one-shot watch is removed after firing.
|
|
773
|
+
const fired = schedulerHarness(createSchedulerService, [watchJob({ id: "watch-fire" })], {
|
|
774
|
+
runWatchCheck: async () => ({ code: 0, timedOut: false }),
|
|
775
|
+
outcomes: [successOutcome()],
|
|
776
|
+
});
|
|
777
|
+
const firedResult = await fired.service.fireJob("watch-fire");
|
|
778
|
+
assert.strictEqual(firedResult.ok, true);
|
|
779
|
+
assert.strictEqual(firedResult.status, "succeeded");
|
|
780
|
+
assert.strictEqual(fired.runCalls.length, 1, "passing check wakes the agent exactly once");
|
|
781
|
+
assert.match(fired.runCalls[0].prompt, /Watch triggered/, "watch fire carries the Watch triggered header");
|
|
782
|
+
assert.strictEqual(fired.jobsStore.get("watch-fire"), null, "one-shot watch is removed after firing");
|
|
783
|
+
|
|
784
|
+
// Repeat watch → fires but survives with a fresh occurrence so the next tick polls again.
|
|
785
|
+
const repeating = schedulerHarness(createSchedulerService, [watchJob({ id: "watch-repeat", repeat: true })], {
|
|
786
|
+
runWatchCheck: async () => ({ code: 0, timedOut: false }),
|
|
787
|
+
outcomes: [successOutcome()],
|
|
788
|
+
});
|
|
789
|
+
await repeating.service.fireJob("watch-repeat");
|
|
790
|
+
assert.strictEqual(repeating.runCalls.length, 1);
|
|
791
|
+
const repeatJob = repeating.jobsStore.get("watch-repeat");
|
|
792
|
+
assert.ok(repeatJob, "repeat watch survives firing");
|
|
793
|
+
assert.strictEqual(repeatJob.status, "pending");
|
|
794
|
+
assert.strictEqual(repeatJob.occurrenceId, "next-occ-1", "repeat watch advances to a new occurrence");
|
|
795
|
+
|
|
796
|
+
// Global master switch OFF → skip before the check even runs.
|
|
797
|
+
fs.writeFileSync(watchesEnabledFile, "off");
|
|
798
|
+
let checkRan = false;
|
|
799
|
+
const disabled = schedulerHarness(createSchedulerService, [watchJob({ id: "watch-off" })], {
|
|
800
|
+
runWatchCheck: async () => { checkRan = true; return { code: 0, timedOut: false }; },
|
|
801
|
+
outcomes: [successOutcome()],
|
|
802
|
+
});
|
|
803
|
+
const disabledResult = await disabled.service.fireJob("watch-off");
|
|
804
|
+
assert.strictEqual(disabledResult.status, "watch-disabled");
|
|
805
|
+
assert.strictEqual(disabledResult.skipped, true);
|
|
806
|
+
assert.strictEqual(checkRan, false, "disabled watches never run their check");
|
|
807
|
+
assert.strictEqual(disabled.runCalls.length, 0);
|
|
808
|
+
fs.rmSync(watchesEnabledFile, { force: true });
|
|
809
|
+
}
|
|
810
|
+
|
|
730
811
|
function staticSchedulingProbe() {
|
|
731
812
|
const files = [
|
|
732
813
|
"core/jobs.js",
|
|
@@ -742,6 +823,13 @@ function staticSchedulingProbe() {
|
|
|
742
823
|
assert.ok(!/\bsessionKey\b/.test(source[file]), `${file} must not use the removed sessionKey scheduling field`);
|
|
743
824
|
}
|
|
744
825
|
assert.match(source["bin/cli.js"], /selected provider session/i);
|
|
826
|
+
|
|
827
|
+
// Watch feature must be wired end to end: gate → loopback → CLI surfaces.
|
|
828
|
+
assert.match(source["core/scheduler.js"], /job\.check/, "scheduler must gate firing on a watch check");
|
|
829
|
+
assert.match(source["core/scheduler.js"], /watchesEnabled/, "scheduler must consult the global watch switch");
|
|
830
|
+
assert.match(source["core/loopback.js"], /watch-add/, "loopback must route watch-add");
|
|
831
|
+
assert.match(source["bin/schedule.js"], /runWatchAdd/, "schedule CLI must expose watch-add");
|
|
832
|
+
assert.match(source["bin/cli.js"], /watch-add/, "cli must dispatch watch-add");
|
|
745
833
|
}
|
|
746
834
|
|
|
747
835
|
async function main() {
|
|
@@ -762,9 +850,11 @@ async function main() {
|
|
|
762
850
|
await busyRestartAndMissedProbe(schedulerApi.createSchedulerService);
|
|
763
851
|
await unavailableAndFallbackProbe(schedulerApi.createSchedulerService);
|
|
764
852
|
await cronOccurrenceProbe(schedulerApi.createSchedulerService);
|
|
853
|
+
await watchGateProbe(schedulerApi.createSchedulerService);
|
|
765
854
|
creationEnrichmentProbe(schedulerApi.createSchedulerService);
|
|
766
855
|
loopbackMetadataProbe(require("./core/runner"), require("./bin/loopback-client"));
|
|
767
856
|
staticSchedulingProbe();
|
|
857
|
+
try { fs.rmSync(watchTmpDir, { recursive: true, force: true }); } catch (_) {}
|
|
768
858
|
console.log("provider scheduler OK");
|
|
769
859
|
}
|
|
770
860
|
|
package/test-recall-evolution.js
CHANGED
|
@@ -168,7 +168,7 @@ assert.ok(fs.existsSync(path.join(backupRoot, "packs", "backfill-me")), "deduped
|
|
|
168
168
|
metrics._resetForTest();
|
|
169
169
|
const L = (o) => JSON.stringify(o) + "\n";
|
|
170
170
|
fs.writeFileSync(metrics.LOG_FILE, [
|
|
171
|
-
L({ ts: "2026-06-01T00:00:00Z", v: "2.6.50", model: "haiku", tier: "full", seeds: [{ id: "pack:a" }], kept: [{ id: "pack:a" }, { id: "pack:b" }], latencyMs: 10000, costUsd: 0.01 }),
|
|
171
|
+
L({ ts: "2026-06-01T00:00:00Z", v: "2.6.50", model: "haiku", tier: "full", seeds: [{ id: "pack:a" }], kept: [{ id: "pack:a", score: 80 }, { id: "pack:b", score: 60 }], gate: { min: 55, out: ["pack:z"] }, warmed: [["pack:a", "pack:b"]], latencyMs: 10000, costUsd: 0.01 }),
|
|
172
172
|
L({ ts: "2026-06-01T00:01:00Z", v: "2.6.50", use: ["pack:a"] }),
|
|
173
173
|
L({ ts: "2026-06-02T00:00:00Z", v: "2.6.50", model: "haiku", tier: "full", gated: true, latencyMs: 5 }),
|
|
174
174
|
L({ ts: "2026-07-01T00:00:00Z", v: "2.6.59", model: "haiku", tier: "seeds", seeds: [{ id: "pack:a" }], kept: [{ id: "pack:a" }], latencyMs: 2000, costUsd: 0, auto: { kept: ["pack:a"], dropped: ["pack:z"] } }),
|
|
@@ -183,10 +183,17 @@ assert.strictEqual(v50.gatedPct, 50);
|
|
|
183
183
|
assert.strictEqual(v50.rescuePct, 100, "kept-without-seed counts as a rescue turn");
|
|
184
184
|
assert.strictEqual(v50.noisePct, 50, "one of two kept nodes was used");
|
|
185
185
|
assert.deepStrictEqual(v50.models, ["haiku"]);
|
|
186
|
+
// Pass-4 (WS0) ruler columns: relevance (avg walker score), gate-pass, warmings
|
|
187
|
+
assert.strictEqual(v50.avgKeptScore, 70, "relevance = mean walker score across kept nodes (80,60)");
|
|
188
|
+
assert.strictEqual(v50.gatedNodes, 1, "score-gate dropped one node this window");
|
|
189
|
+
assert.strictEqual(v50.gateFiredPct, 100, "the one gate-active turn fired");
|
|
190
|
+
assert.strictEqual(v50.warmings, 1, "one co-kept warming applied");
|
|
186
191
|
const v59 = rows.find((r) => r.v === "2.6.59");
|
|
187
192
|
assert.strictEqual(v59.rescuePct, 0);
|
|
188
193
|
assert.strictEqual(v59.autoKept, 1);
|
|
189
194
|
assert.strictEqual(v59.autoDropped, 1);
|
|
195
|
+
assert.strictEqual(v59.avgKeptScore, null, "a window with no scored keeps reports null relevance, not 0");
|
|
196
|
+
assert.strictEqual(v59.warmings, 0, "no warmings logged → zero");
|
|
190
197
|
assert.strictEqual(rows.find((r) => r.v === kpi.LEGACY_VERSION).walkerFailPct, 100);
|
|
191
198
|
|
|
192
199
|
metrics.appendKpi({ trigger: "test", health: { rescueRate: 0.25, noiseRate: 0.5, avgLatencyMs: 8000 }, window: { turns: 40 } });
|
|
@@ -197,6 +204,7 @@ assert.ok(snaps[0].v && snaps[0].ts, "snapshots are version-stamped");
|
|
|
197
204
|
|
|
198
205
|
const table = kpi.renderTable(rows);
|
|
199
206
|
assert.ok(/2\.6\.50/.test(table) && /rescue/.test(table), "table renders versions and metrics");
|
|
207
|
+
assert.ok(/relevance/.test(table) && /warm/.test(table), "table surfaces the Pass-4 ruler columns");
|
|
200
208
|
const html = kpi.htmlReport();
|
|
201
209
|
assert.ok(/<svg/.test(html) && /2\.6\.59/.test(html) && /rescue/i.test(html), "HTML report has charts and versions");
|
|
202
210
|
const outPath = kpi.writeHtmlReport(path.join(tmp, "kpi.html"));
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
// WS4: injection score-gate (the keystone). A walker-kept node scoring below
|
|
2
|
+
// the relevance floor (minRelevanceScore, default 55) is NOT injected — this is
|
|
3
|
+
// what stops a weakly-linked/incidental pack from riding along (Defect B, the
|
|
4
|
+
// cascade precision leak). The gate only fires on a full-tier successful walk;
|
|
5
|
+
// a fail-open walk (garbage/empty output → scores null) bypasses it and takes
|
|
6
|
+
// the classic top-3 user-seed path. Crucially, a successful walk that gates
|
|
7
|
+
// EVERY node leaves an empty (non-null) kept map → inject nothing — which is a
|
|
8
|
+
// different signal from fail-open (keep the strongest seeds). This test drives
|
|
9
|
+
// the gate end-to-end through disc.run() with an injected fake walker.
|
|
10
|
+
const assert = require("assert");
|
|
11
|
+
const fs = require("fs");
|
|
12
|
+
const os = require("os");
|
|
13
|
+
const path = require("path");
|
|
14
|
+
|
|
15
|
+
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "recall-gate-"));
|
|
16
|
+
process.env.RECALL_GRAPH_DB = path.join(tmp, "graph.db");
|
|
17
|
+
process.env.RECALL_DISCOVERER_WALKER = "on"; // walker ON — we inject a fake one
|
|
18
|
+
process.env.RECALL_METRICS = "off";
|
|
19
|
+
process.env.TOOLS_DIR = path.join(tmp, "tools");
|
|
20
|
+
process.env.RECALL_CASCADE = "off"; // send both candidates straight to the judge (no history/shadow randomness)
|
|
21
|
+
delete process.env.RECALL_MIN_RELEVANCE_SCORE; // ensure the default floor (55) applies, not an operator pin
|
|
22
|
+
|
|
23
|
+
const disc = require("./core/recall/discoverer");
|
|
24
|
+
|
|
25
|
+
// Two packs; matchPacks returns BOTH for a "widget" query, so both become
|
|
26
|
+
// full-tier candidates that reach the walker as pack:alpha / pack:beta.
|
|
27
|
+
const packs = {
|
|
28
|
+
alpha: { dir: "alpha", name: "Alpha", description: "alpha pack",
|
|
29
|
+
parent: null, tags: [], sections: { Stance: "alpha stance", Procedure: "", State: "", Journal: "" } },
|
|
30
|
+
beta: { dir: "beta", name: "Beta", description: "beta pack",
|
|
31
|
+
parent: null, tags: [], sections: { Stance: "beta stance", Procedure: "", State: "", Journal: "" } },
|
|
32
|
+
};
|
|
33
|
+
const packsLib = {
|
|
34
|
+
matchPacks: (text) => /widget/.test(text)
|
|
35
|
+
? [{ dir: "alpha", name: "Alpha", score: 5 }, { dir: "beta", name: "Beta", score: 4 }]
|
|
36
|
+
: [],
|
|
37
|
+
readPack: (dir) => packs[dir] || null,
|
|
38
|
+
listPacks: () => Object.values(packs),
|
|
39
|
+
searchLines: () => [], // extractive path present but yields no line hits here — keeps the block simple
|
|
40
|
+
PACKS_DIR: tmp,
|
|
41
|
+
};
|
|
42
|
+
const entitiesLib = { matchEntities: () => [], readEntity: () => null, listEntities: () => [], ENTITIES_DIR: tmp };
|
|
43
|
+
const mergeMatches = (primary, secondary, keyOf) => {
|
|
44
|
+
const out = primary.map((m) => ({ ...m, origin: "user" }));
|
|
45
|
+
const seen = new Set(primary.map(keyOf));
|
|
46
|
+
for (const m of secondary) if (!seen.has(keyOf(m))) out.push({ ...m, origin: "context" });
|
|
47
|
+
return out;
|
|
48
|
+
};
|
|
49
|
+
let builtPacks = null;
|
|
50
|
+
const buildPackBlock = (matches) => { builtPacks = matches; return matches.length ? "PACKBLOCK" : ""; };
|
|
51
|
+
const buildEntityBlock = (matches) => (matches.length ? "ENTITYBLOCK" : "");
|
|
52
|
+
const helpers = { packsLib, entitiesLib, mergeMatches, buildPackBlock, buildEntityBlock };
|
|
53
|
+
|
|
54
|
+
// Inject a fake walker: force the cold path (warm disabled), a fixed provider,
|
|
55
|
+
// and a canned agent reply. walk() parses the JSON array out of `text`.
|
|
56
|
+
function walkerReturning(text) {
|
|
57
|
+
return {
|
|
58
|
+
warmWalker: { isEnabled: () => false },
|
|
59
|
+
resolveUtilityProvider: () => ({ provider: {}, providerId: "fake", model: "fake-model" }),
|
|
60
|
+
spawnUtilityAgent: async () => ({ text, provider: "fake", model: "fake-model" }),
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
const QUERY = "tell me about the widget deployment process in detail";
|
|
64
|
+
const runGate = (walkerText, extra = {}) => disc.run({
|
|
65
|
+
userText: QUERY, contextText: "", fullContext: QUERY,
|
|
66
|
+
packLimit: 6, budget: {}, helpers, walkerOptions: walkerReturning(walkerText), ...extra,
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
(async () => {
|
|
70
|
+
// ── Test 1: default floor 55 — alpha@80 kept, beta@40 gated out ──────────────
|
|
71
|
+
builtPacks = null;
|
|
72
|
+
const res1 = await runGate('[{"id":"pack:alpha","why":"directly relevant","score":80},{"id":"pack:beta","why":"weakly linked","score":40}]');
|
|
73
|
+
assert.strictEqual(res1.tier, "full", "substantive widget query lands on the full tier");
|
|
74
|
+
assert.ok(res1.packMatches.some((m) => m.dir === "alpha"), "alpha (score 80 ≥ 55) is kept");
|
|
75
|
+
assert.ok(!res1.packMatches.some((m) => m.dir === "beta"), "beta (score 40 < 55) is gated out of injection");
|
|
76
|
+
assert.ok(res1.why["pack:alpha"], "kept alpha carries its why");
|
|
77
|
+
assert.ok(!res1.why["pack:beta"], "gated beta is absent from the why map");
|
|
78
|
+
assert.ok(builtPacks && builtPacks.length === 1 && builtPacks[0].dir === "alpha", "only alpha reaches the block builder");
|
|
79
|
+
|
|
80
|
+
// ── Test 2: gate disabled (floor 0) — beta@40 now survives ───────────────────
|
|
81
|
+
process.env.RECALL_MIN_RELEVANCE_SCORE = "0";
|
|
82
|
+
const res2 = await runGate('[{"id":"pack:alpha","why":"relevant","score":80},{"id":"pack:beta","why":"weak","score":40}]');
|
|
83
|
+
assert.ok(res2.packMatches.some((m) => m.dir === "alpha") && res2.packMatches.some((m) => m.dir === "beta"),
|
|
84
|
+
"floor 0 → gate off → both kept, including beta@40");
|
|
85
|
+
delete process.env.RECALL_MIN_RELEVANCE_SCORE;
|
|
86
|
+
|
|
87
|
+
// ── Test 3: fail-open — garbage walker output (scores null) bypasses the gate ─
|
|
88
|
+
const res3 = await runGate("this is not json at all, sorry");
|
|
89
|
+
assert.ok(res3.packMatches.some((m) => m.dir === "alpha") && res3.packMatches.some((m) => m.dir === "beta"),
|
|
90
|
+
"unparseable walk → fail-open keeps the strongest user seeds, gate is bypassed");
|
|
91
|
+
|
|
92
|
+
// ── Test 4: all-low scores at floor 55 — inject NOTHING (empty, not fail-open) ─
|
|
93
|
+
// The distinguishing case: a successful walk that gates every node must leave
|
|
94
|
+
// an empty kept set (packMatches [], packBlock ""), NOT fall back to the
|
|
95
|
+
// fail-open top-3 seed path (which would wrongly keep both).
|
|
96
|
+
builtPacks = null;
|
|
97
|
+
const res4 = await runGate('[{"id":"pack:alpha","why":"meh","score":30},{"id":"pack:beta","why":"meh","score":20}]');
|
|
98
|
+
assert.strictEqual(res4.packMatches.length, 0, "all scores below floor → nothing injected (empty non-null kept map)");
|
|
99
|
+
assert.strictEqual(res4.packBlock, "", "empty kept set renders no packBlock");
|
|
100
|
+
assert.deepStrictEqual(res4.why, {}, "empty kept set → empty why map (distinct from fail-open)");
|
|
101
|
+
assert.strictEqual(builtPacks && builtPacks.length, 0, "the block builder is handed zero packs, proving this is NOT the fail-open path");
|
|
102
|
+
|
|
103
|
+
console.log("score gate OK");
|
|
104
|
+
})().catch((e) => { console.error(e); process.exit(1); });
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
// WS2: the walker's score contract. The router now emits {id, why, score};
|
|
2
|
+
// walk() parses score into a parallel id→0-100 map while kept stays id→why, so
|
|
3
|
+
// downstream (injection score-gate, asymmetric warming) gets a relevance signal
|
|
4
|
+
// without reshaping every why-consumer. A missing/garbage score defaults to a
|
|
5
|
+
// neutral 60 so a parse hiccup never silently buries a node the router kept.
|
|
6
|
+
const assert = require("assert");
|
|
7
|
+
|
|
8
|
+
process.env.RECALL_DISCOVERER_WALKER = "on"; // ensure WALKER_ENABLED at module load
|
|
9
|
+
const disc = require("./core/recall/discoverer");
|
|
10
|
+
|
|
11
|
+
const candidates = [
|
|
12
|
+
{ id: "pack:alpha", name: "Alpha" },
|
|
13
|
+
{ id: "pack:beta", name: "Beta" },
|
|
14
|
+
{ id: "pack:gamma", name: "Gamma" },
|
|
15
|
+
{ id: "pack:delta", name: "Delta" },
|
|
16
|
+
{ id: "pack:epsilon", name: "Epsilon" },
|
|
17
|
+
];
|
|
18
|
+
|
|
19
|
+
// Drive the real runWalker → parse path with an injected cold-spawn agent.
|
|
20
|
+
function fakeWalk(replyArray, text) {
|
|
21
|
+
return disc.walk("do the alpha thing", "", candidates, {
|
|
22
|
+
warmWalker: { isEnabled: () => false }, // force the cold spawn path
|
|
23
|
+
resolveUtilityProvider: () => ({ provider: {}, providerId: "fake", model: "fake-model" }),
|
|
24
|
+
spawnUtilityAgent: async () => ({
|
|
25
|
+
text: text != null ? text : JSON.stringify(replyArray),
|
|
26
|
+
provider: "fake",
|
|
27
|
+
model: "fake-model",
|
|
28
|
+
}),
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
(async () => {
|
|
33
|
+
const res = await fakeWalk([
|
|
34
|
+
{ id: "pack:alpha", why: "directly about alpha", score: 92 },
|
|
35
|
+
{ id: "pack:beta", why: "missing score defaults neutral" }, // → 60
|
|
36
|
+
{ id: "pack:gamma", why: "over the top", score: 150 }, // → 100
|
|
37
|
+
{ id: "pack:delta", why: "negative", score: -20 }, // → 0
|
|
38
|
+
{ id: "pack:epsilon", why: "fractional", score: 33.7 }, // → 34
|
|
39
|
+
]);
|
|
40
|
+
|
|
41
|
+
assert.ok(res.kept instanceof Map, "kept is a Map");
|
|
42
|
+
assert.ok(res.scores instanceof Map, "scores is a parallel Map");
|
|
43
|
+
assert.strictEqual(res.kept.get("pack:alpha"), "directly about alpha", "why preserved verbatim (kept stays id→why string)");
|
|
44
|
+
assert.strictEqual(res.scores.get("pack:alpha"), 92, "explicit score parsed");
|
|
45
|
+
assert.strictEqual(res.scores.get("pack:beta"), 60, "missing score → neutral 60 (fail-open, never buried)");
|
|
46
|
+
assert.strictEqual(res.scores.get("pack:gamma"), 100, "score clamped to 100");
|
|
47
|
+
assert.strictEqual(res.scores.get("pack:delta"), 0, "score clamped to 0");
|
|
48
|
+
assert.strictEqual(res.scores.get("pack:epsilon"), 34, "fractional score rounded");
|
|
49
|
+
|
|
50
|
+
// Hallucinated ids (not among candidates) are dropped from BOTH maps.
|
|
51
|
+
const res2 = await fakeWalk([
|
|
52
|
+
{ id: "pack:alpha", why: "ok", score: 70 },
|
|
53
|
+
{ id: "pack:HALLUCINATED", why: "invented", score: 99 },
|
|
54
|
+
]);
|
|
55
|
+
assert.ok(res2.kept.has("pack:alpha"), "known id kept");
|
|
56
|
+
assert.ok(!res2.kept.has("pack:HALLUCINATED"), "unknown id dropped from kept");
|
|
57
|
+
assert.ok(!res2.scores.has("pack:HALLUCINATED"), "unknown id dropped from scores");
|
|
58
|
+
|
|
59
|
+
// Non-numeric score → neutral 60, never NaN.
|
|
60
|
+
const res3 = await fakeWalk([{ id: "pack:alpha", why: "x", score: "high" }]);
|
|
61
|
+
assert.strictEqual(res3.scores.get("pack:alpha"), 60, "non-numeric score → neutral 60");
|
|
62
|
+
|
|
63
|
+
// Unparseable output → fail-open: both maps null, no throw.
|
|
64
|
+
const res4 = await fakeWalk(null, "sorry, I cannot help with that");
|
|
65
|
+
assert.strictEqual(res4.kept, null, "no JSON array → kept null (fail-open)");
|
|
66
|
+
assert.strictEqual(res4.scores, null, "no JSON array → scores null (fail-open)");
|
|
67
|
+
|
|
68
|
+
console.log("walker score OK");
|
|
69
|
+
})().catch((e) => { console.error(e); process.exit(1); });
|