nexus-agents 4.3.5 → 4.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-ZLGQY55I.js → chunk-4KDZ4Q6B.js} +3 -3
- package/dist/{chunk-HI63BMZ6.js → chunk-FLCUFTXF.js} +2 -2
- package/dist/{chunk-ES3CPAON.js → chunk-YBGX57DL.js} +2 -2
- package/dist/cli.js +103 -44
- package/dist/cli.js.map +1 -1
- package/dist/index.js +2 -2
- package/dist/{setup-command-DLIUAXMJ.js → setup-command-4RQUTUDI.js} +3 -3
- package/package.json +1 -1
- /package/dist/{chunk-ZLGQY55I.js.map → chunk-4KDZ4Q6B.js.map} +0 -0
- /package/dist/{chunk-HI63BMZ6.js.map → chunk-FLCUFTXF.js.map} +0 -0
- /package/dist/{chunk-ES3CPAON.js.map → chunk-YBGX57DL.js.map} +0 -0
- /package/dist/{setup-command-DLIUAXMJ.js.map → setup-command-4RQUTUDI.js.map} +0 -0
|
@@ -52,7 +52,7 @@ import {
|
|
|
52
52
|
} from "./chunk-OQLFRJCW.js";
|
|
53
53
|
|
|
54
54
|
// src/version.ts
|
|
55
|
-
var VERSION = true ? "4.
|
|
55
|
+
var VERSION = true ? "4.4.0" : "dev";
|
|
56
56
|
|
|
57
57
|
// src/config/schemas-core.ts
|
|
58
58
|
import { z } from "zod";
|
|
@@ -2339,7 +2339,7 @@ async function runDoctorFix(result) {
|
|
|
2339
2339
|
writeLine2("\u2500".repeat(40));
|
|
2340
2340
|
let fixCount = 0;
|
|
2341
2341
|
if (!result.dataDirectory.rootExists || result.dataDirectory.subdirectories.some((d) => !d.exists || !d.writable)) {
|
|
2342
|
-
const { runSetup } = await import("./setup-command-
|
|
2342
|
+
const { runSetup } = await import("./setup-command-4RQUTUDI.js");
|
|
2343
2343
|
const setupResult = runSetup({
|
|
2344
2344
|
skipMcp: true,
|
|
2345
2345
|
skipRules: true,
|
|
@@ -2452,4 +2452,4 @@ export {
|
|
|
2452
2452
|
startStdioServer,
|
|
2453
2453
|
closeServer
|
|
2454
2454
|
};
|
|
2455
|
-
//# sourceMappingURL=chunk-
|
|
2455
|
+
//# sourceMappingURL=chunk-4KDZ4Q6B.js.map
|
|
@@ -8,7 +8,7 @@ import {
|
|
|
8
8
|
checkSqlite,
|
|
9
9
|
defaultConfig,
|
|
10
10
|
initDataDirectories
|
|
11
|
-
} from "./chunk-
|
|
11
|
+
} from "./chunk-4KDZ4Q6B.js";
|
|
12
12
|
import {
|
|
13
13
|
BUILT_IN_EXPERTS
|
|
14
14
|
} from "./chunk-YPNQPT2F.js";
|
|
@@ -2001,4 +2001,4 @@ export {
|
|
|
2001
2001
|
setupCommand,
|
|
2002
2002
|
setupCommandAsync
|
|
2003
2003
|
};
|
|
2004
|
-
//# sourceMappingURL=chunk-
|
|
2004
|
+
//# sourceMappingURL=chunk-FLCUFTXF.js.map
|
|
@@ -22,7 +22,7 @@ import {
|
|
|
22
22
|
DEFAULT_TASK_TTL_MS,
|
|
23
23
|
DEFAULT_TOOL_RATE_LIMITS,
|
|
24
24
|
clampTaskTtl
|
|
25
|
-
} from "./chunk-
|
|
25
|
+
} from "./chunk-4KDZ4Q6B.js";
|
|
26
26
|
import {
|
|
27
27
|
executeExpert
|
|
28
28
|
} from "./chunk-QTP4AIVR.js";
|
|
@@ -54497,4 +54497,4 @@ export {
|
|
|
54497
54497
|
shutdownFeedbackSubscriber,
|
|
54498
54498
|
createEventBusBridge
|
|
54499
54499
|
};
|
|
54500
|
-
//# sourceMappingURL=chunk-
|
|
54500
|
+
//# sourceMappingURL=chunk-YBGX57DL.js.map
|
package/dist/cli.js
CHANGED
|
@@ -18,7 +18,7 @@ import "./chunk-TKFWYFBD.js";
|
|
|
18
18
|
import {
|
|
19
19
|
setupCommandAsync,
|
|
20
20
|
verifyCommand
|
|
21
|
-
} from "./chunk-
|
|
21
|
+
} from "./chunk-FLCUFTXF.js";
|
|
22
22
|
import "./chunk-YIV4GQHK.js";
|
|
23
23
|
import {
|
|
24
24
|
AuthHandler,
|
|
@@ -139,7 +139,7 @@ import {
|
|
|
139
139
|
validateCommand,
|
|
140
140
|
validateWorkflow,
|
|
141
141
|
wrapInMarkdownFence
|
|
142
|
-
} from "./chunk-
|
|
142
|
+
} from "./chunk-YBGX57DL.js";
|
|
143
143
|
import "./chunk-SHG3MYMD.js";
|
|
144
144
|
import "./chunk-HFOQKCD2.js";
|
|
145
145
|
import "./chunk-RSMG2DPB.js";
|
|
@@ -166,7 +166,7 @@ import {
|
|
|
166
166
|
loadConfig,
|
|
167
167
|
runDoctor,
|
|
168
168
|
validateNexusEnv
|
|
169
|
-
} from "./chunk-
|
|
169
|
+
} from "./chunk-4KDZ4Q6B.js";
|
|
170
170
|
import {
|
|
171
171
|
buildOpenAICompatAdapters,
|
|
172
172
|
readOpenAICompatEnv
|
|
@@ -4279,6 +4279,14 @@ function validateCommandWithPolicy(commandString, options = {}) {
|
|
|
4279
4279
|
}
|
|
4280
4280
|
return null;
|
|
4281
4281
|
}
|
|
4282
|
+
function readErrorStdout(error) {
|
|
4283
|
+
if (typeof error !== "object" || error === null || !("stdout" in error)) return null;
|
|
4284
|
+
const { stdout } = error;
|
|
4285
|
+
const text = typeof stdout === "string" ? stdout : Buffer.isBuffer(stdout) ? stdout.toString("utf-8") : null;
|
|
4286
|
+
if (text === null) return null;
|
|
4287
|
+
const trimmed = text.trim();
|
|
4288
|
+
return trimmed === "" ? null : trimmed;
|
|
4289
|
+
}
|
|
4282
4290
|
function safeExecSandboxed(commandString, options = {}) {
|
|
4283
4291
|
const violation = validateCommandWithPolicy(commandString, options);
|
|
4284
4292
|
if (violation !== null) {
|
|
@@ -4302,6 +4310,13 @@ function safeExecSandboxed(commandString, options = {}) {
|
|
|
4302
4310
|
const result = execSync(commandString, execOptions);
|
|
4303
4311
|
return typeof result === "string" ? result.trim() : result.toString("utf-8").trim();
|
|
4304
4312
|
} catch (error) {
|
|
4313
|
+
if (options.allowNonZeroExit === true) {
|
|
4314
|
+
const stdout = readErrorStdout(error);
|
|
4315
|
+
if (stdout !== null) {
|
|
4316
|
+
logger4.debug("Command exited non-zero but produced output", { command: commandString });
|
|
4317
|
+
return stdout;
|
|
4318
|
+
}
|
|
4319
|
+
}
|
|
4305
4320
|
logger4.debug("Command execution failed", {
|
|
4306
4321
|
command: commandString,
|
|
4307
4322
|
error: getErrorMessage(error)
|
|
@@ -6329,6 +6344,12 @@ var SYSTEM_REVIEW_CONSTANTS = {
|
|
|
6329
6344
|
/** Security severity penalties */
|
|
6330
6345
|
SECURITY_HIGH_PENALTY: 20,
|
|
6331
6346
|
SECURITY_MODERATE_PENALTY: 5,
|
|
6347
|
+
/**
|
|
6348
|
+
* Applied when the audit could not run (#4838). An unmeasured audit is not
|
|
6349
|
+
* evidence of health, so it must not score the same as a clean one; it is
|
|
6350
|
+
* set below SECURITY_HIGH_PENALTY because "unknown" is not "known bad".
|
|
6351
|
+
*/
|
|
6352
|
+
SECURITY_UNMEASURED_PENALTY: 10,
|
|
6332
6353
|
/** Code quality penalties */
|
|
6333
6354
|
TYPECHECK_FAIL_PENALTY: 15,
|
|
6334
6355
|
LINT_FAIL_PENALTY: 15,
|
|
@@ -6347,6 +6368,7 @@ var {
|
|
|
6347
6368
|
DOC_REVIEW_PENALTY,
|
|
6348
6369
|
SECURITY_HIGH_PENALTY,
|
|
6349
6370
|
SECURITY_MODERATE_PENALTY,
|
|
6371
|
+
SECURITY_UNMEASURED_PENALTY,
|
|
6350
6372
|
TYPECHECK_FAIL_PENALTY,
|
|
6351
6373
|
LINT_FAIL_PENALTY,
|
|
6352
6374
|
LOW_COVERAGE_PENALTY,
|
|
@@ -6357,7 +6379,7 @@ function calculateHealthScore(r) {
|
|
|
6357
6379
|
for (const d of r.docs) {
|
|
6358
6380
|
score -= d.status === "stale" ? DOC_STALE_PENALTY : d.status === "review" ? DOC_REVIEW_PENALTY : 0;
|
|
6359
6381
|
}
|
|
6360
|
-
score -= r.security.high * SECURITY_HIGH_PENALTY + r.security.moderate * SECURITY_MODERATE_PENALTY;
|
|
6382
|
+
score -= r.security.parseError === true ? SECURITY_UNMEASURED_PENALTY : r.security.high * SECURITY_HIGH_PENALTY + r.security.moderate * SECURITY_MODERATE_PENALTY;
|
|
6361
6383
|
if (!r.quality.typecheckPass) score -= TYPECHECK_FAIL_PENALTY;
|
|
6362
6384
|
if (!r.quality.lintPass) score -= LINT_FAIL_PENALTY;
|
|
6363
6385
|
if (r.quality.coveragePercent !== null && r.quality.coveragePercent < COVERAGE_TARGET_PERCENT) {
|
|
@@ -6402,6 +6424,11 @@ function printPhase3(i) {
|
|
|
6402
6424
|
function printPhase4(s) {
|
|
6403
6425
|
writeLine(`${colors.cyan}Phase 4: Security Audit${colors.reset}
|
|
6404
6426
|
`);
|
|
6427
|
+
if (s.parseError === true) {
|
|
6428
|
+
writeLine(` ${formatStatus("warn")} Unmeasured \u2014 the audit did not run`);
|
|
6429
|
+
writeLine("");
|
|
6430
|
+
return;
|
|
6431
|
+
}
|
|
6405
6432
|
if (s.totalVulns === 0) {
|
|
6406
6433
|
writeLine(` ${formatStatus("pass")} No vulnerabilities found`);
|
|
6407
6434
|
} else {
|
|
@@ -6428,6 +6455,9 @@ function formatDocRow(d) {
|
|
|
6428
6455
|
const newerInfo = newerCount > 0 ? ` (${String(newerCount)} newer deps)` : "";
|
|
6429
6456
|
return `| ${d.file} | ${String(d.daysSinceUpdate)} | ${statusIcon}${newerInfo} |`;
|
|
6430
6457
|
}
|
|
6458
|
+
function formatSecurityCaveat(s) {
|
|
6459
|
+
return s.parseError === true ? "> **Unmeasured:** the audit did not run \u2014 the counts below are defaults, not findings.\n\n" : "";
|
|
6460
|
+
}
|
|
6431
6461
|
function createIssueBody(r) {
|
|
6432
6462
|
const now = new Date(getTimeProvider().now());
|
|
6433
6463
|
const date = now.toISOString().split("T")[0] ?? "unknown";
|
|
@@ -6462,7 +6492,7 @@ ${r.docs.map(formatDocRow).join("\n")}
|
|
|
6462
6492
|
|
|
6463
6493
|
### Phase 4: Security
|
|
6464
6494
|
|
|
6465
|
-
- High: ${String(r.security.high)}
|
|
6495
|
+
${formatSecurityCaveat(r.security)}- High: ${String(r.security.high)}
|
|
6466
6496
|
- Moderate: ${String(r.security.moderate)}
|
|
6467
6497
|
- Low: ${String(r.security.low)}
|
|
6468
6498
|
|
|
@@ -6750,9 +6780,9 @@ var {
|
|
|
6750
6780
|
NOT_STARTED_TECHNIQUE_THRESHOLD,
|
|
6751
6781
|
HEALTH_SCORE_WARN_THRESHOLD: HEALTH_SCORE_WARN_THRESHOLD2
|
|
6752
6782
|
} = SYSTEM_REVIEW_CONSTANTS;
|
|
6753
|
-
function safeExec(command, cwd) {
|
|
6783
|
+
function safeExec(command, cwd, allowNonZeroExit = false) {
|
|
6754
6784
|
const context = command.startsWith("gh ") ? "gh" : command.startsWith("git ") ? "git" : command.startsWith("pnpm ") ? "write" : "read";
|
|
6755
|
-
return cwd !== void 0 ? safeExecSandboxed(command, { context, cwd }) : safeExecSandboxed(command, { context });
|
|
6785
|
+
return cwd !== void 0 ? safeExecSandboxed(command, { context, cwd, allowNonZeroExit }) : safeExecSandboxed(command, { context, allowNonZeroExit });
|
|
6756
6786
|
}
|
|
6757
6787
|
function parseGhIssueList(json, context) {
|
|
6758
6788
|
try {
|
|
@@ -6828,7 +6858,7 @@ function runPhase3() {
|
|
|
6828
6858
|
}
|
|
6829
6859
|
function runPhase4(projectRoot) {
|
|
6830
6860
|
const def = { totalVulns: 0, high: 0, moderate: 0, low: 0, parseError: false };
|
|
6831
|
-
const out = safeExec("pnpm audit --json", projectRoot);
|
|
6861
|
+
const out = safeExec("pnpm audit --json", projectRoot, true);
|
|
6832
6862
|
if (out === null) {
|
|
6833
6863
|
logger7.warn("pnpm audit command failed or returned null", { projectRoot });
|
|
6834
6864
|
return { ...def, parseError: true };
|
|
@@ -14776,13 +14806,36 @@ async function releaseNotesCommand(args) {
|
|
|
14776
14806
|
import { readFileSync as readFileSync10 } from "fs";
|
|
14777
14807
|
|
|
14778
14808
|
// src/cli/release-validate-helpers.ts
|
|
14779
|
-
import { execSync as
|
|
14809
|
+
import { execSync as execSync3 } from "child_process";
|
|
14780
14810
|
import { existsSync as existsSync14, readFileSync as readFileSync9 } from "fs";
|
|
14811
|
+
|
|
14812
|
+
// src/cli/release-secret-scan.ts
|
|
14813
|
+
import { execSync as execSync2 } from "child_process";
|
|
14814
|
+
var SECRET_LINE_PATTERN = /api[_-]?key|secret|password|token/i;
|
|
14815
|
+
var MAX_REPORTED_MATCHES = 5;
|
|
14816
|
+
function scanRecentCommitsForSecrets(options = {}) {
|
|
14817
|
+
const range = options.range ?? "HEAD~10..HEAD";
|
|
14818
|
+
let diff;
|
|
14819
|
+
try {
|
|
14820
|
+
diff = execSync2(`git diff ${range} -- "*.ts" "*.js"`, {
|
|
14821
|
+
encoding: "utf-8",
|
|
14822
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
14823
|
+
timeout: CLI_SUBPROCESS_TIMEOUTS.ghCommandMs,
|
|
14824
|
+
...options.cwd === void 0 ? {} : { cwd: options.cwd }
|
|
14825
|
+
});
|
|
14826
|
+
} catch (error) {
|
|
14827
|
+
return { ok: false, reason: error instanceof Error ? error.message : String(error) };
|
|
14828
|
+
}
|
|
14829
|
+
const matches = diff.split("\n").filter((line) => SECRET_LINE_PATTERN.test(line)).slice(0, MAX_REPORTED_MATCHES);
|
|
14830
|
+
return { ok: true, matches };
|
|
14831
|
+
}
|
|
14832
|
+
|
|
14833
|
+
// src/cli/release-validate-helpers.ts
|
|
14781
14834
|
async function validateSecurity(options) {
|
|
14782
14835
|
const startTime = Date.now();
|
|
14783
14836
|
const findings = [];
|
|
14784
14837
|
try {
|
|
14785
|
-
|
|
14838
|
+
execSync3("npm audit --audit-level=high 2>/dev/null", {
|
|
14786
14839
|
encoding: "utf-8",
|
|
14787
14840
|
stdio: ["pipe", "pipe", "pipe"],
|
|
14788
14841
|
timeout: CLI_SUBPROCESS_TIMEOUTS.ghCommandMs
|
|
@@ -14805,37 +14858,29 @@ async function validateSecurity(options) {
|
|
|
14805
14858
|
remediation: "Ensure .env is in .gitignore and not committed."
|
|
14806
14859
|
});
|
|
14807
14860
|
}
|
|
14808
|
-
|
|
14809
|
-
|
|
14810
|
-
'git diff HEAD~10..HEAD -- "*.ts" "*.js" | grep -iE "(api[_-]?key|secret|password|token)" | head -5',
|
|
14811
|
-
{
|
|
14812
|
-
encoding: "utf-8",
|
|
14813
|
-
stdio: ["pipe", "pipe", "pipe"],
|
|
14814
|
-
timeout: CLI_SUBPROCESS_TIMEOUTS.ghCommandMs
|
|
14815
|
-
}
|
|
14816
|
-
);
|
|
14817
|
-
if (result.trim()) {
|
|
14818
|
-
findings.push({
|
|
14819
|
-
severity: "warning",
|
|
14820
|
-
category: "security",
|
|
14821
|
-
title: "Potential secrets in recent commits",
|
|
14822
|
-
description: "Recent commits may contain hardcoded secrets.",
|
|
14823
|
-
remediation: "Review commits for any exposed credentials."
|
|
14824
|
-
});
|
|
14825
|
-
}
|
|
14826
|
-
} catch {
|
|
14861
|
+
const scan = scanRecentCommitsForSecrets();
|
|
14862
|
+
if (!scan.ok) {
|
|
14827
14863
|
findings.push({
|
|
14828
14864
|
severity: "warning",
|
|
14829
14865
|
category: "security",
|
|
14830
14866
|
title: "Secret scan did not run",
|
|
14831
|
-
description:
|
|
14867
|
+
description: `The hardcoded-secret scan over recent commits failed to execute: ${scan.reason}`,
|
|
14832
14868
|
remediation: "Re-run with a full git history available, then review the output."
|
|
14833
14869
|
});
|
|
14870
|
+
} else if (scan.matches.length > 0) {
|
|
14871
|
+
findings.push({
|
|
14872
|
+
severity: "warning",
|
|
14873
|
+
category: "security",
|
|
14874
|
+
title: "Potential secrets in recent commits",
|
|
14875
|
+
description: "Recent commits may contain hardcoded secrets.",
|
|
14876
|
+
remediation: "Review commits for any exposed credentials."
|
|
14877
|
+
});
|
|
14834
14878
|
}
|
|
14835
14879
|
return {
|
|
14836
14880
|
expert: "security",
|
|
14837
|
-
// whenEmpty = false:
|
|
14838
|
-
//
|
|
14881
|
+
// whenEmpty = false: an empty list genuinely means "scanned, nothing
|
|
14882
|
+
// found" — every check above records a finding when it cannot run, which
|
|
14883
|
+
// for the secret scan required #4839 as well as #4581.
|
|
14839
14884
|
passed: !anyOf(findings, (f) => f.severity === "error", false),
|
|
14840
14885
|
confidence: 0.85,
|
|
14841
14886
|
findings,
|
|
@@ -14846,7 +14891,7 @@ async function validateArchitecture2(options) {
|
|
|
14846
14891
|
const startTime = Date.now();
|
|
14847
14892
|
const findings = [];
|
|
14848
14893
|
try {
|
|
14849
|
-
const result =
|
|
14894
|
+
const result = execSync3("npx nexus-agents fitness-audit --format=json 2>/dev/null", {
|
|
14850
14895
|
encoding: "utf-8",
|
|
14851
14896
|
stdio: ["pipe", "pipe", "pipe"],
|
|
14852
14897
|
timeout: CLI_SUBPROCESS_TIMEOUTS.releaseValidateMs
|
|
@@ -14967,7 +15012,7 @@ async function validateDevOps(options) {
|
|
|
14967
15012
|
const startTime = Date.now();
|
|
14968
15013
|
const findings = [];
|
|
14969
15014
|
try {
|
|
14970
|
-
|
|
15015
|
+
execSync3("pnpm build 2>/dev/null", {
|
|
14971
15016
|
encoding: "utf-8",
|
|
14972
15017
|
stdio: ["pipe", "pipe", "pipe"],
|
|
14973
15018
|
timeout: CLI_SUBPROCESS_TIMEOUTS.releaseValidateMs
|
|
@@ -14988,7 +15033,7 @@ async function validateDevOps(options) {
|
|
|
14988
15033
|
});
|
|
14989
15034
|
}
|
|
14990
15035
|
try {
|
|
14991
|
-
|
|
15036
|
+
execSync3("pnpm lint 2>/dev/null", {
|
|
14992
15037
|
encoding: "utf-8",
|
|
14993
15038
|
stdio: ["pipe", "pipe", "pipe"],
|
|
14994
15039
|
timeout: CLI_SUBPROCESS_TIMEOUTS.releaseBuildMs
|
|
@@ -15009,7 +15054,7 @@ async function validateDevOps(options) {
|
|
|
15009
15054
|
});
|
|
15010
15055
|
}
|
|
15011
15056
|
try {
|
|
15012
|
-
|
|
15057
|
+
execSync3("pnpm typecheck 2>/dev/null", {
|
|
15013
15058
|
encoding: "utf-8",
|
|
15014
15059
|
stdio: ["pipe", "pipe", "pipe"],
|
|
15015
15060
|
timeout: CLI_SUBPROCESS_TIMEOUTS.releaseValidateMs
|
|
@@ -16270,6 +16315,7 @@ Winner (by success rate): ${report.winnerBySuccessRate}`);
|
|
|
16270
16315
|
}
|
|
16271
16316
|
|
|
16272
16317
|
// src/cli/memory-eval.ts
|
|
16318
|
+
var TOP_K = 5;
|
|
16273
16319
|
function generateEvalDataset(size = 50) {
|
|
16274
16320
|
const pairs = [];
|
|
16275
16321
|
const sources = ["session", "belief", "agentic", "typed"];
|
|
@@ -16285,16 +16331,19 @@ function generateEvalDataset(size = 50) {
|
|
|
16285
16331
|
{ query: "logging best practices", content: "Structured JSON logs with correlation IDs" },
|
|
16286
16332
|
{ query: "security vulnerability scan", content: "Dependabot and CodeQL automated scans" }
|
|
16287
16333
|
];
|
|
16334
|
+
const queryCount = Math.max(1, Math.min(topics.length, Math.floor(size / (TOP_K * 2))));
|
|
16288
16335
|
for (let i = 0; i < size; i++) {
|
|
16289
|
-
const topicIdx = i %
|
|
16336
|
+
const topicIdx = i % queryCount;
|
|
16290
16337
|
const topic = topics[topicIdx];
|
|
16291
16338
|
const source = sources[i % sources.length] ?? "session";
|
|
16292
16339
|
const isRelevant = i % 3 !== 0;
|
|
16293
|
-
|
|
16340
|
+
const offset = 1 + i % (topics.length - 1);
|
|
16341
|
+
const contentTopic = isRelevant ? topic : topics[(topicIdx + offset) % topics.length];
|
|
16342
|
+
if (topic !== void 0 && contentTopic !== void 0) {
|
|
16294
16343
|
pairs.push({
|
|
16295
|
-
query:
|
|
16344
|
+
query: topic.query,
|
|
16296
16345
|
memoryKey: `mem-${String(i)}-${source}`,
|
|
16297
|
-
memoryContent:
|
|
16346
|
+
memoryContent: contentTopic.content,
|
|
16298
16347
|
source,
|
|
16299
16348
|
expectedRelevant: isRelevant
|
|
16300
16349
|
});
|
|
@@ -16344,8 +16393,9 @@ function expandKeywords(query) {
|
|
|
16344
16393
|
return result;
|
|
16345
16394
|
}
|
|
16346
16395
|
function evaluateScorer(dataset, scorer) {
|
|
16347
|
-
const k =
|
|
16396
|
+
const k = TOP_K;
|
|
16348
16397
|
let totalRecall = 0;
|
|
16398
|
+
let recallQueries = 0;
|
|
16349
16399
|
let totalPrecision = 0;
|
|
16350
16400
|
let totalMrr = 0;
|
|
16351
16401
|
const latencies = [];
|
|
@@ -16363,7 +16413,10 @@ function evaluateScorer(dataset, scorer) {
|
|
|
16363
16413
|
const topK = scored.slice(0, k);
|
|
16364
16414
|
const relevantInTopK = topK.filter((s) => s.relevant).length;
|
|
16365
16415
|
const totalRelevant = pairs.filter((p) => p.expectedRelevant).length;
|
|
16366
|
-
|
|
16416
|
+
if (totalRelevant > 0) {
|
|
16417
|
+
totalRecall += relevantInTopK / totalRelevant;
|
|
16418
|
+
recallQueries += 1;
|
|
16419
|
+
}
|
|
16367
16420
|
totalPrecision += topK.length > 0 ? relevantInTopK / topK.length : 0;
|
|
16368
16421
|
const firstRelevantIdx = scored.findIndex((s) => s.relevant);
|
|
16369
16422
|
totalMrr += firstRelevantIdx >= 0 ? 1 / (firstRelevantIdx + 1) : 0;
|
|
@@ -16372,12 +16425,13 @@ function evaluateScorer(dataset, scorer) {
|
|
|
16372
16425
|
latencies.sort((a, b) => a - b);
|
|
16373
16426
|
const p95Idx = Math.floor(latencies.length * 0.95);
|
|
16374
16427
|
return {
|
|
16375
|
-
recallAt5:
|
|
16428
|
+
recallAt5: recallQueries > 0 ? round(totalRecall / recallQueries) : 0,
|
|
16376
16429
|
precisionAt5: n > 0 ? round(totalPrecision / n) : 0,
|
|
16377
16430
|
mrr: n > 0 ? round(totalMrr / n) : 0,
|
|
16378
16431
|
avgLatencyMs: round(average(latencies)),
|
|
16379
16432
|
p95LatencyMs: round(latencies[p95Idx] ?? 0),
|
|
16380
|
-
totalQueries: n
|
|
16433
|
+
totalQueries: n,
|
|
16434
|
+
recallQueries
|
|
16381
16435
|
};
|
|
16382
16436
|
}
|
|
16383
16437
|
function groupByQuery(dataset) {
|
|
@@ -16416,6 +16470,11 @@ function formatMemoryEvalReport(report) {
|
|
|
16416
16470
|
}
|
|
16417
16471
|
function formatEvalDetails(baseline, reflective, improvement, datasetSize) {
|
|
16418
16472
|
const lines = [`Dataset: ${String(datasetSize)} query-memory pairs`];
|
|
16473
|
+
if (baseline.recallQueries < baseline.totalQueries) {
|
|
16474
|
+
lines.push(
|
|
16475
|
+
`Recall measured over ${String(baseline.recallQueries)} of ${String(baseline.totalQueries)} queries \u2014 the rest had no relevant memory.`
|
|
16476
|
+
);
|
|
16477
|
+
}
|
|
16419
16478
|
lines.push("\nBaseline (keyword search):");
|
|
16420
16479
|
lines.push(` Recall@5: ${baseline.recallAt5.toFixed(3)}`);
|
|
16421
16480
|
lines.push(` Precision@5: ${baseline.precisionAt5.toFixed(3)}`);
|