@orquentra/codebuddy 2.5.0 → 2.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. package/README.md +8 -0
  2. package/dist/{chunk-KYK6RPAJ.js → chunk-JFDRBPYF.js} +2 -2
  3. package/dist/{chunk-JMWJ7V6Y.js → chunk-JPHJBCZB.js} +2 -2
  4. package/dist/{chunk-JMWJ7V6Y.js.map → chunk-JPHJBCZB.js.map} +1 -1
  5. package/dist/{chunk-ROQTT5ZO.js → chunk-N34XHWBA.js} +2 -2
  6. package/dist/{chunk-CQPCHMBS.js → chunk-SYWJG5NA.js} +2 -2
  7. package/dist/{chunk-3XYQGFPB.js → chunk-U4HAJLIB.js} +2 -2
  8. package/dist/{chunk-JDZ3VK26.js → chunk-UN57FKQX.js} +32 -5
  9. package/dist/chunk-UN57FKQX.js.map +1 -0
  10. package/dist/{chunk-Z5QB367W.js → chunk-VAKBJV26.js} +106 -22
  11. package/dist/chunk-VAKBJV26.js.map +1 -0
  12. package/dist/{chunk-4BYGTEJT.js → chunk-XBJQKCLU.js} +2 -2
  13. package/dist/cli/index.js +2 -2
  14. package/dist/{cloud-GP6G3EAK.js → cloud-7WM3YLI4.js} +3 -3
  15. package/dist/index.js +7 -7
  16. package/dist/{lean-command-DA4ZNKG3.js → lean-command-C5YPFKHP.js} +5 -5
  17. package/dist/{local-3NGNESNB.js → local-OTOY56B4.js} +3 -3
  18. package/dist/{main-BDVQAGAS.js → main-HFHLGID4.js} +201 -55
  19. package/dist/main-HFHLGID4.js.map +1 -0
  20. package/dist/{operations-EQGOFCBD.js → operations-LOBAQRLO.js} +6 -6
  21. package/package.json +1 -1
  22. package/dist/chunk-JDZ3VK26.js.map +0 -1
  23. package/dist/chunk-Z5QB367W.js.map +0 -1
  24. package/dist/main-BDVQAGAS.js.map +0 -1
  25. /package/dist/{chunk-KYK6RPAJ.js.map → chunk-JFDRBPYF.js.map} +0 -0
  26. /package/dist/{chunk-ROQTT5ZO.js.map → chunk-N34XHWBA.js.map} +0 -0
  27. /package/dist/{chunk-CQPCHMBS.js.map → chunk-SYWJG5NA.js.map} +0 -0
  28. /package/dist/{chunk-3XYQGFPB.js.map → chunk-U4HAJLIB.js.map} +0 -0
  29. /package/dist/{chunk-4BYGTEJT.js.map → chunk-XBJQKCLU.js.map} +0 -0
  30. /package/dist/{cloud-GP6G3EAK.js.map → cloud-7WM3YLI4.js.map} +0 -0
  31. /package/dist/{lean-command-DA4ZNKG3.js.map → lean-command-C5YPFKHP.js.map} +0 -0
  32. /package/dist/{local-3NGNESNB.js.map → local-OTOY56B4.js.map} +0 -0
  33. /package/dist/{operations-EQGOFCBD.js.map → operations-LOBAQRLO.js.map} +0 -0
@@ -1,6 +1,6 @@
1
1
  import {
2
2
  getModelContextWindow
3
- } from "./chunk-JMWJ7V6Y.js";
3
+ } from "./chunk-JPHJBCZB.js";
4
4
 
5
5
  // src/planner/budget.ts
6
6
  import { Tiktoken } from "js-tiktoken/lite";
@@ -100,4 +100,4 @@ export {
100
100
  computeContentHash,
101
101
  computeFactHash
102
102
  };
103
- //# sourceMappingURL=chunk-4BYGTEJT.js.map
103
+ //# sourceMappingURL=chunk-XBJQKCLU.js.map
package/dist/cli/index.js CHANGED
@@ -3,13 +3,13 @@
3
3
  // src/cli/index.ts
4
4
  var [command, subcommand] = process.argv.slice(2);
5
5
  if (command === "hooks" && subcommand === "lean") {
6
- const { runLeanHookCommand } = await import("../lean-command-DA4ZNKG3.js");
6
+ const { runLeanHookCommand } = await import("../lean-command-C5YPFKHP.js");
7
7
  await runLeanHookCommand();
8
8
  } else if (command === "hooks" && subcommand === "pretool") {
9
9
  const { runPreToolHookCommand } = await import("../pretool-command-MQQ2UNQM.js");
10
10
  await runPreToolHookCommand();
11
11
  } else {
12
- const { createCli } = await import("../main-BDVQAGAS.js");
12
+ const { createCli } = await import("../main-HFHLGID4.js");
13
13
  await createCli().parseAsync();
14
14
  }
15
15
  //# sourceMappingURL=index.js.map
@@ -4,8 +4,8 @@ import {
4
4
  OpenRouterEmbeddingProvider,
5
5
  rerankHuggingFace,
6
6
  rerankOpenRouter
7
- } from "./chunk-KYK6RPAJ.js";
8
- import "./chunk-JMWJ7V6Y.js";
7
+ } from "./chunk-JFDRBPYF.js";
8
+ import "./chunk-JPHJBCZB.js";
9
9
  import "./chunk-PZ5AY32C.js";
10
10
  export {
11
11
  HF_ROUTER_BASE_URL,
@@ -14,4 +14,4 @@ export {
14
14
  rerankHuggingFace,
15
15
  rerankOpenRouter
16
16
  };
17
- //# sourceMappingURL=cloud-GP6G3EAK.js.map
17
+ //# sourceMappingURL=cloud-7WM3YLI4.js.map
package/dist/index.js CHANGED
@@ -19,9 +19,9 @@ import {
19
19
  registerCodeBuddyTools,
20
20
  resolveRiskPaths,
21
21
  startMcpServer
22
- } from "./chunk-Z5QB367W.js";
22
+ } from "./chunk-VAKBJV26.js";
23
23
  import "./chunk-IHQM2ZDV.js";
24
- import "./chunk-CQPCHMBS.js";
24
+ import "./chunk-SYWJG5NA.js";
25
25
  import {
26
26
  buildArchitectureMap,
27
27
  neighbours,
@@ -45,17 +45,17 @@ import {
45
45
  isLangSmithTracingEnabled,
46
46
  listFactsPage,
47
47
  runDoctor
48
- } from "./chunk-JDZ3VK26.js";
48
+ } from "./chunk-UN57FKQX.js";
49
49
  import "./chunk-GDJH7XJC.js";
50
50
  import {
51
51
  clampBudgetForCallerModel,
52
52
  countTokens
53
- } from "./chunk-4BYGTEJT.js";
54
- import "./chunk-KYK6RPAJ.js";
53
+ } from "./chunk-XBJQKCLU.js";
54
+ import "./chunk-JFDRBPYF.js";
55
55
  import {
56
56
  LocalModelProvider
57
- } from "./chunk-3XYQGFPB.js";
58
- import "./chunk-JMWJ7V6Y.js";
57
+ } from "./chunk-U4HAJLIB.js";
58
+ import "./chunk-JPHJBCZB.js";
59
59
  import "./chunk-QDBDYCM2.js";
60
60
  import {
61
61
  PLAN_POLICIES,
@@ -1,15 +1,15 @@
1
1
  import {
2
2
  runLeanHookCommand
3
- } from "./chunk-ROQTT5ZO.js";
3
+ } from "./chunk-N34XHWBA.js";
4
4
  import "./chunk-PYLH4MH5.js";
5
- import "./chunk-CQPCHMBS.js";
5
+ import "./chunk-SYWJG5NA.js";
6
6
  import "./chunk-L5AYZC3F.js";
7
7
  import "./chunk-PQK3GKZH.js";
8
- import "./chunk-4BYGTEJT.js";
9
- import "./chunk-JMWJ7V6Y.js";
8
+ import "./chunk-XBJQKCLU.js";
9
+ import "./chunk-JPHJBCZB.js";
10
10
  import "./chunk-776EQXDV.js";
11
11
  import "./chunk-PZ5AY32C.js";
12
12
  export {
13
13
  runLeanHookCommand
14
14
  };
15
- //# sourceMappingURL=lean-command-DA4ZNKG3.js.map
15
+ //# sourceMappingURL=lean-command-C5YPFKHP.js.map
@@ -2,12 +2,12 @@ import {
2
2
  LocalModelProvider,
3
3
  modelCacheDir,
4
4
  rerankLocally
5
- } from "./chunk-3XYQGFPB.js";
6
- import "./chunk-JMWJ7V6Y.js";
5
+ } from "./chunk-U4HAJLIB.js";
6
+ import "./chunk-JPHJBCZB.js";
7
7
  import "./chunk-PZ5AY32C.js";
8
8
  export {
9
9
  LocalModelProvider,
10
10
  modelCacheDir,
11
11
  rerankLocally
12
12
  };
13
- //# sourceMappingURL=local-3NGNESNB.js.map
13
+ //# sourceMappingURL=local-OTOY56B4.js.map
@@ -10,7 +10,7 @@ import {
10
10
  import {
11
11
  renderLeanBlock,
12
12
  runLeanHookCommand
13
- } from "./chunk-ROQTT5ZO.js";
13
+ } from "./chunk-N34XHWBA.js";
14
14
  import {
15
15
  runPreToolHookCommand
16
16
  } from "./chunk-WHZSVY2L.js";
@@ -45,12 +45,12 @@ import {
45
45
  startMcpServer,
46
46
  tracePath,
47
47
  watchRepository
48
- } from "./chunk-Z5QB367W.js";
48
+ } from "./chunk-VAKBJV26.js";
49
49
  import "./chunk-IHQM2ZDV.js";
50
50
  import {
51
51
  SessionCapsuleLedger,
52
52
  estimateTokens
53
- } from "./chunk-CQPCHMBS.js";
53
+ } from "./chunk-SYWJG5NA.js";
54
54
  import {
55
55
  buildArchitectureMap,
56
56
  generateGraphifyGraph,
@@ -77,24 +77,24 @@ import {
77
77
  pingDatabase,
78
78
  runDoctor,
79
79
  runMigrations
80
- } from "./chunk-JDZ3VK26.js";
80
+ } from "./chunk-UN57FKQX.js";
81
81
  import {
82
82
  redactPersisted
83
83
  } from "./chunk-GDJH7XJC.js";
84
84
  import {
85
85
  computeFactHash,
86
86
  generateEntityId
87
- } from "./chunk-4BYGTEJT.js";
88
- import "./chunk-KYK6RPAJ.js";
87
+ } from "./chunk-XBJQKCLU.js";
88
+ import "./chunk-JFDRBPYF.js";
89
89
  import {
90
90
  modelCacheDir
91
- } from "./chunk-3XYQGFPB.js";
91
+ } from "./chunk-U4HAJLIB.js";
92
92
  import {
93
93
  DEFAULT_CLOUD_EMBEDDING_MODELS,
94
94
  DEFAULT_RERANK_MODELS,
95
95
  LOCAL_EMBEDDING_MODEL,
96
96
  getModelDefinition
97
- } from "./chunk-JMWJ7V6Y.js";
97
+ } from "./chunk-JPHJBCZB.js";
98
98
  import {
99
99
  DEFAULT_ZVEC_MODEL,
100
100
  buildZvecIndex,
@@ -106,6 +106,7 @@ import {
106
106
  import {
107
107
  PLAN_POLICIES,
108
108
  configPath,
109
+ defaultNamespaceForCwd,
109
110
  initConfigFile,
110
111
  initProjectScaffold,
111
112
  loadPlanPolicy,
@@ -633,9 +634,9 @@ function resolveCliEntrypoint() {
633
634
  }
634
635
  function buildEntry(options) {
635
636
  const env = {
636
- CODEBUDDY_NAMESPACE: options.namespace
637
+ CODEBUDDY_NAMESPACE: options.namespace,
638
+ CODEBUDDY_PROJECT_ROOT: resolve2(options.projectRoot ?? process.cwd())
637
639
  };
638
- if (options.projectRoot) env.CODEBUDDY_PROJECT_ROOT = options.projectRoot;
639
640
  if (options.hfToken) env.HF_TOKEN = options.hfToken;
640
641
  if (options.groqApiKey) env.GROQ_API_KEY = options.groqApiKey;
641
642
  if (options.databaseUrl) env.DATABASE_URL = options.databaseUrl;
@@ -3540,6 +3541,7 @@ async function pingOnce(databaseUrl) {
3540
3541
  }
3541
3542
  }
3542
3543
  async function runInitWizard() {
3544
+ const projectRoot = process.cwd();
3543
3545
  const sigintHandler = () => bail();
3544
3546
  process.on("SIGINT", sigintHandler);
3545
3547
  try {
@@ -3562,7 +3564,7 @@ async function runInitWizard() {
3562
3564
  const namespace = unwrap(
3563
3565
  await text({
3564
3566
  message: "Default namespace",
3565
- initialValue: process.env.CODEBUDDY_NAMESPACE ?? "default",
3567
+ initialValue: process.env.CODEBUDDY_NAMESPACE ?? defaultNamespaceForCwd(projectRoot),
3566
3568
  validate: (value) => value && /^[a-zA-Z0-9_.:-]+$/.test(value) ? void 0 : "Letters, digits, _ . : - only."
3567
3569
  })
3568
3570
  );
@@ -3638,6 +3640,7 @@ async function runInitWizard() {
3638
3640
  try {
3639
3641
  const install = await installClaudeEntry({
3640
3642
  namespace,
3643
+ projectRoot,
3641
3644
  hfToken,
3642
3645
  databaseUrl
3643
3646
  });
@@ -3883,8 +3886,8 @@ import pc13 from "picocolors";
3883
3886
 
3884
3887
  // evals/fact-relevance.json
3885
3888
  var fact_relevance_default = {
3886
- version: 1,
3887
- note: "Task prompts vs durable project facts. Queries are paraphrased so most share few or no words with their fact; every unlisted (query, fact) pair is irrelevant.",
3889
+ version: 2,
3890
+ note: "Task prompts vs durable project facts. Queries are paraphrased so most share few or no words with their fact; every unlisted (query, fact) pair is irrelevant. Hard negatives share a subsystem with a relevant fact but do not answer its task. Queries alternate dev/test by position (even index = dev): floors are chosen on dev and reported on test.",
3888
3891
  facts: {
3889
3892
  cookie: "OAuth callback breaks unless the session cookie is SameSite=Lax; Strict drops it on the cross-site redirect.",
3890
3893
  pool: "Postgres connections go through one bounded pool of 10; never open ad-hoc clients in request handlers.",
@@ -3905,43 +3908,174 @@ var fact_relevance_default = {
3905
3908
  avatar: "Profile images are resized to 256px and stored as WebP in the media bucket.",
3906
3909
  flags: "Feature flags are read once at boot; changing one requires a restart of the worker.",
3907
3910
  pagination: "List endpoints use cursor pagination with an opaque base64url cursor of createdAt and id.",
3908
- i18n: "UI strings live in locales/en.json; never hard-code user-facing text in components."
3911
+ i18n: "UI strings live in locales/en.json; never hard-code user-facing text in components.",
3912
+ cookieDomain: "Session cookies are scoped to the app subdomain; never set them on the apex domain.",
3913
+ poolMetrics: "Database pool metrics are exported to Prometheus under the codebuddy_pool_ prefix.",
3914
+ releaseNotes: "Release notes are drafted from merged pull request titles by the changelog script.",
3915
+ hookInstall: "Hooks are written to .claude/settings.json and merged idempotently with the user's own hooks.",
3916
+ migrationNaming: "Migration files are numbered by drizzle-kit; never rename or edit a migration that has been applied.",
3917
+ testsParallel: "Vitest runs test files in parallel, so every test must use its own temporary directory.",
3918
+ logsStderr: "Server logs go to stderr through pino; never log full request bodies.",
3919
+ retry5xx: "The HTTP client retries 5xx responses twice with jittered backoff.",
3920
+ dateFormat: "Dates in the UI are formatted with the user's locale through Intl.DateTimeFormat.",
3921
+ flagNames: "Feature flag names use kebab-case and are declared in flags.yaml."
3909
3922
  },
3910
3923
  queries: [
3911
3924
  {
3912
3925
  q: "users get bounced back to sign-in forever after the identity provider returns",
3913
3926
  relevant: ["cookie"]
3914
3927
  },
3915
- { q: "handlers are exhausting connections to the database under load", relevant: ["pool"] },
3916
- { q: "cut a new version and push the package to the registry", relevant: ["release"] },
3917
- { q: "the injected block sometimes never shows up in the chat", relevant: ["hook"] },
3918
- { q: "fresh database setup fails on the embedding column", relevant: ["migration"] },
3919
- { q: "an API key ended up in a saved note", relevant: ["redaction"] },
3920
- { q: "context is sent again after I switch tasks and come back", relevant: ["ledger"] },
3921
- { q: "semantic code search crashes and the agent gets no files", relevant: ["zvec"] },
3922
- { q: "why do the db tests not run on my machine", relevant: ["tests"] },
3923
- { q: "module not found when importing a sibling ts file at runtime", relevant: ["esm"] },
3924
- { q: "Claude Desktop says Unexpected token in the server response", relevant: ["stdout"] },
3925
- { q: "the summarizer quietly stopped calling Groq in the afternoon", relevant: ["budget"] },
3926
- { q: "my uncommitted changes vanished after another session ran", relevant: ["worktree"] },
3928
+ {
3929
+ q: "handlers are exhausting connections to the database under load",
3930
+ relevant: ["pool"]
3931
+ },
3932
+ {
3933
+ q: "cut a new version and push the package to the registry",
3934
+ relevant: ["release"]
3935
+ },
3936
+ {
3937
+ q: "the injected block sometimes never shows up in the chat",
3938
+ relevant: ["hook"]
3939
+ },
3940
+ {
3941
+ q: "fresh database setup fails on the embedding column",
3942
+ relevant: ["migration"]
3943
+ },
3944
+ {
3945
+ q: "an API key ended up in a saved note",
3946
+ relevant: ["redaction"]
3947
+ },
3948
+ {
3949
+ q: "context is sent again after I switch tasks and come back",
3950
+ relevant: ["ledger"]
3951
+ },
3952
+ {
3953
+ q: "semantic code search crashes and the agent gets no files",
3954
+ relevant: ["zvec"]
3955
+ },
3956
+ {
3957
+ q: "why do the db tests not run on my machine",
3958
+ relevant: ["tests"]
3959
+ },
3960
+ {
3961
+ q: "module not found when importing a sibling ts file at runtime",
3962
+ relevant: ["esm"]
3963
+ },
3964
+ {
3965
+ q: "Claude Desktop says Unexpected token in the server response",
3966
+ relevant: ["stdout"]
3967
+ },
3968
+ {
3969
+ q: "the summarizer quietly stopped calling Groq in the afternoon",
3970
+ relevant: ["budget"]
3971
+ },
3972
+ {
3973
+ q: "my uncommitted changes vanished after another session ran",
3974
+ relevant: ["worktree"]
3975
+ },
3927
3976
  {
3928
3977
  q: "clients see too many requests errors during the import job",
3929
3978
  relevant: ["rateLimit"]
3930
3979
  },
3931
- { q: "meeting times show up one hour off for users in Berlin", relevant: ["timezone"] },
3932
- { q: "uploaded profile pictures look blurry", relevant: ["avatar"] },
3933
- { q: "I toggled the new checkout switch but nothing changed", relevant: ["flags"] },
3934
- { q: "infinite scroll repeats items on the second page", relevant: ["pagination"] },
3935
- { q: "translate the onboarding screen to German", relevant: ["i18n"] },
3936
- { q: "format the code before I commit", relevant: ["biome"] },
3937
- { q: "make the sidebar collapsible", relevant: [] },
3938
- { q: "add dark mode to the dashboard", relevant: [] },
3939
- { q: "write a haiku about the project", relevant: [] },
3940
- { q: "rename the button label on the settings page", relevant: [] }
3980
+ {
3981
+ q: "meeting times show up one hour off for users in Berlin",
3982
+ relevant: ["timezone"]
3983
+ },
3984
+ {
3985
+ q: "uploaded profile pictures look blurry",
3986
+ relevant: ["avatar"]
3987
+ },
3988
+ {
3989
+ q: "I toggled the new checkout switch but nothing changed",
3990
+ relevant: ["flags"]
3991
+ },
3992
+ {
3993
+ q: "infinite scroll repeats items on the second page",
3994
+ relevant: ["pagination"]
3995
+ },
3996
+ {
3997
+ q: "translate the onboarding screen to German",
3998
+ relevant: ["i18n"]
3999
+ },
4000
+ {
4001
+ q: "format the code before I commit",
4002
+ relevant: ["biome"]
4003
+ },
4004
+ {
4005
+ q: "make the sidebar collapsible",
4006
+ relevant: []
4007
+ },
4008
+ {
4009
+ q: "add dark mode to the dashboard",
4010
+ relevant: []
4011
+ },
4012
+ {
4013
+ q: "write a haiku about the project",
4014
+ relevant: []
4015
+ },
4016
+ {
4017
+ q: "rename the button label on the settings page",
4018
+ relevant: []
4019
+ },
4020
+ {
4021
+ q: "where do I declare a new feature flag",
4022
+ relevant: ["flagNames"]
4023
+ },
4024
+ {
4025
+ q: "show dates in the user's own language and order",
4026
+ relevant: ["dateFormat"]
4027
+ },
4028
+ {
4029
+ q: "can I tidy up the name of the 0004 migration file",
4030
+ relevant: ["migrationNaming"]
4031
+ },
4032
+ {
4033
+ q: "tests interfere with each other when the whole suite runs",
4034
+ relevant: ["testsParallel"]
4035
+ },
4036
+ {
4037
+ q: "where can I see how busy the connection pool is",
4038
+ relevant: ["poolMetrics"]
4039
+ },
4040
+ {
4041
+ q: "how is the text for the changelog produced",
4042
+ relevant: ["releaseNotes"]
4043
+ },
4044
+ {
4045
+ q: "the login cookie shows up on example.com instead of app.example.com",
4046
+ relevant: ["cookieDomain"]
4047
+ },
4048
+ {
4049
+ q: "is it fine to print the incoming payload for debugging",
4050
+ relevant: ["logsStderr"]
4051
+ },
4052
+ {
4053
+ q: "our hook setup appears twice in the Claude settings file",
4054
+ relevant: ["hookInstall"]
4055
+ },
4056
+ {
4057
+ q: "flaky upstream 502s should not fail the request on the first try",
4058
+ relevant: ["retry5xx"]
4059
+ },
4060
+ {
4061
+ q: "rewrite the marketing copy on the landing page",
4062
+ relevant: []
4063
+ },
4064
+ {
4065
+ q: "upgrade React to the next major version",
4066
+ relevant: []
4067
+ },
4068
+ {
4069
+ q: "add a CSV export button to the reports table",
4070
+ relevant: []
4071
+ }
3941
4072
  ]
3942
4073
  };
3943
4074
 
3944
4075
  // src/providers/calibration.ts
4076
+ function splitOf(index) {
4077
+ return index % 2 === 0 ? "dev" : "test";
4078
+ }
3945
4079
  async function calibrateEmbedder(embed) {
3946
4080
  const data = fact_relevance_default;
3947
4081
  const factIds = Object.keys(data.facts);
@@ -3950,47 +4084,59 @@ async function calibrateEmbedder(embed) {
3950
4084
  const dot = (a, b) => a.reduce((sum, value, index) => sum + value * (b[index] ?? 0), 0);
3951
4085
  const scores = queryVectors.map((query) => factVectors.map((fact) => dot(query, fact)));
3952
4086
  const relevant = (qi, fi) => data.queries[qi]?.relevant.includes(factIds[fi] ?? "") ?? false;
4087
+ const answerable = (qi) => (data.queries[qi]?.relevant.length ?? 0) > 0;
4088
+ const rows = (split) => scores.flatMap((row2, qi) => splitOf(qi) === split ? [{ row: row2, qi }] : []);
3953
4089
  const positives = [];
3954
4090
  const negatives = [];
3955
- for (const [qi, row2] of scores.entries()) {
3956
- for (const [fi, score] of row2.entries()) (relevant(qi, fi) ? positives : negatives).push(score);
3957
- }
3958
- const auc = positives.reduce((wins, p) => wins + negatives.filter((n) => p > n).length, 0) / (positives.length * negatives.length);
3959
4091
  let hits = 0;
3960
4092
  let reciprocal = 0;
3961
- let answerable = 0;
3962
- for (const [qi, row2] of scores.entries()) {
3963
- if (!data.queries[qi]?.relevant.length) continue;
3964
- answerable += 1;
4093
+ let answered = 0;
4094
+ for (const { row: row2, qi } of rows("test")) {
4095
+ for (const [fi, score] of row2.entries()) (relevant(qi, fi) ? positives : negatives).push(score);
4096
+ if (!answerable(qi)) continue;
4097
+ answered += 1;
3965
4098
  const order = row2.map((score, fi) => ({ score, fi })).sort((a, b) => b.score - a.score);
3966
4099
  const rank = order.findIndex(({ fi }) => relevant(qi, fi)) + 1;
3967
4100
  if (rank === 1) hits += 1;
3968
4101
  reciprocal += 1 / rank;
3969
4102
  }
3970
- const at = (floor) => {
4103
+ const auc = positives.reduce((wins, p) => wins + negatives.filter((n) => p > n).length, 0) / (positives.length * negatives.length);
4104
+ const at = (floor, split = "test") => {
3971
4105
  let tp = 0;
3972
4106
  let fp = 0;
4107
+ let relevantPairs = 0;
3973
4108
  let abstained = 0;
3974
4109
  let unrelated = 0;
3975
- for (const [qi, row2] of scores.entries()) {
4110
+ for (const { row: row2, qi } of rows(split)) {
4111
+ relevantPairs += data.queries[qi]?.relevant.length ?? 0;
3976
4112
  const admitted = row2.flatMap((score, fi) => score >= floor ? [fi] : []);
3977
4113
  for (const fi of admitted) {
3978
4114
  if (relevant(qi, fi)) tp += 1;
3979
4115
  else fp += 1;
3980
4116
  }
3981
- if (!data.queries[qi]?.relevant.length) {
4117
+ if (!answerable(qi)) {
3982
4118
  unrelated += 1;
3983
4119
  if (admitted.length === 0) abstained += 1;
3984
4120
  }
3985
4121
  }
3986
4122
  const precision = tp + fp === 0 ? 1 : tp / (tp + fp);
3987
- const recall = tp / positives.length;
4123
+ const recall = relevantPairs === 0 ? 0 : tp / relevantPairs;
3988
4124
  const f05 = precision + recall === 0 ? 0 : 1.25 * precision * recall / (0.25 * precision + recall);
3989
- return { floor, precision, recall, f05, abstain: abstained / unrelated };
4125
+ return { floor, precision, recall, f05, abstain: unrelated === 0 ? 1 : abstained / unrelated };
3990
4126
  };
3991
- const sweep = Array.from({ length: 96 }, (_, i) => at(Math.round((0.05 + i * 0.01) * 100) / 100));
4127
+ const sweep = Array.from(
4128
+ { length: 96 },
4129
+ (_, i) => at(Math.round((0.05 + i * 0.01) * 100) / 100, "dev")
4130
+ );
3992
4131
  const best = sweep.reduce((left, right) => right.f05 > left.f05 ? right : left);
3993
- return { auc, recallAt1: hits / answerable, mrr: reciprocal / answerable, best, at };
4132
+ return {
4133
+ auc,
4134
+ recallAt1: answered === 0 ? 0 : hits / answered,
4135
+ mrr: answered === 0 ? 0 : reciprocal / answered,
4136
+ best,
4137
+ test: at(best.floor, "test"),
4138
+ at
4139
+ };
3994
4140
  }
3995
4141
 
3996
4142
  // src/cli/commands/models.ts
@@ -4225,7 +4371,7 @@ before keeping one on. A reranker never admits a fact, it only orders them.`
4225
4371
  });
4226
4372
  models.command("pull").description("Download the local models now instead of on first use.").action(async () => {
4227
4373
  await runSafely2(async () => {
4228
- const { LocalModelProvider, rerankLocally } = await import("./local-3NGNESNB.js");
4374
+ const { LocalModelProvider, rerankLocally } = await import("./local-OTOY56B4.js");
4229
4375
  const file = await readConfigFile();
4230
4376
  let started = Date.now();
4231
4377
  await withSpinner(
@@ -4261,13 +4407,13 @@ before keeping one on. A reranker never admits a fact, it only orders them.`
4261
4407
  console.log(
4262
4408
  row(
4263
4409
  "ranking",
4264
- `AUC ${result.auc.toFixed(3)} \xB7 right fact first ${pct(result.recallAt1)}`
4410
+ `AUC ${result.auc.toFixed(3)} \xB7 right fact first ${pct(result.recallAt1)} ${pc13.dim("(held-out tasks)")}`
4265
4411
  )
4266
4412
  );
4267
4413
  console.log(
4268
4414
  row(
4269
4415
  "floor",
4270
- `${result.best.floor}: precision ${pct(result.best.precision)}, recall ${pct(result.best.recall)}, silent on ${pct(result.best.abstain)} of unrelated tasks`
4416
+ `${result.best.floor}: precision ${pct(result.test.precision)}, recall ${pct(result.test.recall)}, silent on ${pct(result.test.abstain)} of unrelated tasks ${pc13.dim("(chosen on dev, measured on test)")}`
4271
4417
  )
4272
4418
  );
4273
4419
  if (opts.dryRun) return;
@@ -5591,4 +5737,4 @@ function printDbDoctor(report) {
5591
5737
  export {
5592
5738
  createCli
5593
5739
  };
5594
- //# sourceMappingURL=main-BDVQAGAS.js.map
5740
+ //# sourceMappingURL=main-HFHLGID4.js.map