codetrap 0.1.6 → 0.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/README.md +159 -51
  2. package/docs/installation.md +113 -29
  3. package/package.json +4 -3
  4. package/plugins/codetrap-agent/.codex-plugin/plugin.json +1 -2
  5. package/plugins/codetrap-agent/hooks/post-flight-capture.example.md +19 -17
  6. package/plugins/codetrap-agent/hooks.json +2 -2
  7. package/{skills → plugins/codetrap-agent/skills}/codetrap-add/SKILL.md +10 -4
  8. package/plugins/codetrap-agent/skills/codetrap-capture/SKILL.md +14 -3
  9. package/plugins/codetrap-agent/skills/codetrap-capture-external/SKILL.md +52 -9
  10. package/plugins/codetrap-agent/skills/codetrap-check/SKILL.md +74 -6
  11. package/{skills → plugins/codetrap-agent/skills}/codetrap-search/SKILL.md +6 -5
  12. package/plugins/codetrap-agent/templates/AGENTS.codetrap.md +31 -5
  13. package/scripts/search-policy-sweep.ts +131 -0
  14. package/src/commands/workflow.ts +186 -68
  15. package/src/db/connection.ts +6 -6
  16. package/src/db/embedding-queries.ts +230 -48
  17. package/src/db/queries.ts +0 -25
  18. package/src/db/repository.ts +32 -21
  19. package/src/db/schema.ts +80 -0
  20. package/src/index.ts +32 -7
  21. package/src/lib/command-requests.ts +134 -1
  22. package/src/lib/config.ts +57 -7
  23. package/src/lib/constants.ts +1 -1
  24. package/src/lib/doctor.ts +96 -6
  25. package/src/lib/embed-output.ts +26 -0
  26. package/src/lib/embedder.ts +118 -3
  27. package/src/lib/embedding-health.ts +3 -1
  28. package/src/lib/embedding-job.ts +3 -0
  29. package/src/lib/embedding-management.ts +65 -0
  30. package/src/lib/embedding-runtime.ts +177 -0
  31. package/src/lib/output-json.ts +0 -2
  32. package/src/lib/scope-context.ts +17 -11
  33. package/src/lib/scope-migration.ts +2 -1
  34. package/src/lib/scope.ts +4 -6
  35. package/src/lib/search-eval.ts +136 -23
  36. package/src/lib/search-policy-sweep.ts +563 -0
  37. package/src/lib/search-policy.ts +0 -4
  38. package/src/lib/search-service.ts +14 -15
  39. package/src/lib/session-candidate-document.ts +175 -0
  40. package/src/lib/session-candidate-scope.ts +6 -0
  41. package/src/lib/session-capture.ts +298 -32
  42. package/src/lib/session-codec.ts +1 -8
  43. package/src/lib/session-operations.ts +111 -51
  44. package/src/lib/session-review.ts +327 -0
  45. package/src/lib/session-store.ts +177 -55
  46. package/src/lib/store.ts +79 -11
  47. package/src/lib/string-list.ts +3 -0
  48. package/src/lib/text-lines.ts +7 -0
  49. package/src/lib/trap-search-document.ts +2 -1
  50. package/src/lib/value-types.ts +3 -0
  51. package/src/web/client-review.ts +171 -0
  52. package/src/web/client-script.ts +1543 -0
  53. package/src/web/client-shell.ts +414 -0
  54. package/src/web/client-text.ts +447 -0
  55. package/src/web/project-registry.ts +3 -5
  56. package/src/web/server.ts +184 -111
  57. package/src/web/static.ts +581 -484
  58. package/skills/codetrap-capture-external/SKILL.md +0 -62
  59. package/skills/codetrap-check/SKILL.md +0 -69
  60. package/src/lib/embedding-index.ts +0 -53
@@ -0,0 +1,65 @@
1
+ import type { ConfigWriteResult, EmbeddingSettings } from "./config";
2
+ import type { EmbeddingScopeStatus, TrapEmbeddingProfiles, TrapEmbeddingStatus } from "./store";
3
+ import type { EmbeddingProfileSummary } from "../db/embedding-queries";
4
+
5
+ export type EmbeddingsUseResult = ConfigWriteResult & {
6
+ embeddings: EmbeddingSettings;
7
+ next_action: {
8
+ command: string;
9
+ reason: string;
10
+ };
11
+ };
12
+
13
+ export function formatEmbeddingStatusText(status: TrapEmbeddingStatus): string {
14
+ const runtime = status.runtime.provider
15
+ ? `${status.runtime.provider}/${status.runtime.model} (${status.runtime.dimensions}d)`
16
+ : "unconfigured";
17
+ const lines = [
18
+ `Runtime: ${runtime}`,
19
+ `Available: ${status.runtime.available ? "yes" : "no"}`,
20
+ `Active profile: ${status.runtime.profile_id ?? "(none)"}`,
21
+ ];
22
+
23
+ if (status.runtime.setup_action) {
24
+ lines.push(`Next setup: ${status.runtime.setup_action.command}`);
25
+ }
26
+
27
+ lines.push(...scopeStatusLines("Project", status.project));
28
+ lines.push(...scopeStatusLines("Global", status.global));
29
+ return lines.join("\n");
30
+ }
31
+
32
+ export function formatEmbeddingProfilesText(profiles: TrapEmbeddingProfiles): string {
33
+ const lines = [
34
+ ...profileSectionLines("Project", profiles.project),
35
+ ...profileSectionLines("Global", profiles.global),
36
+ ];
37
+ return lines.length > 0 ? lines.join("\n") : "No stored embedding profiles.";
38
+ }
39
+
40
+ export function formatEmbeddingsUseText(result: EmbeddingsUseResult): string {
41
+ return [
42
+ `Configured embeddings provider: ${result.embeddings.provider}`,
43
+ `Config: ${result.path}`,
44
+ `Next: ${result.next_action.command}`,
45
+ ].join("\n");
46
+ }
47
+
48
+ function scopeStatusLines(label: string, status: EmbeddingScopeStatus | null): string[] {
49
+ if (!status) return [];
50
+ return [
51
+ "",
52
+ `${label}: total ${status.total}, fresh ${status.fresh}, stale ${status.stale}, missing ${status.missing}`,
53
+ ...profileSectionLines("Stored profiles", status.profiles),
54
+ ];
55
+ }
56
+
57
+ function profileSectionLines(label: string, profiles: EmbeddingProfileSummary[] | null): string[] {
58
+ if (!profiles || profiles.length === 0) return [];
59
+ return [
60
+ `${label}:`,
61
+ ...profiles.map((profile) =>
62
+ ` - ${profile.id} (${profile.embedding_count} embeddings, updated ${profile.updated_at ?? "unknown"})`
63
+ ),
64
+ ];
65
+ }
@@ -0,0 +1,177 @@
1
+ import {
2
+ DEFAULT_OLLAMA_DIMENSIONS,
3
+ DEFAULT_OLLAMA_ENDPOINT,
4
+ DEFAULT_OLLAMA_MODEL,
5
+ EmbeddingProviderUnavailableError,
6
+ JinaEmbedder,
7
+ OllamaEmbedder,
8
+ embeddingConfig,
9
+ embeddingProfileId,
10
+ type EmbeddingConfig,
11
+ type EmbeddingProvider,
12
+ } from "./embedder";
13
+ import { loadCodetrapConfig, type CodetrapConfig, type EmbeddingProviderSetting } from "./config";
14
+
15
+ export type EmbeddingRuntimeInput = EmbeddingRuntime | EmbeddingProvider | undefined;
16
+
17
+ export type EmbeddingRuntimeAction = {
18
+ command: string;
19
+ reason: string;
20
+ };
21
+
22
+ export type EmbeddingRuntimeStatus = {
23
+ available: boolean;
24
+ provider: string | null;
25
+ model: string | null;
26
+ dimensions: number | null;
27
+ passage_version: number | null;
28
+ profile_id: string | null;
29
+ setup_action: EmbeddingRuntimeAction | null;
30
+ };
31
+
32
+ export class EmbeddingRuntime {
33
+ constructor(
34
+ private readonly adapter?: EmbeddingProvider,
35
+ private readonly setupProvider: EmbeddingProviderSetting = "ollama",
36
+ private readonly configuredConfig: EmbeddingConfig | null = null
37
+ ) {}
38
+
39
+ static fromEnvironment(
40
+ env: NodeJS.ProcessEnv = process.env,
41
+ config: CodetrapConfig = loadCodetrapConfig()
42
+ ): EmbeddingRuntime {
43
+ const provider = configuredProvider(env, config);
44
+ if (provider === "ollama") {
45
+ return new EmbeddingRuntime(new OllamaEmbedder(ollamaOptions(env, config)), "ollama");
46
+ }
47
+ if (provider === "jina") {
48
+ const apiKey = env.JINA_API_KEY;
49
+ const configuredConfig = embeddingConfig(new JinaEmbedder(apiKey ?? ""));
50
+ return new EmbeddingRuntime(apiKey ? new JinaEmbedder(apiKey) : undefined, "jina", configuredConfig);
51
+ }
52
+
53
+ const apiKey = env.JINA_API_KEY;
54
+ if (apiKey) return new EmbeddingRuntime(new JinaEmbedder(apiKey), "jina");
55
+ return new EmbeddingRuntime(undefined, "ollama");
56
+ }
57
+
58
+ provider(): EmbeddingProvider | undefined {
59
+ return this.adapter;
60
+ }
61
+
62
+ available(): boolean {
63
+ return this.adapter !== undefined;
64
+ }
65
+
66
+ config(): EmbeddingConfig | null {
67
+ return this.adapter ? embeddingConfig(this.adapter) : this.configuredConfig;
68
+ }
69
+
70
+ requireProvider(message?: string): EmbeddingProvider {
71
+ if (!this.adapter) throw this.unavailableError(message);
72
+ return this.adapter;
73
+ }
74
+
75
+ unavailableError(message?: string): EmbeddingProviderUnavailableError {
76
+ return new EmbeddingProviderUnavailableError(
77
+ message ?? "Embedding provider is unavailable. Configure an embedding provider or use --mode fts."
78
+ );
79
+ }
80
+
81
+ setupAction(): EmbeddingRuntimeAction | null {
82
+ if (this.adapter) return null;
83
+ if (this.setupProvider === "jina") {
84
+ return {
85
+ command: "export JINA_API_KEY=<your-jina-api-key>",
86
+ reason: "Enable Jina semantic and hybrid search; otherwise use --mode fts.",
87
+ };
88
+ }
89
+ return {
90
+ command: "export CODETRAP_EMBEDDING_PROVIDER=ollama",
91
+ reason: "Enable local semantic and hybrid search after installing Ollama and pulling qwen3-embedding:0.6b; otherwise use --mode fts.",
92
+ };
93
+ }
94
+
95
+ status(): EmbeddingRuntimeStatus {
96
+ const config = this.config();
97
+ return {
98
+ available: this.available(),
99
+ provider: config?.provider ?? null,
100
+ model: config?.model ?? null,
101
+ dimensions: config?.dimensions ?? null,
102
+ passage_version: config?.passageVersion ?? null,
103
+ profile_id: config ? embeddingProfileId(config) : null,
104
+ setup_action: this.setupAction(),
105
+ };
106
+ }
107
+
108
+ async health(): Promise<EmbeddingRuntimeStatus> {
109
+ const status = this.status();
110
+ if (!this.adapter || !(this.adapter instanceof OllamaEmbedder)) return status;
111
+
112
+ const health = await this.adapter.health();
113
+ if (health.ok) return status;
114
+ return {
115
+ ...status,
116
+ available: false,
117
+ setup_action: {
118
+ command: health.command ?? `ollama pull ${this.adapter.model}`,
119
+ reason: health.message ?? "Ollama embedding provider is unavailable.",
120
+ },
121
+ };
122
+ }
123
+ }
124
+
125
+ export function defaultEmbeddingRuntime(
126
+ env: NodeJS.ProcessEnv = process.env,
127
+ config?: CodetrapConfig
128
+ ): EmbeddingRuntime {
129
+ return EmbeddingRuntime.fromEnvironment(env, config);
130
+ }
131
+
132
+ export function embeddingRuntimeFrom(input: EmbeddingRuntimeInput): EmbeddingRuntime {
133
+ return input instanceof EmbeddingRuntime ? input : new EmbeddingRuntime(input);
134
+ }
135
+
136
+ function configuredProvider(
137
+ env: NodeJS.ProcessEnv,
138
+ config: CodetrapConfig
139
+ ): EmbeddingProviderSetting | undefined {
140
+ if (config.embeddings?.provider) return config.embeddings.provider;
141
+ if (env.CODETRAP_EMBEDDING_PROVIDER) return parseProviderEnv(env.CODETRAP_EMBEDDING_PROVIDER);
142
+ if (config.embeddings?.endpoint || config.embeddings?.model || config.embeddings?.dimensions) return "ollama";
143
+ if (env.CODETRAP_OLLAMA_ENDPOINT || env.CODETRAP_OLLAMA_MODEL || env.CODETRAP_OLLAMA_DIMENSIONS) {
144
+ return "ollama";
145
+ }
146
+ return undefined;
147
+ }
148
+
149
+ function ollamaOptions(env: NodeJS.ProcessEnv, config: CodetrapConfig) {
150
+ return {
151
+ endpoint: normalizeEndpointSetting(
152
+ config.embeddings?.endpoint ?? env.CODETRAP_OLLAMA_ENDPOINT ?? env.OLLAMA_HOST ?? DEFAULT_OLLAMA_ENDPOINT
153
+ ),
154
+ model: config.embeddings?.model ?? env.CODETRAP_OLLAMA_MODEL ?? DEFAULT_OLLAMA_MODEL,
155
+ dimensions:
156
+ config.embeddings?.dimensions ??
157
+ parseOptionalPositiveInt(env.CODETRAP_OLLAMA_DIMENSIONS, "CODETRAP_OLLAMA_DIMENSIONS") ??
158
+ DEFAULT_OLLAMA_DIMENSIONS,
159
+ };
160
+ }
161
+
162
+ function parseProviderEnv(value: string): EmbeddingProviderSetting {
163
+ if (value === "ollama" || value === "jina") return value;
164
+ throw new Error(`Invalid CODETRAP_EMBEDDING_PROVIDER: ${value}. Expected ollama or jina.`);
165
+ }
166
+
167
+ function parseOptionalPositiveInt(value: string | undefined, label: string): number | undefined {
168
+ if (!value) return undefined;
169
+ const parsed = Number.parseInt(value, 10);
170
+ if (Number.isInteger(parsed) && parsed > 0) return parsed;
171
+ throw new Error(`Invalid ${label}: expected a positive integer.`);
172
+ }
173
+
174
+ function normalizeEndpointSetting(endpoint: string): string {
175
+ if (/^https?:\/\//.test(endpoint)) return endpoint;
176
+ return `http://${endpoint}`;
177
+ }
@@ -137,5 +137,3 @@ export function toStatsJson(stats: TrapStatsResult, embeddings?: EmbeddingStatsR
137
137
  export function toDoctorJson(input: DoctorReport): DoctorReport {
138
138
  return input;
139
139
  }
140
-
141
- export { toTrapEvidenceJson, toTrapJson, type JsonTrap, type JsonTrapEvidence };
@@ -2,9 +2,10 @@ import { openGlobal, openProject } from "../db/connection";
2
2
  import { TrapRepository } from "../db/repository";
3
3
  import { CODETRAP_DIR, TRAPS_DB_FILE } from "./constants";
4
4
  import type { Scope } from "./constants";
5
- import type { EmbeddingProvider } from "./embedder";
6
- import { findProjectRoot, getGlobalDB, resolveScopePath } from "./scope";
7
- import { ScopePathResolver } from "./scope-path";
5
+ import type { EmbeddingRuntime } from "./embedding-runtime";
6
+ import type { RankingConfig } from "./search-policy";
7
+ import { findProjectRoot, getGlobalDB } from "./scope";
8
+ import { resolveScopePath, ScopePathResolver } from "./scope-path";
8
9
 
9
10
  const scopePath = new ScopePathResolver();
10
11
 
@@ -15,14 +16,14 @@ export type ScopeContext = {
15
16
  global_db: string;
16
17
  };
17
18
 
18
- export function createScopeContext(cwd = process.cwd()): ScopeContext {
19
- const resolvedCwd = resolveScopePath(cwd);
20
- const projectRoot = findProjectRoot(resolvedCwd);
19
+ export function createScopeContext(cwd = process.cwd(), home?: string): ScopeContext {
20
+ const resolvedCwd = resolveScopePath(cwd, home ?? cwd);
21
+ const projectRoot = findProjectRoot(resolvedCwd, home);
21
22
  return {
22
23
  cwd: resolvedCwd,
23
24
  project_root: projectRoot,
24
25
  project_db: projectRoot ? scopePath.join(projectRoot, CODETRAP_DIR, TRAPS_DB_FILE) : null,
25
- global_db: getGlobalDB(),
26
+ global_db: getGlobalDB(home),
26
27
  };
27
28
  }
28
29
 
@@ -36,8 +37,13 @@ export class ScopedRepositoryContext {
36
37
  private globalRepository?: TrapRepository;
37
38
  private projectRepository?: TrapRepository;
38
39
 
39
- constructor(cwd = process.cwd(), private readonly embedder?: EmbeddingProvider) {
40
- this.context = createScopeContext(cwd);
40
+ constructor(
41
+ cwd = process.cwd(),
42
+ private readonly embeddings?: EmbeddingRuntime,
43
+ private readonly home?: string,
44
+ private readonly ranking?: RankingConfig
45
+ ) {
46
+ this.context = createScopeContext(cwd, home);
41
47
  }
42
48
 
43
49
  hasProject(): boolean {
@@ -87,14 +93,14 @@ export class ScopedRepositoryContext {
87
93
  throw new Error("Not in a project. Run 'codetrap init' first, or use --scope global.");
88
94
  }
89
95
  if (!this.projectRepository) {
90
- this.projectRepository = new TrapRepository(openProject(projectRoot), this.embedder);
96
+ this.projectRepository = new TrapRepository(openProject(projectRoot), this.embeddings, this.ranking);
91
97
  }
92
98
  return this.projectRepository;
93
99
  }
94
100
 
95
101
  private globalRepo(): TrapRepository {
96
102
  if (!this.globalRepository) {
97
- this.globalRepository = new TrapRepository(openGlobal(), this.embedder);
103
+ this.globalRepository = new TrapRepository(openGlobal(this.home), this.embeddings, this.ranking);
98
104
  }
99
105
  return this.globalRepository;
100
106
  }
@@ -3,7 +3,8 @@ import { homedir } from "node:os";
3
3
  import { openDatabase } from "../db/connection";
4
4
  import { TrapRepository } from "../db/repository";
5
5
  import type { TrapExportRecord } from "../domain/trap";
6
- import { findProjectRoot, resolveScopePath } from "./scope";
6
+ import { findProjectRoot } from "./scope";
7
+ import { resolveScopePath } from "./scope-path";
7
8
  import {
8
9
  backupScopeDatabase,
9
10
  buildScopeMaintenancePaths,
package/src/lib/scope.ts CHANGED
@@ -3,16 +3,14 @@ import { homedir } from "node:os";
3
3
  import { CODETRAP_DIR, TRAPS_DB_FILE } from "./constants";
4
4
  import { defaultScopePathResolver, resolveScopePath, ScopePathResolver } from "./scope-path";
5
5
 
6
- export { resolveScopePath, ScopePathResolver } from "./scope-path";
7
-
8
- export function getGlobalDir(): string {
9
- const dir = defaultScopePathResolver.join(homedir(), CODETRAP_DIR);
6
+ export function getGlobalDir(homeDir = homedir()): string {
7
+ const dir = defaultScopePathResolver.join(homeDir, CODETRAP_DIR);
10
8
  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
11
9
  return dir;
12
10
  }
13
11
 
14
- export function getGlobalDB(): string {
15
- return defaultScopePathResolver.join(getGlobalDir(), TRAPS_DB_FILE);
12
+ export function getGlobalDB(homeDir = homedir()): string {
13
+ return defaultScopePathResolver.join(getGlobalDir(homeDir), TRAPS_DB_FILE);
16
14
  }
17
15
 
18
16
  export function findProjectRoot(
@@ -3,13 +3,19 @@ import { openDatabase } from "../db/connection";
3
3
  import { TrapRepository } from "../db/repository";
4
4
  import type { TrapInput, TrapSearchResult } from "../domain/trap";
5
5
  import { SEARCH_MODES, type SearchMode } from "./constants";
6
+ import { isRecord } from "./value-types";
6
7
  import {
7
- createDefaultEmbeddingProvider,
8
- embeddingConfig,
9
8
  type EmbeddingConfig,
10
9
  type EmbeddingProvider,
11
10
  type EmbeddingTask,
12
11
  } from "./embedder";
12
+ import {
13
+ EmbeddingRuntime,
14
+ defaultEmbeddingRuntime,
15
+ embeddingRuntimeFrom,
16
+ type EmbeddingRuntimeInput,
17
+ } from "./embedding-runtime";
18
+ import type { RankingConfig } from "./search-policy";
13
19
 
14
20
  export type PhaseGate = "phase0" | "phase1" | "phase4" | "dogfood";
15
21
  export const DOGFOOD_JUDGMENTS = ["useful_hit", "miss", "noisy_hit", "no_relevant_trap"] as const;
@@ -56,6 +62,11 @@ export type SearchEvalMetrics = {
56
62
  semantic_error_count: number;
57
63
  };
58
64
 
65
+ export type SearchEvalNextAction = {
66
+ command: string;
67
+ reason: string;
68
+ };
69
+
59
70
  export type SearchEvalReport = {
60
71
  mode: "deterministic" | "live";
61
72
  fixture: string;
@@ -71,6 +82,11 @@ export type SearchEvalReport = {
71
82
  failures: EvalCaseReport[];
72
83
  misses: EvalCaseReport[];
73
84
  noisy_hits: EvalCaseReport[];
85
+ next_actions: SearchEvalNextAction[];
86
+ };
87
+
88
+ export type SearchEvalDetailedReport = Omit<SearchEvalReport, "mode" | "fixture" | "next_actions"> & {
89
+ cases: EvalCaseReport[];
74
90
  };
75
91
 
76
92
  export type RecordDogfoodResult = {
@@ -109,30 +125,46 @@ export function recordDogfoodCase(fixturePath: string, jsonInput: string | undef
109
125
 
110
126
  export async function reportDogfood(fixturePath: string, live: boolean): Promise<SearchEvalReport> {
111
127
  const fixture = readEvalFixture(fixturePath);
112
- const provider = live ? createDefaultEmbeddingProvider() : new EvalEmbedder();
113
- const evaluated = await evaluateSearchFixture(fixture, provider);
114
- return {
115
- mode: live ? "live" : "deterministic",
128
+ const runtime = live ? defaultEmbeddingRuntime() : new EmbeddingRuntime(new EvalEmbedder());
129
+ const evaluated = await evaluateSearchFixture(fixture, runtime);
130
+ const mode: SearchEvalReport["mode"] = live ? "live" : "deterministic";
131
+ const report: Omit<SearchEvalReport, "next_actions"> = {
132
+ mode,
116
133
  fixture: fixturePath,
117
134
  ...evaluated,
118
135
  };
136
+ return {
137
+ ...report,
138
+ next_actions: buildSearchEvalNextActions(report),
139
+ };
119
140
  }
120
141
 
121
142
  export async function evaluateSearchFixture(
122
143
  fixture: EvalFixture,
123
- provider: EmbeddingProvider | undefined
124
- ): Promise<Omit<SearchEvalReport, "mode" | "fixture">> {
125
- const repo = fixtureRepository(fixture, provider);
144
+ embeddings: EmbeddingRuntimeInput,
145
+ ranking?: RankingConfig
146
+ ): Promise<Omit<SearchEvalReport, "mode" | "fixture" | "next_actions">> {
147
+ const { cases: _cases, ...report } = await evaluateSearchFixtureCases(fixture, embeddings, ranking);
148
+ return report;
149
+ }
150
+
151
+ export async function evaluateSearchFixtureCases(
152
+ fixture: EvalFixture,
153
+ embeddings: EmbeddingRuntimeInput,
154
+ ranking?: RankingConfig
155
+ ): Promise<SearchEvalDetailedReport> {
156
+ const runtime = embeddingRuntimeFrom(embeddings);
157
+ const repo = fixtureRepository(fixture, runtime, ranking);
126
158
 
127
159
  let providerError: string | null = null;
128
- if (provider) {
160
+ if (runtime.available()) {
129
161
  try {
130
162
  await repo.ensureEmbeddings();
131
163
  } catch (error) {
132
164
  providerError = errorMessage(error);
133
165
  }
134
166
  }
135
- const searchRepo = providerError ? fixtureRepository(fixture, undefined) : repo;
167
+ const searchRepo = providerError ? fixtureRepository(fixture, undefined, ranking) : repo;
136
168
 
137
169
  const cases: EvalCaseReport[] = [];
138
170
  let hybridFallbackCount = 0;
@@ -143,7 +175,7 @@ export async function evaluateSearchFixture(
143
175
  const results = await searchRepo.search(item.query, { mode: item.mode, limit: 5 });
144
176
  const report = caseReport(item, fixture, results);
145
177
  cases.push(report);
146
- if (item.mode === "hybrid" && (!provider || hasSemanticFallback(results))) {
178
+ if (item.mode === "hybrid" && (!runtime.available() || hasSemanticFallback(results))) {
147
179
  hybridFallbackCount++;
148
180
  }
149
181
  } catch (error) {
@@ -153,10 +185,13 @@ export async function evaluateSearchFixture(
153
185
  }
154
186
 
155
187
  const dogfoodCases = cases.filter((item) => item.phaseGate === "dogfood" || item.judgment !== undefined);
188
+ const failures = cases.filter((item) => !item.passed);
189
+ const misses = cases.filter((item) => item.judgment === "miss" || item.recallAt5 < (fixtureQuery(item, fixture)?.minRecallAt5 ?? 1));
190
+ const noisyHits = cases.filter((item) => item.judgment === "noisy_hit");
156
191
  const metrics = aggregateMetrics(cases);
157
192
  return {
158
- provider: provider ? embeddingConfig(provider) : null,
159
- semantic_available: Boolean(provider && providerError === null),
193
+ provider: runtime.config(),
194
+ semantic_available: runtime.available() && providerError === null,
160
195
  provider_error: providerError,
161
196
  total_cases: cases.length,
162
197
  metrics: {
@@ -173,9 +208,10 @@ export async function evaluateSearchFixture(
173
208
  no_relevant_trap: dogfoodCases.filter((item) => item.judgment === "no_relevant_trap").length,
174
209
  },
175
210
  },
176
- failures: cases.filter((item) => !item.passed),
177
- misses: cases.filter((item) => item.judgment === "miss" || item.recallAt5 < (fixtureQuery(item, fixture)?.minRecallAt5 ?? 1)),
178
- noisy_hits: cases.filter((item) => item.judgment === "noisy_hit"),
211
+ failures,
212
+ misses,
213
+ noisy_hits: noisyHits,
214
+ cases,
179
215
  };
180
216
  }
181
217
 
@@ -211,13 +247,93 @@ export function formatSearchEvalReport(report: SearchEvalReport): string {
211
247
  `MRR: ${report.metrics.mrr}`,
212
248
  `Hybrid fallback count: ${report.metrics.hybrid_fallback_count}`,
213
249
  `Semantic error count: ${report.metrics.semantic_error_count}`,
214
- `Dogfood cases: ${report.dogfood.total}`
250
+ `Dogfood cases: ${report.dogfood.total}`,
251
+ `Judgments: ${formatJudgmentCounts(report.dogfood.judgment_counts)}`
215
252
  );
253
+ appendCaseSection(lines, "Failures", report.failures);
254
+ appendCaseSection(lines, "Misses to inspect", report.misses);
255
+ appendCaseSection(lines, "Noisy hits to inspect", report.noisy_hits);
256
+ lines.push("Next actions:", ...formatNextActions(report.next_actions));
216
257
  return lines.join("\n");
217
258
  }
218
259
 
219
- function fixtureRepository(fixture: EvalFixture, provider: EmbeddingProvider | undefined): TrapRepository {
220
- const repo = new TrapRepository(openDatabase(":memory:"), provider);
260
+ function buildSearchEvalNextActions(
261
+ report: Omit<SearchEvalReport, "next_actions">
262
+ ): SearchEvalNextAction[] {
263
+ const actions: SearchEvalNextAction[] = [];
264
+ if (report.mode === "live" && !report.semantic_available) {
265
+ const action = defaultEmbeddingRuntime().setupAction();
266
+ if (action) actions.push({
267
+ command: action.command,
268
+ reason: "Enable live semantic checks, then rerun bun run eval:dogfood -- report --live.",
269
+ });
270
+ }
271
+ if (report.failures.length > 0) {
272
+ actions.push({
273
+ command: "bun run eval:dogfood -- report --json",
274
+ reason: "Inspect expected ids, top results, and errors before changing search behavior or fixture expectations.",
275
+ });
276
+ }
277
+ if (report.misses.length > 0) {
278
+ actions.push({
279
+ command: 'codetrap search "<miss query>" --mode hybrid --ranking-signals --json',
280
+ reason: "Replay a miss with ranking signals before deciding whether to tune search or promote a fixture case.",
281
+ });
282
+ }
283
+ if (report.noisy_hits.length > 0) {
284
+ actions.push({
285
+ command: 'codetrap search "<noisy query>" --mode hybrid --ranking-signals --json',
286
+ reason: "Inspect why noisy results ranked before deciding whether the case belongs in dogfood eval.",
287
+ });
288
+ }
289
+ if (actions.length === 0) {
290
+ actions.push({
291
+ command: 'codetrap search "<task keywords>" --mode hybrid --json',
292
+ reason: "Keep logging real pre-edit searches in dogfood-log.md before automating promotion.",
293
+ });
294
+ }
295
+ return actions;
296
+ }
297
+
298
+ function formatJudgmentCounts(counts: Record<DogfoodJudgment, number>): string {
299
+ return DOGFOOD_JUDGMENTS.map((judgment) => `${judgment}=${counts[judgment]}`).join(", ");
300
+ }
301
+
302
+ function appendCaseSection(lines: string[], title: string, cases: EvalCaseReport[]): void {
303
+ if (cases.length === 0) return;
304
+ lines.push(`${title}:`);
305
+ for (const item of cases.slice(0, 5)) {
306
+ lines.push(` - [${item.mode}] ${item.query}`);
307
+ if (item.goldTrapIds.length > 0) {
308
+ lines.push(` expected: ${formatExpected(item)}`);
309
+ }
310
+ lines.push(` top: ${formatTopResults(item)}`);
311
+ if (item.error) lines.push(` error: ${item.error}`);
312
+ }
313
+ if (cases.length > 5) lines.push(` ... ${cases.length - 5} more`);
314
+ }
315
+
316
+ function formatExpected(item: EvalCaseReport): string {
317
+ return item.goldTrapIds
318
+ .map((id, index) => `#${id} ${item.expectedTitles[index] ?? ""}`.trim())
319
+ .join(", ");
320
+ }
321
+
322
+ function formatTopResults(item: EvalCaseReport): string {
323
+ if (item.topResults.length === 0) return "(none)";
324
+ return item.topResults
325
+ .slice(0, 3)
326
+ .map((result) => `#${result.id} ${result.title}`)
327
+ .join("; ");
328
+ }
329
+
330
+ function formatNextActions(actions: SearchEvalNextAction[]): string[] {
331
+ if (actions.length === 0) return [" (none)"];
332
+ return actions.map((action) => ` - ${action.command} # ${action.reason}`);
333
+ }
334
+
335
+ function fixtureRepository(fixture: EvalFixture, embeddings: EmbeddingRuntimeInput, ranking?: RankingConfig): TrapRepository {
336
+ const repo = new TrapRepository(openDatabase(":memory:"), embeddings, ranking);
221
337
  for (const trap of fixture.traps) repo.add(trap);
222
338
  return repo;
223
339
  }
@@ -378,9 +494,6 @@ function providerLabel(provider: EmbeddingConfig | null): string {
378
494
  return `${provider.provider}/${provider.model}`;
379
495
  }
380
496
 
381
- function isRecord(value: unknown): value is Record<string, unknown> {
382
- return typeof value === "object" && value !== null && !Array.isArray(value);
383
- }
384
497
 
385
498
  function round(value: number): number {
386
499
  return Math.round(value * 10000) / 10000;