codetrap 0.1.6 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +159 -51
- package/docs/installation.md +113 -29
- package/package.json +4 -3
- package/plugins/codetrap-agent/.codex-plugin/plugin.json +1 -2
- package/plugins/codetrap-agent/hooks/post-flight-capture.example.md +19 -17
- package/plugins/codetrap-agent/hooks.json +2 -2
- package/{skills → plugins/codetrap-agent/skills}/codetrap-add/SKILL.md +10 -4
- package/plugins/codetrap-agent/skills/codetrap-capture/SKILL.md +14 -3
- package/plugins/codetrap-agent/skills/codetrap-capture-external/SKILL.md +52 -9
- package/plugins/codetrap-agent/skills/codetrap-check/SKILL.md +74 -6
- package/{skills → plugins/codetrap-agent/skills}/codetrap-search/SKILL.md +6 -5
- package/plugins/codetrap-agent/templates/AGENTS.codetrap.md +31 -5
- package/scripts/search-policy-sweep.ts +131 -0
- package/src/commands/workflow.ts +186 -68
- package/src/db/connection.ts +6 -6
- package/src/db/embedding-queries.ts +230 -48
- package/src/db/queries.ts +0 -25
- package/src/db/repository.ts +32 -21
- package/src/db/schema.ts +80 -0
- package/src/index.ts +32 -7
- package/src/lib/command-requests.ts +134 -1
- package/src/lib/config.ts +57 -7
- package/src/lib/constants.ts +1 -1
- package/src/lib/doctor.ts +96 -6
- package/src/lib/embed-output.ts +26 -0
- package/src/lib/embedder.ts +118 -3
- package/src/lib/embedding-health.ts +3 -1
- package/src/lib/embedding-job.ts +3 -0
- package/src/lib/embedding-management.ts +65 -0
- package/src/lib/embedding-runtime.ts +177 -0
- package/src/lib/output-json.ts +0 -2
- package/src/lib/scope-context.ts +17 -11
- package/src/lib/scope-migration.ts +2 -1
- package/src/lib/scope.ts +4 -6
- package/src/lib/search-eval.ts +136 -23
- package/src/lib/search-policy-sweep.ts +563 -0
- package/src/lib/search-policy.ts +0 -4
- package/src/lib/search-service.ts +14 -15
- package/src/lib/session-candidate-document.ts +175 -0
- package/src/lib/session-candidate-scope.ts +6 -0
- package/src/lib/session-capture.ts +298 -32
- package/src/lib/session-codec.ts +1 -8
- package/src/lib/session-operations.ts +111 -51
- package/src/lib/session-review.ts +327 -0
- package/src/lib/session-store.ts +177 -55
- package/src/lib/store.ts +79 -11
- package/src/lib/string-list.ts +3 -0
- package/src/lib/text-lines.ts +7 -0
- package/src/lib/trap-search-document.ts +2 -1
- package/src/lib/value-types.ts +3 -0
- package/src/web/client-review.ts +171 -0
- package/src/web/client-script.ts +1543 -0
- package/src/web/client-shell.ts +414 -0
- package/src/web/client-text.ts +447 -0
- package/src/web/project-registry.ts +3 -5
- package/src/web/server.ts +184 -111
- package/src/web/static.ts +581 -484
- package/skills/codetrap-capture-external/SKILL.md +0 -62
- package/skills/codetrap-check/SKILL.md +0 -69
- package/src/lib/embedding-index.ts +0 -53
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import type { ConfigWriteResult, EmbeddingSettings } from "./config";
|
|
2
|
+
import type { EmbeddingScopeStatus, TrapEmbeddingProfiles, TrapEmbeddingStatus } from "./store";
|
|
3
|
+
import type { EmbeddingProfileSummary } from "../db/embedding-queries";
|
|
4
|
+
|
|
5
|
+
export type EmbeddingsUseResult = ConfigWriteResult & {
|
|
6
|
+
embeddings: EmbeddingSettings;
|
|
7
|
+
next_action: {
|
|
8
|
+
command: string;
|
|
9
|
+
reason: string;
|
|
10
|
+
};
|
|
11
|
+
};
|
|
12
|
+
|
|
13
|
+
export function formatEmbeddingStatusText(status: TrapEmbeddingStatus): string {
|
|
14
|
+
const runtime = status.runtime.provider
|
|
15
|
+
? `${status.runtime.provider}/${status.runtime.model} (${status.runtime.dimensions}d)`
|
|
16
|
+
: "unconfigured";
|
|
17
|
+
const lines = [
|
|
18
|
+
`Runtime: ${runtime}`,
|
|
19
|
+
`Available: ${status.runtime.available ? "yes" : "no"}`,
|
|
20
|
+
`Active profile: ${status.runtime.profile_id ?? "(none)"}`,
|
|
21
|
+
];
|
|
22
|
+
|
|
23
|
+
if (status.runtime.setup_action) {
|
|
24
|
+
lines.push(`Next setup: ${status.runtime.setup_action.command}`);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
lines.push(...scopeStatusLines("Project", status.project));
|
|
28
|
+
lines.push(...scopeStatusLines("Global", status.global));
|
|
29
|
+
return lines.join("\n");
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export function formatEmbeddingProfilesText(profiles: TrapEmbeddingProfiles): string {
|
|
33
|
+
const lines = [
|
|
34
|
+
...profileSectionLines("Project", profiles.project),
|
|
35
|
+
...profileSectionLines("Global", profiles.global),
|
|
36
|
+
];
|
|
37
|
+
return lines.length > 0 ? lines.join("\n") : "No stored embedding profiles.";
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export function formatEmbeddingsUseText(result: EmbeddingsUseResult): string {
|
|
41
|
+
return [
|
|
42
|
+
`Configured embeddings provider: ${result.embeddings.provider}`,
|
|
43
|
+
`Config: ${result.path}`,
|
|
44
|
+
`Next: ${result.next_action.command}`,
|
|
45
|
+
].join("\n");
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function scopeStatusLines(label: string, status: EmbeddingScopeStatus | null): string[] {
|
|
49
|
+
if (!status) return [];
|
|
50
|
+
return [
|
|
51
|
+
"",
|
|
52
|
+
`${label}: total ${status.total}, fresh ${status.fresh}, stale ${status.stale}, missing ${status.missing}`,
|
|
53
|
+
...profileSectionLines("Stored profiles", status.profiles),
|
|
54
|
+
];
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function profileSectionLines(label: string, profiles: EmbeddingProfileSummary[] | null): string[] {
|
|
58
|
+
if (!profiles || profiles.length === 0) return [];
|
|
59
|
+
return [
|
|
60
|
+
`${label}:`,
|
|
61
|
+
...profiles.map((profile) =>
|
|
62
|
+
` - ${profile.id} (${profile.embedding_count} embeddings, updated ${profile.updated_at ?? "unknown"})`
|
|
63
|
+
),
|
|
64
|
+
];
|
|
65
|
+
}
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
import {
|
|
2
|
+
DEFAULT_OLLAMA_DIMENSIONS,
|
|
3
|
+
DEFAULT_OLLAMA_ENDPOINT,
|
|
4
|
+
DEFAULT_OLLAMA_MODEL,
|
|
5
|
+
EmbeddingProviderUnavailableError,
|
|
6
|
+
JinaEmbedder,
|
|
7
|
+
OllamaEmbedder,
|
|
8
|
+
embeddingConfig,
|
|
9
|
+
embeddingProfileId,
|
|
10
|
+
type EmbeddingConfig,
|
|
11
|
+
type EmbeddingProvider,
|
|
12
|
+
} from "./embedder";
|
|
13
|
+
import { loadCodetrapConfig, type CodetrapConfig, type EmbeddingProviderSetting } from "./config";
|
|
14
|
+
|
|
15
|
+
export type EmbeddingRuntimeInput = EmbeddingRuntime | EmbeddingProvider | undefined;
|
|
16
|
+
|
|
17
|
+
export type EmbeddingRuntimeAction = {
|
|
18
|
+
command: string;
|
|
19
|
+
reason: string;
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
export type EmbeddingRuntimeStatus = {
|
|
23
|
+
available: boolean;
|
|
24
|
+
provider: string | null;
|
|
25
|
+
model: string | null;
|
|
26
|
+
dimensions: number | null;
|
|
27
|
+
passage_version: number | null;
|
|
28
|
+
profile_id: string | null;
|
|
29
|
+
setup_action: EmbeddingRuntimeAction | null;
|
|
30
|
+
};
|
|
31
|
+
|
|
32
|
+
export class EmbeddingRuntime {
|
|
33
|
+
constructor(
|
|
34
|
+
private readonly adapter?: EmbeddingProvider,
|
|
35
|
+
private readonly setupProvider: EmbeddingProviderSetting = "ollama",
|
|
36
|
+
private readonly configuredConfig: EmbeddingConfig | null = null
|
|
37
|
+
) {}
|
|
38
|
+
|
|
39
|
+
static fromEnvironment(
|
|
40
|
+
env: NodeJS.ProcessEnv = process.env,
|
|
41
|
+
config: CodetrapConfig = loadCodetrapConfig()
|
|
42
|
+
): EmbeddingRuntime {
|
|
43
|
+
const provider = configuredProvider(env, config);
|
|
44
|
+
if (provider === "ollama") {
|
|
45
|
+
return new EmbeddingRuntime(new OllamaEmbedder(ollamaOptions(env, config)), "ollama");
|
|
46
|
+
}
|
|
47
|
+
if (provider === "jina") {
|
|
48
|
+
const apiKey = env.JINA_API_KEY;
|
|
49
|
+
const configuredConfig = embeddingConfig(new JinaEmbedder(apiKey ?? ""));
|
|
50
|
+
return new EmbeddingRuntime(apiKey ? new JinaEmbedder(apiKey) : undefined, "jina", configuredConfig);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const apiKey = env.JINA_API_KEY;
|
|
54
|
+
if (apiKey) return new EmbeddingRuntime(new JinaEmbedder(apiKey), "jina");
|
|
55
|
+
return new EmbeddingRuntime(undefined, "ollama");
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
provider(): EmbeddingProvider | undefined {
|
|
59
|
+
return this.adapter;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
available(): boolean {
|
|
63
|
+
return this.adapter !== undefined;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
config(): EmbeddingConfig | null {
|
|
67
|
+
return this.adapter ? embeddingConfig(this.adapter) : this.configuredConfig;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
requireProvider(message?: string): EmbeddingProvider {
|
|
71
|
+
if (!this.adapter) throw this.unavailableError(message);
|
|
72
|
+
return this.adapter;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
unavailableError(message?: string): EmbeddingProviderUnavailableError {
|
|
76
|
+
return new EmbeddingProviderUnavailableError(
|
|
77
|
+
message ?? "Embedding provider is unavailable. Configure an embedding provider or use --mode fts."
|
|
78
|
+
);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
setupAction(): EmbeddingRuntimeAction | null {
|
|
82
|
+
if (this.adapter) return null;
|
|
83
|
+
if (this.setupProvider === "jina") {
|
|
84
|
+
return {
|
|
85
|
+
command: "export JINA_API_KEY=<your-jina-api-key>",
|
|
86
|
+
reason: "Enable Jina semantic and hybrid search; otherwise use --mode fts.",
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
return {
|
|
90
|
+
command: "export CODETRAP_EMBEDDING_PROVIDER=ollama",
|
|
91
|
+
reason: "Enable local semantic and hybrid search after installing Ollama and pulling qwen3-embedding:0.6b; otherwise use --mode fts.",
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
status(): EmbeddingRuntimeStatus {
|
|
96
|
+
const config = this.config();
|
|
97
|
+
return {
|
|
98
|
+
available: this.available(),
|
|
99
|
+
provider: config?.provider ?? null,
|
|
100
|
+
model: config?.model ?? null,
|
|
101
|
+
dimensions: config?.dimensions ?? null,
|
|
102
|
+
passage_version: config?.passageVersion ?? null,
|
|
103
|
+
profile_id: config ? embeddingProfileId(config) : null,
|
|
104
|
+
setup_action: this.setupAction(),
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
async health(): Promise<EmbeddingRuntimeStatus> {
|
|
109
|
+
const status = this.status();
|
|
110
|
+
if (!this.adapter || !(this.adapter instanceof OllamaEmbedder)) return status;
|
|
111
|
+
|
|
112
|
+
const health = await this.adapter.health();
|
|
113
|
+
if (health.ok) return status;
|
|
114
|
+
return {
|
|
115
|
+
...status,
|
|
116
|
+
available: false,
|
|
117
|
+
setup_action: {
|
|
118
|
+
command: health.command ?? `ollama pull ${this.adapter.model}`,
|
|
119
|
+
reason: health.message ?? "Ollama embedding provider is unavailable.",
|
|
120
|
+
},
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
export function defaultEmbeddingRuntime(
|
|
126
|
+
env: NodeJS.ProcessEnv = process.env,
|
|
127
|
+
config?: CodetrapConfig
|
|
128
|
+
): EmbeddingRuntime {
|
|
129
|
+
return EmbeddingRuntime.fromEnvironment(env, config);
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
export function embeddingRuntimeFrom(input: EmbeddingRuntimeInput): EmbeddingRuntime {
|
|
133
|
+
return input instanceof EmbeddingRuntime ? input : new EmbeddingRuntime(input);
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
function configuredProvider(
|
|
137
|
+
env: NodeJS.ProcessEnv,
|
|
138
|
+
config: CodetrapConfig
|
|
139
|
+
): EmbeddingProviderSetting | undefined {
|
|
140
|
+
if (config.embeddings?.provider) return config.embeddings.provider;
|
|
141
|
+
if (env.CODETRAP_EMBEDDING_PROVIDER) return parseProviderEnv(env.CODETRAP_EMBEDDING_PROVIDER);
|
|
142
|
+
if (config.embeddings?.endpoint || config.embeddings?.model || config.embeddings?.dimensions) return "ollama";
|
|
143
|
+
if (env.CODETRAP_OLLAMA_ENDPOINT || env.CODETRAP_OLLAMA_MODEL || env.CODETRAP_OLLAMA_DIMENSIONS) {
|
|
144
|
+
return "ollama";
|
|
145
|
+
}
|
|
146
|
+
return undefined;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
function ollamaOptions(env: NodeJS.ProcessEnv, config: CodetrapConfig) {
|
|
150
|
+
return {
|
|
151
|
+
endpoint: normalizeEndpointSetting(
|
|
152
|
+
config.embeddings?.endpoint ?? env.CODETRAP_OLLAMA_ENDPOINT ?? env.OLLAMA_HOST ?? DEFAULT_OLLAMA_ENDPOINT
|
|
153
|
+
),
|
|
154
|
+
model: config.embeddings?.model ?? env.CODETRAP_OLLAMA_MODEL ?? DEFAULT_OLLAMA_MODEL,
|
|
155
|
+
dimensions:
|
|
156
|
+
config.embeddings?.dimensions ??
|
|
157
|
+
parseOptionalPositiveInt(env.CODETRAP_OLLAMA_DIMENSIONS, "CODETRAP_OLLAMA_DIMENSIONS") ??
|
|
158
|
+
DEFAULT_OLLAMA_DIMENSIONS,
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
function parseProviderEnv(value: string): EmbeddingProviderSetting {
|
|
163
|
+
if (value === "ollama" || value === "jina") return value;
|
|
164
|
+
throw new Error(`Invalid CODETRAP_EMBEDDING_PROVIDER: ${value}. Expected ollama or jina.`);
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
function parseOptionalPositiveInt(value: string | undefined, label: string): number | undefined {
|
|
168
|
+
if (!value) return undefined;
|
|
169
|
+
const parsed = Number.parseInt(value, 10);
|
|
170
|
+
if (Number.isInteger(parsed) && parsed > 0) return parsed;
|
|
171
|
+
throw new Error(`Invalid ${label}: expected a positive integer.`);
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
function normalizeEndpointSetting(endpoint: string): string {
|
|
175
|
+
if (/^https?:\/\//.test(endpoint)) return endpoint;
|
|
176
|
+
return `http://${endpoint}`;
|
|
177
|
+
}
|
package/src/lib/output-json.ts
CHANGED
package/src/lib/scope-context.ts
CHANGED
|
@@ -2,9 +2,10 @@ import { openGlobal, openProject } from "../db/connection";
|
|
|
2
2
|
import { TrapRepository } from "../db/repository";
|
|
3
3
|
import { CODETRAP_DIR, TRAPS_DB_FILE } from "./constants";
|
|
4
4
|
import type { Scope } from "./constants";
|
|
5
|
-
import type {
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
5
|
+
import type { EmbeddingRuntime } from "./embedding-runtime";
|
|
6
|
+
import type { RankingConfig } from "./search-policy";
|
|
7
|
+
import { findProjectRoot, getGlobalDB } from "./scope";
|
|
8
|
+
import { resolveScopePath, ScopePathResolver } from "./scope-path";
|
|
8
9
|
|
|
9
10
|
const scopePath = new ScopePathResolver();
|
|
10
11
|
|
|
@@ -15,14 +16,14 @@ export type ScopeContext = {
|
|
|
15
16
|
global_db: string;
|
|
16
17
|
};
|
|
17
18
|
|
|
18
|
-
export function createScopeContext(cwd = process.cwd()): ScopeContext {
|
|
19
|
-
const resolvedCwd = resolveScopePath(cwd);
|
|
20
|
-
const projectRoot = findProjectRoot(resolvedCwd);
|
|
19
|
+
export function createScopeContext(cwd = process.cwd(), home?: string): ScopeContext {
|
|
20
|
+
const resolvedCwd = resolveScopePath(cwd, home ?? cwd);
|
|
21
|
+
const projectRoot = findProjectRoot(resolvedCwd, home);
|
|
21
22
|
return {
|
|
22
23
|
cwd: resolvedCwd,
|
|
23
24
|
project_root: projectRoot,
|
|
24
25
|
project_db: projectRoot ? scopePath.join(projectRoot, CODETRAP_DIR, TRAPS_DB_FILE) : null,
|
|
25
|
-
global_db: getGlobalDB(),
|
|
26
|
+
global_db: getGlobalDB(home),
|
|
26
27
|
};
|
|
27
28
|
}
|
|
28
29
|
|
|
@@ -36,8 +37,13 @@ export class ScopedRepositoryContext {
|
|
|
36
37
|
private globalRepository?: TrapRepository;
|
|
37
38
|
private projectRepository?: TrapRepository;
|
|
38
39
|
|
|
39
|
-
constructor(
|
|
40
|
-
|
|
40
|
+
constructor(
|
|
41
|
+
cwd = process.cwd(),
|
|
42
|
+
private readonly embeddings?: EmbeddingRuntime,
|
|
43
|
+
private readonly home?: string,
|
|
44
|
+
private readonly ranking?: RankingConfig
|
|
45
|
+
) {
|
|
46
|
+
this.context = createScopeContext(cwd, home);
|
|
41
47
|
}
|
|
42
48
|
|
|
43
49
|
hasProject(): boolean {
|
|
@@ -87,14 +93,14 @@ export class ScopedRepositoryContext {
|
|
|
87
93
|
throw new Error("Not in a project. Run 'codetrap init' first, or use --scope global.");
|
|
88
94
|
}
|
|
89
95
|
if (!this.projectRepository) {
|
|
90
|
-
this.projectRepository = new TrapRepository(openProject(projectRoot), this.
|
|
96
|
+
this.projectRepository = new TrapRepository(openProject(projectRoot), this.embeddings, this.ranking);
|
|
91
97
|
}
|
|
92
98
|
return this.projectRepository;
|
|
93
99
|
}
|
|
94
100
|
|
|
95
101
|
private globalRepo(): TrapRepository {
|
|
96
102
|
if (!this.globalRepository) {
|
|
97
|
-
this.globalRepository = new TrapRepository(openGlobal(), this.
|
|
103
|
+
this.globalRepository = new TrapRepository(openGlobal(this.home), this.embeddings, this.ranking);
|
|
98
104
|
}
|
|
99
105
|
return this.globalRepository;
|
|
100
106
|
}
|
|
@@ -3,7 +3,8 @@ import { homedir } from "node:os";
|
|
|
3
3
|
import { openDatabase } from "../db/connection";
|
|
4
4
|
import { TrapRepository } from "../db/repository";
|
|
5
5
|
import type { TrapExportRecord } from "../domain/trap";
|
|
6
|
-
import { findProjectRoot
|
|
6
|
+
import { findProjectRoot } from "./scope";
|
|
7
|
+
import { resolveScopePath } from "./scope-path";
|
|
7
8
|
import {
|
|
8
9
|
backupScopeDatabase,
|
|
9
10
|
buildScopeMaintenancePaths,
|
package/src/lib/scope.ts
CHANGED
|
@@ -3,16 +3,14 @@ import { homedir } from "node:os";
|
|
|
3
3
|
import { CODETRAP_DIR, TRAPS_DB_FILE } from "./constants";
|
|
4
4
|
import { defaultScopePathResolver, resolveScopePath, ScopePathResolver } from "./scope-path";
|
|
5
5
|
|
|
6
|
-
export
|
|
7
|
-
|
|
8
|
-
export function getGlobalDir(): string {
|
|
9
|
-
const dir = defaultScopePathResolver.join(homedir(), CODETRAP_DIR);
|
|
6
|
+
export function getGlobalDir(homeDir = homedir()): string {
|
|
7
|
+
const dir = defaultScopePathResolver.join(homeDir, CODETRAP_DIR);
|
|
10
8
|
if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
|
|
11
9
|
return dir;
|
|
12
10
|
}
|
|
13
11
|
|
|
14
|
-
export function getGlobalDB(): string {
|
|
15
|
-
return defaultScopePathResolver.join(getGlobalDir(), TRAPS_DB_FILE);
|
|
12
|
+
export function getGlobalDB(homeDir = homedir()): string {
|
|
13
|
+
return defaultScopePathResolver.join(getGlobalDir(homeDir), TRAPS_DB_FILE);
|
|
16
14
|
}
|
|
17
15
|
|
|
18
16
|
export function findProjectRoot(
|
package/src/lib/search-eval.ts
CHANGED
|
@@ -3,13 +3,19 @@ import { openDatabase } from "../db/connection";
|
|
|
3
3
|
import { TrapRepository } from "../db/repository";
|
|
4
4
|
import type { TrapInput, TrapSearchResult } from "../domain/trap";
|
|
5
5
|
import { SEARCH_MODES, type SearchMode } from "./constants";
|
|
6
|
+
import { isRecord } from "./value-types";
|
|
6
7
|
import {
|
|
7
|
-
createDefaultEmbeddingProvider,
|
|
8
|
-
embeddingConfig,
|
|
9
8
|
type EmbeddingConfig,
|
|
10
9
|
type EmbeddingProvider,
|
|
11
10
|
type EmbeddingTask,
|
|
12
11
|
} from "./embedder";
|
|
12
|
+
import {
|
|
13
|
+
EmbeddingRuntime,
|
|
14
|
+
defaultEmbeddingRuntime,
|
|
15
|
+
embeddingRuntimeFrom,
|
|
16
|
+
type EmbeddingRuntimeInput,
|
|
17
|
+
} from "./embedding-runtime";
|
|
18
|
+
import type { RankingConfig } from "./search-policy";
|
|
13
19
|
|
|
14
20
|
export type PhaseGate = "phase0" | "phase1" | "phase4" | "dogfood";
|
|
15
21
|
export const DOGFOOD_JUDGMENTS = ["useful_hit", "miss", "noisy_hit", "no_relevant_trap"] as const;
|
|
@@ -56,6 +62,11 @@ export type SearchEvalMetrics = {
|
|
|
56
62
|
semantic_error_count: number;
|
|
57
63
|
};
|
|
58
64
|
|
|
65
|
+
export type SearchEvalNextAction = {
|
|
66
|
+
command: string;
|
|
67
|
+
reason: string;
|
|
68
|
+
};
|
|
69
|
+
|
|
59
70
|
export type SearchEvalReport = {
|
|
60
71
|
mode: "deterministic" | "live";
|
|
61
72
|
fixture: string;
|
|
@@ -71,6 +82,11 @@ export type SearchEvalReport = {
|
|
|
71
82
|
failures: EvalCaseReport[];
|
|
72
83
|
misses: EvalCaseReport[];
|
|
73
84
|
noisy_hits: EvalCaseReport[];
|
|
85
|
+
next_actions: SearchEvalNextAction[];
|
|
86
|
+
};
|
|
87
|
+
|
|
88
|
+
export type SearchEvalDetailedReport = Omit<SearchEvalReport, "mode" | "fixture" | "next_actions"> & {
|
|
89
|
+
cases: EvalCaseReport[];
|
|
74
90
|
};
|
|
75
91
|
|
|
76
92
|
export type RecordDogfoodResult = {
|
|
@@ -109,30 +125,46 @@ export function recordDogfoodCase(fixturePath: string, jsonInput: string | undef
|
|
|
109
125
|
|
|
110
126
|
export async function reportDogfood(fixturePath: string, live: boolean): Promise<SearchEvalReport> {
|
|
111
127
|
const fixture = readEvalFixture(fixturePath);
|
|
112
|
-
const
|
|
113
|
-
const evaluated = await evaluateSearchFixture(fixture,
|
|
114
|
-
|
|
115
|
-
|
|
128
|
+
const runtime = live ? defaultEmbeddingRuntime() : new EmbeddingRuntime(new EvalEmbedder());
|
|
129
|
+
const evaluated = await evaluateSearchFixture(fixture, runtime);
|
|
130
|
+
const mode: SearchEvalReport["mode"] = live ? "live" : "deterministic";
|
|
131
|
+
const report: Omit<SearchEvalReport, "next_actions"> = {
|
|
132
|
+
mode,
|
|
116
133
|
fixture: fixturePath,
|
|
117
134
|
...evaluated,
|
|
118
135
|
};
|
|
136
|
+
return {
|
|
137
|
+
...report,
|
|
138
|
+
next_actions: buildSearchEvalNextActions(report),
|
|
139
|
+
};
|
|
119
140
|
}
|
|
120
141
|
|
|
121
142
|
export async function evaluateSearchFixture(
|
|
122
143
|
fixture: EvalFixture,
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
144
|
+
embeddings: EmbeddingRuntimeInput,
|
|
145
|
+
ranking?: RankingConfig
|
|
146
|
+
): Promise<Omit<SearchEvalReport, "mode" | "fixture" | "next_actions">> {
|
|
147
|
+
const { cases: _cases, ...report } = await evaluateSearchFixtureCases(fixture, embeddings, ranking);
|
|
148
|
+
return report;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
export async function evaluateSearchFixtureCases(
|
|
152
|
+
fixture: EvalFixture,
|
|
153
|
+
embeddings: EmbeddingRuntimeInput,
|
|
154
|
+
ranking?: RankingConfig
|
|
155
|
+
): Promise<SearchEvalDetailedReport> {
|
|
156
|
+
const runtime = embeddingRuntimeFrom(embeddings);
|
|
157
|
+
const repo = fixtureRepository(fixture, runtime, ranking);
|
|
126
158
|
|
|
127
159
|
let providerError: string | null = null;
|
|
128
|
-
if (
|
|
160
|
+
if (runtime.available()) {
|
|
129
161
|
try {
|
|
130
162
|
await repo.ensureEmbeddings();
|
|
131
163
|
} catch (error) {
|
|
132
164
|
providerError = errorMessage(error);
|
|
133
165
|
}
|
|
134
166
|
}
|
|
135
|
-
const searchRepo = providerError ? fixtureRepository(fixture, undefined) : repo;
|
|
167
|
+
const searchRepo = providerError ? fixtureRepository(fixture, undefined, ranking) : repo;
|
|
136
168
|
|
|
137
169
|
const cases: EvalCaseReport[] = [];
|
|
138
170
|
let hybridFallbackCount = 0;
|
|
@@ -143,7 +175,7 @@ export async function evaluateSearchFixture(
|
|
|
143
175
|
const results = await searchRepo.search(item.query, { mode: item.mode, limit: 5 });
|
|
144
176
|
const report = caseReport(item, fixture, results);
|
|
145
177
|
cases.push(report);
|
|
146
|
-
if (item.mode === "hybrid" && (!
|
|
178
|
+
if (item.mode === "hybrid" && (!runtime.available() || hasSemanticFallback(results))) {
|
|
147
179
|
hybridFallbackCount++;
|
|
148
180
|
}
|
|
149
181
|
} catch (error) {
|
|
@@ -153,10 +185,13 @@ export async function evaluateSearchFixture(
|
|
|
153
185
|
}
|
|
154
186
|
|
|
155
187
|
const dogfoodCases = cases.filter((item) => item.phaseGate === "dogfood" || item.judgment !== undefined);
|
|
188
|
+
const failures = cases.filter((item) => !item.passed);
|
|
189
|
+
const misses = cases.filter((item) => item.judgment === "miss" || item.recallAt5 < (fixtureQuery(item, fixture)?.minRecallAt5 ?? 1));
|
|
190
|
+
const noisyHits = cases.filter((item) => item.judgment === "noisy_hit");
|
|
156
191
|
const metrics = aggregateMetrics(cases);
|
|
157
192
|
return {
|
|
158
|
-
provider:
|
|
159
|
-
semantic_available:
|
|
193
|
+
provider: runtime.config(),
|
|
194
|
+
semantic_available: runtime.available() && providerError === null,
|
|
160
195
|
provider_error: providerError,
|
|
161
196
|
total_cases: cases.length,
|
|
162
197
|
metrics: {
|
|
@@ -173,9 +208,10 @@ export async function evaluateSearchFixture(
|
|
|
173
208
|
no_relevant_trap: dogfoodCases.filter((item) => item.judgment === "no_relevant_trap").length,
|
|
174
209
|
},
|
|
175
210
|
},
|
|
176
|
-
failures
|
|
177
|
-
misses
|
|
178
|
-
noisy_hits:
|
|
211
|
+
failures,
|
|
212
|
+
misses,
|
|
213
|
+
noisy_hits: noisyHits,
|
|
214
|
+
cases,
|
|
179
215
|
};
|
|
180
216
|
}
|
|
181
217
|
|
|
@@ -211,13 +247,93 @@ export function formatSearchEvalReport(report: SearchEvalReport): string {
|
|
|
211
247
|
`MRR: ${report.metrics.mrr}`,
|
|
212
248
|
`Hybrid fallback count: ${report.metrics.hybrid_fallback_count}`,
|
|
213
249
|
`Semantic error count: ${report.metrics.semantic_error_count}`,
|
|
214
|
-
`Dogfood cases: ${report.dogfood.total}
|
|
250
|
+
`Dogfood cases: ${report.dogfood.total}`,
|
|
251
|
+
`Judgments: ${formatJudgmentCounts(report.dogfood.judgment_counts)}`
|
|
215
252
|
);
|
|
253
|
+
appendCaseSection(lines, "Failures", report.failures);
|
|
254
|
+
appendCaseSection(lines, "Misses to inspect", report.misses);
|
|
255
|
+
appendCaseSection(lines, "Noisy hits to inspect", report.noisy_hits);
|
|
256
|
+
lines.push("Next actions:", ...formatNextActions(report.next_actions));
|
|
216
257
|
return lines.join("\n");
|
|
217
258
|
}
|
|
218
259
|
|
|
219
|
-
function
|
|
220
|
-
|
|
260
|
+
function buildSearchEvalNextActions(
|
|
261
|
+
report: Omit<SearchEvalReport, "next_actions">
|
|
262
|
+
): SearchEvalNextAction[] {
|
|
263
|
+
const actions: SearchEvalNextAction[] = [];
|
|
264
|
+
if (report.mode === "live" && !report.semantic_available) {
|
|
265
|
+
const action = defaultEmbeddingRuntime().setupAction();
|
|
266
|
+
if (action) actions.push({
|
|
267
|
+
command: action.command,
|
|
268
|
+
reason: "Enable live semantic checks, then rerun bun run eval:dogfood -- report --live.",
|
|
269
|
+
});
|
|
270
|
+
}
|
|
271
|
+
if (report.failures.length > 0) {
|
|
272
|
+
actions.push({
|
|
273
|
+
command: "bun run eval:dogfood -- report --json",
|
|
274
|
+
reason: "Inspect expected ids, top results, and errors before changing search behavior or fixture expectations.",
|
|
275
|
+
});
|
|
276
|
+
}
|
|
277
|
+
if (report.misses.length > 0) {
|
|
278
|
+
actions.push({
|
|
279
|
+
command: 'codetrap search "<miss query>" --mode hybrid --ranking-signals --json',
|
|
280
|
+
reason: "Replay a miss with ranking signals before deciding whether to tune search or promote a fixture case.",
|
|
281
|
+
});
|
|
282
|
+
}
|
|
283
|
+
if (report.noisy_hits.length > 0) {
|
|
284
|
+
actions.push({
|
|
285
|
+
command: 'codetrap search "<noisy query>" --mode hybrid --ranking-signals --json',
|
|
286
|
+
reason: "Inspect why noisy results ranked before deciding whether the case belongs in dogfood eval.",
|
|
287
|
+
});
|
|
288
|
+
}
|
|
289
|
+
if (actions.length === 0) {
|
|
290
|
+
actions.push({
|
|
291
|
+
command: 'codetrap search "<task keywords>" --mode hybrid --json',
|
|
292
|
+
reason: "Keep logging real pre-edit searches in dogfood-log.md before automating promotion.",
|
|
293
|
+
});
|
|
294
|
+
}
|
|
295
|
+
return actions;
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
function formatJudgmentCounts(counts: Record<DogfoodJudgment, number>): string {
|
|
299
|
+
return DOGFOOD_JUDGMENTS.map((judgment) => `${judgment}=${counts[judgment]}`).join(", ");
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
function appendCaseSection(lines: string[], title: string, cases: EvalCaseReport[]): void {
|
|
303
|
+
if (cases.length === 0) return;
|
|
304
|
+
lines.push(`${title}:`);
|
|
305
|
+
for (const item of cases.slice(0, 5)) {
|
|
306
|
+
lines.push(` - [${item.mode}] ${item.query}`);
|
|
307
|
+
if (item.goldTrapIds.length > 0) {
|
|
308
|
+
lines.push(` expected: ${formatExpected(item)}`);
|
|
309
|
+
}
|
|
310
|
+
lines.push(` top: ${formatTopResults(item)}`);
|
|
311
|
+
if (item.error) lines.push(` error: ${item.error}`);
|
|
312
|
+
}
|
|
313
|
+
if (cases.length > 5) lines.push(` ... ${cases.length - 5} more`);
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
function formatExpected(item: EvalCaseReport): string {
|
|
317
|
+
return item.goldTrapIds
|
|
318
|
+
.map((id, index) => `#${id} ${item.expectedTitles[index] ?? ""}`.trim())
|
|
319
|
+
.join(", ");
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
function formatTopResults(item: EvalCaseReport): string {
|
|
323
|
+
if (item.topResults.length === 0) return "(none)";
|
|
324
|
+
return item.topResults
|
|
325
|
+
.slice(0, 3)
|
|
326
|
+
.map((result) => `#${result.id} ${result.title}`)
|
|
327
|
+
.join("; ");
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
function formatNextActions(actions: SearchEvalNextAction[]): string[] {
|
|
331
|
+
if (actions.length === 0) return [" (none)"];
|
|
332
|
+
return actions.map((action) => ` - ${action.command} # ${action.reason}`);
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
function fixtureRepository(fixture: EvalFixture, embeddings: EmbeddingRuntimeInput, ranking?: RankingConfig): TrapRepository {
|
|
336
|
+
const repo = new TrapRepository(openDatabase(":memory:"), embeddings, ranking);
|
|
221
337
|
for (const trap of fixture.traps) repo.add(trap);
|
|
222
338
|
return repo;
|
|
223
339
|
}
|
|
@@ -378,9 +494,6 @@ function providerLabel(provider: EmbeddingConfig | null): string {
|
|
|
378
494
|
return `${provider.provider}/${provider.model}`;
|
|
379
495
|
}
|
|
380
496
|
|
|
381
|
-
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
382
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
383
|
-
}
|
|
384
497
|
|
|
385
498
|
function round(value: number): number {
|
|
386
499
|
return Math.round(value * 10000) / 10000;
|