codetrap 0.1.7 → 0.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/README.md +151 -52
  2. package/docs/installation.md +113 -29
  3. package/package.json +4 -3
  4. package/plugins/codetrap-agent/.codex-plugin/plugin.json +1 -2
  5. package/plugins/codetrap-agent/hooks/post-flight-capture.example.md +19 -17
  6. package/plugins/codetrap-agent/hooks.json +2 -2
  7. package/{skills → plugins/codetrap-agent/skills}/codetrap-add/SKILL.md +10 -4
  8. package/plugins/codetrap-agent/skills/codetrap-capture/SKILL.md +14 -3
  9. package/plugins/codetrap-agent/skills/codetrap-capture-external/SKILL.md +52 -9
  10. package/plugins/codetrap-agent/skills/codetrap-check/SKILL.md +74 -6
  11. package/{skills → plugins/codetrap-agent/skills}/codetrap-search/SKILL.md +6 -5
  12. package/plugins/codetrap-agent/templates/AGENTS.codetrap.md +31 -5
  13. package/scripts/search-policy-sweep.ts +131 -0
  14. package/src/commands/workflow.ts +144 -68
  15. package/src/db/embedding-queries.ts +230 -48
  16. package/src/db/queries.ts +0 -25
  17. package/src/db/repository.ts +32 -21
  18. package/src/db/schema.ts +80 -0
  19. package/src/index.ts +28 -3
  20. package/src/lib/command-requests.ts +112 -1
  21. package/src/lib/config.ts +57 -7
  22. package/src/lib/constants.ts +1 -1
  23. package/src/lib/doctor.ts +42 -12
  24. package/src/lib/embedder.ts +118 -3
  25. package/src/lib/embedding-health.ts +3 -1
  26. package/src/lib/embedding-job.ts +3 -0
  27. package/src/lib/embedding-management.ts +65 -0
  28. package/src/lib/embedding-runtime.ts +177 -0
  29. package/src/lib/output-json.ts +0 -2
  30. package/src/lib/scope-context.ts +12 -6
  31. package/src/lib/scope-migration.ts +2 -1
  32. package/src/lib/scope.ts +0 -2
  33. package/src/lib/search-eval.ts +38 -18
  34. package/src/lib/search-policy-sweep.ts +563 -0
  35. package/src/lib/search-policy.ts +0 -4
  36. package/src/lib/search-service.ts +14 -15
  37. package/src/lib/session-candidate-document.ts +175 -0
  38. package/src/lib/session-candidate-scope.ts +6 -0
  39. package/src/lib/session-capture.ts +298 -32
  40. package/src/lib/session-codec.ts +1 -8
  41. package/src/lib/session-operations.ts +83 -60
  42. package/src/lib/session-review.ts +327 -0
  43. package/src/lib/session-store.ts +87 -73
  44. package/src/lib/store.ts +74 -10
  45. package/src/lib/string-list.ts +3 -0
  46. package/src/lib/text-lines.ts +7 -0
  47. package/src/lib/trap-search-document.ts +2 -1
  48. package/src/lib/value-types.ts +3 -0
  49. package/src/web/client-review.ts +171 -0
  50. package/src/web/client-script.ts +426 -51
  51. package/src/web/client-shell.ts +414 -0
  52. package/src/web/client-text.ts +112 -0
  53. package/src/web/project-registry.ts +3 -5
  54. package/src/web/server.ts +117 -103
  55. package/src/web/static.ts +364 -19
  56. package/skills/codetrap-capture-external/SKILL.md +0 -62
  57. package/skills/codetrap-check/SKILL.md +0 -69
  58. package/src/lib/embedding-index.ts +0 -53
@@ -2,7 +2,6 @@ import { readFileSync } from "node:fs";
2
2
  import { TrapStore } from "../lib/store";
3
3
  import { formatTrapShort, formatTrapDetails, formatTrapActionCard } from "../lib/format";
4
4
  import type { Trap } from "../domain/trap";
5
- import type { SessionMetadata } from "../domain/session";
6
5
  import {
7
6
  formatScopeMigrationText,
8
7
  runScopeMigration,
@@ -11,6 +10,12 @@ import {
11
10
  import { TrapOperations } from "../lib/trap-operations";
12
11
  import { buildDoctorReport, formatDoctorText } from "../lib/doctor";
13
12
  import { formatEmbedText } from "../lib/embed-output";
13
+ import {
14
+ formatEmbeddingProfilesText,
15
+ formatEmbeddingStatusText,
16
+ formatEmbeddingsUseText,
17
+ type EmbeddingsUseResult,
18
+ } from "../lib/embedding-management";
14
19
  import { searchDefaultsFromConfig } from "../lib/config";
15
20
  import { SessionStore } from "../lib/session-store";
16
21
  import { SessionOperations, type SessionConflictResult } from "../lib/session-operations";
@@ -18,9 +23,17 @@ import {
18
23
  CANDIDATES_FILE,
19
24
  NOTES_FILE,
20
25
  RECAP_FILE,
21
- sessionRelativeDir,
22
26
  sessionRelativeFile,
23
27
  } from "../lib/session-codec";
28
+ import {
29
+ sessionAcceptPayload,
30
+ sessionCliConflictPayload,
31
+ sessionCleanupPayload,
32
+ sessionConflictPayload,
33
+ sessionConflictText,
34
+ sessionPayload,
35
+ sessionRejectPayload,
36
+ } from "../lib/session-review";
24
37
  import {
25
38
  toCliSearchJson,
26
39
  toListJson,
@@ -36,10 +49,12 @@ import {
36
49
  import { mutationJsonPayload } from "../lib/trap-mutation-result";
37
50
  import {
38
51
  embedRequestFromArgs,
52
+ embeddingsUseRequestFromArgs,
39
53
  evidenceRequestFromArgs,
40
54
  listRequestFromArgs,
41
55
  searchRequestFromArgs,
42
56
  sessionAcceptRequestFromArgs,
57
+ sessionCaptureRequestFromArgs,
43
58
  sessionCandidateRequestFromArgs,
44
59
  sessionCloseRequestFromArgs,
45
60
  sessionIdRequestFromArgs,
@@ -101,12 +116,14 @@ export async function executeCommand(strip: string[], store: TrapStore): Promise
101
116
  return cmdScopeMigration("migrate-project", args, operations);
102
117
  case "embed":
103
118
  return cmdEmbed(args, store);
119
+ case "embeddings":
120
+ return cmdEmbeddings(args, store);
104
121
  case "session":
105
122
  return cmdSession(args, store, operations);
106
123
  default:
107
124
  return errorResult([
108
125
  `Unknown command: ${sub}`,
109
- "Commands: init, add, search, list, show, edit, delete, add_trap_evidence, archive_trap, supersede_trap, export, import, stats, doctor, repair-scope, migrate-project, embed, session",
126
+ "Commands: init, add, search, list, show, edit, delete, add_trap_evidence, archive_trap, supersede_trap, export, import, stats, doctor, repair-scope, migrate-project, embed, embeddings, session",
110
127
  ].join("\n"));
111
128
  }
112
129
  }
@@ -341,9 +358,13 @@ function cmdStats(args: string[], operations: TrapOperations): CommandResult {
341
358
  : textResult(formatStatsText(stats));
342
359
  }
343
360
 
344
- function cmdDoctor(args: string[], store: TrapStore, operations: TrapOperations): CommandResult {
361
+ async function cmdDoctor(args: string[], store: TrapStore, operations: TrapOperations): Promise<CommandResult> {
345
362
  const { opts } = parseArgs(args);
346
- const report = buildDoctorReport(store, operations);
363
+ const projectRoot = store.getProjectRoot();
364
+ const candidateReview = projectRoot
365
+ ? new SessionOperations(new SessionStore(projectRoot), operations).candidateReviewSummary()
366
+ : null;
367
+ const report = await buildDoctorReport(store, operations, process.cwd(), candidateReview);
347
368
  return opts.json !== undefined
348
369
  ? jsonResult(report)
349
370
  : textResult(formatDoctorText(report));
@@ -388,6 +409,59 @@ async function cmdEmbed(args: string[], store: TrapStore): Promise<CommandResult
388
409
  }
389
410
  }
390
411
 
412
+ async function cmdEmbeddings(args: string[], store: TrapStore): Promise<CommandResult> {
413
+ const sub = args[0] ?? "status";
414
+ const rest = args.length === 0 ? [] : args.slice(1);
415
+
416
+ try {
417
+ switch (sub) {
418
+ case "status": {
419
+ const { opts } = parseArgs(rest);
420
+ const status = await store.embeddingStatus({ scope: opts.scope });
421
+ return opts.json !== undefined
422
+ ? jsonResult(status)
423
+ : textResult(formatEmbeddingStatusText(status));
424
+ }
425
+ case "list":
426
+ case "profiles": {
427
+ const { opts } = parseArgs(rest);
428
+ const profiles = store.embeddingProfiles({ scope: opts.scope });
429
+ const payload = {
430
+ active_profile_id: store.embeddingRuntimeStatus().profile_id,
431
+ ...profiles,
432
+ };
433
+ return opts.json !== undefined
434
+ ? jsonResult(payload)
435
+ : textResult(formatEmbeddingProfilesText(profiles));
436
+ }
437
+ case "use": {
438
+ const { opts, positionals } = parseArgs(rest);
439
+ const request = embeddingsUseRequestFromArgs(positionals, opts);
440
+ const written = store.configureEmbeddings(request.embeddings);
441
+ const scope = store.hasProject() ? "project" : "global";
442
+ const result: EmbeddingsUseResult = {
443
+ ...written,
444
+ embeddings: written.config.embeddings ?? request.embeddings,
445
+ next_action: {
446
+ command: `codetrap embeddings reindex --scope ${scope}`,
447
+ reason: "Generate embeddings for the selected profile.",
448
+ },
449
+ };
450
+ return opts.json !== undefined
451
+ ? jsonResult(result)
452
+ : textResult(formatEmbeddingsUseText(result));
453
+ }
454
+ case "reindex":
455
+ case "embed":
456
+ return cmdEmbed(rest, store);
457
+ default:
458
+ return errorResult("Usage: codetrap embeddings <status|list|profiles|use|reindex> [--json]");
459
+ }
460
+ } catch (error) {
461
+ return errorFrom(error);
462
+ }
463
+ }
464
+
391
465
  async function cmdSession(args: string[], store: TrapStore, trapOperations: TrapOperations): Promise<CommandResult> {
392
466
  const sub = args[0];
393
467
  const rest = args.slice(1);
@@ -413,6 +487,8 @@ async function cmdSession(args: string[], store: TrapStore, trapOperations: Trap
413
487
  return cmdSessionNotes(rest, sessions);
414
488
  case "close":
415
489
  return cmdSessionClose(rest, sessions);
490
+ case "capture":
491
+ return cmdSessionCapture(rest, sessions);
416
492
  case "candidates":
417
493
  return cmdSessionCandidates(rest, sessions);
418
494
  case "candidate":
@@ -428,7 +504,7 @@ async function cmdSession(args: string[], store: TrapStore, trapOperations: Trap
428
504
  case "cleanup":
429
505
  return cmdSessionCleanup(rest, sessions);
430
506
  default:
431
- return errorResult("Usage: codetrap session <start|note|status|list|show|notes|close|candidates|candidate|accept|reject|delete|prune|cleanup>");
507
+ return errorResult("Usage: codetrap session <start|note|status|list|show|notes|close|capture|candidates|candidate|accept|reject|delete|prune|cleanup>");
432
508
  }
433
509
  } catch (error) {
434
510
  return errorFrom(error);
@@ -472,14 +548,31 @@ function cmdSessionStatus(args: string[], sessions: SessionOperations): CommandR
472
548
  return jsonResult({
473
549
  active_session_id: status.active_session_id,
474
550
  session: status.session ? sessionPayload(status.session) : null,
551
+ candidate_review: status.candidate_review,
475
552
  });
476
553
  }
477
- if (!status.session) return textResult("No active session.");
478
- return textResult([
554
+ if (!status.session) {
555
+ const lines = ["No active session."];
556
+ if (status.candidate_review.pending_count > 0) {
557
+ lines.push(
558
+ `Pending candidate review: ${status.candidate_review.pending_count} candidate(s) across ${status.candidate_review.pending_session_count} session(s).`
559
+ );
560
+ if (status.candidate_review.next_session_id) {
561
+ lines.push(`Review: codetrap session candidates ${status.candidate_review.next_session_id}`);
562
+ }
563
+ lines.push("Open web review: codetrap web");
564
+ }
565
+ return textResult(lines.join("\n"));
566
+ }
567
+ const lines = [
479
568
  `Active session ${status.session.id}`,
480
569
  `Goal: ${status.session.goal}`,
481
570
  `Notes: ${sessionRelativeFile(status.session.id, NOTES_FILE)}`,
482
- ].join("\n"));
571
+ ];
572
+ if (status.candidate_review.pending_count > 0) {
573
+ lines.push(`Pending candidate review: ${status.candidate_review.pending_count} candidate(s).`);
574
+ }
575
+ return textResult(lines.join("\n"));
483
576
  }
484
577
 
485
578
  function cmdSessionList(args: string[], sessions: SessionOperations): CommandResult {
@@ -487,7 +580,9 @@ function cmdSessionList(args: string[], sessions: SessionOperations): CommandRes
487
580
  const entries = sessions.listSessions(sessionListRequestFromArgs(opts));
488
581
  if (opts.json !== undefined) return jsonResult(entries);
489
582
  if (entries.length === 0) return textResult("No sessions found.");
490
- return textResult(entries.map((entry) => `${entry.id} [${entry.status}] ${entry.goal}`).join("\n"));
583
+ return textResult(entries.map((entry) =>
584
+ `${entry.id} [${entry.status}] ${entry.goal} (${entry.pending_count} pending, ${entry.reviewed_count} reviewed)`
585
+ ).join("\n"));
491
586
  }
492
587
 
493
588
  function cmdSessionShow(args: string[], sessions: SessionOperations): CommandResult {
@@ -525,7 +620,6 @@ function cmdSessionClose(args: string[], sessions: SessionOperations): CommandRe
525
620
  ...sessionPayload(result.session),
526
621
  recap_path: result.recap_path,
527
622
  candidate_count: result.candidate_count,
528
- traps_written: result.traps_written,
529
623
  };
530
624
  if (opts.json !== undefined) return jsonResult(payload);
531
625
  const lines = [
@@ -539,6 +633,39 @@ function cmdSessionClose(args: string[], sessions: SessionOperations): CommandRe
539
633
  return textResult(lines.join("\n"));
540
634
  }
541
635
 
636
+ function cmdSessionCapture(args: string[], sessions: SessionOperations): CommandResult {
637
+ const { opts } = parseArgs(args);
638
+ const result = sessions.captureCandidate(sessionCaptureRequestFromArgs(opts, {
639
+ isTTY: process.stdin.isTTY === true,
640
+ readStdin: () => readFileSync(0, "utf-8"),
641
+ readFile: (path) => readFileSync(path, "utf-8"),
642
+ }));
643
+ const nextAction = `codetrap session candidate ${result.candidate.id} --session ${result.session.id} --json`;
644
+ const payload = {
645
+ success: true,
646
+ session_id: result.session.id,
647
+ candidate_id: result.candidate.id,
648
+ status: result.candidate.status,
649
+ quality_score: result.candidate.quality_score,
650
+ candidate_count: result.candidate_count,
651
+ created_session: result.created_session,
652
+ closed_session: result.closed_session,
653
+ duplicate: result.duplicate,
654
+ candidate_traps_path: sessionRelativeFile(result.session.id, CANDIDATES_FILE),
655
+ recap_path: result.recap_path,
656
+ next_action: {
657
+ command: nextAction,
658
+ },
659
+ };
660
+ if (opts.json !== undefined) return jsonResult(payload);
661
+ return textResult([
662
+ `${result.duplicate ? "Reused" : "Captured"} candidate ${result.candidate.id} in session ${result.session.id}.`,
663
+ result.created_session ? "Created and closed a post-flight session." : "Session remains active.",
664
+ `Candidate inbox: ${payload.candidate_traps_path}`,
665
+ `Review: ${nextAction}`,
666
+ ].join("\n"));
667
+ }
668
+
542
669
  function cmdSessionCandidates(args: string[], sessions: SessionOperations): CommandResult {
543
670
  const { opts, positionals } = parseArgs(args);
544
671
  const request = sessionIdRequestFromArgs(positionals);
@@ -562,16 +689,7 @@ async function cmdSessionAccept(args: string[], sessions: SessionOperations): Pr
562
689
  const { opts, positionals } = parseArgs(args);
563
690
  const accepted = await sessions.acceptCandidate(sessionAcceptRequestFromArgs(positionals, opts));
564
691
  if (!accepted.success) return possibleConflictResult(accepted, opts.json !== undefined);
565
- const payload = {
566
- success: true,
567
- session_id: accepted.session.id,
568
- candidate_id: accepted.candidate.id,
569
- status: accepted.candidate.status,
570
- trap_id: accepted.trap_id,
571
- scope: accepted.scope,
572
- evidence_id: accepted.evidence_id,
573
- superseded_id: accepted.superseded_id,
574
- };
692
+ const payload = sessionAcceptPayload(accepted);
575
693
  if (opts.json !== undefined) return jsonResult(payload);
576
694
  const lines = [`Accepted ${accepted.candidate.id}; wrote trap #${accepted.trap_id} to ${accepted.scope} scope.`];
577
695
  if (accepted.superseded_id !== null) lines.push(`Superseded trap #${accepted.superseded_id}.`);
@@ -581,13 +699,7 @@ async function cmdSessionAccept(args: string[], sessions: SessionOperations): Pr
581
699
  function cmdSessionReject(args: string[], sessions: SessionOperations): CommandResult {
582
700
  const { opts, positionals } = parseArgs(args);
583
701
  const rejected = sessions.rejectCandidate(sessionRejectRequestFromArgs(positionals, opts));
584
- const payload = {
585
- success: true,
586
- session_id: rejected.session.id,
587
- candidate_id: rejected.candidate.id,
588
- status: rejected.candidate.status,
589
- reason: rejected.candidate.rejection_reason ?? null,
590
- };
702
+ const payload = sessionRejectPayload(rejected);
591
703
  if (opts.json !== undefined) return jsonResult(payload);
592
704
  return textResult(`Rejected ${rejected.candidate.id}.`);
593
705
  }
@@ -621,12 +733,7 @@ function cmdSessionCleanup(args: string[], sessions: SessionOperations): Command
621
733
  }
622
734
  const request = sessionIdRequestFromArgs(positionals);
623
735
  const result = sessions.cleanupDeletedTrapCandidates(request.sessionId);
624
- const payload = {
625
- success: true,
626
- session_id: result.session.id,
627
- removed_count: result.removed_count,
628
- removed_candidate_ids: result.removed_candidate_ids,
629
- };
736
+ const payload = sessionCleanupPayload(result);
630
737
  if (opts.json !== undefined) return jsonResult(payload);
631
738
  return textResult(`Removed ${result.removed_count} deleted-trap candidate(s) from session ${result.session.id}.`);
632
739
  }
@@ -679,42 +786,11 @@ function errorMessage(error: unknown): string {
679
786
  return error instanceof Error ? error.message : String(error);
680
787
  }
681
788
 
682
- function sessionPayload(session: SessionMetadata) {
683
- return {
684
- ...session,
685
- session_dir: sessionRelativeDir(session.id),
686
- notes_path: sessionRelativeFile(session.id, NOTES_FILE),
687
- recap_path: sessionRelativeFile(session.id, RECAP_FILE),
688
- candidate_traps_path: sessionRelativeFile(session.id, CANDIDATES_FILE),
689
- };
690
- }
691
-
692
789
  function possibleConflictResult(
693
790
  result: SessionConflictResult,
694
791
  asJson: boolean
695
792
  ): CommandResult {
696
- const payload = {
697
- success: false,
698
- error: "Possible active trap conflict found.",
699
- session_id: result.session_id,
700
- candidate_id: result.candidate_id,
701
- possible_conflicts: result.possible_conflicts,
702
- next_actions: [
703
- `codetrap session accept ${result.candidate_id} --session ${result.session_id} --accept-anyway`,
704
- `codetrap session accept ${result.candidate_id} --session ${result.session_id} --supersedes <trap-id>`,
705
- `codetrap session reject ${result.candidate_id} --session ${result.session_id} --reason <reason>`,
706
- ],
707
- };
708
- if (asJson) return jsonResult(payload, 1);
709
-
710
- return errorResult([
711
- "Possible active trap conflict found:",
712
- ...result.possible_conflicts.map((conflict) => [
713
- `#${conflict.trap_id} ${conflict.title}`,
714
- ` reason: ${conflict.reason}`,
715
- ` fix: ${conflict.fix}`,
716
- ].join("\n")),
717
- "",
718
- `Use --accept-anyway to save as a new trap, or --supersedes <trap-id> to preserve lifecycle history.`,
719
- ].join("\n"));
793
+ const payload = sessionConflictPayload(result);
794
+ if (asJson) return jsonResult(sessionCliConflictPayload(payload), 1);
795
+ return errorResult(sessionConflictText(payload));
720
796
  }
@@ -5,15 +5,17 @@ import {
5
5
  decodeEmbedding,
6
6
  encodeEmbedding,
7
7
  type EmbeddingConfig,
8
+ embeddingProfileId,
8
9
  type FreshEmbedding,
9
10
  type StoredEmbedding,
10
11
  } from "../lib/embedder";
11
12
  import type { EmbeddingStateCounts } from "../lib/embedding-health";
12
- import { embeddingIsFresh, passageHashForTrap } from "../lib/trap-search-document";
13
- import { countTraps, listTraps } from "./queries";
13
+ import { passageHashForTrap } from "../lib/trap-search-document";
14
+ import { countTraps } from "./queries";
14
15
 
15
16
  type TrapEmbeddingRow = {
16
17
  trap_id: number;
18
+ profile_id: string;
17
19
  provider: string;
18
20
  model: string;
19
21
  dimensions: number;
@@ -23,33 +25,98 @@ type TrapEmbeddingRow = {
23
25
  updated_at: string;
24
26
  };
25
27
 
26
- export function getEmbedding(db: Database, trapId: number): StoredEmbedding | null {
28
+ export type EmbeddingProfileSummary = {
29
+ id: string;
30
+ provider: string;
31
+ model: string;
32
+ dimensions: number;
33
+ passage_version: number;
34
+ embedding_count: number;
35
+ updated_at: string | null;
36
+ };
37
+
38
+ type EmbeddingProfileRow = Omit<EmbeddingProfileSummary, "embedding_count"> & {
39
+ embedding_count: number;
40
+ };
41
+
42
+ type TrapEmbeddingStateRow = Trap & {
43
+ embedding_passage_hash: string | null;
44
+ };
45
+
46
+ type TrapAnyEmbeddingStateRow = Trap & {
47
+ has_embedding: number;
48
+ };
49
+
50
+ export function getEmbedding(
51
+ db: Database,
52
+ trapId: number,
53
+ config?: EmbeddingConfig
54
+ ): StoredEmbedding | null {
55
+ const profileClause = config ? "AND e.profile_id = ?" : "";
56
+ const params: SQLQueryBindings[] = [trapId];
57
+ if (config) params.push(embeddingProfileId(config));
58
+
27
59
  const row = db
28
- .query("SELECT * FROM trap_embeddings WHERE trap_id = ?")
29
- .get(trapId) as TrapEmbeddingRow | null;
60
+ .query(`
61
+ SELECT
62
+ e.trap_id,
63
+ e.profile_id,
64
+ p.provider,
65
+ p.model,
66
+ p.dimensions,
67
+ p.passage_version,
68
+ e.passage_hash,
69
+ e.embedding,
70
+ e.updated_at
71
+ FROM trap_embeddings e
72
+ JOIN embedding_profiles p ON p.id = e.profile_id
73
+ WHERE e.trap_id = ? ${profileClause}
74
+ ORDER BY e.updated_at DESC
75
+ LIMIT 1
76
+ `)
77
+ .get(...params) as TrapEmbeddingRow | null;
30
78
  return row ? rowToStoredEmbedding(row) : null;
31
79
  }
32
80
 
33
81
  export function upsertEmbedding(db: Database, record: StoredEmbedding): void {
82
+ const profileId = record.profile_id || embeddingProfileId({
83
+ provider: record.provider,
84
+ model: record.model,
85
+ dimensions: record.dimensions,
86
+ passageVersion: record.passage_version,
87
+ });
88
+
34
89
  db.prepare(`
35
- INSERT INTO trap_embeddings (
36
- trap_id, provider, model, dimensions, passage_version, passage_hash, embedding, updated_at
90
+ INSERT INTO embedding_profiles (
91
+ id, provider, model, dimensions, passage_version, created_at, updated_at
37
92
  )
38
- VALUES (?, ?, ?, ?, ?, ?, ?, datetime('now'))
39
- ON CONFLICT(trap_id) DO UPDATE SET
93
+ VALUES (?, ?, ?, ?, ?, datetime('now'), datetime('now'))
94
+ ON CONFLICT(id) DO UPDATE SET
40
95
  provider = excluded.provider,
41
96
  model = excluded.model,
42
97
  dimensions = excluded.dimensions,
43
98
  passage_version = excluded.passage_version,
44
- passage_hash = excluded.passage_hash,
45
- embedding = excluded.embedding,
46
99
  updated_at = datetime('now')
47
100
  `).run(
48
- record.trap_id,
101
+ profileId,
49
102
  record.provider,
50
103
  record.model,
51
104
  record.dimensions,
52
- record.passage_version,
105
+ record.passage_version
106
+ );
107
+
108
+ db.prepare(`
109
+ INSERT INTO trap_embeddings (
110
+ trap_id, profile_id, passage_hash, embedding, updated_at
111
+ )
112
+ VALUES (?, ?, ?, ?, datetime('now'))
113
+ ON CONFLICT(trap_id, profile_id) DO UPDATE SET
114
+ passage_hash = excluded.passage_hash,
115
+ embedding = excluded.embedding,
116
+ updated_at = datetime('now')
117
+ `).run(
118
+ record.trap_id,
119
+ profileId,
53
120
  record.passage_hash,
54
121
  encodeEmbedding(record.embedding)
55
122
  );
@@ -65,16 +132,10 @@ export function getAllFreshEmbeddings(
65
132
  opts: { category?: string; scope?: string; status?: TrapStatus | "all" } = {}
66
133
  ): FreshEmbedding[] {
67
134
  const conditions = [
68
- "e.provider = ?",
69
- "e.model = ?",
70
- "e.dimensions = ?",
71
- "e.passage_version = ?",
135
+ "e.profile_id = ?",
72
136
  ];
73
137
  const params: SQLQueryBindings[] = [
74
- config.provider,
75
- config.model,
76
- config.dimensions,
77
- config.passageVersion,
138
+ embeddingProfileId(config),
78
139
  ];
79
140
 
80
141
  if (opts.category) {
@@ -116,22 +177,11 @@ export function getTrapsNeedingEmbeddings(
116
177
  config: EmbeddingConfig,
117
178
  opts: { scope?: string; category?: string; status?: TrapStatus | "all"; force?: boolean; limit?: number } = {}
118
179
  ): Trap[] {
119
- const traps = listTraps(db, {
120
- scope: opts.scope,
121
- category: opts.category,
122
- status: opts.status,
123
- limit: 100000,
124
- });
125
- const needed: Trap[] = [];
126
-
127
- for (const trap of traps) {
128
- const embedding = getEmbedding(db, trap.id);
129
- if (opts.force || !embeddingIsFresh(trap, embedding, config)) {
130
- needed.push(trap);
131
- }
132
- }
133
-
134
- return needed.slice(0, opts.limit ?? needed.length);
180
+ const rows = trapEmbeddingStateRows(db, config, opts);
181
+ const needed = opts.force
182
+ ? rows
183
+ : rows.filter((row) => row.embedding_passage_hash !== passageHashForTrap(row));
184
+ return needed.map(rowToTrap).slice(0, opts.limit ?? needed.length);
135
185
  }
136
186
 
137
187
  export function countEmbeddableTraps(
@@ -146,24 +196,20 @@ export function getEmbeddingStateCounts(
146
196
  config: EmbeddingConfig | null,
147
197
  opts: { scope?: string; category?: string; status?: TrapStatus | "all" } = {}
148
198
  ): EmbeddingStateCounts {
149
- const traps = listTraps(db, {
150
- scope: opts.scope,
151
- category: opts.category,
152
- status: opts.status,
153
- limit: 100000,
154
- });
199
+ if (!config) return getAnyEmbeddingStateCounts(db, opts);
200
+
201
+ const rows = trapEmbeddingStateRows(db, config, opts);
155
202
  const counts: EmbeddingStateCounts = {
156
- total: traps.length,
203
+ total: rows.length,
157
204
  fresh: 0,
158
205
  stale: 0,
159
206
  missing: 0,
160
207
  };
161
208
 
162
- for (const trap of traps) {
163
- const embedding = getEmbedding(db, trap.id);
164
- if (!embedding) {
209
+ for (const row of rows) {
210
+ if (!row.embedding_passage_hash) {
165
211
  counts.missing++;
166
- } else if (config && embeddingIsFresh(trap, embedding, config)) {
212
+ } else if (row.embedding_passage_hash === passageHashForTrap(row)) {
167
213
  counts.fresh++;
168
214
  } else {
169
215
  counts.stale++;
@@ -173,9 +219,56 @@ export function getEmbeddingStateCounts(
173
219
  return counts;
174
220
  }
175
221
 
222
+ export function listEmbeddingProfiles(
223
+ db: Database,
224
+ opts: { scope?: string; category?: string; status?: TrapStatus | "all" } = {}
225
+ ): EmbeddingProfileSummary[] {
226
+ const conditions: string[] = [];
227
+ const params: SQLQueryBindings[] = [];
228
+
229
+ if (opts.category) {
230
+ conditions.push("t.category = ?");
231
+ params.push(opts.category);
232
+ }
233
+ if (opts.scope) {
234
+ conditions.push("t.scope = ?");
235
+ params.push(opts.scope);
236
+ }
237
+ if (opts.status !== "all") {
238
+ conditions.push("t.status = ?");
239
+ params.push(opts.status ?? DEFAULT_TRAP_STATUS);
240
+ }
241
+
242
+ const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
243
+ const rows = db
244
+ .query(`
245
+ SELECT
246
+ p.id,
247
+ p.provider,
248
+ p.model,
249
+ p.dimensions,
250
+ p.passage_version,
251
+ COUNT(e.trap_id) AS embedding_count,
252
+ MAX(e.updated_at) AS updated_at
253
+ FROM embedding_profiles p
254
+ JOIN trap_embeddings e ON e.profile_id = p.id
255
+ JOIN traps t ON t.id = e.trap_id
256
+ ${where}
257
+ GROUP BY p.id, p.provider, p.model, p.dimensions, p.passage_version
258
+ ORDER BY updated_at DESC, p.provider ASC, p.model ASC
259
+ `)
260
+ .all(...params) as EmbeddingProfileRow[];
261
+
262
+ return rows.map((row) => ({
263
+ ...row,
264
+ embedding_count: Number(row.embedding_count),
265
+ }));
266
+ }
267
+
176
268
  function rowToStoredEmbedding(row: TrapEmbeddingRow): StoredEmbedding {
177
269
  return {
178
270
  trap_id: row.trap_id,
271
+ profile_id: row.profile_id,
179
272
  provider: row.provider,
180
273
  model: row.model,
181
274
  dimensions: row.dimensions,
@@ -185,3 +278,92 @@ function rowToStoredEmbedding(row: TrapEmbeddingRow): StoredEmbedding {
185
278
  updated_at: row.updated_at,
186
279
  };
187
280
  }
281
+
282
+ function getAnyEmbeddingStateCounts(
283
+ db: Database,
284
+ opts: { scope?: string; category?: string; status?: TrapStatus | "all" } = {}
285
+ ): EmbeddingStateCounts {
286
+ const rows = trapAnyEmbeddingStateRows(db, opts);
287
+ const stale = rows.filter((row) => row.has_embedding > 0).length;
288
+ return {
289
+ total: rows.length,
290
+ fresh: 0,
291
+ stale,
292
+ missing: rows.length - stale,
293
+ };
294
+ }
295
+
296
+ function trapEmbeddingStateRows(
297
+ db: Database,
298
+ config: EmbeddingConfig,
299
+ opts: { scope?: string; category?: string; status?: TrapStatus | "all" } = {}
300
+ ): TrapEmbeddingStateRow[] {
301
+ const conditions: string[] = [];
302
+ const filterParams = trapFilterParams(conditions, opts, "t");
303
+ const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
304
+ return db
305
+ .query(`
306
+ SELECT
307
+ t.*,
308
+ e.passage_hash AS embedding_passage_hash
309
+ FROM traps t
310
+ LEFT JOIN trap_embeddings e
311
+ ON e.trap_id = t.id
312
+ AND e.profile_id = ?
313
+ ${where}
314
+ ORDER BY t.updated_at DESC
315
+ LIMIT 100000
316
+ `)
317
+ .all(embeddingProfileId(config), ...filterParams) as TrapEmbeddingStateRow[];
318
+ }
319
+
320
+ function trapAnyEmbeddingStateRows(
321
+ db: Database,
322
+ opts: { scope?: string; category?: string; status?: TrapStatus | "all" } = {}
323
+ ): TrapAnyEmbeddingStateRow[] {
324
+ const conditions: string[] = [];
325
+ const params = trapFilterParams(conditions, opts, "t");
326
+ const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
327
+ return db
328
+ .query(`
329
+ SELECT
330
+ t.*,
331
+ EXISTS (
332
+ SELECT 1
333
+ FROM trap_embeddings e
334
+ WHERE e.trap_id = t.id
335
+ ) AS has_embedding
336
+ FROM traps t
337
+ ${where}
338
+ ORDER BY t.updated_at DESC
339
+ LIMIT 100000
340
+ `)
341
+ .all(...params) as TrapAnyEmbeddingStateRow[];
342
+ }
343
+
344
+ function trapFilterParams(
345
+ conditions: string[],
346
+ opts: { scope?: string; category?: string; status?: TrapStatus | "all" },
347
+ alias: string
348
+ ): SQLQueryBindings[] {
349
+ const params: SQLQueryBindings[] = [];
350
+ const prefix = `${alias}.`;
351
+ if (opts.category) {
352
+ conditions.push(`${prefix}category = ?`);
353
+ params.push(opts.category);
354
+ }
355
+ if (opts.scope) {
356
+ conditions.push(`${prefix}scope = ?`);
357
+ params.push(opts.scope);
358
+ }
359
+ if (opts.status !== "all") {
360
+ conditions.push(`${prefix}status = ?`);
361
+ params.push(opts.status ?? DEFAULT_TRAP_STATUS);
362
+ }
363
+ return params;
364
+ }
365
+
366
+ function rowToTrap(row: TrapEmbeddingStateRow): Trap {
367
+ const { embedding_passage_hash: _embeddingPassageHash, ...trap } = row;
368
+ return trap as Trap;
369
+ }