@loreai/core 0.18.0 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. package/dist/bun/curator.d.ts.map +1 -1
  2. package/dist/bun/db.d.ts +73 -0
  3. package/dist/bun/db.d.ts.map +1 -1
  4. package/dist/bun/distillation.d.ts +2 -13
  5. package/dist/bun/distillation.d.ts.map +1 -1
  6. package/dist/bun/embedding.d.ts +5 -1
  7. package/dist/bun/embedding.d.ts.map +1 -1
  8. package/dist/bun/gradient.d.ts +9 -0
  9. package/dist/bun/gradient.d.ts.map +1 -1
  10. package/dist/bun/index.d.ts +2 -2
  11. package/dist/bun/index.d.ts.map +1 -1
  12. package/dist/bun/index.js +817 -99
  13. package/dist/bun/index.js.map +4 -4
  14. package/dist/bun/ltm.d.ts +99 -5
  15. package/dist/bun/ltm.d.ts.map +1 -1
  16. package/dist/bun/session-limiter.d.ts +26 -0
  17. package/dist/bun/session-limiter.d.ts.map +1 -0
  18. package/dist/bun/temporal.d.ts +2 -0
  19. package/dist/bun/temporal.d.ts.map +1 -1
  20. package/dist/node/curator.d.ts.map +1 -1
  21. package/dist/node/db.d.ts +73 -0
  22. package/dist/node/db.d.ts.map +1 -1
  23. package/dist/node/distillation.d.ts +2 -13
  24. package/dist/node/distillation.d.ts.map +1 -1
  25. package/dist/node/embedding.d.ts +5 -1
  26. package/dist/node/embedding.d.ts.map +1 -1
  27. package/dist/node/gradient.d.ts +9 -0
  28. package/dist/node/gradient.d.ts.map +1 -1
  29. package/dist/node/index.d.ts +2 -2
  30. package/dist/node/index.d.ts.map +1 -1
  31. package/dist/node/index.js +817 -99
  32. package/dist/node/index.js.map +4 -4
  33. package/dist/node/ltm.d.ts +99 -5
  34. package/dist/node/ltm.d.ts.map +1 -1
  35. package/dist/node/session-limiter.d.ts +26 -0
  36. package/dist/node/session-limiter.d.ts.map +1 -0
  37. package/dist/node/temporal.d.ts +2 -0
  38. package/dist/node/temporal.d.ts.map +1 -1
  39. package/dist/types/curator.d.ts.map +1 -1
  40. package/dist/types/db.d.ts +73 -0
  41. package/dist/types/db.d.ts.map +1 -1
  42. package/dist/types/distillation.d.ts +2 -13
  43. package/dist/types/distillation.d.ts.map +1 -1
  44. package/dist/types/embedding.d.ts +5 -1
  45. package/dist/types/embedding.d.ts.map +1 -1
  46. package/dist/types/gradient.d.ts +9 -0
  47. package/dist/types/gradient.d.ts.map +1 -1
  48. package/dist/types/index.d.ts +2 -2
  49. package/dist/types/index.d.ts.map +1 -1
  50. package/dist/types/ltm.d.ts +99 -5
  51. package/dist/types/ltm.d.ts.map +1 -1
  52. package/dist/types/session-limiter.d.ts +26 -0
  53. package/dist/types/session-limiter.d.ts.map +1 -0
  54. package/dist/types/temporal.d.ts +2 -0
  55. package/dist/types/temporal.d.ts.map +1 -1
  56. package/package.json +2 -1
  57. package/src/curator.ts +54 -2
  58. package/src/db.ts +347 -0
  59. package/src/distillation.ts +55 -14
  60. package/src/embedding.ts +28 -3
  61. package/src/gradient.ts +183 -74
  62. package/src/index.ts +8 -0
  63. package/src/ltm.ts +480 -45
  64. package/src/session-limiter.ts +47 -0
  65. package/src/temporal.ts +10 -0
package/dist/bun/index.js CHANGED
@@ -125,6 +125,7 @@ __export(temporal_exports, {
125
125
  CHUNK_TERMINATOR: () => CHUNK_TERMINATOR,
126
126
  bySession: () => bySession,
127
127
  count: () => count,
128
+ hasMessages: () => hasMessages,
128
129
  markDistilled: () => markDistilled,
129
130
  partsToText: () => partsToText,
130
131
  prune: () => prune,
@@ -737,6 +738,55 @@ var MIGRATIONS = [
737
738
  WHERE ih.project_id = projects.id
738
739
  AND ih.source_id = '__declined__'
739
740
  );
741
+ `,
742
+ `
743
+ -- Version 23: Persist volatile session tracking state across restarts.
744
+ -- Previously these were in-memory only, causing duplicate processing,
745
+ -- false compaction detection, and expensive prompt cache busts on restart.
746
+ ALTER TABLE session_state ADD COLUMN last_curated_at INTEGER NOT NULL DEFAULT 0;
747
+ ALTER TABLE session_state ADD COLUMN message_count INTEGER NOT NULL DEFAULT 0;
748
+ ALTER TABLE session_state ADD COLUMN turns_since_curation INTEGER NOT NULL DEFAULT 0;
749
+ ALTER TABLE session_state ADD COLUMN ltm_cache_text TEXT;
750
+ ALTER TABLE session_state ADD COLUMN ltm_cache_tokens INTEGER;
751
+ ALTER TABLE session_state ADD COLUMN ltm_pin_text TEXT;
752
+ ALTER TABLE session_state ADD COLUMN ltm_pin_tokens INTEGER;
753
+ ALTER TABLE session_state ADD COLUMN consecutive_text_only_turns INTEGER NOT NULL DEFAULT 0;
754
+ `,
755
+ `
756
+ -- Version 24: Persist remaining volatile session state across restarts.
757
+ -- Session identity (Tier 1/2/3 session correlation)
758
+ ALTER TABLE session_state ADD COLUMN fingerprint TEXT NOT NULL DEFAULT '';
759
+ ALTER TABLE session_state ADD COLUMN header_session_id TEXT;
760
+ ALTER TABLE session_state ADD COLUMN header_name TEXT;
761
+ -- Cache warming state
762
+ ALTER TABLE session_state ADD COLUMN resolved_conversation_ttl TEXT NOT NULL DEFAULT '5m';
763
+ ALTER TABLE session_state ADD COLUMN warmup_state TEXT;
764
+ -- Gradient calibration state (survives restarts to avoid uncalibrated busts)
765
+ ALTER TABLE session_state ADD COLUMN dynamic_context_cap REAL NOT NULL DEFAULT 0;
766
+ ALTER TABLE session_state ADD COLUMN bust_rate_ema REAL NOT NULL DEFAULT -1;
767
+ ALTER TABLE session_state ADD COLUMN inter_bust_interval_ema REAL NOT NULL DEFAULT -1;
768
+ ALTER TABLE session_state ADD COLUMN last_layer INTEGER NOT NULL DEFAULT 0;
769
+ ALTER TABLE session_state ADD COLUMN last_known_input INTEGER NOT NULL DEFAULT 0;
770
+ ALTER TABLE session_state ADD COLUMN last_turn_at INTEGER NOT NULL DEFAULT 0;
771
+ ALTER TABLE session_state ADD COLUMN last_bust_at INTEGER NOT NULL DEFAULT 0;
772
+ `,
773
+ `
774
+ -- Version 25: Adaptive dedup threshold \u2014 store accept/reject feedback
775
+ -- on embedding-based duplicate pairs for per-project threshold calibration.
776
+ -- Titles stored instead of FK IDs because entries are deleted during dedup;
777
+ -- the similarity float is the actual calibration input.
778
+ CREATE TABLE IF NOT EXISTS dedup_feedback (
779
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
780
+ project_id TEXT,
781
+ entry_a_title TEXT NOT NULL,
782
+ entry_b_title TEXT NOT NULL,
783
+ similarity REAL NOT NULL,
784
+ accepted INTEGER NOT NULL,
785
+ source TEXT NOT NULL DEFAULT 'manual',
786
+ created_at INTEGER NOT NULL
787
+ );
788
+ CREATE INDEX IF NOT EXISTS idx_dedup_feedback_project
789
+ ON dedup_feedback(project_id);
740
790
  `
741
791
  ];
742
792
  function dbPath() {
@@ -873,6 +923,11 @@ function close() {
873
923
  }
874
924
  }
875
925
  function ensureProject(path, name) {
926
+ if (!process.env.LORE_DB_PATH && /^\/test\//.test(path)) {
927
+ throw new Error(
928
+ `Refusing to create project with test path "${path}" in the production DB. Set LORE_DB_PATH to a temp path, or run tests via \`bun test\` from the repo root.`
929
+ );
930
+ }
876
931
  const existing = db().query("SELECT id, git_remote FROM projects WHERE path = ?").get(path);
877
932
  if (existing) {
878
933
  if (!existing.git_remote) {
@@ -1041,6 +1096,153 @@ function loadAllSessionCosts() {
1041
1096
  }
1042
1097
  return result;
1043
1098
  }
1099
+ function saveSessionTracking(sessionID, state) {
1100
+ const now = Date.now();
1101
+ db().query(
1102
+ "INSERT OR IGNORE INTO session_state (session_id, force_min_layer, updated_at) VALUES (?, 0, ?)"
1103
+ ).run(sessionID, now);
1104
+ const sets = ["updated_at = ?"];
1105
+ const vals = [now];
1106
+ if (state.lastCuratedAt !== void 0) {
1107
+ sets.push("last_curated_at = ?");
1108
+ vals.push(state.lastCuratedAt);
1109
+ }
1110
+ if (state.messageCount !== void 0) {
1111
+ sets.push("message_count = ?");
1112
+ vals.push(state.messageCount);
1113
+ }
1114
+ if (state.turnsSinceCuration !== void 0) {
1115
+ sets.push("turns_since_curation = ?");
1116
+ vals.push(state.turnsSinceCuration);
1117
+ }
1118
+ if (state.consecutiveTextOnlyTurns !== void 0) {
1119
+ sets.push("consecutive_text_only_turns = ?");
1120
+ vals.push(state.consecutiveTextOnlyTurns);
1121
+ }
1122
+ if (state.ltmCacheText !== void 0) {
1123
+ sets.push("ltm_cache_text = ?");
1124
+ vals.push(state.ltmCacheText);
1125
+ }
1126
+ if (state.ltmCacheTokens !== void 0) {
1127
+ sets.push("ltm_cache_tokens = ?");
1128
+ vals.push(state.ltmCacheTokens);
1129
+ }
1130
+ if (state.ltmPinText !== void 0) {
1131
+ sets.push("ltm_pin_text = ?");
1132
+ vals.push(state.ltmPinText);
1133
+ }
1134
+ if (state.ltmPinTokens !== void 0) {
1135
+ sets.push("ltm_pin_tokens = ?");
1136
+ vals.push(state.ltmPinTokens);
1137
+ }
1138
+ if (state.fingerprint !== void 0) {
1139
+ sets.push("fingerprint = ?");
1140
+ vals.push(state.fingerprint);
1141
+ }
1142
+ if (state.headerSessionId !== void 0) {
1143
+ sets.push("header_session_id = ?");
1144
+ vals.push(state.headerSessionId);
1145
+ }
1146
+ if (state.headerName !== void 0) {
1147
+ sets.push("header_name = ?");
1148
+ vals.push(state.headerName);
1149
+ }
1150
+ if (state.resolvedConversationTTL !== void 0) {
1151
+ sets.push("resolved_conversation_ttl = ?");
1152
+ vals.push(state.resolvedConversationTTL);
1153
+ }
1154
+ if (state.warmupState !== void 0) {
1155
+ sets.push("warmup_state = ?");
1156
+ vals.push(state.warmupState);
1157
+ }
1158
+ if (state.dynamicContextCap !== void 0) {
1159
+ sets.push("dynamic_context_cap = ?");
1160
+ vals.push(state.dynamicContextCap);
1161
+ }
1162
+ if (state.bustRateEMA !== void 0) {
1163
+ sets.push("bust_rate_ema = ?");
1164
+ vals.push(state.bustRateEMA);
1165
+ }
1166
+ if (state.interBustIntervalEMA !== void 0) {
1167
+ sets.push("inter_bust_interval_ema = ?");
1168
+ vals.push(state.interBustIntervalEMA);
1169
+ }
1170
+ if (state.lastLayer !== void 0) {
1171
+ sets.push("last_layer = ?");
1172
+ vals.push(state.lastLayer);
1173
+ }
1174
+ if (state.lastKnownInput !== void 0) {
1175
+ sets.push("last_known_input = ?");
1176
+ vals.push(state.lastKnownInput);
1177
+ }
1178
+ if (state.lastTurnAt !== void 0) {
1179
+ sets.push("last_turn_at = ?");
1180
+ vals.push(state.lastTurnAt);
1181
+ }
1182
+ if (state.lastBustAt !== void 0) {
1183
+ sets.push("last_bust_at = ?");
1184
+ vals.push(state.lastBustAt);
1185
+ }
1186
+ db().query(
1187
+ "UPDATE session_state SET " + sets.join(", ") + " WHERE session_id = ?"
1188
+ ).run(...vals, sessionID);
1189
+ }
1190
+ function loadSessionTracking(sessionID) {
1191
+ const row = db().query(
1192
+ `SELECT last_curated_at, message_count, turns_since_curation,
1193
+ consecutive_text_only_turns,
1194
+ ltm_cache_text, ltm_cache_tokens, ltm_pin_text, ltm_pin_tokens,
1195
+ fingerprint, header_session_id, header_name,
1196
+ resolved_conversation_ttl, warmup_state,
1197
+ dynamic_context_cap, bust_rate_ema, inter_bust_interval_ema,
1198
+ last_layer, last_known_input, last_turn_at, last_bust_at
1199
+ FROM session_state WHERE session_id = ?`
1200
+ ).get(sessionID);
1201
+ if (!row) return null;
1202
+ return {
1203
+ lastCuratedAt: row.last_curated_at,
1204
+ messageCount: row.message_count,
1205
+ turnsSinceCuration: row.turns_since_curation,
1206
+ consecutiveTextOnlyTurns: row.consecutive_text_only_turns,
1207
+ ltmCacheText: row.ltm_cache_text,
1208
+ ltmCacheTokens: row.ltm_cache_tokens,
1209
+ ltmPinText: row.ltm_pin_text,
1210
+ ltmPinTokens: row.ltm_pin_tokens,
1211
+ fingerprint: row.fingerprint,
1212
+ headerSessionId: row.header_session_id,
1213
+ headerName: row.header_name,
1214
+ resolvedConversationTTL: row.resolved_conversation_ttl,
1215
+ warmupState: row.warmup_state,
1216
+ dynamicContextCap: row.dynamic_context_cap,
1217
+ bustRateEMA: row.bust_rate_ema,
1218
+ interBustIntervalEMA: row.inter_bust_interval_ema,
1219
+ lastLayer: row.last_layer,
1220
+ lastKnownInput: row.last_known_input,
1221
+ lastTurnAt: row.last_turn_at,
1222
+ lastBustAt: row.last_bust_at
1223
+ };
1224
+ }
1225
+ function loadHeaderSessionIndex() {
1226
+ const rows = db().query(
1227
+ `SELECT session_id, header_session_id, header_name
1228
+ FROM session_state
1229
+ WHERE header_session_id IS NOT NULL AND header_name IS NOT NULL`
1230
+ ).all();
1231
+ return rows.map((row) => ({
1232
+ sessionId: row.session_id,
1233
+ headerSessionId: row.header_session_id,
1234
+ headerName: row.header_name
1235
+ }));
1236
+ }
1237
+ function getKV(key) {
1238
+ const row = db().query("SELECT value FROM kv_meta WHERE key = ?").get(key);
1239
+ return row?.value ?? null;
1240
+ }
1241
+ function setKV(key, value) {
1242
+ db().query(
1243
+ "INSERT INTO kv_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = ?"
1244
+ ).run(key, value, value);
1245
+ }
1044
1246
  function getMeta(key) {
1045
1247
  const row = db().query("SELECT value FROM metadata WHERE key = ?").get(key);
1046
1248
  return row?.value ?? null;
@@ -26753,7 +26955,16 @@ var LocalProvider = class {
26753
26955
  workerUrl = vendorWorkerUrl;
26754
26956
  }
26755
26957
  } else {
26756
- workerUrl = new URL(`./embedding-worker${import.meta.url.endsWith(".ts") ? ".ts" : ".js"}`, import.meta.url);
26958
+ const selfUrl = typeof import.meta.url === "string" ? import.meta.url : void 0;
26959
+ if (selfUrl) {
26960
+ workerUrl = new URL(
26961
+ `./embedding-worker${selfUrl.endsWith(".ts") ? ".ts" : ".js"}`,
26962
+ selfUrl
26963
+ );
26964
+ } else {
26965
+ const { pathToFileURL } = await import("node:url");
26966
+ workerUrl = new URL("./embedding-worker.cjs", pathToFileURL(__filename));
26967
+ }
26757
26968
  }
26758
26969
  const vendor = vendorModelInfo();
26759
26970
  const workerInitData = {
@@ -27028,8 +27239,14 @@ function fromBlob(blob) {
27028
27239
  const bytes = new Uint8Array(blob);
27029
27240
  return new Float32Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / 4);
27030
27241
  }
27031
- function vectorSearch(queryEmbedding, limit = 10) {
27032
- const rows = db().query("SELECT id, embedding FROM knowledge WHERE embedding IS NOT NULL AND confidence > 0.2").all();
27242
+ function vectorSearch(queryEmbedding, limit = 10, excludeCategories) {
27243
+ let sql = "SELECT id, embedding FROM knowledge WHERE embedding IS NOT NULL AND confidence > 0.2";
27244
+ const params = [];
27245
+ if (excludeCategories?.length) {
27246
+ sql += ` AND category NOT IN (${excludeCategories.map(() => "?").join(",")})`;
27247
+ params.push(...excludeCategories);
27248
+ }
27249
+ const rows = db().query(sql).all(...params);
27033
27250
  const scored = [];
27034
27251
  for (const row of rows) {
27035
27252
  const vec = fromBlob(row.embedding);
@@ -27418,6 +27635,12 @@ function count(projectPath, sessionID) {
27418
27635
  const params = sessionID ? [pid, sessionID] : [pid];
27419
27636
  return db().query(query).get(...params).count;
27420
27637
  }
27638
+ function hasMessages(projectPath, sessionID) {
27639
+ const pid = ensureProject(projectPath);
27640
+ return !!db().query(
27641
+ "SELECT 1 FROM temporal_messages WHERE project_id = ? AND session_id = ? LIMIT 1"
27642
+ ).get(pid, sessionID);
27643
+ }
27421
27644
  function undistilledCount(projectPath, sessionID) {
27422
27645
  const pid = ensureProject(projectPath);
27423
27646
  const query = sessionID ? "SELECT COUNT(*) as count FROM temporal_messages WHERE project_id = ? AND session_id = ? AND distilled = 0" : "SELECT COUNT(*) as count FROM temporal_messages WHERE project_id = ? AND distilled = 0";
@@ -27476,11 +27699,13 @@ function prune(input) {
27476
27699
  var ltm_exports = {};
27477
27700
  __export(ltm_exports, {
27478
27701
  all: () => all2,
27702
+ calibrateDedupThreshold: () => calibrateDedupThreshold,
27479
27703
  cascadeRefReplace: () => cascadeRefReplace,
27480
27704
  check: () => check2,
27481
27705
  cleanDeadRefs: () => cleanDeadRefs,
27482
27706
  create: () => create,
27483
27707
  crossProject: () => crossProject,
27708
+ dedupPairKey: () => dedupPairKey,
27484
27709
  deduplicate: () => deduplicate,
27485
27710
  deduplicateGlobal: () => deduplicateGlobal,
27486
27711
  extractRefs: () => extractRefs,
@@ -27488,9 +27713,17 @@ __export(ltm_exports, {
27488
27713
  forProject: () => forProject,
27489
27714
  forSession: () => forSession,
27490
27715
  get: () => get,
27716
+ getDedupFeedback: () => getDedupFeedback,
27717
+ getDedupFeedbackCount: () => getDedupFeedbackCount,
27718
+ loadCalibratedThreshold: () => loadCalibratedThreshold,
27719
+ pruneDedupFeedback: () => pruneDedupFeedback,
27491
27720
  pruneOversized: () => pruneOversized,
27721
+ recordAutoSignals: () => recordAutoSignals,
27722
+ recordDedupFeedback: () => recordDedupFeedback,
27723
+ recordDedupResultFeedback: () => recordDedupResultFeedback,
27492
27724
  remove: () => remove,
27493
27725
  resolveRef: () => resolveRef2,
27726
+ saveCalibratedThreshold: () => saveCalibratedThreshold,
27494
27727
  search: () => search3,
27495
27728
  searchScored: () => searchScored3,
27496
27729
  searchScoredOtherProjects: () => searchScoredOtherProjects,
@@ -28227,18 +28460,29 @@ function scoreEntriesFTS(sessionContext) {
28227
28460
  return /* @__PURE__ */ new Map();
28228
28461
  }
28229
28462
  }
28230
- function forSession(projectPath, sessionID, maxTokens) {
28463
+ async function forSession(projectPath, sessionID, maxTokens, options) {
28231
28464
  const pid = ensureProject(projectPath);
28465
+ const categoryFilter = options?.categories;
28466
+ const excludeFilter = options?.excludeCategories;
28467
+ let categoryClause = "";
28468
+ let categoryParams = [];
28469
+ if (categoryFilter?.length) {
28470
+ categoryClause = ` AND category IN (${categoryFilter.map(() => "?").join(",")})`;
28471
+ categoryParams = categoryFilter;
28472
+ } else if (excludeFilter?.length) {
28473
+ categoryClause = ` AND category NOT IN (${excludeFilter.map(() => "?").join(",")})`;
28474
+ categoryParams = excludeFilter;
28475
+ }
28232
28476
  const projectEntries = db().query(
28233
28477
  `SELECT ${KNOWLEDGE_COLS} FROM knowledge
28234
- WHERE project_id = ? AND cross_project = 0 AND confidence > 0.2
28478
+ WHERE project_id = ? AND cross_project = 0 AND confidence > 0.2${categoryClause}
28235
28479
  ORDER BY confidence DESC, updated_at DESC`
28236
- ).all(pid);
28480
+ ).all(pid, ...categoryParams);
28237
28481
  const crossEntries = db().query(
28238
28482
  `SELECT ${KNOWLEDGE_COLS} FROM knowledge
28239
- WHERE (project_id IS NULL OR cross_project = 1) AND confidence > 0.2
28483
+ WHERE (project_id IS NULL OR cross_project = 1) AND confidence > 0.2${categoryClause}
28240
28484
  ORDER BY confidence DESC, updated_at DESC`
28241
- ).all();
28485
+ ).all(...categoryParams);
28242
28486
  if (!crossEntries.length && !projectEntries.length) return [];
28243
28487
  let sessionContext = "";
28244
28488
  if (sessionID) {
@@ -28259,22 +28503,52 @@ function forSession(projectPath, sessionID, maxTokens) {
28259
28503
  sessionContext += recentMsgs.map((m) => m.content).join("\n");
28260
28504
  }
28261
28505
  }
28506
+ if (!sessionContext.trim() && options?.contextHint) {
28507
+ sessionContext = options.contextHint;
28508
+ }
28262
28509
  let scoredProject;
28263
28510
  let scoredCross;
28264
- if (sessionContext.trim().length > 20) {
28511
+ if (sessionContext.trim().length > 20 && isAvailable()) {
28512
+ let vectorScores;
28513
+ try {
28514
+ const [contextVec] = await embed([sessionContext], "query");
28515
+ const hits = vectorSearch(contextVec, 50, excludeFilter);
28516
+ vectorScores = new Map(hits.map((h3) => [h3.id, h3.similarity]));
28517
+ } catch (err) {
28518
+ warn("Vector scoring failed, falling back to FTS5:", err);
28519
+ vectorScores = /* @__PURE__ */ new Map();
28520
+ }
28521
+ if (vectorScores.size > 0) {
28522
+ const ftsScores = scoreEntriesFTS(sessionContext);
28523
+ const rawScored = projectEntries.map((entry) => {
28524
+ const vecScore = vectorScores.get(entry.id);
28525
+ const score = vecScore != null ? vecScore * entry.confidence : (ftsScores.get(entry.id) ?? 0) * entry.confidence;
28526
+ return { entry, score };
28527
+ });
28528
+ const matched = rawScored.filter((s) => s.score > 0);
28529
+ const matchedIds = new Set(matched.map((s) => s.entry.id));
28530
+ const safetyNet = projectEntries.filter((e) => !matchedIds.has(e.id)).slice(0, PROJECT_SAFETY_NET).map((e) => ({ entry: e, score: 1e-3 * e.confidence }));
28531
+ scoredProject = [...matched, ...safetyNet];
28532
+ scoredCross = crossEntries.filter((e) => vectorScores.has(e.id) || ftsScores.has(e.id)).map((e) => {
28533
+ const vecScore = vectorScores.get(e.id);
28534
+ const score = vecScore != null ? vecScore * e.confidence : (ftsScores.get(e.id) ?? 0) * e.confidence;
28535
+ return { entry: e, score };
28536
+ });
28537
+ } else {
28538
+ const ftsScores = scoreEntriesFTS(sessionContext);
28539
+ ({ scoredProject, scoredCross } = scoreFTS(
28540
+ projectEntries,
28541
+ crossEntries,
28542
+ ftsScores
28543
+ ));
28544
+ }
28545
+ } else if (sessionContext.trim().length > 20) {
28265
28546
  const ftsScores = scoreEntriesFTS(sessionContext);
28266
- const rawScored = projectEntries.map((entry) => ({
28267
- entry,
28268
- score: (ftsScores.get(entry.id) ?? 0) * entry.confidence
28269
- }));
28270
- const matched = rawScored.filter((s) => s.score > 0);
28271
- const matchedIds = new Set(matched.map((s) => s.entry.id));
28272
- const safetyNet = projectEntries.filter((e) => !matchedIds.has(e.id)).slice(0, PROJECT_SAFETY_NET).map((e) => ({ entry: e, score: 1e-3 * e.confidence }));
28273
- scoredProject = [...matched, ...safetyNet];
28274
- scoredCross = crossEntries.filter((e) => ftsScores.has(e.id)).map((e) => ({
28275
- entry: e,
28276
- score: (ftsScores.get(e.id) ?? 0) * e.confidence
28277
- }));
28547
+ ({ scoredProject, scoredCross } = scoreFTS(
28548
+ projectEntries,
28549
+ crossEntries,
28550
+ ftsScores
28551
+ ));
28278
28552
  } else {
28279
28553
  scoredProject = projectEntries.slice(0, NO_CONTEXT_FALLBACK_CAP).map((entry) => ({ entry, score: entry.confidence }));
28280
28554
  scoredCross = crossEntries.slice(0, NO_CONTEXT_FALLBACK_CAP).map((entry) => ({ entry, score: entry.confidence }));
@@ -28320,6 +28594,21 @@ function forSession(projectPath, sessionID, maxTokens) {
28320
28594
  }
28321
28595
  return result;
28322
28596
  }
28597
+ function scoreFTS(projectEntries, crossEntries, ftsScores) {
28598
+ const rawScored = projectEntries.map((entry) => ({
28599
+ entry,
28600
+ score: (ftsScores.get(entry.id) ?? 0) * entry.confidence
28601
+ }));
28602
+ const matched = rawScored.filter((s) => s.score > 0);
28603
+ const matchedIds = new Set(matched.map((s) => s.entry.id));
28604
+ const safetyNet = projectEntries.filter((e) => !matchedIds.has(e.id)).slice(0, PROJECT_SAFETY_NET).map((e) => ({ entry: e, score: 1e-3 * e.confidence }));
28605
+ const scoredProject = [...matched, ...safetyNet];
28606
+ const scoredCross = crossEntries.filter((e) => ftsScores.has(e.id)).map((e) => ({
28607
+ entry: e,
28608
+ score: (ftsScores.get(e.id) ?? 0) * e.confidence
28609
+ }));
28610
+ return { scoredProject, scoredCross };
28611
+ }
28323
28612
  function all2() {
28324
28613
  return db().query(
28325
28614
  `SELECT ${KNOWLEDGE_COLS} FROM knowledge WHERE confidence > 0.2 ORDER BY confidence DESC, updated_at DESC`
@@ -28563,8 +28852,11 @@ function check2(projectPath) {
28563
28852
  }
28564
28853
  return issues;
28565
28854
  }
28566
- function _dedup(entries, dryRun) {
28567
- if (entries.length < 2) return { clusters: [], totalRemoved: 0 };
28855
+ function dedupPairKey(idA, idB) {
28856
+ return idA < idB ? `${idA}:${idB}` : `${idB}:${idA}`;
28857
+ }
28858
+ function _dedup(entries, dryRun, embeddingThreshold = EMBEDDING_DEDUP_THRESHOLD) {
28859
+ if (entries.length < 2) return { clusters: [], totalRemoved: 0, pairSimilarities: /* @__PURE__ */ new Map(), entryTitles: /* @__PURE__ */ new Map() };
28568
28860
  const embeddingMap = /* @__PURE__ */ new Map();
28569
28861
  {
28570
28862
  const entryIds = entries.map((e) => e.id);
@@ -28579,6 +28871,7 @@ function _dedup(entries, dryRun) {
28579
28871
  }
28580
28872
  }
28581
28873
  const neighborMap = /* @__PURE__ */ new Map();
28874
+ const pairSimilarities = /* @__PURE__ */ new Map();
28582
28875
  for (const entry of entries) {
28583
28876
  const neighbors = [];
28584
28877
  const entryVec = embeddingMap.get(entry.id);
@@ -28592,7 +28885,13 @@ function _dedup(entries, dryRun) {
28592
28885
  const otherVec = embeddingMap.get(other.id);
28593
28886
  if (otherVec && entryVec.length === otherVec.length) {
28594
28887
  similarity = cosineSimilarity(entryVec, otherVec);
28595
- embeddingMatch = similarity >= EMBEDDING_DEDUP_THRESHOLD;
28888
+ embeddingMatch = similarity >= embeddingThreshold;
28889
+ }
28890
+ }
28891
+ if (similarity > 0) {
28892
+ const pk = dedupPairKey(entry.id, other.id);
28893
+ if (!pairSimilarities.has(pk)) {
28894
+ pairSimilarities.set(pk, similarity);
28596
28895
  }
28597
28896
  }
28598
28897
  if (titleMatch || embeddingMatch) {
@@ -28644,20 +28943,178 @@ function _dedup(entries, dryRun) {
28644
28943
  totalRemoved += merged.length;
28645
28944
  }
28646
28945
  result.sort((a, b) => b.merged.length - a.merged.length);
28647
- return { clusters: result, totalRemoved };
28946
+ const entryTitles = new Map(entries.map((e) => [e.id, e.title]));
28947
+ return { clusters: result, totalRemoved, pairSimilarities, entryTitles };
28648
28948
  }
28649
28949
  async function deduplicate(projectPath, opts) {
28950
+ const pid = ensureProject(projectPath);
28951
+ const threshold = loadCalibratedThreshold(pid) ?? EMBEDDING_DEDUP_THRESHOLD;
28650
28952
  const entries = forProject(projectPath, false);
28651
- return _dedup(entries, opts?.dryRun ?? true);
28953
+ return _dedup(entries, opts?.dryRun ?? true, threshold);
28652
28954
  }
28653
28955
  async function deduplicateGlobal(opts) {
28956
+ const threshold = loadCalibratedThreshold(null) ?? EMBEDDING_DEDUP_THRESHOLD;
28654
28957
  const entries = db().query(
28655
28958
  `SELECT ${KNOWLEDGE_COLS} FROM knowledge
28656
28959
  WHERE project_id IS NULL
28657
28960
  AND confidence > 0.2
28658
28961
  ORDER BY confidence DESC, updated_at DESC`
28659
28962
  ).all();
28660
- return _dedup(entries, opts?.dryRun ?? true);
28963
+ return _dedup(entries, opts?.dryRun ?? true, threshold);
28964
+ }
28965
+ var MIN_CALIBRATION_SAMPLES = 20;
28966
+ var DEFAULT_EMBEDDING_DEDUP_THRESHOLD = EMBEDDING_DEDUP_THRESHOLD;
28967
+ var AUTO_SIGNAL_MIN_SIMILARITY = 0.8;
28968
+ var AUTO_SIGNAL_MAX_PAIRS = 50;
28969
+ function recordDedupFeedback(input) {
28970
+ db().query(
28971
+ `INSERT INTO dedup_feedback
28972
+ (project_id, entry_a_title, entry_b_title, similarity, accepted, source, created_at)
28973
+ VALUES (?, ?, ?, ?, ?, ?, ?)`
28974
+ ).run(
28975
+ input.projectId,
28976
+ input.entryATitle,
28977
+ input.entryBTitle,
28978
+ input.similarity,
28979
+ input.accepted ? 1 : 0,
28980
+ input.source,
28981
+ Date.now()
28982
+ );
28983
+ }
28984
+ function recordDedupResultFeedback(projectId2, result, accepted, source) {
28985
+ for (const cluster of result.clusters) {
28986
+ for (const merged of cluster.merged) {
28987
+ const pk = dedupPairKey(cluster.surviving.id, merged.id);
28988
+ const similarity = result.pairSimilarities.get(pk);
28989
+ if (similarity != null && similarity > 0) {
28990
+ recordDedupFeedback({
28991
+ projectId: projectId2,
28992
+ entryATitle: cluster.surviving.title,
28993
+ entryBTitle: merged.title,
28994
+ similarity,
28995
+ accepted,
28996
+ source
28997
+ });
28998
+ }
28999
+ }
29000
+ }
29001
+ }
29002
+ function recordAutoSignals(projectId2, result) {
29003
+ const mergedPairs = /* @__PURE__ */ new Set();
29004
+ for (const cluster of result.clusters) {
29005
+ for (const merged of cluster.merged) {
29006
+ mergedPairs.add(dedupPairKey(cluster.surviving.id, merged.id));
29007
+ }
29008
+ }
29009
+ const titleMap = new Map(result.entryTitles);
29010
+ for (const cluster of result.clusters) {
29011
+ if (!titleMap.has(cluster.surviving.id)) {
29012
+ titleMap.set(cluster.surviving.id, cluster.surviving.title);
29013
+ }
29014
+ for (const m of cluster.merged) {
29015
+ if (!titleMap.has(m.id)) titleMap.set(m.id, m.title);
29016
+ }
29017
+ }
29018
+ const signals = [];
29019
+ for (const [pk, sim] of result.pairSimilarities) {
29020
+ if (sim < AUTO_SIGNAL_MIN_SIMILARITY) continue;
29021
+ if (mergedPairs.has(pk)) continue;
29022
+ const [idA, idB] = pk.split(":");
29023
+ const titleA = titleMap.get(idA);
29024
+ const titleB = titleMap.get(idB);
29025
+ if (!titleA || !titleB) continue;
29026
+ signals.push({ entryATitle: titleA, entryBTitle: titleB, similarity: sim });
29027
+ }
29028
+ const currentThreshold = loadCalibratedThreshold(projectId2) ?? DEFAULT_EMBEDDING_DEDUP_THRESHOLD;
29029
+ signals.sort((a, b) => Math.abs(a.similarity - currentThreshold) - Math.abs(b.similarity - currentThreshold));
29030
+ const capped = signals.slice(0, AUTO_SIGNAL_MAX_PAIRS);
29031
+ pruneDedupFeedback(projectId2);
29032
+ for (const s of capped) {
29033
+ recordDedupFeedback({
29034
+ projectId: projectId2,
29035
+ entryATitle: s.entryATitle,
29036
+ entryBTitle: s.entryBTitle,
29037
+ similarity: s.similarity,
29038
+ accepted: false,
29039
+ source: "auto_dedup"
29040
+ });
29041
+ }
29042
+ }
29043
+ function getDedupFeedback(projectId2) {
29044
+ const rows = projectId2 !== null ? db().query(
29045
+ "SELECT similarity, accepted, source FROM dedup_feedback WHERE project_id = ? ORDER BY similarity"
29046
+ ).all(projectId2) : db().query(
29047
+ "SELECT similarity, accepted, source FROM dedup_feedback WHERE project_id IS NULL ORDER BY similarity"
29048
+ ).all();
29049
+ return rows.map((r) => ({ similarity: r.similarity, accepted: r.accepted === 1, source: r.source }));
29050
+ }
29051
+ function getDedupFeedbackCount(projectId2) {
29052
+ const row = projectId2 !== null ? db().query("SELECT COUNT(*) as cnt FROM dedup_feedback WHERE project_id = ?").get(projectId2) : db().query("SELECT COUNT(*) as cnt FROM dedup_feedback WHERE project_id IS NULL").get();
29053
+ return row?.cnt ?? 0;
29054
+ }
29055
+ var MAX_FEEDBACK_ROWS_PER_PROJECT = 500;
29056
+ function pruneDedupFeedback(projectId2) {
29057
+ const count3 = getDedupFeedbackCount(projectId2);
29058
+ if (count3 <= MAX_FEEDBACK_ROWS_PER_PROJECT) return;
29059
+ const excess = count3 - MAX_FEEDBACK_ROWS_PER_PROJECT;
29060
+ if (projectId2 !== null) {
29061
+ db().query(
29062
+ `DELETE FROM dedup_feedback WHERE id IN (
29063
+ SELECT id FROM dedup_feedback WHERE project_id = ?
29064
+ ORDER BY created_at ASC LIMIT ?
29065
+ )`
29066
+ ).run(projectId2, excess);
29067
+ } else {
29068
+ db().query(
29069
+ `DELETE FROM dedup_feedback WHERE id IN (
29070
+ SELECT id FROM dedup_feedback WHERE project_id IS NULL
29071
+ ORDER BY created_at ASC LIMIT ?
29072
+ )`
29073
+ ).run(excess);
29074
+ }
29075
+ }
29076
+ function calibrateDedupThreshold(projectId2) {
29077
+ const feedback = getDedupFeedback(projectId2);
29078
+ if (feedback.length < MIN_CALIBRATION_SAMPLES) return null;
29079
+ const accepted = feedback.filter((f) => f.accepted);
29080
+ const rejected = feedback.filter((f) => !f.accepted);
29081
+ if (rejected.length === 0) {
29082
+ const minAccepted = Math.min(...accepted.map((f) => f.similarity));
29083
+ return Math.max(0.85, minAccepted - 5e-3);
29084
+ }
29085
+ if (accepted.length === 0) {
29086
+ warn("dedup calibration: all feedback is reject \u2014 keeping default threshold");
29087
+ return null;
29088
+ }
29089
+ const allSims = [...new Set(feedback.map((f) => f.similarity))].sort((a, b) => a - b);
29090
+ let bestThreshold = DEFAULT_EMBEDDING_DEDUP_THRESHOLD;
29091
+ let bestAccuracy = -1;
29092
+ for (let i = 0; i < allSims.length - 1; i++) {
29093
+ const candidate = (allSims[i] + allSims[i + 1]) / 2;
29094
+ const correctAccepted = accepted.filter((f) => f.similarity >= candidate).length;
29095
+ const correctRejected = rejected.filter((f) => f.similarity < candidate).length;
29096
+ const accuracy = (correctAccepted + correctRejected) / feedback.length;
29097
+ if (accuracy > bestAccuracy || accuracy === bestAccuracy && candidate > bestThreshold) {
29098
+ bestAccuracy = accuracy;
29099
+ bestThreshold = candidate;
29100
+ }
29101
+ }
29102
+ return Math.max(0.85, Math.min(0.98, bestThreshold));
29103
+ }
29104
+ function saveCalibratedThreshold(projectId2, threshold, sampleSize) {
29105
+ const key = `dedup_threshold:${projectId2 ?? "global"}`;
29106
+ setKV(key, JSON.stringify({ threshold, sampleSize, calibratedAt: Date.now() }));
29107
+ }
29108
+ function loadCalibratedThreshold(projectId2) {
29109
+ const key = `dedup_threshold:${projectId2 ?? "global"}`;
29110
+ const raw = getKV(key);
29111
+ if (!raw) return null;
29112
+ try {
29113
+ const parsed = JSON.parse(raw);
29114
+ return typeof parsed.threshold === "number" ? parsed.threshold : null;
29115
+ } catch {
29116
+ return null;
29117
+ }
28661
29118
  }
28662
29119
 
28663
29120
  // src/data.ts
@@ -29506,6 +29963,16 @@ function getSessionState(sessionID) {
29506
29963
  if (!state) {
29507
29964
  state = makeSessionState();
29508
29965
  state.forceMinLayer = loadForceMinLayer(sessionID);
29966
+ const persisted = loadSessionTracking(sessionID);
29967
+ if (persisted && persisted.lastTurnAt > 0) {
29968
+ state.dynamicContextCap = persisted.dynamicContextCap;
29969
+ state.bustRateEMA = persisted.bustRateEMA;
29970
+ state.interBustIntervalEMA = persisted.interBustIntervalEMA;
29971
+ state.lastLayer = persisted.lastLayer;
29972
+ state.lastKnownInput = persisted.lastKnownInput;
29973
+ state.lastTurnAt = persisted.lastTurnAt;
29974
+ state.lastBustAt = persisted.lastBustAt;
29975
+ }
29509
29976
  sessionStates.set(sessionID, state);
29510
29977
  }
29511
29978
  return state;
@@ -29611,6 +30078,19 @@ function inspectSessionState(sessionID) {
29611
30078
  function setLastTurnAtForTest(sessionID, ms) {
29612
30079
  getSessionState(sessionID).lastTurnAt = ms;
29613
30080
  }
30081
+ function saveGradientState(sessionID) {
30082
+ const state = sessionStates.get(sessionID);
30083
+ if (!state) return;
30084
+ saveSessionTracking(sessionID, {
30085
+ dynamicContextCap: state.dynamicContextCap,
30086
+ bustRateEMA: state.bustRateEMA,
30087
+ interBustIntervalEMA: state.interBustIntervalEMA,
30088
+ lastLayer: state.lastLayer,
30089
+ lastKnownInput: state.lastKnownInput,
30090
+ lastTurnAt: state.lastTurnAt,
30091
+ lastBustAt: state.lastBustAt
30092
+ });
30093
+ }
29614
30094
  function loadDistillations(projectPath, sessionID) {
29615
30095
  const pid = ensureProject(projectPath);
29616
30096
  const query = sessionID ? "SELECT id, observations, generation, token_count, created_at, session_id FROM distillations WHERE project_id = ? AND session_id = ? AND archived = 0 ORDER BY created_at ASC" : "SELECT id, observations, generation, token_count, created_at, session_id FROM distillations WHERE project_id = ? AND archived = 0 ORDER BY created_at ASC";
@@ -29895,6 +30375,26 @@ function buildPrefixMessages(formatted) {
29895
30375
  }
29896
30376
  ];
29897
30377
  }
30378
+ var DECISION_RE = /\b(?:decision|decided|chose|chosen|agreed)\b/i;
30379
+ var GOTCHA_RE = /\b(?:gotcha|(?:critical|known|subtle)\s+bug|broken|crash(?:ed|es)?|regression)\b/i;
30380
+ var ARCH_RE = /\b(?:architecture|design.(?:decision|pattern)|system.design)\b/i;
30381
+ function importanceBonus(d) {
30382
+ let bonus = 0;
30383
+ if (DECISION_RE.test(d.observations)) bonus += 0.3;
30384
+ if (GOTCHA_RE.test(d.observations)) bonus += 0.2;
30385
+ if (ARCH_RE.test(d.observations)) bonus += 0.1;
30386
+ if (d.generation >= 1) bonus += 0.2;
30387
+ return Math.min(bonus, 1);
30388
+ }
30389
+ function selectDistillations(all3, limit) {
30390
+ if (all3.length <= limit) return all3;
30391
+ const maxIdx = all3.length - 1;
30392
+ const scored = all3.map((d, i) => ({
30393
+ d,
30394
+ score: (maxIdx > 0 ? i / maxIdx : 1) * 0.7 + importanceBonus(d) * 0.3
30395
+ }));
30396
+ return scored.sort((a, b) => b.score - a.score).slice(0, limit).map((s) => s.d).sort((a, b) => a.created_at - b.created_at);
30397
+ }
29898
30398
  function distilledPrefix(distillations) {
29899
30399
  if (!distillations.length) return [];
29900
30400
  const formatted = formatDistillations(distillations);
@@ -30012,6 +30512,11 @@ function tryFitStable(input) {
30012
30512
  }
30013
30513
  return result;
30014
30514
  }
30515
+ var COMPRESSION_STAGES = [
30516
+ { strip: "none", rawFrac: null, distFrac: null, distLimit: Infinity, protectedTurns: 0, useStableWindow: true },
30517
+ { strip: "old-tools", rawFrac: 0.5, distFrac: null, distLimit: Infinity, protectedTurns: 2, useStableWindow: false },
30518
+ { strip: "all-tools", rawFrac: 0.55, distFrac: 0.15, distLimit: 5, protectedTurns: 0, useStableWindow: false }
30519
+ ];
30015
30520
  var urgentDistillationMap = /* @__PURE__ */ new Map();
30016
30521
  function needsUrgentDistillation(sessionID) {
30017
30522
  const v = urgentDistillationMap.get(sessionID) ?? false;
@@ -30043,7 +30548,7 @@ function transformInner(input) {
30043
30548
  if (calibrated) return true;
30044
30549
  return result.totalTokens * UNCALIBRATED_SAFETY <= maxInput;
30045
30550
  }
30046
- if (calibrated && sessState.lastLayer >= 1 && input.messages.length >= sessState.lastKnownMessageCount) {
30551
+ if (calibrated && sessState.lastLayer >= 1 && sessState.lastLayer <= 3 && input.messages.length >= sessState.lastKnownMessageCount) {
30047
30552
  effectiveMinLayer = Math.max(effectiveMinLayer, sessState.lastLayer);
30048
30553
  }
30049
30554
  const postIdleCompact = sessState.postIdleCompact;
@@ -30081,7 +30586,8 @@ function transformInner(input) {
30081
30586
  totalTokens: Math.max(0, messageTokens),
30082
30587
  usable,
30083
30588
  distilledBudget,
30084
- rawBudget
30589
+ rawBudget,
30590
+ refreshLtm: false
30085
30591
  };
30086
30592
  }
30087
30593
  const turnStart = currentTurnStart(input.messages);
@@ -30091,67 +30597,52 @@ function transformInner(input) {
30091
30597
  const msgs = distilledPrefix(distillations);
30092
30598
  return { messages: msgs, tokens: msgs.reduce((sum, m) => sum + estimateMessage(m), 0) };
30093
30599
  })();
30094
- if (effectiveMinLayer <= 1) {
30095
- const layer1 = sid ? tryFitStable({
30096
- messages: dedupMessages,
30097
- prefix: cached2.messages,
30098
- prefixTokens: cached2.tokens,
30099
- distilledBudget,
30100
- rawBudget,
30101
- sessionID: sid,
30102
- sessState
30103
- }) : tryFit({
30104
- messages: dedupMessages,
30105
- prefix: cached2.messages,
30106
- prefixTokens: cached2.tokens,
30107
- distilledBudget,
30108
- rawBudget,
30109
- strip: "none"
30110
- });
30111
- if (fitsWithSafetyMargin(layer1)) {
30112
- if (cached2.tokens === 0 && sid) {
30600
+ for (let s = 0; s < COMPRESSION_STAGES.length; s++) {
30601
+ const stageLayer = s + 1;
30602
+ if (effectiveMinLayer > stageLayer) continue;
30603
+ const stage = COMPRESSION_STAGES[s];
30604
+ const stageRawBudget = stage.rawFrac !== null ? Math.floor(usable * stage.rawFrac) : rawBudget;
30605
+ const stageDistBudget = stage.distFrac !== null ? Math.floor(usable * stage.distFrac) : distilledBudget;
30606
+ let stagePrefix = cached2.messages;
30607
+ let stagePrefixTokens = cached2.tokens;
30608
+ if (stage.distLimit !== Infinity && distillations.length > stage.distLimit) {
30609
+ const trimmed = selectDistillations(distillations, stage.distLimit);
30610
+ stagePrefix = distilledPrefix(trimmed);
30611
+ stagePrefixTokens = stagePrefix.reduce((sum, m) => sum + estimateMessage(m), 0);
30612
+ }
30613
+ let result;
30614
+ if (stage.useStableWindow && sid) {
30615
+ result = tryFitStable({
30616
+ messages: dedupMessages,
30617
+ prefix: stagePrefix,
30618
+ prefixTokens: stagePrefixTokens,
30619
+ distilledBudget: stageDistBudget,
30620
+ rawBudget: stageRawBudget,
30621
+ sessionID: sid,
30622
+ sessState
30623
+ });
30624
+ } else {
30625
+ sessState.rawWindowCache = null;
30626
+ result = tryFit({
30627
+ messages: dedupMessages,
30628
+ prefix: stagePrefix,
30629
+ prefixTokens: stagePrefixTokens,
30630
+ distilledBudget: stageDistBudget,
30631
+ rawBudget: stageRawBudget,
30632
+ strip: stage.strip,
30633
+ protectedTurns: stage.protectedTurns
30634
+ });
30635
+ }
30636
+ if (fitsWithSafetyMargin(result)) {
30637
+ if (sid && (s > 0 || cached2.tokens === 0)) {
30113
30638
  urgentDistillationMap.set(sid, true);
30114
30639
  }
30115
- return { ...layer1, layer: 1, usable, distilledBudget, rawBudget };
30640
+ return { ...result, layer: stageLayer, usable, distilledBudget, rawBudget, refreshLtm: false };
30116
30641
  }
30117
30642
  }
30118
30643
  sessState.rawWindowCache = null;
30119
- if (effectiveMinLayer <= 2) {
30120
- const layer2 = tryFit({
30121
- messages: dedupMessages,
30122
- prefix: cached2.messages,
30123
- prefixTokens: cached2.tokens,
30124
- distilledBudget,
30125
- rawBudget: Math.floor(usable * 0.5),
30126
- // give raw more room
30127
- strip: "old-tools",
30128
- protectedTurns: 2
30129
- });
30130
- if (fitsWithSafetyMargin(layer2)) {
30131
- if (sid) urgentDistillationMap.set(sid, true);
30132
- return { ...layer2, layer: 2, usable, distilledBudget, rawBudget };
30133
- }
30134
- }
30135
- const trimmedDistillations = distillations.slice(-5);
30136
- const trimmedPrefix = distilledPrefix(trimmedDistillations);
30137
- const trimmedPrefixTokens = trimmedPrefix.reduce(
30138
- (sum, m) => sum + estimateMessage(m),
30139
- 0
30140
- );
30141
- const layer3 = tryFit({
30142
- messages: dedupMessages,
30143
- prefix: trimmedPrefix,
30144
- prefixTokens: trimmedPrefixTokens,
30145
- distilledBudget: Math.floor(usable * 0.15),
30146
- rawBudget: Math.floor(usable * 0.55),
30147
- strip: "all-tools"
30148
- });
30149
- if (fitsWithSafetyMargin(layer3)) {
30150
- if (sid) urgentDistillationMap.set(sid, true);
30151
- return { ...layer3, layer: 3, usable, distilledBudget, rawBudget };
30152
- }
30153
30644
  if (sid) urgentDistillationMap.set(sid, true);
30154
- const nuclearDistillations = distillations.slice(-2);
30645
+ const nuclearDistillations = selectDistillations(distillations, 2);
30155
30646
  const nuclearPrefix = distilledPrefix(nuclearDistillations);
30156
30647
  const nuclearPrefixTokens = nuclearPrefix.reduce(
30157
30648
  (sum, m) => sum + estimateMessage(m),
@@ -30190,7 +30681,8 @@ function transformInner(input) {
30190
30681
  totalTokens: nuclearPrefixTokens + nuclearRawTokens,
30191
30682
  usable,
30192
30683
  distilledBudget,
30193
- rawBudget
30684
+ rawBudget,
30685
+ refreshLtm: true
30194
30686
  };
30195
30687
  }
30196
30688
  function transform2(input) {
@@ -30297,6 +30789,185 @@ function isWorkerSession(sessionID) {
30297
30789
  return workerSessionIDs.has(sessionID);
30298
30790
  }
30299
30791
 
30792
+ // ../../node_modules/.bun/yocto-queue@1.2.2/node_modules/yocto-queue/index.js
30793
+ var Node = class {
30794
+ value;
30795
+ next;
30796
+ constructor(value) {
30797
+ this.value = value;
30798
+ }
30799
+ };
30800
+ var Queue = class {
30801
+ #head;
30802
+ #tail;
30803
+ #size;
30804
+ constructor() {
30805
+ this.clear();
30806
+ }
30807
+ enqueue(value) {
30808
+ const node2 = new Node(value);
30809
+ if (this.#head) {
30810
+ this.#tail.next = node2;
30811
+ this.#tail = node2;
30812
+ } else {
30813
+ this.#head = node2;
30814
+ this.#tail = node2;
30815
+ }
30816
+ this.#size++;
30817
+ }
30818
+ dequeue() {
30819
+ const current2 = this.#head;
30820
+ if (!current2) {
30821
+ return;
30822
+ }
30823
+ this.#head = this.#head.next;
30824
+ this.#size--;
30825
+ if (!this.#head) {
30826
+ this.#tail = void 0;
30827
+ }
30828
+ return current2.value;
30829
+ }
30830
+ peek() {
30831
+ if (!this.#head) {
30832
+ return;
30833
+ }
30834
+ return this.#head.value;
30835
+ }
30836
+ clear() {
30837
+ this.#head = void 0;
30838
+ this.#tail = void 0;
30839
+ this.#size = 0;
30840
+ }
30841
+ get size() {
30842
+ return this.#size;
30843
+ }
30844
+ *[Symbol.iterator]() {
30845
+ let current2 = this.#head;
30846
+ while (current2) {
30847
+ yield current2.value;
30848
+ current2 = current2.next;
30849
+ }
30850
+ }
30851
+ *drain() {
30852
+ while (this.#head) {
30853
+ yield this.dequeue();
30854
+ }
30855
+ }
30856
+ };
30857
+
30858
+ // ../../node_modules/.bun/p-limit@7.3.0/node_modules/p-limit/index.js
30859
+ function pLimit(concurrency) {
30860
+ let rejectOnClear = false;
30861
+ if (typeof concurrency === "object") {
30862
+ ({ concurrency, rejectOnClear = false } = concurrency);
30863
+ }
30864
+ validateConcurrency(concurrency);
30865
+ if (typeof rejectOnClear !== "boolean") {
30866
+ throw new TypeError("Expected `rejectOnClear` to be a boolean");
30867
+ }
30868
+ const queue = new Queue();
30869
+ let activeCount = 0;
30870
+ const resumeNext = () => {
30871
+ if (activeCount < concurrency && queue.size > 0) {
30872
+ activeCount++;
30873
+ queue.dequeue().run();
30874
+ }
30875
+ };
30876
+ const next = () => {
30877
+ activeCount--;
30878
+ resumeNext();
30879
+ };
30880
+ const run3 = async (function_, resolve, arguments_) => {
30881
+ const result = (async () => function_(...arguments_))();
30882
+ resolve(result);
30883
+ try {
30884
+ await result;
30885
+ } catch {
30886
+ }
30887
+ next();
30888
+ };
30889
+ const enqueue = (function_, resolve, reject, arguments_) => {
30890
+ const queueItem = { reject };
30891
+ new Promise((internalResolve) => {
30892
+ queueItem.run = internalResolve;
30893
+ queue.enqueue(queueItem);
30894
+ }).then(run3.bind(void 0, function_, resolve, arguments_));
30895
+ if (activeCount < concurrency) {
30896
+ resumeNext();
30897
+ }
30898
+ };
30899
+ const generator = (function_, ...arguments_) => new Promise((resolve, reject) => {
30900
+ enqueue(function_, resolve, reject, arguments_);
30901
+ });
30902
+ Object.defineProperties(generator, {
30903
+ activeCount: {
30904
+ get: () => activeCount
30905
+ },
30906
+ pendingCount: {
30907
+ get: () => queue.size
30908
+ },
30909
+ clearQueue: {
30910
+ value() {
30911
+ if (!rejectOnClear) {
30912
+ queue.clear();
30913
+ return;
30914
+ }
30915
+ const abortError = AbortSignal.abort().reason;
30916
+ while (queue.size > 0) {
30917
+ queue.dequeue().reject(abortError);
30918
+ }
30919
+ }
30920
+ },
30921
+ concurrency: {
30922
+ get: () => concurrency,
30923
+ set(newConcurrency) {
30924
+ validateConcurrency(newConcurrency);
30925
+ concurrency = newConcurrency;
30926
+ queueMicrotask(() => {
30927
+ while (activeCount < concurrency && queue.size > 0) {
30928
+ resumeNext();
30929
+ }
30930
+ });
30931
+ }
30932
+ },
30933
+ map: {
30934
+ async value(iterable, function_) {
30935
+ const promises = Array.from(iterable, (value, index2) => this(function_, value, index2));
30936
+ return Promise.all(promises);
30937
+ }
30938
+ }
30939
+ });
30940
+ return generator;
30941
+ }
30942
+ function validateConcurrency(concurrency) {
30943
+ if (!((Number.isInteger(concurrency) || concurrency === Number.POSITIVE_INFINITY) && concurrency > 0)) {
30944
+ throw new TypeError("Expected `concurrency` to be a number from 1 and up");
30945
+ }
30946
+ }
30947
+
30948
+ // src/session-limiter.ts
30949
+ function createLimiterPool() {
30950
+ const limiters = /* @__PURE__ */ new Map();
30951
+ function get2(key) {
30952
+ let limiter = limiters.get(key);
30953
+ if (!limiter) {
30954
+ limiter = pLimit(1);
30955
+ limiters.set(key, limiter);
30956
+ }
30957
+ return limiter;
30958
+ }
30959
+ function isBusy(key) {
30960
+ const limiter = limiters.get(key);
30961
+ return limiter ? limiter.activeCount + limiter.pendingCount > 0 : false;
30962
+ }
30963
+ function clear() {
30964
+ limiters.clear();
30965
+ }
30966
+ return { get: get2, isBusy, clear };
30967
+ }
30968
+ var distillLimiter = createLimiterPool();
30969
+ var curatorLimiter = createLimiterPool();
30970
+
30300
30971
  // src/distillation.ts
30301
30972
  function compressionRatio(distilledTokens, sourceTokens) {
30302
30973
  if (sourceTokens <= 0) return 0;
@@ -30541,6 +31212,9 @@ function resetOrphans(projectPath, sessionID) {
30541
31212
  return orphans.length;
30542
31213
  }
30543
31214
  async function run(input) {
31215
+ return distillLimiter.get(input.sessionID)(() => runInner(input));
31216
+ }
31217
+ async function runInner(input) {
30544
31218
  const orphans = resetOrphans(input.projectPath, input.sessionID);
30545
31219
  if (orphans > 0) {
30546
31220
  info(
@@ -30584,7 +31258,7 @@ async function run(input) {
30584
31258
  }
30585
31259
  }
30586
31260
  if (!input.skipMeta && gen0Count(input.projectPath, input.sessionID) >= cfg.distillation.metaThreshold) {
30587
- await metaDistill({
31261
+ await metaDistillInner({
30588
31262
  llm: input.llm,
30589
31263
  projectPath: input.projectPath,
30590
31264
  sessionID: input.sessionID,
@@ -30634,17 +31308,25 @@ async function distillSegment(input) {
30634
31308
  );
30635
31309
  return null;
30636
31310
  }
30637
- const distillId = storeDistillation({
30638
- projectPath: input.projectPath,
30639
- sessionID: input.sessionID,
30640
- observations: result.observations,
30641
- sourceIDs: input.messages.map((m) => m.id),
30642
- generation: 0,
30643
- rCompression: rComp,
30644
- cNorm,
30645
- callType: input.callType
30646
- });
30647
- markDistilled(input.messages.map((m) => m.id));
31311
+ let distillId;
31312
+ db().exec("BEGIN IMMEDIATE");
31313
+ try {
31314
+ distillId = storeDistillation({
31315
+ projectPath: input.projectPath,
31316
+ sessionID: input.sessionID,
31317
+ observations: result.observations,
31318
+ sourceIDs: input.messages.map((m) => m.id),
31319
+ generation: 0,
31320
+ rCompression: rComp,
31321
+ cNorm,
31322
+ callType: input.callType
31323
+ });
31324
+ markDistilled(input.messages.map((m) => m.id));
31325
+ db().exec("COMMIT");
31326
+ } catch (e) {
31327
+ db().exec("ROLLBACK");
31328
+ throw e;
31329
+ }
30648
31330
  info(
30649
31331
  `distill segment: ${input.messages.length} msgs, ${sourceTokens}\u2192${distilledTokens} tokens, R=${rComp.toFixed(2)}, C_norm=${cNorm.toFixed(3)}`
30650
31332
  );
@@ -30678,6 +31360,9 @@ async function distillSegment(input) {
30678
31360
  return result;
30679
31361
  }
30680
31362
  async function metaDistill(input) {
31363
+ return distillLimiter.get(input.sessionID)(() => metaDistillInner(input));
31364
+ }
31365
+ async function metaDistillInner(input) {
30681
31366
  const existing = loadGen0(input.projectPath, input.sessionID);
30682
31367
  const priorMeta = latestMeta(input.projectPath, input.sessionID);
30683
31368
  if (priorMeta) {
@@ -30977,11 +31662,27 @@ function applyOps(ops, input) {
30977
31662
  return { created, updated, deleted };
30978
31663
  }
30979
31664
  var lastCuratedAt = /* @__PURE__ */ new Map();
31665
+ function getLastCuratedAt(sessionID) {
31666
+ const cached2 = lastCuratedAt.get(sessionID);
31667
+ if (cached2 !== void 0) return cached2;
31668
+ const persisted = loadSessionTracking(sessionID);
31669
+ const ts = persisted?.lastCuratedAt ?? 0;
31670
+ lastCuratedAt.set(sessionID, ts);
31671
+ return ts;
31672
+ }
30980
31673
  async function run2(input) {
30981
31674
  const cfg = config2();
30982
31675
  if (!cfg.curator.enabled) return { created: 0, updated: 0, deleted: 0 };
31676
+ if (curatorLimiter.isBusy(input.sessionID)) {
31677
+ info(`curation skipped: already running for session ${input.sessionID.slice(0, 16)}`);
31678
+ return { created: 0, updated: 0, deleted: 0 };
31679
+ }
31680
+ return curatorLimiter.get(input.sessionID)(() => runInner2(input));
31681
+ }
31682
+ async function runInner2(input) {
31683
+ const cfg = config2();
30983
31684
  const all3 = bySession(input.projectPath, input.sessionID);
30984
- const sessionCuratedAt = lastCuratedAt.get(input.sessionID) ?? 0;
31685
+ const sessionCuratedAt = getLastCuratedAt(input.sessionID);
30985
31686
  const recent = all3.filter((m) => m.created_at > sessionCuratedAt);
30986
31687
  if (recent.length < 3) return { created: 0, updated: 0, deleted: 0 };
30987
31688
  const text4 = recent.map((m) => `[${m.role}] ${m.content}`).join("\n\n");
@@ -31025,11 +31726,22 @@ async function run2(input) {
31025
31726
  info(`post-curation dedup: merged ${dupes.totalRemoved} duplicate entries`);
31026
31727
  result.deleted += dupes.totalRemoved;
31027
31728
  }
31729
+ if (dupes.pairSimilarities.size > 0) {
31730
+ const pid = ensureProject(input.projectPath);
31731
+ recordAutoSignals(pid, dupes);
31732
+ const newThreshold = calibrateDedupThreshold(pid);
31733
+ if (newThreshold !== null) {
31734
+ const count3 = getDedupFeedbackCount(pid);
31735
+ saveCalibratedThreshold(pid, newThreshold, count3);
31736
+ }
31737
+ }
31028
31738
  } catch (err) {
31029
31739
  warn("post-curation dedup failed (non-fatal):", err);
31030
31740
  }
31031
31741
  }
31032
- lastCuratedAt.set(input.sessionID, Date.now());
31742
+ const now = Date.now();
31743
+ lastCuratedAt.set(input.sessionID, now);
31744
+ saveSessionTracking(input.sessionID, { lastCuratedAt: now });
31033
31745
  return result;
31034
31746
  }
31035
31747
  function resetCurationTracker(sessionID) {
@@ -33152,6 +33864,7 @@ export {
33152
33864
  ftsQueryRelaxed,
33153
33865
  getGitRemote,
33154
33866
  getInstanceId,
33867
+ getKV,
33155
33868
  getLastImportAt,
33156
33869
  getLastTransformEstimate,
33157
33870
  getLastTransformedCount,
@@ -33177,7 +33890,9 @@ export {
33177
33890
  load,
33178
33891
  loadAllSessionCosts,
33179
33892
  loadForceMinLayer,
33893
+ loadHeaderSessionIndex,
33180
33894
  loadSessionCosts,
33895
+ loadSessionTracking,
33181
33896
  log_exports as log,
33182
33897
  loreFileExists,
33183
33898
  ltm_exports as ltm,
@@ -33198,10 +33913,13 @@ export {
33198
33913
  runRecall,
33199
33914
  sanitizeSurrogates,
33200
33915
  saveForceMinLayer,
33916
+ saveGradientState,
33201
33917
  saveSessionCosts,
33918
+ saveSessionTracking,
33202
33919
  searchRecall,
33203
33920
  serialize,
33204
33921
  setForceMinLayer,
33922
+ setKV,
33205
33923
  setLastImportAt,
33206
33924
  setLastTurnAtForTest,
33207
33925
  setLtmTokens,