@loreai/core 0.18.0 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. package/dist/bun/curator.d.ts.map +1 -1
  2. package/dist/bun/db.d.ts +73 -0
  3. package/dist/bun/db.d.ts.map +1 -1
  4. package/dist/bun/distillation.d.ts +2 -13
  5. package/dist/bun/distillation.d.ts.map +1 -1
  6. package/dist/bun/embedding.d.ts +5 -1
  7. package/dist/bun/embedding.d.ts.map +1 -1
  8. package/dist/bun/gradient.d.ts +9 -0
  9. package/dist/bun/gradient.d.ts.map +1 -1
  10. package/dist/bun/index.d.ts +2 -2
  11. package/dist/bun/index.d.ts.map +1 -1
  12. package/dist/bun/index.js +817 -99
  13. package/dist/bun/index.js.map +4 -4
  14. package/dist/bun/ltm.d.ts +99 -5
  15. package/dist/bun/ltm.d.ts.map +1 -1
  16. package/dist/bun/session-limiter.d.ts +26 -0
  17. package/dist/bun/session-limiter.d.ts.map +1 -0
  18. package/dist/bun/temporal.d.ts +2 -0
  19. package/dist/bun/temporal.d.ts.map +1 -1
  20. package/dist/node/curator.d.ts.map +1 -1
  21. package/dist/node/db.d.ts +73 -0
  22. package/dist/node/db.d.ts.map +1 -1
  23. package/dist/node/distillation.d.ts +2 -13
  24. package/dist/node/distillation.d.ts.map +1 -1
  25. package/dist/node/embedding.d.ts +5 -1
  26. package/dist/node/embedding.d.ts.map +1 -1
  27. package/dist/node/gradient.d.ts +9 -0
  28. package/dist/node/gradient.d.ts.map +1 -1
  29. package/dist/node/index.d.ts +2 -2
  30. package/dist/node/index.d.ts.map +1 -1
  31. package/dist/node/index.js +817 -99
  32. package/dist/node/index.js.map +4 -4
  33. package/dist/node/ltm.d.ts +99 -5
  34. package/dist/node/ltm.d.ts.map +1 -1
  35. package/dist/node/session-limiter.d.ts +26 -0
  36. package/dist/node/session-limiter.d.ts.map +1 -0
  37. package/dist/node/temporal.d.ts +2 -0
  38. package/dist/node/temporal.d.ts.map +1 -1
  39. package/dist/types/curator.d.ts.map +1 -1
  40. package/dist/types/db.d.ts +73 -0
  41. package/dist/types/db.d.ts.map +1 -1
  42. package/dist/types/distillation.d.ts +2 -13
  43. package/dist/types/distillation.d.ts.map +1 -1
  44. package/dist/types/embedding.d.ts +5 -1
  45. package/dist/types/embedding.d.ts.map +1 -1
  46. package/dist/types/gradient.d.ts +9 -0
  47. package/dist/types/gradient.d.ts.map +1 -1
  48. package/dist/types/index.d.ts +2 -2
  49. package/dist/types/index.d.ts.map +1 -1
  50. package/dist/types/ltm.d.ts +99 -5
  51. package/dist/types/ltm.d.ts.map +1 -1
  52. package/dist/types/session-limiter.d.ts +26 -0
  53. package/dist/types/session-limiter.d.ts.map +1 -0
  54. package/dist/types/temporal.d.ts +2 -0
  55. package/dist/types/temporal.d.ts.map +1 -1
  56. package/package.json +2 -1
  57. package/src/curator.ts +54 -2
  58. package/src/db.ts +347 -0
  59. package/src/distillation.ts +55 -14
  60. package/src/embedding.ts +28 -3
  61. package/src/gradient.ts +183 -74
  62. package/src/index.ts +8 -0
  63. package/src/ltm.ts +480 -45
  64. package/src/session-limiter.ts +47 -0
  65. package/src/temporal.ts +10 -0
@@ -125,6 +125,7 @@ __export(temporal_exports, {
125
125
  CHUNK_TERMINATOR: () => CHUNK_TERMINATOR,
126
126
  bySession: () => bySession,
127
127
  count: () => count,
128
+ hasMessages: () => hasMessages,
128
129
  markDistilled: () => markDistilled,
129
130
  partsToText: () => partsToText,
130
131
  prune: () => prune,
@@ -754,6 +755,55 @@ var MIGRATIONS = [
754
755
  WHERE ih.project_id = projects.id
755
756
  AND ih.source_id = '__declined__'
756
757
  );
758
+ `,
759
+ `
760
+ -- Version 23: Persist volatile session tracking state across restarts.
761
+ -- Previously these were in-memory only, causing duplicate processing,
762
+ -- false compaction detection, and expensive prompt cache busts on restart.
763
+ ALTER TABLE session_state ADD COLUMN last_curated_at INTEGER NOT NULL DEFAULT 0;
764
+ ALTER TABLE session_state ADD COLUMN message_count INTEGER NOT NULL DEFAULT 0;
765
+ ALTER TABLE session_state ADD COLUMN turns_since_curation INTEGER NOT NULL DEFAULT 0;
766
+ ALTER TABLE session_state ADD COLUMN ltm_cache_text TEXT;
767
+ ALTER TABLE session_state ADD COLUMN ltm_cache_tokens INTEGER;
768
+ ALTER TABLE session_state ADD COLUMN ltm_pin_text TEXT;
769
+ ALTER TABLE session_state ADD COLUMN ltm_pin_tokens INTEGER;
770
+ ALTER TABLE session_state ADD COLUMN consecutive_text_only_turns INTEGER NOT NULL DEFAULT 0;
771
+ `,
772
+ `
773
+ -- Version 24: Persist remaining volatile session state across restarts.
774
+ -- Session identity (Tier 1/2/3 session correlation)
775
+ ALTER TABLE session_state ADD COLUMN fingerprint TEXT NOT NULL DEFAULT '';
776
+ ALTER TABLE session_state ADD COLUMN header_session_id TEXT;
777
+ ALTER TABLE session_state ADD COLUMN header_name TEXT;
778
+ -- Cache warming state
779
+ ALTER TABLE session_state ADD COLUMN resolved_conversation_ttl TEXT NOT NULL DEFAULT '5m';
780
+ ALTER TABLE session_state ADD COLUMN warmup_state TEXT;
781
+ -- Gradient calibration state (survives restarts to avoid uncalibrated busts)
782
+ ALTER TABLE session_state ADD COLUMN dynamic_context_cap REAL NOT NULL DEFAULT 0;
783
+ ALTER TABLE session_state ADD COLUMN bust_rate_ema REAL NOT NULL DEFAULT -1;
784
+ ALTER TABLE session_state ADD COLUMN inter_bust_interval_ema REAL NOT NULL DEFAULT -1;
785
+ ALTER TABLE session_state ADD COLUMN last_layer INTEGER NOT NULL DEFAULT 0;
786
+ ALTER TABLE session_state ADD COLUMN last_known_input INTEGER NOT NULL DEFAULT 0;
787
+ ALTER TABLE session_state ADD COLUMN last_turn_at INTEGER NOT NULL DEFAULT 0;
788
+ ALTER TABLE session_state ADD COLUMN last_bust_at INTEGER NOT NULL DEFAULT 0;
789
+ `,
790
+ `
791
+ -- Version 25: Adaptive dedup threshold \u2014 store accept/reject feedback
792
+ -- on embedding-based duplicate pairs for per-project threshold calibration.
793
+ -- Titles stored instead of FK IDs because entries are deleted during dedup;
794
+ -- the similarity float is the actual calibration input.
795
+ CREATE TABLE IF NOT EXISTS dedup_feedback (
796
+ id INTEGER PRIMARY KEY AUTOINCREMENT,
797
+ project_id TEXT,
798
+ entry_a_title TEXT NOT NULL,
799
+ entry_b_title TEXT NOT NULL,
800
+ similarity REAL NOT NULL,
801
+ accepted INTEGER NOT NULL,
802
+ source TEXT NOT NULL DEFAULT 'manual',
803
+ created_at INTEGER NOT NULL
804
+ );
805
+ CREATE INDEX IF NOT EXISTS idx_dedup_feedback_project
806
+ ON dedup_feedback(project_id);
757
807
  `
758
808
  ];
759
809
  function dbPath() {
@@ -890,6 +940,11 @@ function close() {
890
940
  }
891
941
  }
892
942
  function ensureProject(path, name) {
943
+ if (!process.env.LORE_DB_PATH && /^\/test\//.test(path)) {
944
+ throw new Error(
945
+ `Refusing to create project with test path "${path}" in the production DB. Set LORE_DB_PATH to a temp path, or run tests via \`bun test\` from the repo root.`
946
+ );
947
+ }
893
948
  const existing = db().query("SELECT id, git_remote FROM projects WHERE path = ?").get(path);
894
949
  if (existing) {
895
950
  if (!existing.git_remote) {
@@ -1058,6 +1113,153 @@ function loadAllSessionCosts() {
1058
1113
  }
1059
1114
  return result;
1060
1115
  }
1116
+ function saveSessionTracking(sessionID, state) {
1117
+ const now = Date.now();
1118
+ db().query(
1119
+ "INSERT OR IGNORE INTO session_state (session_id, force_min_layer, updated_at) VALUES (?, 0, ?)"
1120
+ ).run(sessionID, now);
1121
+ const sets = ["updated_at = ?"];
1122
+ const vals = [now];
1123
+ if (state.lastCuratedAt !== void 0) {
1124
+ sets.push("last_curated_at = ?");
1125
+ vals.push(state.lastCuratedAt);
1126
+ }
1127
+ if (state.messageCount !== void 0) {
1128
+ sets.push("message_count = ?");
1129
+ vals.push(state.messageCount);
1130
+ }
1131
+ if (state.turnsSinceCuration !== void 0) {
1132
+ sets.push("turns_since_curation = ?");
1133
+ vals.push(state.turnsSinceCuration);
1134
+ }
1135
+ if (state.consecutiveTextOnlyTurns !== void 0) {
1136
+ sets.push("consecutive_text_only_turns = ?");
1137
+ vals.push(state.consecutiveTextOnlyTurns);
1138
+ }
1139
+ if (state.ltmCacheText !== void 0) {
1140
+ sets.push("ltm_cache_text = ?");
1141
+ vals.push(state.ltmCacheText);
1142
+ }
1143
+ if (state.ltmCacheTokens !== void 0) {
1144
+ sets.push("ltm_cache_tokens = ?");
1145
+ vals.push(state.ltmCacheTokens);
1146
+ }
1147
+ if (state.ltmPinText !== void 0) {
1148
+ sets.push("ltm_pin_text = ?");
1149
+ vals.push(state.ltmPinText);
1150
+ }
1151
+ if (state.ltmPinTokens !== void 0) {
1152
+ sets.push("ltm_pin_tokens = ?");
1153
+ vals.push(state.ltmPinTokens);
1154
+ }
1155
+ if (state.fingerprint !== void 0) {
1156
+ sets.push("fingerprint = ?");
1157
+ vals.push(state.fingerprint);
1158
+ }
1159
+ if (state.headerSessionId !== void 0) {
1160
+ sets.push("header_session_id = ?");
1161
+ vals.push(state.headerSessionId);
1162
+ }
1163
+ if (state.headerName !== void 0) {
1164
+ sets.push("header_name = ?");
1165
+ vals.push(state.headerName);
1166
+ }
1167
+ if (state.resolvedConversationTTL !== void 0) {
1168
+ sets.push("resolved_conversation_ttl = ?");
1169
+ vals.push(state.resolvedConversationTTL);
1170
+ }
1171
+ if (state.warmupState !== void 0) {
1172
+ sets.push("warmup_state = ?");
1173
+ vals.push(state.warmupState);
1174
+ }
1175
+ if (state.dynamicContextCap !== void 0) {
1176
+ sets.push("dynamic_context_cap = ?");
1177
+ vals.push(state.dynamicContextCap);
1178
+ }
1179
+ if (state.bustRateEMA !== void 0) {
1180
+ sets.push("bust_rate_ema = ?");
1181
+ vals.push(state.bustRateEMA);
1182
+ }
1183
+ if (state.interBustIntervalEMA !== void 0) {
1184
+ sets.push("inter_bust_interval_ema = ?");
1185
+ vals.push(state.interBustIntervalEMA);
1186
+ }
1187
+ if (state.lastLayer !== void 0) {
1188
+ sets.push("last_layer = ?");
1189
+ vals.push(state.lastLayer);
1190
+ }
1191
+ if (state.lastKnownInput !== void 0) {
1192
+ sets.push("last_known_input = ?");
1193
+ vals.push(state.lastKnownInput);
1194
+ }
1195
+ if (state.lastTurnAt !== void 0) {
1196
+ sets.push("last_turn_at = ?");
1197
+ vals.push(state.lastTurnAt);
1198
+ }
1199
+ if (state.lastBustAt !== void 0) {
1200
+ sets.push("last_bust_at = ?");
1201
+ vals.push(state.lastBustAt);
1202
+ }
1203
+ db().query(
1204
+ "UPDATE session_state SET " + sets.join(", ") + " WHERE session_id = ?"
1205
+ ).run(...vals, sessionID);
1206
+ }
1207
+ function loadSessionTracking(sessionID) {
1208
+ const row = db().query(
1209
+ `SELECT last_curated_at, message_count, turns_since_curation,
1210
+ consecutive_text_only_turns,
1211
+ ltm_cache_text, ltm_cache_tokens, ltm_pin_text, ltm_pin_tokens,
1212
+ fingerprint, header_session_id, header_name,
1213
+ resolved_conversation_ttl, warmup_state,
1214
+ dynamic_context_cap, bust_rate_ema, inter_bust_interval_ema,
1215
+ last_layer, last_known_input, last_turn_at, last_bust_at
1216
+ FROM session_state WHERE session_id = ?`
1217
+ ).get(sessionID);
1218
+ if (!row) return null;
1219
+ return {
1220
+ lastCuratedAt: row.last_curated_at,
1221
+ messageCount: row.message_count,
1222
+ turnsSinceCuration: row.turns_since_curation,
1223
+ consecutiveTextOnlyTurns: row.consecutive_text_only_turns,
1224
+ ltmCacheText: row.ltm_cache_text,
1225
+ ltmCacheTokens: row.ltm_cache_tokens,
1226
+ ltmPinText: row.ltm_pin_text,
1227
+ ltmPinTokens: row.ltm_pin_tokens,
1228
+ fingerprint: row.fingerprint,
1229
+ headerSessionId: row.header_session_id,
1230
+ headerName: row.header_name,
1231
+ resolvedConversationTTL: row.resolved_conversation_ttl,
1232
+ warmupState: row.warmup_state,
1233
+ dynamicContextCap: row.dynamic_context_cap,
1234
+ bustRateEMA: row.bust_rate_ema,
1235
+ interBustIntervalEMA: row.inter_bust_interval_ema,
1236
+ lastLayer: row.last_layer,
1237
+ lastKnownInput: row.last_known_input,
1238
+ lastTurnAt: row.last_turn_at,
1239
+ lastBustAt: row.last_bust_at
1240
+ };
1241
+ }
1242
+ function loadHeaderSessionIndex() {
1243
+ const rows = db().query(
1244
+ `SELECT session_id, header_session_id, header_name
1245
+ FROM session_state
1246
+ WHERE header_session_id IS NOT NULL AND header_name IS NOT NULL`
1247
+ ).all();
1248
+ return rows.map((row) => ({
1249
+ sessionId: row.session_id,
1250
+ headerSessionId: row.header_session_id,
1251
+ headerName: row.header_name
1252
+ }));
1253
+ }
1254
+ function getKV(key) {
1255
+ const row = db().query("SELECT value FROM kv_meta WHERE key = ?").get(key);
1256
+ return row?.value ?? null;
1257
+ }
1258
+ function setKV(key, value) {
1259
+ db().query(
1260
+ "INSERT INTO kv_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = ?"
1261
+ ).run(key, value, value);
1262
+ }
1061
1263
  function getMeta(key) {
1062
1264
  const row = db().query("SELECT value FROM metadata WHERE key = ?").get(key);
1063
1265
  return row?.value ?? null;
@@ -26770,7 +26972,16 @@ var LocalProvider = class {
26770
26972
  workerUrl = vendorWorkerUrl;
26771
26973
  }
26772
26974
  } else {
26773
- workerUrl = new URL(`./embedding-worker${import.meta.url.endsWith(".ts") ? ".ts" : ".js"}`, import.meta.url);
26975
+ const selfUrl = typeof import.meta.url === "string" ? import.meta.url : void 0;
26976
+ if (selfUrl) {
26977
+ workerUrl = new URL(
26978
+ `./embedding-worker${selfUrl.endsWith(".ts") ? ".ts" : ".js"}`,
26979
+ selfUrl
26980
+ );
26981
+ } else {
26982
+ const { pathToFileURL } = await import("node:url");
26983
+ workerUrl = new URL("./embedding-worker.cjs", pathToFileURL(__filename));
26984
+ }
26774
26985
  }
26775
26986
  const vendor = vendorModelInfo();
26776
26987
  const workerInitData = {
@@ -27045,8 +27256,14 @@ function fromBlob(blob) {
27045
27256
  const bytes = new Uint8Array(blob);
27046
27257
  return new Float32Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / 4);
27047
27258
  }
27048
- function vectorSearch(queryEmbedding, limit = 10) {
27049
- const rows = db().query("SELECT id, embedding FROM knowledge WHERE embedding IS NOT NULL AND confidence > 0.2").all();
27259
+ function vectorSearch(queryEmbedding, limit = 10, excludeCategories) {
27260
+ let sql = "SELECT id, embedding FROM knowledge WHERE embedding IS NOT NULL AND confidence > 0.2";
27261
+ const params = [];
27262
+ if (excludeCategories?.length) {
27263
+ sql += ` AND category NOT IN (${excludeCategories.map(() => "?").join(",")})`;
27264
+ params.push(...excludeCategories);
27265
+ }
27266
+ const rows = db().query(sql).all(...params);
27050
27267
  const scored = [];
27051
27268
  for (const row of rows) {
27052
27269
  const vec = fromBlob(row.embedding);
@@ -27435,6 +27652,12 @@ function count(projectPath, sessionID) {
27435
27652
  const params = sessionID ? [pid, sessionID] : [pid];
27436
27653
  return db().query(query).get(...params).count;
27437
27654
  }
27655
+ function hasMessages(projectPath, sessionID) {
27656
+ const pid = ensureProject(projectPath);
27657
+ return !!db().query(
27658
+ "SELECT 1 FROM temporal_messages WHERE project_id = ? AND session_id = ? LIMIT 1"
27659
+ ).get(pid, sessionID);
27660
+ }
27438
27661
  function undistilledCount(projectPath, sessionID) {
27439
27662
  const pid = ensureProject(projectPath);
27440
27663
  const query = sessionID ? "SELECT COUNT(*) as count FROM temporal_messages WHERE project_id = ? AND session_id = ? AND distilled = 0" : "SELECT COUNT(*) as count FROM temporal_messages WHERE project_id = ? AND distilled = 0";
@@ -27493,11 +27716,13 @@ function prune(input) {
27493
27716
  var ltm_exports = {};
27494
27717
  __export(ltm_exports, {
27495
27718
  all: () => all2,
27719
+ calibrateDedupThreshold: () => calibrateDedupThreshold,
27496
27720
  cascadeRefReplace: () => cascadeRefReplace,
27497
27721
  check: () => check2,
27498
27722
  cleanDeadRefs: () => cleanDeadRefs,
27499
27723
  create: () => create,
27500
27724
  crossProject: () => crossProject,
27725
+ dedupPairKey: () => dedupPairKey,
27501
27726
  deduplicate: () => deduplicate,
27502
27727
  deduplicateGlobal: () => deduplicateGlobal,
27503
27728
  extractRefs: () => extractRefs,
@@ -27505,9 +27730,17 @@ __export(ltm_exports, {
27505
27730
  forProject: () => forProject,
27506
27731
  forSession: () => forSession,
27507
27732
  get: () => get,
27733
+ getDedupFeedback: () => getDedupFeedback,
27734
+ getDedupFeedbackCount: () => getDedupFeedbackCount,
27735
+ loadCalibratedThreshold: () => loadCalibratedThreshold,
27736
+ pruneDedupFeedback: () => pruneDedupFeedback,
27508
27737
  pruneOversized: () => pruneOversized,
27738
+ recordAutoSignals: () => recordAutoSignals,
27739
+ recordDedupFeedback: () => recordDedupFeedback,
27740
+ recordDedupResultFeedback: () => recordDedupResultFeedback,
27509
27741
  remove: () => remove,
27510
27742
  resolveRef: () => resolveRef2,
27743
+ saveCalibratedThreshold: () => saveCalibratedThreshold,
27511
27744
  search: () => search3,
27512
27745
  searchScored: () => searchScored3,
27513
27746
  searchScoredOtherProjects: () => searchScoredOtherProjects,
@@ -28244,18 +28477,29 @@ function scoreEntriesFTS(sessionContext) {
28244
28477
  return /* @__PURE__ */ new Map();
28245
28478
  }
28246
28479
  }
28247
- function forSession(projectPath, sessionID, maxTokens) {
28480
+ async function forSession(projectPath, sessionID, maxTokens, options) {
28248
28481
  const pid = ensureProject(projectPath);
28482
+ const categoryFilter = options?.categories;
28483
+ const excludeFilter = options?.excludeCategories;
28484
+ let categoryClause = "";
28485
+ let categoryParams = [];
28486
+ if (categoryFilter?.length) {
28487
+ categoryClause = ` AND category IN (${categoryFilter.map(() => "?").join(",")})`;
28488
+ categoryParams = categoryFilter;
28489
+ } else if (excludeFilter?.length) {
28490
+ categoryClause = ` AND category NOT IN (${excludeFilter.map(() => "?").join(",")})`;
28491
+ categoryParams = excludeFilter;
28492
+ }
28249
28493
  const projectEntries = db().query(
28250
28494
  `SELECT ${KNOWLEDGE_COLS} FROM knowledge
28251
- WHERE project_id = ? AND cross_project = 0 AND confidence > 0.2
28495
+ WHERE project_id = ? AND cross_project = 0 AND confidence > 0.2${categoryClause}
28252
28496
  ORDER BY confidence DESC, updated_at DESC`
28253
- ).all(pid);
28497
+ ).all(pid, ...categoryParams);
28254
28498
  const crossEntries = db().query(
28255
28499
  `SELECT ${KNOWLEDGE_COLS} FROM knowledge
28256
- WHERE (project_id IS NULL OR cross_project = 1) AND confidence > 0.2
28500
+ WHERE (project_id IS NULL OR cross_project = 1) AND confidence > 0.2${categoryClause}
28257
28501
  ORDER BY confidence DESC, updated_at DESC`
28258
- ).all();
28502
+ ).all(...categoryParams);
28259
28503
  if (!crossEntries.length && !projectEntries.length) return [];
28260
28504
  let sessionContext = "";
28261
28505
  if (sessionID) {
@@ -28276,22 +28520,52 @@ function forSession(projectPath, sessionID, maxTokens) {
28276
28520
  sessionContext += recentMsgs.map((m) => m.content).join("\n");
28277
28521
  }
28278
28522
  }
28523
+ if (!sessionContext.trim() && options?.contextHint) {
28524
+ sessionContext = options.contextHint;
28525
+ }
28279
28526
  let scoredProject;
28280
28527
  let scoredCross;
28281
- if (sessionContext.trim().length > 20) {
28528
+ if (sessionContext.trim().length > 20 && isAvailable()) {
28529
+ let vectorScores;
28530
+ try {
28531
+ const [contextVec] = await embed([sessionContext], "query");
28532
+ const hits = vectorSearch(contextVec, 50, excludeFilter);
28533
+ vectorScores = new Map(hits.map((h3) => [h3.id, h3.similarity]));
28534
+ } catch (err) {
28535
+ warn("Vector scoring failed, falling back to FTS5:", err);
28536
+ vectorScores = /* @__PURE__ */ new Map();
28537
+ }
28538
+ if (vectorScores.size > 0) {
28539
+ const ftsScores = scoreEntriesFTS(sessionContext);
28540
+ const rawScored = projectEntries.map((entry) => {
28541
+ const vecScore = vectorScores.get(entry.id);
28542
+ const score = vecScore != null ? vecScore * entry.confidence : (ftsScores.get(entry.id) ?? 0) * entry.confidence;
28543
+ return { entry, score };
28544
+ });
28545
+ const matched = rawScored.filter((s) => s.score > 0);
28546
+ const matchedIds = new Set(matched.map((s) => s.entry.id));
28547
+ const safetyNet = projectEntries.filter((e) => !matchedIds.has(e.id)).slice(0, PROJECT_SAFETY_NET).map((e) => ({ entry: e, score: 1e-3 * e.confidence }));
28548
+ scoredProject = [...matched, ...safetyNet];
28549
+ scoredCross = crossEntries.filter((e) => vectorScores.has(e.id) || ftsScores.has(e.id)).map((e) => {
28550
+ const vecScore = vectorScores.get(e.id);
28551
+ const score = vecScore != null ? vecScore * e.confidence : (ftsScores.get(e.id) ?? 0) * e.confidence;
28552
+ return { entry: e, score };
28553
+ });
28554
+ } else {
28555
+ const ftsScores = scoreEntriesFTS(sessionContext);
28556
+ ({ scoredProject, scoredCross } = scoreFTS(
28557
+ projectEntries,
28558
+ crossEntries,
28559
+ ftsScores
28560
+ ));
28561
+ }
28562
+ } else if (sessionContext.trim().length > 20) {
28282
28563
  const ftsScores = scoreEntriesFTS(sessionContext);
28283
- const rawScored = projectEntries.map((entry) => ({
28284
- entry,
28285
- score: (ftsScores.get(entry.id) ?? 0) * entry.confidence
28286
- }));
28287
- const matched = rawScored.filter((s) => s.score > 0);
28288
- const matchedIds = new Set(matched.map((s) => s.entry.id));
28289
- const safetyNet = projectEntries.filter((e) => !matchedIds.has(e.id)).slice(0, PROJECT_SAFETY_NET).map((e) => ({ entry: e, score: 1e-3 * e.confidence }));
28290
- scoredProject = [...matched, ...safetyNet];
28291
- scoredCross = crossEntries.filter((e) => ftsScores.has(e.id)).map((e) => ({
28292
- entry: e,
28293
- score: (ftsScores.get(e.id) ?? 0) * e.confidence
28294
- }));
28564
+ ({ scoredProject, scoredCross } = scoreFTS(
28565
+ projectEntries,
28566
+ crossEntries,
28567
+ ftsScores
28568
+ ));
28295
28569
  } else {
28296
28570
  scoredProject = projectEntries.slice(0, NO_CONTEXT_FALLBACK_CAP).map((entry) => ({ entry, score: entry.confidence }));
28297
28571
  scoredCross = crossEntries.slice(0, NO_CONTEXT_FALLBACK_CAP).map((entry) => ({ entry, score: entry.confidence }));
@@ -28337,6 +28611,21 @@ function forSession(projectPath, sessionID, maxTokens) {
28337
28611
  }
28338
28612
  return result;
28339
28613
  }
28614
+ function scoreFTS(projectEntries, crossEntries, ftsScores) {
28615
+ const rawScored = projectEntries.map((entry) => ({
28616
+ entry,
28617
+ score: (ftsScores.get(entry.id) ?? 0) * entry.confidence
28618
+ }));
28619
+ const matched = rawScored.filter((s) => s.score > 0);
28620
+ const matchedIds = new Set(matched.map((s) => s.entry.id));
28621
+ const safetyNet = projectEntries.filter((e) => !matchedIds.has(e.id)).slice(0, PROJECT_SAFETY_NET).map((e) => ({ entry: e, score: 1e-3 * e.confidence }));
28622
+ const scoredProject = [...matched, ...safetyNet];
28623
+ const scoredCross = crossEntries.filter((e) => ftsScores.has(e.id)).map((e) => ({
28624
+ entry: e,
28625
+ score: (ftsScores.get(e.id) ?? 0) * e.confidence
28626
+ }));
28627
+ return { scoredProject, scoredCross };
28628
+ }
28340
28629
  function all2() {
28341
28630
  return db().query(
28342
28631
  `SELECT ${KNOWLEDGE_COLS} FROM knowledge WHERE confidence > 0.2 ORDER BY confidence DESC, updated_at DESC`
@@ -28580,8 +28869,11 @@ function check2(projectPath) {
28580
28869
  }
28581
28870
  return issues;
28582
28871
  }
28583
- function _dedup(entries, dryRun) {
28584
- if (entries.length < 2) return { clusters: [], totalRemoved: 0 };
28872
+ function dedupPairKey(idA, idB) {
28873
+ return idA < idB ? `${idA}:${idB}` : `${idB}:${idA}`;
28874
+ }
28875
+ function _dedup(entries, dryRun, embeddingThreshold = EMBEDDING_DEDUP_THRESHOLD) {
28876
+ if (entries.length < 2) return { clusters: [], totalRemoved: 0, pairSimilarities: /* @__PURE__ */ new Map(), entryTitles: /* @__PURE__ */ new Map() };
28585
28877
  const embeddingMap = /* @__PURE__ */ new Map();
28586
28878
  {
28587
28879
  const entryIds = entries.map((e) => e.id);
@@ -28596,6 +28888,7 @@ function _dedup(entries, dryRun) {
28596
28888
  }
28597
28889
  }
28598
28890
  const neighborMap = /* @__PURE__ */ new Map();
28891
+ const pairSimilarities = /* @__PURE__ */ new Map();
28599
28892
  for (const entry of entries) {
28600
28893
  const neighbors = [];
28601
28894
  const entryVec = embeddingMap.get(entry.id);
@@ -28609,7 +28902,13 @@ function _dedup(entries, dryRun) {
28609
28902
  const otherVec = embeddingMap.get(other.id);
28610
28903
  if (otherVec && entryVec.length === otherVec.length) {
28611
28904
  similarity = cosineSimilarity(entryVec, otherVec);
28612
- embeddingMatch = similarity >= EMBEDDING_DEDUP_THRESHOLD;
28905
+ embeddingMatch = similarity >= embeddingThreshold;
28906
+ }
28907
+ }
28908
+ if (similarity > 0) {
28909
+ const pk = dedupPairKey(entry.id, other.id);
28910
+ if (!pairSimilarities.has(pk)) {
28911
+ pairSimilarities.set(pk, similarity);
28613
28912
  }
28614
28913
  }
28615
28914
  if (titleMatch || embeddingMatch) {
@@ -28661,20 +28960,178 @@ function _dedup(entries, dryRun) {
28661
28960
  totalRemoved += merged.length;
28662
28961
  }
28663
28962
  result.sort((a, b) => b.merged.length - a.merged.length);
28664
- return { clusters: result, totalRemoved };
28963
+ const entryTitles = new Map(entries.map((e) => [e.id, e.title]));
28964
+ return { clusters: result, totalRemoved, pairSimilarities, entryTitles };
28665
28965
  }
28666
28966
  async function deduplicate(projectPath, opts) {
28967
+ const pid = ensureProject(projectPath);
28968
+ const threshold = loadCalibratedThreshold(pid) ?? EMBEDDING_DEDUP_THRESHOLD;
28667
28969
  const entries = forProject(projectPath, false);
28668
- return _dedup(entries, opts?.dryRun ?? true);
28970
+ return _dedup(entries, opts?.dryRun ?? true, threshold);
28669
28971
  }
28670
28972
  async function deduplicateGlobal(opts) {
28973
+ const threshold = loadCalibratedThreshold(null) ?? EMBEDDING_DEDUP_THRESHOLD;
28671
28974
  const entries = db().query(
28672
28975
  `SELECT ${KNOWLEDGE_COLS} FROM knowledge
28673
28976
  WHERE project_id IS NULL
28674
28977
  AND confidence > 0.2
28675
28978
  ORDER BY confidence DESC, updated_at DESC`
28676
28979
  ).all();
28677
- return _dedup(entries, opts?.dryRun ?? true);
28980
+ return _dedup(entries, opts?.dryRun ?? true, threshold);
28981
+ }
28982
+ var MIN_CALIBRATION_SAMPLES = 20;
28983
+ var DEFAULT_EMBEDDING_DEDUP_THRESHOLD = EMBEDDING_DEDUP_THRESHOLD;
28984
+ var AUTO_SIGNAL_MIN_SIMILARITY = 0.8;
28985
+ var AUTO_SIGNAL_MAX_PAIRS = 50;
28986
+ function recordDedupFeedback(input) {
28987
+ db().query(
28988
+ `INSERT INTO dedup_feedback
28989
+ (project_id, entry_a_title, entry_b_title, similarity, accepted, source, created_at)
28990
+ VALUES (?, ?, ?, ?, ?, ?, ?)`
28991
+ ).run(
28992
+ input.projectId,
28993
+ input.entryATitle,
28994
+ input.entryBTitle,
28995
+ input.similarity,
28996
+ input.accepted ? 1 : 0,
28997
+ input.source,
28998
+ Date.now()
28999
+ );
29000
+ }
29001
+ function recordDedupResultFeedback(projectId2, result, accepted, source) {
29002
+ for (const cluster of result.clusters) {
29003
+ for (const merged of cluster.merged) {
29004
+ const pk = dedupPairKey(cluster.surviving.id, merged.id);
29005
+ const similarity = result.pairSimilarities.get(pk);
29006
+ if (similarity != null && similarity > 0) {
29007
+ recordDedupFeedback({
29008
+ projectId: projectId2,
29009
+ entryATitle: cluster.surviving.title,
29010
+ entryBTitle: merged.title,
29011
+ similarity,
29012
+ accepted,
29013
+ source
29014
+ });
29015
+ }
29016
+ }
29017
+ }
29018
+ }
29019
+ function recordAutoSignals(projectId2, result) {
29020
+ const mergedPairs = /* @__PURE__ */ new Set();
29021
+ for (const cluster of result.clusters) {
29022
+ for (const merged of cluster.merged) {
29023
+ mergedPairs.add(dedupPairKey(cluster.surviving.id, merged.id));
29024
+ }
29025
+ }
29026
+ const titleMap = new Map(result.entryTitles);
29027
+ for (const cluster of result.clusters) {
29028
+ if (!titleMap.has(cluster.surviving.id)) {
29029
+ titleMap.set(cluster.surviving.id, cluster.surviving.title);
29030
+ }
29031
+ for (const m of cluster.merged) {
29032
+ if (!titleMap.has(m.id)) titleMap.set(m.id, m.title);
29033
+ }
29034
+ }
29035
+ const signals = [];
29036
+ for (const [pk, sim] of result.pairSimilarities) {
29037
+ if (sim < AUTO_SIGNAL_MIN_SIMILARITY) continue;
29038
+ if (mergedPairs.has(pk)) continue;
29039
+ const [idA, idB] = pk.split(":");
29040
+ const titleA = titleMap.get(idA);
29041
+ const titleB = titleMap.get(idB);
29042
+ if (!titleA || !titleB) continue;
29043
+ signals.push({ entryATitle: titleA, entryBTitle: titleB, similarity: sim });
29044
+ }
29045
+ const currentThreshold = loadCalibratedThreshold(projectId2) ?? DEFAULT_EMBEDDING_DEDUP_THRESHOLD;
29046
+ signals.sort((a, b) => Math.abs(a.similarity - currentThreshold) - Math.abs(b.similarity - currentThreshold));
29047
+ const capped = signals.slice(0, AUTO_SIGNAL_MAX_PAIRS);
29048
+ pruneDedupFeedback(projectId2);
29049
+ for (const s of capped) {
29050
+ recordDedupFeedback({
29051
+ projectId: projectId2,
29052
+ entryATitle: s.entryATitle,
29053
+ entryBTitle: s.entryBTitle,
29054
+ similarity: s.similarity,
29055
+ accepted: false,
29056
+ source: "auto_dedup"
29057
+ });
29058
+ }
29059
+ }
29060
+ function getDedupFeedback(projectId2) {
29061
+ const rows = projectId2 !== null ? db().query(
29062
+ "SELECT similarity, accepted, source FROM dedup_feedback WHERE project_id = ? ORDER BY similarity"
29063
+ ).all(projectId2) : db().query(
29064
+ "SELECT similarity, accepted, source FROM dedup_feedback WHERE project_id IS NULL ORDER BY similarity"
29065
+ ).all();
29066
+ return rows.map((r) => ({ similarity: r.similarity, accepted: r.accepted === 1, source: r.source }));
29067
+ }
29068
+ function getDedupFeedbackCount(projectId2) {
29069
+ const row = projectId2 !== null ? db().query("SELECT COUNT(*) as cnt FROM dedup_feedback WHERE project_id = ?").get(projectId2) : db().query("SELECT COUNT(*) as cnt FROM dedup_feedback WHERE project_id IS NULL").get();
29070
+ return row?.cnt ?? 0;
29071
+ }
29072
+ var MAX_FEEDBACK_ROWS_PER_PROJECT = 500;
29073
+ function pruneDedupFeedback(projectId2) {
29074
+ const count3 = getDedupFeedbackCount(projectId2);
29075
+ if (count3 <= MAX_FEEDBACK_ROWS_PER_PROJECT) return;
29076
+ const excess = count3 - MAX_FEEDBACK_ROWS_PER_PROJECT;
29077
+ if (projectId2 !== null) {
29078
+ db().query(
29079
+ `DELETE FROM dedup_feedback WHERE id IN (
29080
+ SELECT id FROM dedup_feedback WHERE project_id = ?
29081
+ ORDER BY created_at ASC LIMIT ?
29082
+ )`
29083
+ ).run(projectId2, excess);
29084
+ } else {
29085
+ db().query(
29086
+ `DELETE FROM dedup_feedback WHERE id IN (
29087
+ SELECT id FROM dedup_feedback WHERE project_id IS NULL
29088
+ ORDER BY created_at ASC LIMIT ?
29089
+ )`
29090
+ ).run(excess);
29091
+ }
29092
+ }
29093
+ function calibrateDedupThreshold(projectId2) {
29094
+ const feedback = getDedupFeedback(projectId2);
29095
+ if (feedback.length < MIN_CALIBRATION_SAMPLES) return null;
29096
+ const accepted = feedback.filter((f) => f.accepted);
29097
+ const rejected = feedback.filter((f) => !f.accepted);
29098
+ if (rejected.length === 0) {
29099
+ const minAccepted = Math.min(...accepted.map((f) => f.similarity));
29100
+ return Math.max(0.85, minAccepted - 5e-3);
29101
+ }
29102
+ if (accepted.length === 0) {
29103
+ warn("dedup calibration: all feedback is reject \u2014 keeping default threshold");
29104
+ return null;
29105
+ }
29106
+ const allSims = [...new Set(feedback.map((f) => f.similarity))].sort((a, b) => a - b);
29107
+ let bestThreshold = DEFAULT_EMBEDDING_DEDUP_THRESHOLD;
29108
+ let bestAccuracy = -1;
29109
+ for (let i = 0; i < allSims.length - 1; i++) {
29110
+ const candidate = (allSims[i] + allSims[i + 1]) / 2;
29111
+ const correctAccepted = accepted.filter((f) => f.similarity >= candidate).length;
29112
+ const correctRejected = rejected.filter((f) => f.similarity < candidate).length;
29113
+ const accuracy = (correctAccepted + correctRejected) / feedback.length;
29114
+ if (accuracy > bestAccuracy || accuracy === bestAccuracy && candidate > bestThreshold) {
29115
+ bestAccuracy = accuracy;
29116
+ bestThreshold = candidate;
29117
+ }
29118
+ }
29119
+ return Math.max(0.85, Math.min(0.98, bestThreshold));
29120
+ }
29121
+ function saveCalibratedThreshold(projectId2, threshold, sampleSize) {
29122
+ const key = `dedup_threshold:${projectId2 ?? "global"}`;
29123
+ setKV(key, JSON.stringify({ threshold, sampleSize, calibratedAt: Date.now() }));
29124
+ }
29125
+ function loadCalibratedThreshold(projectId2) {
29126
+ const key = `dedup_threshold:${projectId2 ?? "global"}`;
29127
+ const raw = getKV(key);
29128
+ if (!raw) return null;
29129
+ try {
29130
+ const parsed = JSON.parse(raw);
29131
+ return typeof parsed.threshold === "number" ? parsed.threshold : null;
29132
+ } catch {
29133
+ return null;
29134
+ }
28678
29135
  }
28679
29136
 
28680
29137
  // src/data.ts
@@ -29523,6 +29980,16 @@ function getSessionState(sessionID) {
29523
29980
  if (!state) {
29524
29981
  state = makeSessionState();
29525
29982
  state.forceMinLayer = loadForceMinLayer(sessionID);
29983
+ const persisted = loadSessionTracking(sessionID);
29984
+ if (persisted && persisted.lastTurnAt > 0) {
29985
+ state.dynamicContextCap = persisted.dynamicContextCap;
29986
+ state.bustRateEMA = persisted.bustRateEMA;
29987
+ state.interBustIntervalEMA = persisted.interBustIntervalEMA;
29988
+ state.lastLayer = persisted.lastLayer;
29989
+ state.lastKnownInput = persisted.lastKnownInput;
29990
+ state.lastTurnAt = persisted.lastTurnAt;
29991
+ state.lastBustAt = persisted.lastBustAt;
29992
+ }
29526
29993
  sessionStates.set(sessionID, state);
29527
29994
  }
29528
29995
  return state;
@@ -29628,6 +30095,19 @@ function inspectSessionState(sessionID) {
29628
30095
  function setLastTurnAtForTest(sessionID, ms) {
29629
30096
  getSessionState(sessionID).lastTurnAt = ms;
29630
30097
  }
30098
+ function saveGradientState(sessionID) {
30099
+ const state = sessionStates.get(sessionID);
30100
+ if (!state) return;
30101
+ saveSessionTracking(sessionID, {
30102
+ dynamicContextCap: state.dynamicContextCap,
30103
+ bustRateEMA: state.bustRateEMA,
30104
+ interBustIntervalEMA: state.interBustIntervalEMA,
30105
+ lastLayer: state.lastLayer,
30106
+ lastKnownInput: state.lastKnownInput,
30107
+ lastTurnAt: state.lastTurnAt,
30108
+ lastBustAt: state.lastBustAt
30109
+ });
30110
+ }
29631
30111
  function loadDistillations(projectPath, sessionID) {
29632
30112
  const pid = ensureProject(projectPath);
29633
30113
  const query = sessionID ? "SELECT id, observations, generation, token_count, created_at, session_id FROM distillations WHERE project_id = ? AND session_id = ? AND archived = 0 ORDER BY created_at ASC" : "SELECT id, observations, generation, token_count, created_at, session_id FROM distillations WHERE project_id = ? AND archived = 0 ORDER BY created_at ASC";
@@ -29912,6 +30392,26 @@ function buildPrefixMessages(formatted) {
29912
30392
  }
29913
30393
  ];
29914
30394
  }
30395
+ var DECISION_RE = /\b(?:decision|decided|chose|chosen|agreed)\b/i;
30396
+ var GOTCHA_RE = /\b(?:gotcha|(?:critical|known|subtle)\s+bug|broken|crash(?:ed|es)?|regression)\b/i;
30397
+ var ARCH_RE = /\b(?:architecture|design.(?:decision|pattern)|system.design)\b/i;
30398
+ function importanceBonus(d) {
30399
+ let bonus = 0;
30400
+ if (DECISION_RE.test(d.observations)) bonus += 0.3;
30401
+ if (GOTCHA_RE.test(d.observations)) bonus += 0.2;
30402
+ if (ARCH_RE.test(d.observations)) bonus += 0.1;
30403
+ if (d.generation >= 1) bonus += 0.2;
30404
+ return Math.min(bonus, 1);
30405
+ }
30406
+ function selectDistillations(all3, limit) {
30407
+ if (all3.length <= limit) return all3;
30408
+ const maxIdx = all3.length - 1;
30409
+ const scored = all3.map((d, i) => ({
30410
+ d,
30411
+ score: (maxIdx > 0 ? i / maxIdx : 1) * 0.7 + importanceBonus(d) * 0.3
30412
+ }));
30413
+ return scored.sort((a, b) => b.score - a.score).slice(0, limit).map((s) => s.d).sort((a, b) => a.created_at - b.created_at);
30414
+ }
29915
30415
  function distilledPrefix(distillations) {
29916
30416
  if (!distillations.length) return [];
29917
30417
  const formatted = formatDistillations(distillations);
@@ -30029,6 +30529,11 @@ function tryFitStable(input) {
30029
30529
  }
30030
30530
  return result;
30031
30531
  }
30532
+ var COMPRESSION_STAGES = [
30533
+ { strip: "none", rawFrac: null, distFrac: null, distLimit: Infinity, protectedTurns: 0, useStableWindow: true },
30534
+ { strip: "old-tools", rawFrac: 0.5, distFrac: null, distLimit: Infinity, protectedTurns: 2, useStableWindow: false },
30535
+ { strip: "all-tools", rawFrac: 0.55, distFrac: 0.15, distLimit: 5, protectedTurns: 0, useStableWindow: false }
30536
+ ];
30032
30537
  var urgentDistillationMap = /* @__PURE__ */ new Map();
30033
30538
  function needsUrgentDistillation(sessionID) {
30034
30539
  const v = urgentDistillationMap.get(sessionID) ?? false;
@@ -30060,7 +30565,7 @@ function transformInner(input) {
30060
30565
  if (calibrated) return true;
30061
30566
  return result.totalTokens * UNCALIBRATED_SAFETY <= maxInput;
30062
30567
  }
30063
- if (calibrated && sessState.lastLayer >= 1 && input.messages.length >= sessState.lastKnownMessageCount) {
30568
+ if (calibrated && sessState.lastLayer >= 1 && sessState.lastLayer <= 3 && input.messages.length >= sessState.lastKnownMessageCount) {
30064
30569
  effectiveMinLayer = Math.max(effectiveMinLayer, sessState.lastLayer);
30065
30570
  }
30066
30571
  const postIdleCompact = sessState.postIdleCompact;
@@ -30098,7 +30603,8 @@ function transformInner(input) {
30098
30603
  totalTokens: Math.max(0, messageTokens),
30099
30604
  usable,
30100
30605
  distilledBudget,
30101
- rawBudget
30606
+ rawBudget,
30607
+ refreshLtm: false
30102
30608
  };
30103
30609
  }
30104
30610
  const turnStart = currentTurnStart(input.messages);
@@ -30108,67 +30614,52 @@ function transformInner(input) {
30108
30614
  const msgs = distilledPrefix(distillations);
30109
30615
  return { messages: msgs, tokens: msgs.reduce((sum, m) => sum + estimateMessage(m), 0) };
30110
30616
  })();
30111
- if (effectiveMinLayer <= 1) {
30112
- const layer1 = sid ? tryFitStable({
30113
- messages: dedupMessages,
30114
- prefix: cached2.messages,
30115
- prefixTokens: cached2.tokens,
30116
- distilledBudget,
30117
- rawBudget,
30118
- sessionID: sid,
30119
- sessState
30120
- }) : tryFit({
30121
- messages: dedupMessages,
30122
- prefix: cached2.messages,
30123
- prefixTokens: cached2.tokens,
30124
- distilledBudget,
30125
- rawBudget,
30126
- strip: "none"
30127
- });
30128
- if (fitsWithSafetyMargin(layer1)) {
30129
- if (cached2.tokens === 0 && sid) {
30617
+ for (let s = 0; s < COMPRESSION_STAGES.length; s++) {
30618
+ const stageLayer = s + 1;
30619
+ if (effectiveMinLayer > stageLayer) continue;
30620
+ const stage = COMPRESSION_STAGES[s];
30621
+ const stageRawBudget = stage.rawFrac !== null ? Math.floor(usable * stage.rawFrac) : rawBudget;
30622
+ const stageDistBudget = stage.distFrac !== null ? Math.floor(usable * stage.distFrac) : distilledBudget;
30623
+ let stagePrefix = cached2.messages;
30624
+ let stagePrefixTokens = cached2.tokens;
30625
+ if (stage.distLimit !== Infinity && distillations.length > stage.distLimit) {
30626
+ const trimmed = selectDistillations(distillations, stage.distLimit);
30627
+ stagePrefix = distilledPrefix(trimmed);
30628
+ stagePrefixTokens = stagePrefix.reduce((sum, m) => sum + estimateMessage(m), 0);
30629
+ }
30630
+ let result;
30631
+ if (stage.useStableWindow && sid) {
30632
+ result = tryFitStable({
30633
+ messages: dedupMessages,
30634
+ prefix: stagePrefix,
30635
+ prefixTokens: stagePrefixTokens,
30636
+ distilledBudget: stageDistBudget,
30637
+ rawBudget: stageRawBudget,
30638
+ sessionID: sid,
30639
+ sessState
30640
+ });
30641
+ } else {
30642
+ sessState.rawWindowCache = null;
30643
+ result = tryFit({
30644
+ messages: dedupMessages,
30645
+ prefix: stagePrefix,
30646
+ prefixTokens: stagePrefixTokens,
30647
+ distilledBudget: stageDistBudget,
30648
+ rawBudget: stageRawBudget,
30649
+ strip: stage.strip,
30650
+ protectedTurns: stage.protectedTurns
30651
+ });
30652
+ }
30653
+ if (fitsWithSafetyMargin(result)) {
30654
+ if (sid && (s > 0 || cached2.tokens === 0)) {
30130
30655
  urgentDistillationMap.set(sid, true);
30131
30656
  }
30132
- return { ...layer1, layer: 1, usable, distilledBudget, rawBudget };
30657
+ return { ...result, layer: stageLayer, usable, distilledBudget, rawBudget, refreshLtm: false };
30133
30658
  }
30134
30659
  }
30135
30660
  sessState.rawWindowCache = null;
30136
- if (effectiveMinLayer <= 2) {
30137
- const layer2 = tryFit({
30138
- messages: dedupMessages,
30139
- prefix: cached2.messages,
30140
- prefixTokens: cached2.tokens,
30141
- distilledBudget,
30142
- rawBudget: Math.floor(usable * 0.5),
30143
- // give raw more room
30144
- strip: "old-tools",
30145
- protectedTurns: 2
30146
- });
30147
- if (fitsWithSafetyMargin(layer2)) {
30148
- if (sid) urgentDistillationMap.set(sid, true);
30149
- return { ...layer2, layer: 2, usable, distilledBudget, rawBudget };
30150
- }
30151
- }
30152
- const trimmedDistillations = distillations.slice(-5);
30153
- const trimmedPrefix = distilledPrefix(trimmedDistillations);
30154
- const trimmedPrefixTokens = trimmedPrefix.reduce(
30155
- (sum, m) => sum + estimateMessage(m),
30156
- 0
30157
- );
30158
- const layer3 = tryFit({
30159
- messages: dedupMessages,
30160
- prefix: trimmedPrefix,
30161
- prefixTokens: trimmedPrefixTokens,
30162
- distilledBudget: Math.floor(usable * 0.15),
30163
- rawBudget: Math.floor(usable * 0.55),
30164
- strip: "all-tools"
30165
- });
30166
- if (fitsWithSafetyMargin(layer3)) {
30167
- if (sid) urgentDistillationMap.set(sid, true);
30168
- return { ...layer3, layer: 3, usable, distilledBudget, rawBudget };
30169
- }
30170
30661
  if (sid) urgentDistillationMap.set(sid, true);
30171
- const nuclearDistillations = distillations.slice(-2);
30662
+ const nuclearDistillations = selectDistillations(distillations, 2);
30172
30663
  const nuclearPrefix = distilledPrefix(nuclearDistillations);
30173
30664
  const nuclearPrefixTokens = nuclearPrefix.reduce(
30174
30665
  (sum, m) => sum + estimateMessage(m),
@@ -30207,7 +30698,8 @@ function transformInner(input) {
30207
30698
  totalTokens: nuclearPrefixTokens + nuclearRawTokens,
30208
30699
  usable,
30209
30700
  distilledBudget,
30210
- rawBudget
30701
+ rawBudget,
30702
+ refreshLtm: true
30211
30703
  };
30212
30704
  }
30213
30705
  function transform2(input) {
@@ -30314,6 +30806,185 @@ function isWorkerSession(sessionID) {
30314
30806
  return workerSessionIDs.has(sessionID);
30315
30807
  }
30316
30808
 
30809
+ // ../../node_modules/.bun/yocto-queue@1.2.2/node_modules/yocto-queue/index.js
30810
+ var Node = class {
30811
+ value;
30812
+ next;
30813
+ constructor(value) {
30814
+ this.value = value;
30815
+ }
30816
+ };
30817
+ var Queue = class {
30818
+ #head;
30819
+ #tail;
30820
+ #size;
30821
+ constructor() {
30822
+ this.clear();
30823
+ }
30824
+ enqueue(value) {
30825
+ const node2 = new Node(value);
30826
+ if (this.#head) {
30827
+ this.#tail.next = node2;
30828
+ this.#tail = node2;
30829
+ } else {
30830
+ this.#head = node2;
30831
+ this.#tail = node2;
30832
+ }
30833
+ this.#size++;
30834
+ }
30835
+ dequeue() {
30836
+ const current2 = this.#head;
30837
+ if (!current2) {
30838
+ return;
30839
+ }
30840
+ this.#head = this.#head.next;
30841
+ this.#size--;
30842
+ if (!this.#head) {
30843
+ this.#tail = void 0;
30844
+ }
30845
+ return current2.value;
30846
+ }
30847
+ peek() {
30848
+ if (!this.#head) {
30849
+ return;
30850
+ }
30851
+ return this.#head.value;
30852
+ }
30853
+ clear() {
30854
+ this.#head = void 0;
30855
+ this.#tail = void 0;
30856
+ this.#size = 0;
30857
+ }
30858
+ get size() {
30859
+ return this.#size;
30860
+ }
30861
+ *[Symbol.iterator]() {
30862
+ let current2 = this.#head;
30863
+ while (current2) {
30864
+ yield current2.value;
30865
+ current2 = current2.next;
30866
+ }
30867
+ }
30868
+ *drain() {
30869
+ while (this.#head) {
30870
+ yield this.dequeue();
30871
+ }
30872
+ }
30873
+ };
30874
+
30875
+ // ../../node_modules/.bun/p-limit@7.3.0/node_modules/p-limit/index.js
30876
+ function pLimit(concurrency) {
30877
+ let rejectOnClear = false;
30878
+ if (typeof concurrency === "object") {
30879
+ ({ concurrency, rejectOnClear = false } = concurrency);
30880
+ }
30881
+ validateConcurrency(concurrency);
30882
+ if (typeof rejectOnClear !== "boolean") {
30883
+ throw new TypeError("Expected `rejectOnClear` to be a boolean");
30884
+ }
30885
+ const queue = new Queue();
30886
+ let activeCount = 0;
30887
+ const resumeNext = () => {
30888
+ if (activeCount < concurrency && queue.size > 0) {
30889
+ activeCount++;
30890
+ queue.dequeue().run();
30891
+ }
30892
+ };
30893
+ const next = () => {
30894
+ activeCount--;
30895
+ resumeNext();
30896
+ };
30897
+ const run3 = async (function_, resolve, arguments_) => {
30898
+ const result = (async () => function_(...arguments_))();
30899
+ resolve(result);
30900
+ try {
30901
+ await result;
30902
+ } catch {
30903
+ }
30904
+ next();
30905
+ };
30906
+ const enqueue = (function_, resolve, reject, arguments_) => {
30907
+ const queueItem = { reject };
30908
+ new Promise((internalResolve) => {
30909
+ queueItem.run = internalResolve;
30910
+ queue.enqueue(queueItem);
30911
+ }).then(run3.bind(void 0, function_, resolve, arguments_));
30912
+ if (activeCount < concurrency) {
30913
+ resumeNext();
30914
+ }
30915
+ };
30916
+ const generator = (function_, ...arguments_) => new Promise((resolve, reject) => {
30917
+ enqueue(function_, resolve, reject, arguments_);
30918
+ });
30919
+ Object.defineProperties(generator, {
30920
+ activeCount: {
30921
+ get: () => activeCount
30922
+ },
30923
+ pendingCount: {
30924
+ get: () => queue.size
30925
+ },
30926
+ clearQueue: {
30927
+ value() {
30928
+ if (!rejectOnClear) {
30929
+ queue.clear();
30930
+ return;
30931
+ }
30932
+ const abortError = AbortSignal.abort().reason;
30933
+ while (queue.size > 0) {
30934
+ queue.dequeue().reject(abortError);
30935
+ }
30936
+ }
30937
+ },
30938
+ concurrency: {
30939
+ get: () => concurrency,
30940
+ set(newConcurrency) {
30941
+ validateConcurrency(newConcurrency);
30942
+ concurrency = newConcurrency;
30943
+ queueMicrotask(() => {
30944
+ while (activeCount < concurrency && queue.size > 0) {
30945
+ resumeNext();
30946
+ }
30947
+ });
30948
+ }
30949
+ },
30950
+ map: {
30951
+ async value(iterable, function_) {
30952
+ const promises = Array.from(iterable, (value, index2) => this(function_, value, index2));
30953
+ return Promise.all(promises);
30954
+ }
30955
+ }
30956
+ });
30957
+ return generator;
30958
+ }
30959
+ function validateConcurrency(concurrency) {
30960
+ if (!((Number.isInteger(concurrency) || concurrency === Number.POSITIVE_INFINITY) && concurrency > 0)) {
30961
+ throw new TypeError("Expected `concurrency` to be a number from 1 and up");
30962
+ }
30963
+ }
30964
+
30965
+ // src/session-limiter.ts
30966
+ function createLimiterPool() {
30967
+ const limiters = /* @__PURE__ */ new Map();
30968
+ function get2(key) {
30969
+ let limiter = limiters.get(key);
30970
+ if (!limiter) {
30971
+ limiter = pLimit(1);
30972
+ limiters.set(key, limiter);
30973
+ }
30974
+ return limiter;
30975
+ }
30976
+ function isBusy(key) {
30977
+ const limiter = limiters.get(key);
30978
+ return limiter ? limiter.activeCount + limiter.pendingCount > 0 : false;
30979
+ }
30980
+ function clear() {
30981
+ limiters.clear();
30982
+ }
30983
+ return { get: get2, isBusy, clear };
30984
+ }
30985
+ var distillLimiter = createLimiterPool();
30986
+ var curatorLimiter = createLimiterPool();
30987
+
30317
30988
  // src/distillation.ts
30318
30989
  function compressionRatio(distilledTokens, sourceTokens) {
30319
30990
  if (sourceTokens <= 0) return 0;
@@ -30558,6 +31229,9 @@ function resetOrphans(projectPath, sessionID) {
30558
31229
  return orphans.length;
30559
31230
  }
30560
31231
  async function run(input) {
31232
+ return distillLimiter.get(input.sessionID)(() => runInner(input));
31233
+ }
31234
+ async function runInner(input) {
30561
31235
  const orphans = resetOrphans(input.projectPath, input.sessionID);
30562
31236
  if (orphans > 0) {
30563
31237
  info(
@@ -30601,7 +31275,7 @@ async function run(input) {
30601
31275
  }
30602
31276
  }
30603
31277
  if (!input.skipMeta && gen0Count(input.projectPath, input.sessionID) >= cfg.distillation.metaThreshold) {
30604
- await metaDistill({
31278
+ await metaDistillInner({
30605
31279
  llm: input.llm,
30606
31280
  projectPath: input.projectPath,
30607
31281
  sessionID: input.sessionID,
@@ -30651,17 +31325,25 @@ async function distillSegment(input) {
30651
31325
  );
30652
31326
  return null;
30653
31327
  }
30654
- const distillId = storeDistillation({
30655
- projectPath: input.projectPath,
30656
- sessionID: input.sessionID,
30657
- observations: result.observations,
30658
- sourceIDs: input.messages.map((m) => m.id),
30659
- generation: 0,
30660
- rCompression: rComp,
30661
- cNorm,
30662
- callType: input.callType
30663
- });
30664
- markDistilled(input.messages.map((m) => m.id));
31328
+ let distillId;
31329
+ db().exec("BEGIN IMMEDIATE");
31330
+ try {
31331
+ distillId = storeDistillation({
31332
+ projectPath: input.projectPath,
31333
+ sessionID: input.sessionID,
31334
+ observations: result.observations,
31335
+ sourceIDs: input.messages.map((m) => m.id),
31336
+ generation: 0,
31337
+ rCompression: rComp,
31338
+ cNorm,
31339
+ callType: input.callType
31340
+ });
31341
+ markDistilled(input.messages.map((m) => m.id));
31342
+ db().exec("COMMIT");
31343
+ } catch (e) {
31344
+ db().exec("ROLLBACK");
31345
+ throw e;
31346
+ }
30665
31347
  info(
30666
31348
  `distill segment: ${input.messages.length} msgs, ${sourceTokens}\u2192${distilledTokens} tokens, R=${rComp.toFixed(2)}, C_norm=${cNorm.toFixed(3)}`
30667
31349
  );
@@ -30695,6 +31377,9 @@ async function distillSegment(input) {
30695
31377
  return result;
30696
31378
  }
30697
31379
  async function metaDistill(input) {
31380
+ return distillLimiter.get(input.sessionID)(() => metaDistillInner(input));
31381
+ }
31382
+ async function metaDistillInner(input) {
30698
31383
  const existing = loadGen0(input.projectPath, input.sessionID);
30699
31384
  const priorMeta = latestMeta(input.projectPath, input.sessionID);
30700
31385
  if (priorMeta) {
@@ -30994,11 +31679,27 @@ function applyOps(ops, input) {
30994
31679
  return { created, updated, deleted };
30995
31680
  }
30996
31681
  var lastCuratedAt = /* @__PURE__ */ new Map();
31682
+ function getLastCuratedAt(sessionID) {
31683
+ const cached2 = lastCuratedAt.get(sessionID);
31684
+ if (cached2 !== void 0) return cached2;
31685
+ const persisted = loadSessionTracking(sessionID);
31686
+ const ts = persisted?.lastCuratedAt ?? 0;
31687
+ lastCuratedAt.set(sessionID, ts);
31688
+ return ts;
31689
+ }
30997
31690
  async function run2(input) {
30998
31691
  const cfg = config2();
30999
31692
  if (!cfg.curator.enabled) return { created: 0, updated: 0, deleted: 0 };
31693
+ if (curatorLimiter.isBusy(input.sessionID)) {
31694
+ info(`curation skipped: already running for session ${input.sessionID.slice(0, 16)}`);
31695
+ return { created: 0, updated: 0, deleted: 0 };
31696
+ }
31697
+ return curatorLimiter.get(input.sessionID)(() => runInner2(input));
31698
+ }
31699
+ async function runInner2(input) {
31700
+ const cfg = config2();
31000
31701
  const all3 = bySession(input.projectPath, input.sessionID);
31001
- const sessionCuratedAt = lastCuratedAt.get(input.sessionID) ?? 0;
31702
+ const sessionCuratedAt = getLastCuratedAt(input.sessionID);
31002
31703
  const recent = all3.filter((m) => m.created_at > sessionCuratedAt);
31003
31704
  if (recent.length < 3) return { created: 0, updated: 0, deleted: 0 };
31004
31705
  const text4 = recent.map((m) => `[${m.role}] ${m.content}`).join("\n\n");
@@ -31042,11 +31743,22 @@ async function run2(input) {
31042
31743
  info(`post-curation dedup: merged ${dupes.totalRemoved} duplicate entries`);
31043
31744
  result.deleted += dupes.totalRemoved;
31044
31745
  }
31746
+ if (dupes.pairSimilarities.size > 0) {
31747
+ const pid = ensureProject(input.projectPath);
31748
+ recordAutoSignals(pid, dupes);
31749
+ const newThreshold = calibrateDedupThreshold(pid);
31750
+ if (newThreshold !== null) {
31751
+ const count3 = getDedupFeedbackCount(pid);
31752
+ saveCalibratedThreshold(pid, newThreshold, count3);
31753
+ }
31754
+ }
31045
31755
  } catch (err) {
31046
31756
  warn("post-curation dedup failed (non-fatal):", err);
31047
31757
  }
31048
31758
  }
31049
- lastCuratedAt.set(input.sessionID, Date.now());
31759
+ const now = Date.now();
31760
+ lastCuratedAt.set(input.sessionID, now);
31761
+ saveSessionTracking(input.sessionID, { lastCuratedAt: now });
31050
31762
  return result;
31051
31763
  }
31052
31764
  function resetCurationTracker(sessionID) {
@@ -33169,6 +33881,7 @@ export {
33169
33881
  ftsQueryRelaxed,
33170
33882
  getGitRemote,
33171
33883
  getInstanceId,
33884
+ getKV,
33172
33885
  getLastImportAt,
33173
33886
  getLastTransformEstimate,
33174
33887
  getLastTransformedCount,
@@ -33194,7 +33907,9 @@ export {
33194
33907
  load,
33195
33908
  loadAllSessionCosts,
33196
33909
  loadForceMinLayer,
33910
+ loadHeaderSessionIndex,
33197
33911
  loadSessionCosts,
33912
+ loadSessionTracking,
33198
33913
  log_exports as log,
33199
33914
  loreFileExists,
33200
33915
  ltm_exports as ltm,
@@ -33215,10 +33930,13 @@ export {
33215
33930
  runRecall,
33216
33931
  sanitizeSurrogates,
33217
33932
  saveForceMinLayer,
33933
+ saveGradientState,
33218
33934
  saveSessionCosts,
33935
+ saveSessionTracking,
33219
33936
  searchRecall,
33220
33937
  serialize,
33221
33938
  setForceMinLayer,
33939
+ setKV,
33222
33940
  setLastImportAt,
33223
33941
  setLastTurnAtForTest,
33224
33942
  setLtmTokens,