@loreai/core 0.18.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bun/curator.d.ts.map +1 -1
- package/dist/bun/db.d.ts +73 -0
- package/dist/bun/db.d.ts.map +1 -1
- package/dist/bun/distillation.d.ts +2 -13
- package/dist/bun/distillation.d.ts.map +1 -1
- package/dist/bun/embedding.d.ts +5 -1
- package/dist/bun/embedding.d.ts.map +1 -1
- package/dist/bun/gradient.d.ts +9 -0
- package/dist/bun/gradient.d.ts.map +1 -1
- package/dist/bun/index.d.ts +2 -2
- package/dist/bun/index.d.ts.map +1 -1
- package/dist/bun/index.js +817 -99
- package/dist/bun/index.js.map +4 -4
- package/dist/bun/ltm.d.ts +99 -5
- package/dist/bun/ltm.d.ts.map +1 -1
- package/dist/bun/session-limiter.d.ts +26 -0
- package/dist/bun/session-limiter.d.ts.map +1 -0
- package/dist/bun/temporal.d.ts +2 -0
- package/dist/bun/temporal.d.ts.map +1 -1
- package/dist/node/curator.d.ts.map +1 -1
- package/dist/node/db.d.ts +73 -0
- package/dist/node/db.d.ts.map +1 -1
- package/dist/node/distillation.d.ts +2 -13
- package/dist/node/distillation.d.ts.map +1 -1
- package/dist/node/embedding.d.ts +5 -1
- package/dist/node/embedding.d.ts.map +1 -1
- package/dist/node/gradient.d.ts +9 -0
- package/dist/node/gradient.d.ts.map +1 -1
- package/dist/node/index.d.ts +2 -2
- package/dist/node/index.d.ts.map +1 -1
- package/dist/node/index.js +817 -99
- package/dist/node/index.js.map +4 -4
- package/dist/node/ltm.d.ts +99 -5
- package/dist/node/ltm.d.ts.map +1 -1
- package/dist/node/session-limiter.d.ts +26 -0
- package/dist/node/session-limiter.d.ts.map +1 -0
- package/dist/node/temporal.d.ts +2 -0
- package/dist/node/temporal.d.ts.map +1 -1
- package/dist/types/curator.d.ts.map +1 -1
- package/dist/types/db.d.ts +73 -0
- package/dist/types/db.d.ts.map +1 -1
- package/dist/types/distillation.d.ts +2 -13
- package/dist/types/distillation.d.ts.map +1 -1
- package/dist/types/embedding.d.ts +5 -1
- package/dist/types/embedding.d.ts.map +1 -1
- package/dist/types/gradient.d.ts +9 -0
- package/dist/types/gradient.d.ts.map +1 -1
- package/dist/types/index.d.ts +2 -2
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/ltm.d.ts +99 -5
- package/dist/types/ltm.d.ts.map +1 -1
- package/dist/types/session-limiter.d.ts +26 -0
- package/dist/types/session-limiter.d.ts.map +1 -0
- package/dist/types/temporal.d.ts +2 -0
- package/dist/types/temporal.d.ts.map +1 -1
- package/package.json +2 -1
- package/src/curator.ts +54 -2
- package/src/db.ts +347 -0
- package/src/distillation.ts +55 -14
- package/src/embedding.ts +28 -3
- package/src/gradient.ts +183 -74
- package/src/index.ts +8 -0
- package/src/ltm.ts +480 -45
- package/src/session-limiter.ts +47 -0
- package/src/temporal.ts +10 -0
package/dist/bun/index.js
CHANGED
|
@@ -125,6 +125,7 @@ __export(temporal_exports, {
|
|
|
125
125
|
CHUNK_TERMINATOR: () => CHUNK_TERMINATOR,
|
|
126
126
|
bySession: () => bySession,
|
|
127
127
|
count: () => count,
|
|
128
|
+
hasMessages: () => hasMessages,
|
|
128
129
|
markDistilled: () => markDistilled,
|
|
129
130
|
partsToText: () => partsToText,
|
|
130
131
|
prune: () => prune,
|
|
@@ -737,6 +738,55 @@ var MIGRATIONS = [
|
|
|
737
738
|
WHERE ih.project_id = projects.id
|
|
738
739
|
AND ih.source_id = '__declined__'
|
|
739
740
|
);
|
|
741
|
+
`,
|
|
742
|
+
`
|
|
743
|
+
-- Version 23: Persist volatile session tracking state across restarts.
|
|
744
|
+
-- Previously these were in-memory only, causing duplicate processing,
|
|
745
|
+
-- false compaction detection, and expensive prompt cache busts on restart.
|
|
746
|
+
ALTER TABLE session_state ADD COLUMN last_curated_at INTEGER NOT NULL DEFAULT 0;
|
|
747
|
+
ALTER TABLE session_state ADD COLUMN message_count INTEGER NOT NULL DEFAULT 0;
|
|
748
|
+
ALTER TABLE session_state ADD COLUMN turns_since_curation INTEGER NOT NULL DEFAULT 0;
|
|
749
|
+
ALTER TABLE session_state ADD COLUMN ltm_cache_text TEXT;
|
|
750
|
+
ALTER TABLE session_state ADD COLUMN ltm_cache_tokens INTEGER;
|
|
751
|
+
ALTER TABLE session_state ADD COLUMN ltm_pin_text TEXT;
|
|
752
|
+
ALTER TABLE session_state ADD COLUMN ltm_pin_tokens INTEGER;
|
|
753
|
+
ALTER TABLE session_state ADD COLUMN consecutive_text_only_turns INTEGER NOT NULL DEFAULT 0;
|
|
754
|
+
`,
|
|
755
|
+
`
|
|
756
|
+
-- Version 24: Persist remaining volatile session state across restarts.
|
|
757
|
+
-- Session identity (Tier 1/2/3 session correlation)
|
|
758
|
+
ALTER TABLE session_state ADD COLUMN fingerprint TEXT NOT NULL DEFAULT '';
|
|
759
|
+
ALTER TABLE session_state ADD COLUMN header_session_id TEXT;
|
|
760
|
+
ALTER TABLE session_state ADD COLUMN header_name TEXT;
|
|
761
|
+
-- Cache warming state
|
|
762
|
+
ALTER TABLE session_state ADD COLUMN resolved_conversation_ttl TEXT NOT NULL DEFAULT '5m';
|
|
763
|
+
ALTER TABLE session_state ADD COLUMN warmup_state TEXT;
|
|
764
|
+
-- Gradient calibration state (survives restarts to avoid uncalibrated busts)
|
|
765
|
+
ALTER TABLE session_state ADD COLUMN dynamic_context_cap REAL NOT NULL DEFAULT 0;
|
|
766
|
+
ALTER TABLE session_state ADD COLUMN bust_rate_ema REAL NOT NULL DEFAULT -1;
|
|
767
|
+
ALTER TABLE session_state ADD COLUMN inter_bust_interval_ema REAL NOT NULL DEFAULT -1;
|
|
768
|
+
ALTER TABLE session_state ADD COLUMN last_layer INTEGER NOT NULL DEFAULT 0;
|
|
769
|
+
ALTER TABLE session_state ADD COLUMN last_known_input INTEGER NOT NULL DEFAULT 0;
|
|
770
|
+
ALTER TABLE session_state ADD COLUMN last_turn_at INTEGER NOT NULL DEFAULT 0;
|
|
771
|
+
ALTER TABLE session_state ADD COLUMN last_bust_at INTEGER NOT NULL DEFAULT 0;
|
|
772
|
+
`,
|
|
773
|
+
`
|
|
774
|
+
-- Version 25: Adaptive dedup threshold \u2014 store accept/reject feedback
|
|
775
|
+
-- on embedding-based duplicate pairs for per-project threshold calibration.
|
|
776
|
+
-- Titles stored instead of FK IDs because entries are deleted during dedup;
|
|
777
|
+
-- the similarity float is the actual calibration input.
|
|
778
|
+
CREATE TABLE IF NOT EXISTS dedup_feedback (
|
|
779
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
780
|
+
project_id TEXT,
|
|
781
|
+
entry_a_title TEXT NOT NULL,
|
|
782
|
+
entry_b_title TEXT NOT NULL,
|
|
783
|
+
similarity REAL NOT NULL,
|
|
784
|
+
accepted INTEGER NOT NULL,
|
|
785
|
+
source TEXT NOT NULL DEFAULT 'manual',
|
|
786
|
+
created_at INTEGER NOT NULL
|
|
787
|
+
);
|
|
788
|
+
CREATE INDEX IF NOT EXISTS idx_dedup_feedback_project
|
|
789
|
+
ON dedup_feedback(project_id);
|
|
740
790
|
`
|
|
741
791
|
];
|
|
742
792
|
function dbPath() {
|
|
@@ -873,6 +923,11 @@ function close() {
|
|
|
873
923
|
}
|
|
874
924
|
}
|
|
875
925
|
function ensureProject(path, name) {
|
|
926
|
+
if (!process.env.LORE_DB_PATH && /^\/test\//.test(path)) {
|
|
927
|
+
throw new Error(
|
|
928
|
+
`Refusing to create project with test path "${path}" in the production DB. Set LORE_DB_PATH to a temp path, or run tests via \`bun test\` from the repo root.`
|
|
929
|
+
);
|
|
930
|
+
}
|
|
876
931
|
const existing = db().query("SELECT id, git_remote FROM projects WHERE path = ?").get(path);
|
|
877
932
|
if (existing) {
|
|
878
933
|
if (!existing.git_remote) {
|
|
@@ -1041,6 +1096,153 @@ function loadAllSessionCosts() {
|
|
|
1041
1096
|
}
|
|
1042
1097
|
return result;
|
|
1043
1098
|
}
|
|
1099
|
+
function saveSessionTracking(sessionID, state) {
|
|
1100
|
+
const now = Date.now();
|
|
1101
|
+
db().query(
|
|
1102
|
+
"INSERT OR IGNORE INTO session_state (session_id, force_min_layer, updated_at) VALUES (?, 0, ?)"
|
|
1103
|
+
).run(sessionID, now);
|
|
1104
|
+
const sets = ["updated_at = ?"];
|
|
1105
|
+
const vals = [now];
|
|
1106
|
+
if (state.lastCuratedAt !== void 0) {
|
|
1107
|
+
sets.push("last_curated_at = ?");
|
|
1108
|
+
vals.push(state.lastCuratedAt);
|
|
1109
|
+
}
|
|
1110
|
+
if (state.messageCount !== void 0) {
|
|
1111
|
+
sets.push("message_count = ?");
|
|
1112
|
+
vals.push(state.messageCount);
|
|
1113
|
+
}
|
|
1114
|
+
if (state.turnsSinceCuration !== void 0) {
|
|
1115
|
+
sets.push("turns_since_curation = ?");
|
|
1116
|
+
vals.push(state.turnsSinceCuration);
|
|
1117
|
+
}
|
|
1118
|
+
if (state.consecutiveTextOnlyTurns !== void 0) {
|
|
1119
|
+
sets.push("consecutive_text_only_turns = ?");
|
|
1120
|
+
vals.push(state.consecutiveTextOnlyTurns);
|
|
1121
|
+
}
|
|
1122
|
+
if (state.ltmCacheText !== void 0) {
|
|
1123
|
+
sets.push("ltm_cache_text = ?");
|
|
1124
|
+
vals.push(state.ltmCacheText);
|
|
1125
|
+
}
|
|
1126
|
+
if (state.ltmCacheTokens !== void 0) {
|
|
1127
|
+
sets.push("ltm_cache_tokens = ?");
|
|
1128
|
+
vals.push(state.ltmCacheTokens);
|
|
1129
|
+
}
|
|
1130
|
+
if (state.ltmPinText !== void 0) {
|
|
1131
|
+
sets.push("ltm_pin_text = ?");
|
|
1132
|
+
vals.push(state.ltmPinText);
|
|
1133
|
+
}
|
|
1134
|
+
if (state.ltmPinTokens !== void 0) {
|
|
1135
|
+
sets.push("ltm_pin_tokens = ?");
|
|
1136
|
+
vals.push(state.ltmPinTokens);
|
|
1137
|
+
}
|
|
1138
|
+
if (state.fingerprint !== void 0) {
|
|
1139
|
+
sets.push("fingerprint = ?");
|
|
1140
|
+
vals.push(state.fingerprint);
|
|
1141
|
+
}
|
|
1142
|
+
if (state.headerSessionId !== void 0) {
|
|
1143
|
+
sets.push("header_session_id = ?");
|
|
1144
|
+
vals.push(state.headerSessionId);
|
|
1145
|
+
}
|
|
1146
|
+
if (state.headerName !== void 0) {
|
|
1147
|
+
sets.push("header_name = ?");
|
|
1148
|
+
vals.push(state.headerName);
|
|
1149
|
+
}
|
|
1150
|
+
if (state.resolvedConversationTTL !== void 0) {
|
|
1151
|
+
sets.push("resolved_conversation_ttl = ?");
|
|
1152
|
+
vals.push(state.resolvedConversationTTL);
|
|
1153
|
+
}
|
|
1154
|
+
if (state.warmupState !== void 0) {
|
|
1155
|
+
sets.push("warmup_state = ?");
|
|
1156
|
+
vals.push(state.warmupState);
|
|
1157
|
+
}
|
|
1158
|
+
if (state.dynamicContextCap !== void 0) {
|
|
1159
|
+
sets.push("dynamic_context_cap = ?");
|
|
1160
|
+
vals.push(state.dynamicContextCap);
|
|
1161
|
+
}
|
|
1162
|
+
if (state.bustRateEMA !== void 0) {
|
|
1163
|
+
sets.push("bust_rate_ema = ?");
|
|
1164
|
+
vals.push(state.bustRateEMA);
|
|
1165
|
+
}
|
|
1166
|
+
if (state.interBustIntervalEMA !== void 0) {
|
|
1167
|
+
sets.push("inter_bust_interval_ema = ?");
|
|
1168
|
+
vals.push(state.interBustIntervalEMA);
|
|
1169
|
+
}
|
|
1170
|
+
if (state.lastLayer !== void 0) {
|
|
1171
|
+
sets.push("last_layer = ?");
|
|
1172
|
+
vals.push(state.lastLayer);
|
|
1173
|
+
}
|
|
1174
|
+
if (state.lastKnownInput !== void 0) {
|
|
1175
|
+
sets.push("last_known_input = ?");
|
|
1176
|
+
vals.push(state.lastKnownInput);
|
|
1177
|
+
}
|
|
1178
|
+
if (state.lastTurnAt !== void 0) {
|
|
1179
|
+
sets.push("last_turn_at = ?");
|
|
1180
|
+
vals.push(state.lastTurnAt);
|
|
1181
|
+
}
|
|
1182
|
+
if (state.lastBustAt !== void 0) {
|
|
1183
|
+
sets.push("last_bust_at = ?");
|
|
1184
|
+
vals.push(state.lastBustAt);
|
|
1185
|
+
}
|
|
1186
|
+
db().query(
|
|
1187
|
+
"UPDATE session_state SET " + sets.join(", ") + " WHERE session_id = ?"
|
|
1188
|
+
).run(...vals, sessionID);
|
|
1189
|
+
}
|
|
1190
|
+
function loadSessionTracking(sessionID) {
|
|
1191
|
+
const row = db().query(
|
|
1192
|
+
`SELECT last_curated_at, message_count, turns_since_curation,
|
|
1193
|
+
consecutive_text_only_turns,
|
|
1194
|
+
ltm_cache_text, ltm_cache_tokens, ltm_pin_text, ltm_pin_tokens,
|
|
1195
|
+
fingerprint, header_session_id, header_name,
|
|
1196
|
+
resolved_conversation_ttl, warmup_state,
|
|
1197
|
+
dynamic_context_cap, bust_rate_ema, inter_bust_interval_ema,
|
|
1198
|
+
last_layer, last_known_input, last_turn_at, last_bust_at
|
|
1199
|
+
FROM session_state WHERE session_id = ?`
|
|
1200
|
+
).get(sessionID);
|
|
1201
|
+
if (!row) return null;
|
|
1202
|
+
return {
|
|
1203
|
+
lastCuratedAt: row.last_curated_at,
|
|
1204
|
+
messageCount: row.message_count,
|
|
1205
|
+
turnsSinceCuration: row.turns_since_curation,
|
|
1206
|
+
consecutiveTextOnlyTurns: row.consecutive_text_only_turns,
|
|
1207
|
+
ltmCacheText: row.ltm_cache_text,
|
|
1208
|
+
ltmCacheTokens: row.ltm_cache_tokens,
|
|
1209
|
+
ltmPinText: row.ltm_pin_text,
|
|
1210
|
+
ltmPinTokens: row.ltm_pin_tokens,
|
|
1211
|
+
fingerprint: row.fingerprint,
|
|
1212
|
+
headerSessionId: row.header_session_id,
|
|
1213
|
+
headerName: row.header_name,
|
|
1214
|
+
resolvedConversationTTL: row.resolved_conversation_ttl,
|
|
1215
|
+
warmupState: row.warmup_state,
|
|
1216
|
+
dynamicContextCap: row.dynamic_context_cap,
|
|
1217
|
+
bustRateEMA: row.bust_rate_ema,
|
|
1218
|
+
interBustIntervalEMA: row.inter_bust_interval_ema,
|
|
1219
|
+
lastLayer: row.last_layer,
|
|
1220
|
+
lastKnownInput: row.last_known_input,
|
|
1221
|
+
lastTurnAt: row.last_turn_at,
|
|
1222
|
+
lastBustAt: row.last_bust_at
|
|
1223
|
+
};
|
|
1224
|
+
}
|
|
1225
|
+
function loadHeaderSessionIndex() {
|
|
1226
|
+
const rows = db().query(
|
|
1227
|
+
`SELECT session_id, header_session_id, header_name
|
|
1228
|
+
FROM session_state
|
|
1229
|
+
WHERE header_session_id IS NOT NULL AND header_name IS NOT NULL`
|
|
1230
|
+
).all();
|
|
1231
|
+
return rows.map((row) => ({
|
|
1232
|
+
sessionId: row.session_id,
|
|
1233
|
+
headerSessionId: row.header_session_id,
|
|
1234
|
+
headerName: row.header_name
|
|
1235
|
+
}));
|
|
1236
|
+
}
|
|
1237
|
+
function getKV(key) {
|
|
1238
|
+
const row = db().query("SELECT value FROM kv_meta WHERE key = ?").get(key);
|
|
1239
|
+
return row?.value ?? null;
|
|
1240
|
+
}
|
|
1241
|
+
function setKV(key, value) {
|
|
1242
|
+
db().query(
|
|
1243
|
+
"INSERT INTO kv_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = ?"
|
|
1244
|
+
).run(key, value, value);
|
|
1245
|
+
}
|
|
1044
1246
|
function getMeta(key) {
|
|
1045
1247
|
const row = db().query("SELECT value FROM metadata WHERE key = ?").get(key);
|
|
1046
1248
|
return row?.value ?? null;
|
|
@@ -26753,7 +26955,16 @@ var LocalProvider = class {
|
|
|
26753
26955
|
workerUrl = vendorWorkerUrl;
|
|
26754
26956
|
}
|
|
26755
26957
|
} else {
|
|
26756
|
-
|
|
26958
|
+
const selfUrl = typeof import.meta.url === "string" ? import.meta.url : void 0;
|
|
26959
|
+
if (selfUrl) {
|
|
26960
|
+
workerUrl = new URL(
|
|
26961
|
+
`./embedding-worker${selfUrl.endsWith(".ts") ? ".ts" : ".js"}`,
|
|
26962
|
+
selfUrl
|
|
26963
|
+
);
|
|
26964
|
+
} else {
|
|
26965
|
+
const { pathToFileURL } = await import("node:url");
|
|
26966
|
+
workerUrl = new URL("./embedding-worker.cjs", pathToFileURL(__filename));
|
|
26967
|
+
}
|
|
26757
26968
|
}
|
|
26758
26969
|
const vendor = vendorModelInfo();
|
|
26759
26970
|
const workerInitData = {
|
|
@@ -27028,8 +27239,14 @@ function fromBlob(blob) {
|
|
|
27028
27239
|
const bytes = new Uint8Array(blob);
|
|
27029
27240
|
return new Float32Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / 4);
|
|
27030
27241
|
}
|
|
27031
|
-
function vectorSearch(queryEmbedding, limit = 10) {
|
|
27032
|
-
|
|
27242
|
+
function vectorSearch(queryEmbedding, limit = 10, excludeCategories) {
|
|
27243
|
+
let sql = "SELECT id, embedding FROM knowledge WHERE embedding IS NOT NULL AND confidence > 0.2";
|
|
27244
|
+
const params = [];
|
|
27245
|
+
if (excludeCategories?.length) {
|
|
27246
|
+
sql += ` AND category NOT IN (${excludeCategories.map(() => "?").join(",")})`;
|
|
27247
|
+
params.push(...excludeCategories);
|
|
27248
|
+
}
|
|
27249
|
+
const rows = db().query(sql).all(...params);
|
|
27033
27250
|
const scored = [];
|
|
27034
27251
|
for (const row of rows) {
|
|
27035
27252
|
const vec = fromBlob(row.embedding);
|
|
@@ -27418,6 +27635,12 @@ function count(projectPath, sessionID) {
|
|
|
27418
27635
|
const params = sessionID ? [pid, sessionID] : [pid];
|
|
27419
27636
|
return db().query(query).get(...params).count;
|
|
27420
27637
|
}
|
|
27638
|
+
function hasMessages(projectPath, sessionID) {
|
|
27639
|
+
const pid = ensureProject(projectPath);
|
|
27640
|
+
return !!db().query(
|
|
27641
|
+
"SELECT 1 FROM temporal_messages WHERE project_id = ? AND session_id = ? LIMIT 1"
|
|
27642
|
+
).get(pid, sessionID);
|
|
27643
|
+
}
|
|
27421
27644
|
function undistilledCount(projectPath, sessionID) {
|
|
27422
27645
|
const pid = ensureProject(projectPath);
|
|
27423
27646
|
const query = sessionID ? "SELECT COUNT(*) as count FROM temporal_messages WHERE project_id = ? AND session_id = ? AND distilled = 0" : "SELECT COUNT(*) as count FROM temporal_messages WHERE project_id = ? AND distilled = 0";
|
|
@@ -27476,11 +27699,13 @@ function prune(input) {
|
|
|
27476
27699
|
var ltm_exports = {};
|
|
27477
27700
|
__export(ltm_exports, {
|
|
27478
27701
|
all: () => all2,
|
|
27702
|
+
calibrateDedupThreshold: () => calibrateDedupThreshold,
|
|
27479
27703
|
cascadeRefReplace: () => cascadeRefReplace,
|
|
27480
27704
|
check: () => check2,
|
|
27481
27705
|
cleanDeadRefs: () => cleanDeadRefs,
|
|
27482
27706
|
create: () => create,
|
|
27483
27707
|
crossProject: () => crossProject,
|
|
27708
|
+
dedupPairKey: () => dedupPairKey,
|
|
27484
27709
|
deduplicate: () => deduplicate,
|
|
27485
27710
|
deduplicateGlobal: () => deduplicateGlobal,
|
|
27486
27711
|
extractRefs: () => extractRefs,
|
|
@@ -27488,9 +27713,17 @@ __export(ltm_exports, {
|
|
|
27488
27713
|
forProject: () => forProject,
|
|
27489
27714
|
forSession: () => forSession,
|
|
27490
27715
|
get: () => get,
|
|
27716
|
+
getDedupFeedback: () => getDedupFeedback,
|
|
27717
|
+
getDedupFeedbackCount: () => getDedupFeedbackCount,
|
|
27718
|
+
loadCalibratedThreshold: () => loadCalibratedThreshold,
|
|
27719
|
+
pruneDedupFeedback: () => pruneDedupFeedback,
|
|
27491
27720
|
pruneOversized: () => pruneOversized,
|
|
27721
|
+
recordAutoSignals: () => recordAutoSignals,
|
|
27722
|
+
recordDedupFeedback: () => recordDedupFeedback,
|
|
27723
|
+
recordDedupResultFeedback: () => recordDedupResultFeedback,
|
|
27492
27724
|
remove: () => remove,
|
|
27493
27725
|
resolveRef: () => resolveRef2,
|
|
27726
|
+
saveCalibratedThreshold: () => saveCalibratedThreshold,
|
|
27494
27727
|
search: () => search3,
|
|
27495
27728
|
searchScored: () => searchScored3,
|
|
27496
27729
|
searchScoredOtherProjects: () => searchScoredOtherProjects,
|
|
@@ -28227,18 +28460,29 @@ function scoreEntriesFTS(sessionContext) {
|
|
|
28227
28460
|
return /* @__PURE__ */ new Map();
|
|
28228
28461
|
}
|
|
28229
28462
|
}
|
|
28230
|
-
function forSession(projectPath, sessionID, maxTokens) {
|
|
28463
|
+
async function forSession(projectPath, sessionID, maxTokens, options) {
|
|
28231
28464
|
const pid = ensureProject(projectPath);
|
|
28465
|
+
const categoryFilter = options?.categories;
|
|
28466
|
+
const excludeFilter = options?.excludeCategories;
|
|
28467
|
+
let categoryClause = "";
|
|
28468
|
+
let categoryParams = [];
|
|
28469
|
+
if (categoryFilter?.length) {
|
|
28470
|
+
categoryClause = ` AND category IN (${categoryFilter.map(() => "?").join(",")})`;
|
|
28471
|
+
categoryParams = categoryFilter;
|
|
28472
|
+
} else if (excludeFilter?.length) {
|
|
28473
|
+
categoryClause = ` AND category NOT IN (${excludeFilter.map(() => "?").join(",")})`;
|
|
28474
|
+
categoryParams = excludeFilter;
|
|
28475
|
+
}
|
|
28232
28476
|
const projectEntries = db().query(
|
|
28233
28477
|
`SELECT ${KNOWLEDGE_COLS} FROM knowledge
|
|
28234
|
-
WHERE project_id = ? AND cross_project = 0 AND confidence > 0.2
|
|
28478
|
+
WHERE project_id = ? AND cross_project = 0 AND confidence > 0.2${categoryClause}
|
|
28235
28479
|
ORDER BY confidence DESC, updated_at DESC`
|
|
28236
|
-
).all(pid);
|
|
28480
|
+
).all(pid, ...categoryParams);
|
|
28237
28481
|
const crossEntries = db().query(
|
|
28238
28482
|
`SELECT ${KNOWLEDGE_COLS} FROM knowledge
|
|
28239
|
-
WHERE (project_id IS NULL OR cross_project = 1) AND confidence > 0.2
|
|
28483
|
+
WHERE (project_id IS NULL OR cross_project = 1) AND confidence > 0.2${categoryClause}
|
|
28240
28484
|
ORDER BY confidence DESC, updated_at DESC`
|
|
28241
|
-
).all();
|
|
28485
|
+
).all(...categoryParams);
|
|
28242
28486
|
if (!crossEntries.length && !projectEntries.length) return [];
|
|
28243
28487
|
let sessionContext = "";
|
|
28244
28488
|
if (sessionID) {
|
|
@@ -28259,22 +28503,52 @@ function forSession(projectPath, sessionID, maxTokens) {
|
|
|
28259
28503
|
sessionContext += recentMsgs.map((m) => m.content).join("\n");
|
|
28260
28504
|
}
|
|
28261
28505
|
}
|
|
28506
|
+
if (!sessionContext.trim() && options?.contextHint) {
|
|
28507
|
+
sessionContext = options.contextHint;
|
|
28508
|
+
}
|
|
28262
28509
|
let scoredProject;
|
|
28263
28510
|
let scoredCross;
|
|
28264
|
-
if (sessionContext.trim().length > 20) {
|
|
28511
|
+
if (sessionContext.trim().length > 20 && isAvailable()) {
|
|
28512
|
+
let vectorScores;
|
|
28513
|
+
try {
|
|
28514
|
+
const [contextVec] = await embed([sessionContext], "query");
|
|
28515
|
+
const hits = vectorSearch(contextVec, 50, excludeFilter);
|
|
28516
|
+
vectorScores = new Map(hits.map((h3) => [h3.id, h3.similarity]));
|
|
28517
|
+
} catch (err) {
|
|
28518
|
+
warn("Vector scoring failed, falling back to FTS5:", err);
|
|
28519
|
+
vectorScores = /* @__PURE__ */ new Map();
|
|
28520
|
+
}
|
|
28521
|
+
if (vectorScores.size > 0) {
|
|
28522
|
+
const ftsScores = scoreEntriesFTS(sessionContext);
|
|
28523
|
+
const rawScored = projectEntries.map((entry) => {
|
|
28524
|
+
const vecScore = vectorScores.get(entry.id);
|
|
28525
|
+
const score = vecScore != null ? vecScore * entry.confidence : (ftsScores.get(entry.id) ?? 0) * entry.confidence;
|
|
28526
|
+
return { entry, score };
|
|
28527
|
+
});
|
|
28528
|
+
const matched = rawScored.filter((s) => s.score > 0);
|
|
28529
|
+
const matchedIds = new Set(matched.map((s) => s.entry.id));
|
|
28530
|
+
const safetyNet = projectEntries.filter((e) => !matchedIds.has(e.id)).slice(0, PROJECT_SAFETY_NET).map((e) => ({ entry: e, score: 1e-3 * e.confidence }));
|
|
28531
|
+
scoredProject = [...matched, ...safetyNet];
|
|
28532
|
+
scoredCross = crossEntries.filter((e) => vectorScores.has(e.id) || ftsScores.has(e.id)).map((e) => {
|
|
28533
|
+
const vecScore = vectorScores.get(e.id);
|
|
28534
|
+
const score = vecScore != null ? vecScore * e.confidence : (ftsScores.get(e.id) ?? 0) * e.confidence;
|
|
28535
|
+
return { entry: e, score };
|
|
28536
|
+
});
|
|
28537
|
+
} else {
|
|
28538
|
+
const ftsScores = scoreEntriesFTS(sessionContext);
|
|
28539
|
+
({ scoredProject, scoredCross } = scoreFTS(
|
|
28540
|
+
projectEntries,
|
|
28541
|
+
crossEntries,
|
|
28542
|
+
ftsScores
|
|
28543
|
+
));
|
|
28544
|
+
}
|
|
28545
|
+
} else if (sessionContext.trim().length > 20) {
|
|
28265
28546
|
const ftsScores = scoreEntriesFTS(sessionContext);
|
|
28266
|
-
|
|
28267
|
-
|
|
28268
|
-
|
|
28269
|
-
|
|
28270
|
-
|
|
28271
|
-
const matchedIds = new Set(matched.map((s) => s.entry.id));
|
|
28272
|
-
const safetyNet = projectEntries.filter((e) => !matchedIds.has(e.id)).slice(0, PROJECT_SAFETY_NET).map((e) => ({ entry: e, score: 1e-3 * e.confidence }));
|
|
28273
|
-
scoredProject = [...matched, ...safetyNet];
|
|
28274
|
-
scoredCross = crossEntries.filter((e) => ftsScores.has(e.id)).map((e) => ({
|
|
28275
|
-
entry: e,
|
|
28276
|
-
score: (ftsScores.get(e.id) ?? 0) * e.confidence
|
|
28277
|
-
}));
|
|
28547
|
+
({ scoredProject, scoredCross } = scoreFTS(
|
|
28548
|
+
projectEntries,
|
|
28549
|
+
crossEntries,
|
|
28550
|
+
ftsScores
|
|
28551
|
+
));
|
|
28278
28552
|
} else {
|
|
28279
28553
|
scoredProject = projectEntries.slice(0, NO_CONTEXT_FALLBACK_CAP).map((entry) => ({ entry, score: entry.confidence }));
|
|
28280
28554
|
scoredCross = crossEntries.slice(0, NO_CONTEXT_FALLBACK_CAP).map((entry) => ({ entry, score: entry.confidence }));
|
|
@@ -28320,6 +28594,21 @@ function forSession(projectPath, sessionID, maxTokens) {
|
|
|
28320
28594
|
}
|
|
28321
28595
|
return result;
|
|
28322
28596
|
}
|
|
28597
|
+
function scoreFTS(projectEntries, crossEntries, ftsScores) {
|
|
28598
|
+
const rawScored = projectEntries.map((entry) => ({
|
|
28599
|
+
entry,
|
|
28600
|
+
score: (ftsScores.get(entry.id) ?? 0) * entry.confidence
|
|
28601
|
+
}));
|
|
28602
|
+
const matched = rawScored.filter((s) => s.score > 0);
|
|
28603
|
+
const matchedIds = new Set(matched.map((s) => s.entry.id));
|
|
28604
|
+
const safetyNet = projectEntries.filter((e) => !matchedIds.has(e.id)).slice(0, PROJECT_SAFETY_NET).map((e) => ({ entry: e, score: 1e-3 * e.confidence }));
|
|
28605
|
+
const scoredProject = [...matched, ...safetyNet];
|
|
28606
|
+
const scoredCross = crossEntries.filter((e) => ftsScores.has(e.id)).map((e) => ({
|
|
28607
|
+
entry: e,
|
|
28608
|
+
score: (ftsScores.get(e.id) ?? 0) * e.confidence
|
|
28609
|
+
}));
|
|
28610
|
+
return { scoredProject, scoredCross };
|
|
28611
|
+
}
|
|
28323
28612
|
function all2() {
|
|
28324
28613
|
return db().query(
|
|
28325
28614
|
`SELECT ${KNOWLEDGE_COLS} FROM knowledge WHERE confidence > 0.2 ORDER BY confidence DESC, updated_at DESC`
|
|
@@ -28563,8 +28852,11 @@ function check2(projectPath) {
|
|
|
28563
28852
|
}
|
|
28564
28853
|
return issues;
|
|
28565
28854
|
}
|
|
28566
|
-
function
|
|
28567
|
-
|
|
28855
|
+
function dedupPairKey(idA, idB) {
|
|
28856
|
+
return idA < idB ? `${idA}:${idB}` : `${idB}:${idA}`;
|
|
28857
|
+
}
|
|
28858
|
+
function _dedup(entries, dryRun, embeddingThreshold = EMBEDDING_DEDUP_THRESHOLD) {
|
|
28859
|
+
if (entries.length < 2) return { clusters: [], totalRemoved: 0, pairSimilarities: /* @__PURE__ */ new Map(), entryTitles: /* @__PURE__ */ new Map() };
|
|
28568
28860
|
const embeddingMap = /* @__PURE__ */ new Map();
|
|
28569
28861
|
{
|
|
28570
28862
|
const entryIds = entries.map((e) => e.id);
|
|
@@ -28579,6 +28871,7 @@ function _dedup(entries, dryRun) {
|
|
|
28579
28871
|
}
|
|
28580
28872
|
}
|
|
28581
28873
|
const neighborMap = /* @__PURE__ */ new Map();
|
|
28874
|
+
const pairSimilarities = /* @__PURE__ */ new Map();
|
|
28582
28875
|
for (const entry of entries) {
|
|
28583
28876
|
const neighbors = [];
|
|
28584
28877
|
const entryVec = embeddingMap.get(entry.id);
|
|
@@ -28592,7 +28885,13 @@ function _dedup(entries, dryRun) {
|
|
|
28592
28885
|
const otherVec = embeddingMap.get(other.id);
|
|
28593
28886
|
if (otherVec && entryVec.length === otherVec.length) {
|
|
28594
28887
|
similarity = cosineSimilarity(entryVec, otherVec);
|
|
28595
|
-
embeddingMatch = similarity >=
|
|
28888
|
+
embeddingMatch = similarity >= embeddingThreshold;
|
|
28889
|
+
}
|
|
28890
|
+
}
|
|
28891
|
+
if (similarity > 0) {
|
|
28892
|
+
const pk = dedupPairKey(entry.id, other.id);
|
|
28893
|
+
if (!pairSimilarities.has(pk)) {
|
|
28894
|
+
pairSimilarities.set(pk, similarity);
|
|
28596
28895
|
}
|
|
28597
28896
|
}
|
|
28598
28897
|
if (titleMatch || embeddingMatch) {
|
|
@@ -28644,20 +28943,178 @@ function _dedup(entries, dryRun) {
|
|
|
28644
28943
|
totalRemoved += merged.length;
|
|
28645
28944
|
}
|
|
28646
28945
|
result.sort((a, b) => b.merged.length - a.merged.length);
|
|
28647
|
-
|
|
28946
|
+
const entryTitles = new Map(entries.map((e) => [e.id, e.title]));
|
|
28947
|
+
return { clusters: result, totalRemoved, pairSimilarities, entryTitles };
|
|
28648
28948
|
}
|
|
28649
28949
|
async function deduplicate(projectPath, opts) {
|
|
28950
|
+
const pid = ensureProject(projectPath);
|
|
28951
|
+
const threshold = loadCalibratedThreshold(pid) ?? EMBEDDING_DEDUP_THRESHOLD;
|
|
28650
28952
|
const entries = forProject(projectPath, false);
|
|
28651
|
-
return _dedup(entries, opts?.dryRun ?? true);
|
|
28953
|
+
return _dedup(entries, opts?.dryRun ?? true, threshold);
|
|
28652
28954
|
}
|
|
28653
28955
|
async function deduplicateGlobal(opts) {
|
|
28956
|
+
const threshold = loadCalibratedThreshold(null) ?? EMBEDDING_DEDUP_THRESHOLD;
|
|
28654
28957
|
const entries = db().query(
|
|
28655
28958
|
`SELECT ${KNOWLEDGE_COLS} FROM knowledge
|
|
28656
28959
|
WHERE project_id IS NULL
|
|
28657
28960
|
AND confidence > 0.2
|
|
28658
28961
|
ORDER BY confidence DESC, updated_at DESC`
|
|
28659
28962
|
).all();
|
|
28660
|
-
return _dedup(entries, opts?.dryRun ?? true);
|
|
28963
|
+
return _dedup(entries, opts?.dryRun ?? true, threshold);
|
|
28964
|
+
}
|
|
28965
|
+
var MIN_CALIBRATION_SAMPLES = 20;
|
|
28966
|
+
var DEFAULT_EMBEDDING_DEDUP_THRESHOLD = EMBEDDING_DEDUP_THRESHOLD;
|
|
28967
|
+
var AUTO_SIGNAL_MIN_SIMILARITY = 0.8;
|
|
28968
|
+
var AUTO_SIGNAL_MAX_PAIRS = 50;
|
|
28969
|
+
function recordDedupFeedback(input) {
|
|
28970
|
+
db().query(
|
|
28971
|
+
`INSERT INTO dedup_feedback
|
|
28972
|
+
(project_id, entry_a_title, entry_b_title, similarity, accepted, source, created_at)
|
|
28973
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`
|
|
28974
|
+
).run(
|
|
28975
|
+
input.projectId,
|
|
28976
|
+
input.entryATitle,
|
|
28977
|
+
input.entryBTitle,
|
|
28978
|
+
input.similarity,
|
|
28979
|
+
input.accepted ? 1 : 0,
|
|
28980
|
+
input.source,
|
|
28981
|
+
Date.now()
|
|
28982
|
+
);
|
|
28983
|
+
}
|
|
28984
|
+
function recordDedupResultFeedback(projectId2, result, accepted, source) {
|
|
28985
|
+
for (const cluster of result.clusters) {
|
|
28986
|
+
for (const merged of cluster.merged) {
|
|
28987
|
+
const pk = dedupPairKey(cluster.surviving.id, merged.id);
|
|
28988
|
+
const similarity = result.pairSimilarities.get(pk);
|
|
28989
|
+
if (similarity != null && similarity > 0) {
|
|
28990
|
+
recordDedupFeedback({
|
|
28991
|
+
projectId: projectId2,
|
|
28992
|
+
entryATitle: cluster.surviving.title,
|
|
28993
|
+
entryBTitle: merged.title,
|
|
28994
|
+
similarity,
|
|
28995
|
+
accepted,
|
|
28996
|
+
source
|
|
28997
|
+
});
|
|
28998
|
+
}
|
|
28999
|
+
}
|
|
29000
|
+
}
|
|
29001
|
+
}
|
|
29002
|
+
function recordAutoSignals(projectId2, result) {
|
|
29003
|
+
const mergedPairs = /* @__PURE__ */ new Set();
|
|
29004
|
+
for (const cluster of result.clusters) {
|
|
29005
|
+
for (const merged of cluster.merged) {
|
|
29006
|
+
mergedPairs.add(dedupPairKey(cluster.surviving.id, merged.id));
|
|
29007
|
+
}
|
|
29008
|
+
}
|
|
29009
|
+
const titleMap = new Map(result.entryTitles);
|
|
29010
|
+
for (const cluster of result.clusters) {
|
|
29011
|
+
if (!titleMap.has(cluster.surviving.id)) {
|
|
29012
|
+
titleMap.set(cluster.surviving.id, cluster.surviving.title);
|
|
29013
|
+
}
|
|
29014
|
+
for (const m of cluster.merged) {
|
|
29015
|
+
if (!titleMap.has(m.id)) titleMap.set(m.id, m.title);
|
|
29016
|
+
}
|
|
29017
|
+
}
|
|
29018
|
+
const signals = [];
|
|
29019
|
+
for (const [pk, sim] of result.pairSimilarities) {
|
|
29020
|
+
if (sim < AUTO_SIGNAL_MIN_SIMILARITY) continue;
|
|
29021
|
+
if (mergedPairs.has(pk)) continue;
|
|
29022
|
+
const [idA, idB] = pk.split(":");
|
|
29023
|
+
const titleA = titleMap.get(idA);
|
|
29024
|
+
const titleB = titleMap.get(idB);
|
|
29025
|
+
if (!titleA || !titleB) continue;
|
|
29026
|
+
signals.push({ entryATitle: titleA, entryBTitle: titleB, similarity: sim });
|
|
29027
|
+
}
|
|
29028
|
+
const currentThreshold = loadCalibratedThreshold(projectId2) ?? DEFAULT_EMBEDDING_DEDUP_THRESHOLD;
|
|
29029
|
+
signals.sort((a, b) => Math.abs(a.similarity - currentThreshold) - Math.abs(b.similarity - currentThreshold));
|
|
29030
|
+
const capped = signals.slice(0, AUTO_SIGNAL_MAX_PAIRS);
|
|
29031
|
+
pruneDedupFeedback(projectId2);
|
|
29032
|
+
for (const s of capped) {
|
|
29033
|
+
recordDedupFeedback({
|
|
29034
|
+
projectId: projectId2,
|
|
29035
|
+
entryATitle: s.entryATitle,
|
|
29036
|
+
entryBTitle: s.entryBTitle,
|
|
29037
|
+
similarity: s.similarity,
|
|
29038
|
+
accepted: false,
|
|
29039
|
+
source: "auto_dedup"
|
|
29040
|
+
});
|
|
29041
|
+
}
|
|
29042
|
+
}
|
|
29043
|
+
function getDedupFeedback(projectId2) {
|
|
29044
|
+
const rows = projectId2 !== null ? db().query(
|
|
29045
|
+
"SELECT similarity, accepted, source FROM dedup_feedback WHERE project_id = ? ORDER BY similarity"
|
|
29046
|
+
).all(projectId2) : db().query(
|
|
29047
|
+
"SELECT similarity, accepted, source FROM dedup_feedback WHERE project_id IS NULL ORDER BY similarity"
|
|
29048
|
+
).all();
|
|
29049
|
+
return rows.map((r) => ({ similarity: r.similarity, accepted: r.accepted === 1, source: r.source }));
|
|
29050
|
+
}
|
|
29051
|
+
function getDedupFeedbackCount(projectId2) {
|
|
29052
|
+
const row = projectId2 !== null ? db().query("SELECT COUNT(*) as cnt FROM dedup_feedback WHERE project_id = ?").get(projectId2) : db().query("SELECT COUNT(*) as cnt FROM dedup_feedback WHERE project_id IS NULL").get();
|
|
29053
|
+
return row?.cnt ?? 0;
|
|
29054
|
+
}
|
|
29055
|
+
var MAX_FEEDBACK_ROWS_PER_PROJECT = 500;
|
|
29056
|
+
function pruneDedupFeedback(projectId2) {
|
|
29057
|
+
const count3 = getDedupFeedbackCount(projectId2);
|
|
29058
|
+
if (count3 <= MAX_FEEDBACK_ROWS_PER_PROJECT) return;
|
|
29059
|
+
const excess = count3 - MAX_FEEDBACK_ROWS_PER_PROJECT;
|
|
29060
|
+
if (projectId2 !== null) {
|
|
29061
|
+
db().query(
|
|
29062
|
+
`DELETE FROM dedup_feedback WHERE id IN (
|
|
29063
|
+
SELECT id FROM dedup_feedback WHERE project_id = ?
|
|
29064
|
+
ORDER BY created_at ASC LIMIT ?
|
|
29065
|
+
)`
|
|
29066
|
+
).run(projectId2, excess);
|
|
29067
|
+
} else {
|
|
29068
|
+
db().query(
|
|
29069
|
+
`DELETE FROM dedup_feedback WHERE id IN (
|
|
29070
|
+
SELECT id FROM dedup_feedback WHERE project_id IS NULL
|
|
29071
|
+
ORDER BY created_at ASC LIMIT ?
|
|
29072
|
+
)`
|
|
29073
|
+
).run(excess);
|
|
29074
|
+
}
|
|
29075
|
+
}
|
|
29076
|
+
function calibrateDedupThreshold(projectId2) {
|
|
29077
|
+
const feedback = getDedupFeedback(projectId2);
|
|
29078
|
+
if (feedback.length < MIN_CALIBRATION_SAMPLES) return null;
|
|
29079
|
+
const accepted = feedback.filter((f) => f.accepted);
|
|
29080
|
+
const rejected = feedback.filter((f) => !f.accepted);
|
|
29081
|
+
if (rejected.length === 0) {
|
|
29082
|
+
const minAccepted = Math.min(...accepted.map((f) => f.similarity));
|
|
29083
|
+
return Math.max(0.85, minAccepted - 5e-3);
|
|
29084
|
+
}
|
|
29085
|
+
if (accepted.length === 0) {
|
|
29086
|
+
warn("dedup calibration: all feedback is reject \u2014 keeping default threshold");
|
|
29087
|
+
return null;
|
|
29088
|
+
}
|
|
29089
|
+
const allSims = [...new Set(feedback.map((f) => f.similarity))].sort((a, b) => a - b);
|
|
29090
|
+
let bestThreshold = DEFAULT_EMBEDDING_DEDUP_THRESHOLD;
|
|
29091
|
+
let bestAccuracy = -1;
|
|
29092
|
+
for (let i = 0; i < allSims.length - 1; i++) {
|
|
29093
|
+
const candidate = (allSims[i] + allSims[i + 1]) / 2;
|
|
29094
|
+
const correctAccepted = accepted.filter((f) => f.similarity >= candidate).length;
|
|
29095
|
+
const correctRejected = rejected.filter((f) => f.similarity < candidate).length;
|
|
29096
|
+
const accuracy = (correctAccepted + correctRejected) / feedback.length;
|
|
29097
|
+
if (accuracy > bestAccuracy || accuracy === bestAccuracy && candidate > bestThreshold) {
|
|
29098
|
+
bestAccuracy = accuracy;
|
|
29099
|
+
bestThreshold = candidate;
|
|
29100
|
+
}
|
|
29101
|
+
}
|
|
29102
|
+
return Math.max(0.85, Math.min(0.98, bestThreshold));
|
|
29103
|
+
}
|
|
29104
|
+
function saveCalibratedThreshold(projectId2, threshold, sampleSize) {
|
|
29105
|
+
const key = `dedup_threshold:${projectId2 ?? "global"}`;
|
|
29106
|
+
setKV(key, JSON.stringify({ threshold, sampleSize, calibratedAt: Date.now() }));
|
|
29107
|
+
}
|
|
29108
|
+
function loadCalibratedThreshold(projectId2) {
|
|
29109
|
+
const key = `dedup_threshold:${projectId2 ?? "global"}`;
|
|
29110
|
+
const raw = getKV(key);
|
|
29111
|
+
if (!raw) return null;
|
|
29112
|
+
try {
|
|
29113
|
+
const parsed = JSON.parse(raw);
|
|
29114
|
+
return typeof parsed.threshold === "number" ? parsed.threshold : null;
|
|
29115
|
+
} catch {
|
|
29116
|
+
return null;
|
|
29117
|
+
}
|
|
28661
29118
|
}
|
|
28662
29119
|
|
|
28663
29120
|
// src/data.ts
|
|
@@ -29506,6 +29963,16 @@ function getSessionState(sessionID) {
|
|
|
29506
29963
|
if (!state) {
|
|
29507
29964
|
state = makeSessionState();
|
|
29508
29965
|
state.forceMinLayer = loadForceMinLayer(sessionID);
|
|
29966
|
+
const persisted = loadSessionTracking(sessionID);
|
|
29967
|
+
if (persisted && persisted.lastTurnAt > 0) {
|
|
29968
|
+
state.dynamicContextCap = persisted.dynamicContextCap;
|
|
29969
|
+
state.bustRateEMA = persisted.bustRateEMA;
|
|
29970
|
+
state.interBustIntervalEMA = persisted.interBustIntervalEMA;
|
|
29971
|
+
state.lastLayer = persisted.lastLayer;
|
|
29972
|
+
state.lastKnownInput = persisted.lastKnownInput;
|
|
29973
|
+
state.lastTurnAt = persisted.lastTurnAt;
|
|
29974
|
+
state.lastBustAt = persisted.lastBustAt;
|
|
29975
|
+
}
|
|
29509
29976
|
sessionStates.set(sessionID, state);
|
|
29510
29977
|
}
|
|
29511
29978
|
return state;
|
|
@@ -29611,6 +30078,19 @@ function inspectSessionState(sessionID) {
|
|
|
29611
30078
|
function setLastTurnAtForTest(sessionID, ms) {
|
|
29612
30079
|
getSessionState(sessionID).lastTurnAt = ms;
|
|
29613
30080
|
}
|
|
30081
|
+
function saveGradientState(sessionID) {
|
|
30082
|
+
const state = sessionStates.get(sessionID);
|
|
30083
|
+
if (!state) return;
|
|
30084
|
+
saveSessionTracking(sessionID, {
|
|
30085
|
+
dynamicContextCap: state.dynamicContextCap,
|
|
30086
|
+
bustRateEMA: state.bustRateEMA,
|
|
30087
|
+
interBustIntervalEMA: state.interBustIntervalEMA,
|
|
30088
|
+
lastLayer: state.lastLayer,
|
|
30089
|
+
lastKnownInput: state.lastKnownInput,
|
|
30090
|
+
lastTurnAt: state.lastTurnAt,
|
|
30091
|
+
lastBustAt: state.lastBustAt
|
|
30092
|
+
});
|
|
30093
|
+
}
|
|
29614
30094
|
function loadDistillations(projectPath, sessionID) {
|
|
29615
30095
|
const pid = ensureProject(projectPath);
|
|
29616
30096
|
const query = sessionID ? "SELECT id, observations, generation, token_count, created_at, session_id FROM distillations WHERE project_id = ? AND session_id = ? AND archived = 0 ORDER BY created_at ASC" : "SELECT id, observations, generation, token_count, created_at, session_id FROM distillations WHERE project_id = ? AND archived = 0 ORDER BY created_at ASC";
|
|
@@ -29895,6 +30375,26 @@ function buildPrefixMessages(formatted) {
|
|
|
29895
30375
|
}
|
|
29896
30376
|
];
|
|
29897
30377
|
}
|
|
30378
|
+
var DECISION_RE = /\b(?:decision|decided|chose|chosen|agreed)\b/i;
|
|
30379
|
+
var GOTCHA_RE = /\b(?:gotcha|(?:critical|known|subtle)\s+bug|broken|crash(?:ed|es)?|regression)\b/i;
|
|
30380
|
+
var ARCH_RE = /\b(?:architecture|design.(?:decision|pattern)|system.design)\b/i;
|
|
30381
|
+
function importanceBonus(d) {
|
|
30382
|
+
let bonus = 0;
|
|
30383
|
+
if (DECISION_RE.test(d.observations)) bonus += 0.3;
|
|
30384
|
+
if (GOTCHA_RE.test(d.observations)) bonus += 0.2;
|
|
30385
|
+
if (ARCH_RE.test(d.observations)) bonus += 0.1;
|
|
30386
|
+
if (d.generation >= 1) bonus += 0.2;
|
|
30387
|
+
return Math.min(bonus, 1);
|
|
30388
|
+
}
|
|
30389
|
+
function selectDistillations(all3, limit) {
|
|
30390
|
+
if (all3.length <= limit) return all3;
|
|
30391
|
+
const maxIdx = all3.length - 1;
|
|
30392
|
+
const scored = all3.map((d, i) => ({
|
|
30393
|
+
d,
|
|
30394
|
+
score: (maxIdx > 0 ? i / maxIdx : 1) * 0.7 + importanceBonus(d) * 0.3
|
|
30395
|
+
}));
|
|
30396
|
+
return scored.sort((a, b) => b.score - a.score).slice(0, limit).map((s) => s.d).sort((a, b) => a.created_at - b.created_at);
|
|
30397
|
+
}
|
|
29898
30398
|
function distilledPrefix(distillations) {
|
|
29899
30399
|
if (!distillations.length) return [];
|
|
29900
30400
|
const formatted = formatDistillations(distillations);
|
|
@@ -30012,6 +30512,11 @@ function tryFitStable(input) {
|
|
|
30012
30512
|
}
|
|
30013
30513
|
return result;
|
|
30014
30514
|
}
|
|
30515
|
+
var COMPRESSION_STAGES = [
|
|
30516
|
+
{ strip: "none", rawFrac: null, distFrac: null, distLimit: Infinity, protectedTurns: 0, useStableWindow: true },
|
|
30517
|
+
{ strip: "old-tools", rawFrac: 0.5, distFrac: null, distLimit: Infinity, protectedTurns: 2, useStableWindow: false },
|
|
30518
|
+
{ strip: "all-tools", rawFrac: 0.55, distFrac: 0.15, distLimit: 5, protectedTurns: 0, useStableWindow: false }
|
|
30519
|
+
];
|
|
30015
30520
|
var urgentDistillationMap = /* @__PURE__ */ new Map();
|
|
30016
30521
|
function needsUrgentDistillation(sessionID) {
|
|
30017
30522
|
const v = urgentDistillationMap.get(sessionID) ?? false;
|
|
@@ -30043,7 +30548,7 @@ function transformInner(input) {
|
|
|
30043
30548
|
if (calibrated) return true;
|
|
30044
30549
|
return result.totalTokens * UNCALIBRATED_SAFETY <= maxInput;
|
|
30045
30550
|
}
|
|
30046
|
-
if (calibrated && sessState.lastLayer >= 1 && input.messages.length >= sessState.lastKnownMessageCount) {
|
|
30551
|
+
if (calibrated && sessState.lastLayer >= 1 && sessState.lastLayer <= 3 && input.messages.length >= sessState.lastKnownMessageCount) {
|
|
30047
30552
|
effectiveMinLayer = Math.max(effectiveMinLayer, sessState.lastLayer);
|
|
30048
30553
|
}
|
|
30049
30554
|
const postIdleCompact = sessState.postIdleCompact;
|
|
@@ -30081,7 +30586,8 @@ function transformInner(input) {
|
|
|
30081
30586
|
totalTokens: Math.max(0, messageTokens),
|
|
30082
30587
|
usable,
|
|
30083
30588
|
distilledBudget,
|
|
30084
|
-
rawBudget
|
|
30589
|
+
rawBudget,
|
|
30590
|
+
refreshLtm: false
|
|
30085
30591
|
};
|
|
30086
30592
|
}
|
|
30087
30593
|
const turnStart = currentTurnStart(input.messages);
|
|
@@ -30091,67 +30597,52 @@ function transformInner(input) {
|
|
|
30091
30597
|
const msgs = distilledPrefix(distillations);
|
|
30092
30598
|
return { messages: msgs, tokens: msgs.reduce((sum, m) => sum + estimateMessage(m), 0) };
|
|
30093
30599
|
})();
|
|
30094
|
-
|
|
30095
|
-
const
|
|
30096
|
-
|
|
30097
|
-
|
|
30098
|
-
|
|
30099
|
-
|
|
30100
|
-
|
|
30101
|
-
|
|
30102
|
-
|
|
30103
|
-
|
|
30104
|
-
|
|
30105
|
-
|
|
30106
|
-
|
|
30107
|
-
|
|
30108
|
-
|
|
30109
|
-
|
|
30110
|
-
|
|
30111
|
-
|
|
30112
|
-
|
|
30600
|
+
for (let s = 0; s < COMPRESSION_STAGES.length; s++) {
|
|
30601
|
+
const stageLayer = s + 1;
|
|
30602
|
+
if (effectiveMinLayer > stageLayer) continue;
|
|
30603
|
+
const stage = COMPRESSION_STAGES[s];
|
|
30604
|
+
const stageRawBudget = stage.rawFrac !== null ? Math.floor(usable * stage.rawFrac) : rawBudget;
|
|
30605
|
+
const stageDistBudget = stage.distFrac !== null ? Math.floor(usable * stage.distFrac) : distilledBudget;
|
|
30606
|
+
let stagePrefix = cached2.messages;
|
|
30607
|
+
let stagePrefixTokens = cached2.tokens;
|
|
30608
|
+
if (stage.distLimit !== Infinity && distillations.length > stage.distLimit) {
|
|
30609
|
+
const trimmed = selectDistillations(distillations, stage.distLimit);
|
|
30610
|
+
stagePrefix = distilledPrefix(trimmed);
|
|
30611
|
+
stagePrefixTokens = stagePrefix.reduce((sum, m) => sum + estimateMessage(m), 0);
|
|
30612
|
+
}
|
|
30613
|
+
let result;
|
|
30614
|
+
if (stage.useStableWindow && sid) {
|
|
30615
|
+
result = tryFitStable({
|
|
30616
|
+
messages: dedupMessages,
|
|
30617
|
+
prefix: stagePrefix,
|
|
30618
|
+
prefixTokens: stagePrefixTokens,
|
|
30619
|
+
distilledBudget: stageDistBudget,
|
|
30620
|
+
rawBudget: stageRawBudget,
|
|
30621
|
+
sessionID: sid,
|
|
30622
|
+
sessState
|
|
30623
|
+
});
|
|
30624
|
+
} else {
|
|
30625
|
+
sessState.rawWindowCache = null;
|
|
30626
|
+
result = tryFit({
|
|
30627
|
+
messages: dedupMessages,
|
|
30628
|
+
prefix: stagePrefix,
|
|
30629
|
+
prefixTokens: stagePrefixTokens,
|
|
30630
|
+
distilledBudget: stageDistBudget,
|
|
30631
|
+
rawBudget: stageRawBudget,
|
|
30632
|
+
strip: stage.strip,
|
|
30633
|
+
protectedTurns: stage.protectedTurns
|
|
30634
|
+
});
|
|
30635
|
+
}
|
|
30636
|
+
if (fitsWithSafetyMargin(result)) {
|
|
30637
|
+
if (sid && (s > 0 || cached2.tokens === 0)) {
|
|
30113
30638
|
urgentDistillationMap.set(sid, true);
|
|
30114
30639
|
}
|
|
30115
|
-
return { ...
|
|
30640
|
+
return { ...result, layer: stageLayer, usable, distilledBudget, rawBudget, refreshLtm: false };
|
|
30116
30641
|
}
|
|
30117
30642
|
}
|
|
30118
30643
|
sessState.rawWindowCache = null;
|
|
30119
|
-
if (effectiveMinLayer <= 2) {
|
|
30120
|
-
const layer2 = tryFit({
|
|
30121
|
-
messages: dedupMessages,
|
|
30122
|
-
prefix: cached2.messages,
|
|
30123
|
-
prefixTokens: cached2.tokens,
|
|
30124
|
-
distilledBudget,
|
|
30125
|
-
rawBudget: Math.floor(usable * 0.5),
|
|
30126
|
-
// give raw more room
|
|
30127
|
-
strip: "old-tools",
|
|
30128
|
-
protectedTurns: 2
|
|
30129
|
-
});
|
|
30130
|
-
if (fitsWithSafetyMargin(layer2)) {
|
|
30131
|
-
if (sid) urgentDistillationMap.set(sid, true);
|
|
30132
|
-
return { ...layer2, layer: 2, usable, distilledBudget, rawBudget };
|
|
30133
|
-
}
|
|
30134
|
-
}
|
|
30135
|
-
const trimmedDistillations = distillations.slice(-5);
|
|
30136
|
-
const trimmedPrefix = distilledPrefix(trimmedDistillations);
|
|
30137
|
-
const trimmedPrefixTokens = trimmedPrefix.reduce(
|
|
30138
|
-
(sum, m) => sum + estimateMessage(m),
|
|
30139
|
-
0
|
|
30140
|
-
);
|
|
30141
|
-
const layer3 = tryFit({
|
|
30142
|
-
messages: dedupMessages,
|
|
30143
|
-
prefix: trimmedPrefix,
|
|
30144
|
-
prefixTokens: trimmedPrefixTokens,
|
|
30145
|
-
distilledBudget: Math.floor(usable * 0.15),
|
|
30146
|
-
rawBudget: Math.floor(usable * 0.55),
|
|
30147
|
-
strip: "all-tools"
|
|
30148
|
-
});
|
|
30149
|
-
if (fitsWithSafetyMargin(layer3)) {
|
|
30150
|
-
if (sid) urgentDistillationMap.set(sid, true);
|
|
30151
|
-
return { ...layer3, layer: 3, usable, distilledBudget, rawBudget };
|
|
30152
|
-
}
|
|
30153
30644
|
if (sid) urgentDistillationMap.set(sid, true);
|
|
30154
|
-
const nuclearDistillations = distillations
|
|
30645
|
+
const nuclearDistillations = selectDistillations(distillations, 2);
|
|
30155
30646
|
const nuclearPrefix = distilledPrefix(nuclearDistillations);
|
|
30156
30647
|
const nuclearPrefixTokens = nuclearPrefix.reduce(
|
|
30157
30648
|
(sum, m) => sum + estimateMessage(m),
|
|
@@ -30190,7 +30681,8 @@ function transformInner(input) {
|
|
|
30190
30681
|
totalTokens: nuclearPrefixTokens + nuclearRawTokens,
|
|
30191
30682
|
usable,
|
|
30192
30683
|
distilledBudget,
|
|
30193
|
-
rawBudget
|
|
30684
|
+
rawBudget,
|
|
30685
|
+
refreshLtm: true
|
|
30194
30686
|
};
|
|
30195
30687
|
}
|
|
30196
30688
|
function transform2(input) {
|
|
@@ -30297,6 +30789,185 @@ function isWorkerSession(sessionID) {
|
|
|
30297
30789
|
return workerSessionIDs.has(sessionID);
|
|
30298
30790
|
}
|
|
30299
30791
|
|
|
30792
|
+
// ../../node_modules/.bun/yocto-queue@1.2.2/node_modules/yocto-queue/index.js
|
|
30793
|
+
var Node = class {
|
|
30794
|
+
value;
|
|
30795
|
+
next;
|
|
30796
|
+
constructor(value) {
|
|
30797
|
+
this.value = value;
|
|
30798
|
+
}
|
|
30799
|
+
};
|
|
30800
|
+
var Queue = class {
|
|
30801
|
+
#head;
|
|
30802
|
+
#tail;
|
|
30803
|
+
#size;
|
|
30804
|
+
constructor() {
|
|
30805
|
+
this.clear();
|
|
30806
|
+
}
|
|
30807
|
+
enqueue(value) {
|
|
30808
|
+
const node2 = new Node(value);
|
|
30809
|
+
if (this.#head) {
|
|
30810
|
+
this.#tail.next = node2;
|
|
30811
|
+
this.#tail = node2;
|
|
30812
|
+
} else {
|
|
30813
|
+
this.#head = node2;
|
|
30814
|
+
this.#tail = node2;
|
|
30815
|
+
}
|
|
30816
|
+
this.#size++;
|
|
30817
|
+
}
|
|
30818
|
+
dequeue() {
|
|
30819
|
+
const current2 = this.#head;
|
|
30820
|
+
if (!current2) {
|
|
30821
|
+
return;
|
|
30822
|
+
}
|
|
30823
|
+
this.#head = this.#head.next;
|
|
30824
|
+
this.#size--;
|
|
30825
|
+
if (!this.#head) {
|
|
30826
|
+
this.#tail = void 0;
|
|
30827
|
+
}
|
|
30828
|
+
return current2.value;
|
|
30829
|
+
}
|
|
30830
|
+
peek() {
|
|
30831
|
+
if (!this.#head) {
|
|
30832
|
+
return;
|
|
30833
|
+
}
|
|
30834
|
+
return this.#head.value;
|
|
30835
|
+
}
|
|
30836
|
+
clear() {
|
|
30837
|
+
this.#head = void 0;
|
|
30838
|
+
this.#tail = void 0;
|
|
30839
|
+
this.#size = 0;
|
|
30840
|
+
}
|
|
30841
|
+
get size() {
|
|
30842
|
+
return this.#size;
|
|
30843
|
+
}
|
|
30844
|
+
*[Symbol.iterator]() {
|
|
30845
|
+
let current2 = this.#head;
|
|
30846
|
+
while (current2) {
|
|
30847
|
+
yield current2.value;
|
|
30848
|
+
current2 = current2.next;
|
|
30849
|
+
}
|
|
30850
|
+
}
|
|
30851
|
+
*drain() {
|
|
30852
|
+
while (this.#head) {
|
|
30853
|
+
yield this.dequeue();
|
|
30854
|
+
}
|
|
30855
|
+
}
|
|
30856
|
+
};
|
|
30857
|
+
|
|
30858
|
+
// ../../node_modules/.bun/p-limit@7.3.0/node_modules/p-limit/index.js
|
|
30859
|
+
function pLimit(concurrency) {
|
|
30860
|
+
let rejectOnClear = false;
|
|
30861
|
+
if (typeof concurrency === "object") {
|
|
30862
|
+
({ concurrency, rejectOnClear = false } = concurrency);
|
|
30863
|
+
}
|
|
30864
|
+
validateConcurrency(concurrency);
|
|
30865
|
+
if (typeof rejectOnClear !== "boolean") {
|
|
30866
|
+
throw new TypeError("Expected `rejectOnClear` to be a boolean");
|
|
30867
|
+
}
|
|
30868
|
+
const queue = new Queue();
|
|
30869
|
+
let activeCount = 0;
|
|
30870
|
+
const resumeNext = () => {
|
|
30871
|
+
if (activeCount < concurrency && queue.size > 0) {
|
|
30872
|
+
activeCount++;
|
|
30873
|
+
queue.dequeue().run();
|
|
30874
|
+
}
|
|
30875
|
+
};
|
|
30876
|
+
const next = () => {
|
|
30877
|
+
activeCount--;
|
|
30878
|
+
resumeNext();
|
|
30879
|
+
};
|
|
30880
|
+
const run3 = async (function_, resolve, arguments_) => {
|
|
30881
|
+
const result = (async () => function_(...arguments_))();
|
|
30882
|
+
resolve(result);
|
|
30883
|
+
try {
|
|
30884
|
+
await result;
|
|
30885
|
+
} catch {
|
|
30886
|
+
}
|
|
30887
|
+
next();
|
|
30888
|
+
};
|
|
30889
|
+
const enqueue = (function_, resolve, reject, arguments_) => {
|
|
30890
|
+
const queueItem = { reject };
|
|
30891
|
+
new Promise((internalResolve) => {
|
|
30892
|
+
queueItem.run = internalResolve;
|
|
30893
|
+
queue.enqueue(queueItem);
|
|
30894
|
+
}).then(run3.bind(void 0, function_, resolve, arguments_));
|
|
30895
|
+
if (activeCount < concurrency) {
|
|
30896
|
+
resumeNext();
|
|
30897
|
+
}
|
|
30898
|
+
};
|
|
30899
|
+
const generator = (function_, ...arguments_) => new Promise((resolve, reject) => {
|
|
30900
|
+
enqueue(function_, resolve, reject, arguments_);
|
|
30901
|
+
});
|
|
30902
|
+
Object.defineProperties(generator, {
|
|
30903
|
+
activeCount: {
|
|
30904
|
+
get: () => activeCount
|
|
30905
|
+
},
|
|
30906
|
+
pendingCount: {
|
|
30907
|
+
get: () => queue.size
|
|
30908
|
+
},
|
|
30909
|
+
clearQueue: {
|
|
30910
|
+
value() {
|
|
30911
|
+
if (!rejectOnClear) {
|
|
30912
|
+
queue.clear();
|
|
30913
|
+
return;
|
|
30914
|
+
}
|
|
30915
|
+
const abortError = AbortSignal.abort().reason;
|
|
30916
|
+
while (queue.size > 0) {
|
|
30917
|
+
queue.dequeue().reject(abortError);
|
|
30918
|
+
}
|
|
30919
|
+
}
|
|
30920
|
+
},
|
|
30921
|
+
concurrency: {
|
|
30922
|
+
get: () => concurrency,
|
|
30923
|
+
set(newConcurrency) {
|
|
30924
|
+
validateConcurrency(newConcurrency);
|
|
30925
|
+
concurrency = newConcurrency;
|
|
30926
|
+
queueMicrotask(() => {
|
|
30927
|
+
while (activeCount < concurrency && queue.size > 0) {
|
|
30928
|
+
resumeNext();
|
|
30929
|
+
}
|
|
30930
|
+
});
|
|
30931
|
+
}
|
|
30932
|
+
},
|
|
30933
|
+
map: {
|
|
30934
|
+
async value(iterable, function_) {
|
|
30935
|
+
const promises = Array.from(iterable, (value, index2) => this(function_, value, index2));
|
|
30936
|
+
return Promise.all(promises);
|
|
30937
|
+
}
|
|
30938
|
+
}
|
|
30939
|
+
});
|
|
30940
|
+
return generator;
|
|
30941
|
+
}
|
|
30942
|
+
function validateConcurrency(concurrency) {
|
|
30943
|
+
if (!((Number.isInteger(concurrency) || concurrency === Number.POSITIVE_INFINITY) && concurrency > 0)) {
|
|
30944
|
+
throw new TypeError("Expected `concurrency` to be a number from 1 and up");
|
|
30945
|
+
}
|
|
30946
|
+
}
|
|
30947
|
+
|
|
30948
|
+
// src/session-limiter.ts
|
|
30949
|
+
function createLimiterPool() {
|
|
30950
|
+
const limiters = /* @__PURE__ */ new Map();
|
|
30951
|
+
function get2(key) {
|
|
30952
|
+
let limiter = limiters.get(key);
|
|
30953
|
+
if (!limiter) {
|
|
30954
|
+
limiter = pLimit(1);
|
|
30955
|
+
limiters.set(key, limiter);
|
|
30956
|
+
}
|
|
30957
|
+
return limiter;
|
|
30958
|
+
}
|
|
30959
|
+
function isBusy(key) {
|
|
30960
|
+
const limiter = limiters.get(key);
|
|
30961
|
+
return limiter ? limiter.activeCount + limiter.pendingCount > 0 : false;
|
|
30962
|
+
}
|
|
30963
|
+
function clear() {
|
|
30964
|
+
limiters.clear();
|
|
30965
|
+
}
|
|
30966
|
+
return { get: get2, isBusy, clear };
|
|
30967
|
+
}
|
|
30968
|
+
var distillLimiter = createLimiterPool();
|
|
30969
|
+
var curatorLimiter = createLimiterPool();
|
|
30970
|
+
|
|
30300
30971
|
// src/distillation.ts
|
|
30301
30972
|
function compressionRatio(distilledTokens, sourceTokens) {
|
|
30302
30973
|
if (sourceTokens <= 0) return 0;
|
|
@@ -30541,6 +31212,9 @@ function resetOrphans(projectPath, sessionID) {
|
|
|
30541
31212
|
return orphans.length;
|
|
30542
31213
|
}
|
|
30543
31214
|
async function run(input) {
|
|
31215
|
+
return distillLimiter.get(input.sessionID)(() => runInner(input));
|
|
31216
|
+
}
|
|
31217
|
+
async function runInner(input) {
|
|
30544
31218
|
const orphans = resetOrphans(input.projectPath, input.sessionID);
|
|
30545
31219
|
if (orphans > 0) {
|
|
30546
31220
|
info(
|
|
@@ -30584,7 +31258,7 @@ async function run(input) {
|
|
|
30584
31258
|
}
|
|
30585
31259
|
}
|
|
30586
31260
|
if (!input.skipMeta && gen0Count(input.projectPath, input.sessionID) >= cfg.distillation.metaThreshold) {
|
|
30587
|
-
await
|
|
31261
|
+
await metaDistillInner({
|
|
30588
31262
|
llm: input.llm,
|
|
30589
31263
|
projectPath: input.projectPath,
|
|
30590
31264
|
sessionID: input.sessionID,
|
|
@@ -30634,17 +31308,25 @@ async function distillSegment(input) {
|
|
|
30634
31308
|
);
|
|
30635
31309
|
return null;
|
|
30636
31310
|
}
|
|
30637
|
-
|
|
30638
|
-
|
|
30639
|
-
|
|
30640
|
-
|
|
30641
|
-
|
|
30642
|
-
|
|
30643
|
-
|
|
30644
|
-
|
|
30645
|
-
|
|
30646
|
-
|
|
30647
|
-
|
|
31311
|
+
let distillId;
|
|
31312
|
+
db().exec("BEGIN IMMEDIATE");
|
|
31313
|
+
try {
|
|
31314
|
+
distillId = storeDistillation({
|
|
31315
|
+
projectPath: input.projectPath,
|
|
31316
|
+
sessionID: input.sessionID,
|
|
31317
|
+
observations: result.observations,
|
|
31318
|
+
sourceIDs: input.messages.map((m) => m.id),
|
|
31319
|
+
generation: 0,
|
|
31320
|
+
rCompression: rComp,
|
|
31321
|
+
cNorm,
|
|
31322
|
+
callType: input.callType
|
|
31323
|
+
});
|
|
31324
|
+
markDistilled(input.messages.map((m) => m.id));
|
|
31325
|
+
db().exec("COMMIT");
|
|
31326
|
+
} catch (e) {
|
|
31327
|
+
db().exec("ROLLBACK");
|
|
31328
|
+
throw e;
|
|
31329
|
+
}
|
|
30648
31330
|
info(
|
|
30649
31331
|
`distill segment: ${input.messages.length} msgs, ${sourceTokens}\u2192${distilledTokens} tokens, R=${rComp.toFixed(2)}, C_norm=${cNorm.toFixed(3)}`
|
|
30650
31332
|
);
|
|
@@ -30678,6 +31360,9 @@ async function distillSegment(input) {
|
|
|
30678
31360
|
return result;
|
|
30679
31361
|
}
|
|
30680
31362
|
async function metaDistill(input) {
|
|
31363
|
+
return distillLimiter.get(input.sessionID)(() => metaDistillInner(input));
|
|
31364
|
+
}
|
|
31365
|
+
async function metaDistillInner(input) {
|
|
30681
31366
|
const existing = loadGen0(input.projectPath, input.sessionID);
|
|
30682
31367
|
const priorMeta = latestMeta(input.projectPath, input.sessionID);
|
|
30683
31368
|
if (priorMeta) {
|
|
@@ -30977,11 +31662,27 @@ function applyOps(ops, input) {
|
|
|
30977
31662
|
return { created, updated, deleted };
|
|
30978
31663
|
}
|
|
30979
31664
|
var lastCuratedAt = /* @__PURE__ */ new Map();
|
|
31665
|
+
function getLastCuratedAt(sessionID) {
|
|
31666
|
+
const cached2 = lastCuratedAt.get(sessionID);
|
|
31667
|
+
if (cached2 !== void 0) return cached2;
|
|
31668
|
+
const persisted = loadSessionTracking(sessionID);
|
|
31669
|
+
const ts = persisted?.lastCuratedAt ?? 0;
|
|
31670
|
+
lastCuratedAt.set(sessionID, ts);
|
|
31671
|
+
return ts;
|
|
31672
|
+
}
|
|
30980
31673
|
async function run2(input) {
|
|
30981
31674
|
const cfg = config2();
|
|
30982
31675
|
if (!cfg.curator.enabled) return { created: 0, updated: 0, deleted: 0 };
|
|
31676
|
+
if (curatorLimiter.isBusy(input.sessionID)) {
|
|
31677
|
+
info(`curation skipped: already running for session ${input.sessionID.slice(0, 16)}`);
|
|
31678
|
+
return { created: 0, updated: 0, deleted: 0 };
|
|
31679
|
+
}
|
|
31680
|
+
return curatorLimiter.get(input.sessionID)(() => runInner2(input));
|
|
31681
|
+
}
|
|
31682
|
+
async function runInner2(input) {
|
|
31683
|
+
const cfg = config2();
|
|
30983
31684
|
const all3 = bySession(input.projectPath, input.sessionID);
|
|
30984
|
-
const sessionCuratedAt =
|
|
31685
|
+
const sessionCuratedAt = getLastCuratedAt(input.sessionID);
|
|
30985
31686
|
const recent = all3.filter((m) => m.created_at > sessionCuratedAt);
|
|
30986
31687
|
if (recent.length < 3) return { created: 0, updated: 0, deleted: 0 };
|
|
30987
31688
|
const text4 = recent.map((m) => `[${m.role}] ${m.content}`).join("\n\n");
|
|
@@ -31025,11 +31726,22 @@ async function run2(input) {
|
|
|
31025
31726
|
info(`post-curation dedup: merged ${dupes.totalRemoved} duplicate entries`);
|
|
31026
31727
|
result.deleted += dupes.totalRemoved;
|
|
31027
31728
|
}
|
|
31729
|
+
if (dupes.pairSimilarities.size > 0) {
|
|
31730
|
+
const pid = ensureProject(input.projectPath);
|
|
31731
|
+
recordAutoSignals(pid, dupes);
|
|
31732
|
+
const newThreshold = calibrateDedupThreshold(pid);
|
|
31733
|
+
if (newThreshold !== null) {
|
|
31734
|
+
const count3 = getDedupFeedbackCount(pid);
|
|
31735
|
+
saveCalibratedThreshold(pid, newThreshold, count3);
|
|
31736
|
+
}
|
|
31737
|
+
}
|
|
31028
31738
|
} catch (err) {
|
|
31029
31739
|
warn("post-curation dedup failed (non-fatal):", err);
|
|
31030
31740
|
}
|
|
31031
31741
|
}
|
|
31032
|
-
|
|
31742
|
+
const now = Date.now();
|
|
31743
|
+
lastCuratedAt.set(input.sessionID, now);
|
|
31744
|
+
saveSessionTracking(input.sessionID, { lastCuratedAt: now });
|
|
31033
31745
|
return result;
|
|
31034
31746
|
}
|
|
31035
31747
|
function resetCurationTracker(sessionID) {
|
|
@@ -33152,6 +33864,7 @@ export {
|
|
|
33152
33864
|
ftsQueryRelaxed,
|
|
33153
33865
|
getGitRemote,
|
|
33154
33866
|
getInstanceId,
|
|
33867
|
+
getKV,
|
|
33155
33868
|
getLastImportAt,
|
|
33156
33869
|
getLastTransformEstimate,
|
|
33157
33870
|
getLastTransformedCount,
|
|
@@ -33177,7 +33890,9 @@ export {
|
|
|
33177
33890
|
load,
|
|
33178
33891
|
loadAllSessionCosts,
|
|
33179
33892
|
loadForceMinLayer,
|
|
33893
|
+
loadHeaderSessionIndex,
|
|
33180
33894
|
loadSessionCosts,
|
|
33895
|
+
loadSessionTracking,
|
|
33181
33896
|
log_exports as log,
|
|
33182
33897
|
loreFileExists,
|
|
33183
33898
|
ltm_exports as ltm,
|
|
@@ -33198,10 +33913,13 @@ export {
|
|
|
33198
33913
|
runRecall,
|
|
33199
33914
|
sanitizeSurrogates,
|
|
33200
33915
|
saveForceMinLayer,
|
|
33916
|
+
saveGradientState,
|
|
33201
33917
|
saveSessionCosts,
|
|
33918
|
+
saveSessionTracking,
|
|
33202
33919
|
searchRecall,
|
|
33203
33920
|
serialize,
|
|
33204
33921
|
setForceMinLayer,
|
|
33922
|
+
setKV,
|
|
33205
33923
|
setLastImportAt,
|
|
33206
33924
|
setLastTurnAtForTest,
|
|
33207
33925
|
setLtmTokens,
|