@loreai/core 0.18.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bun/curator.d.ts.map +1 -1
- package/dist/bun/db.d.ts +73 -0
- package/dist/bun/db.d.ts.map +1 -1
- package/dist/bun/distillation.d.ts +2 -13
- package/dist/bun/distillation.d.ts.map +1 -1
- package/dist/bun/embedding.d.ts +5 -1
- package/dist/bun/embedding.d.ts.map +1 -1
- package/dist/bun/gradient.d.ts +9 -0
- package/dist/bun/gradient.d.ts.map +1 -1
- package/dist/bun/index.d.ts +2 -2
- package/dist/bun/index.d.ts.map +1 -1
- package/dist/bun/index.js +817 -99
- package/dist/bun/index.js.map +4 -4
- package/dist/bun/ltm.d.ts +99 -5
- package/dist/bun/ltm.d.ts.map +1 -1
- package/dist/bun/session-limiter.d.ts +26 -0
- package/dist/bun/session-limiter.d.ts.map +1 -0
- package/dist/bun/temporal.d.ts +2 -0
- package/dist/bun/temporal.d.ts.map +1 -1
- package/dist/node/curator.d.ts.map +1 -1
- package/dist/node/db.d.ts +73 -0
- package/dist/node/db.d.ts.map +1 -1
- package/dist/node/distillation.d.ts +2 -13
- package/dist/node/distillation.d.ts.map +1 -1
- package/dist/node/embedding.d.ts +5 -1
- package/dist/node/embedding.d.ts.map +1 -1
- package/dist/node/gradient.d.ts +9 -0
- package/dist/node/gradient.d.ts.map +1 -1
- package/dist/node/index.d.ts +2 -2
- package/dist/node/index.d.ts.map +1 -1
- package/dist/node/index.js +817 -99
- package/dist/node/index.js.map +4 -4
- package/dist/node/ltm.d.ts +99 -5
- package/dist/node/ltm.d.ts.map +1 -1
- package/dist/node/session-limiter.d.ts +26 -0
- package/dist/node/session-limiter.d.ts.map +1 -0
- package/dist/node/temporal.d.ts +2 -0
- package/dist/node/temporal.d.ts.map +1 -1
- package/dist/types/curator.d.ts.map +1 -1
- package/dist/types/db.d.ts +73 -0
- package/dist/types/db.d.ts.map +1 -1
- package/dist/types/distillation.d.ts +2 -13
- package/dist/types/distillation.d.ts.map +1 -1
- package/dist/types/embedding.d.ts +5 -1
- package/dist/types/embedding.d.ts.map +1 -1
- package/dist/types/gradient.d.ts +9 -0
- package/dist/types/gradient.d.ts.map +1 -1
- package/dist/types/index.d.ts +2 -2
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/ltm.d.ts +99 -5
- package/dist/types/ltm.d.ts.map +1 -1
- package/dist/types/session-limiter.d.ts +26 -0
- package/dist/types/session-limiter.d.ts.map +1 -0
- package/dist/types/temporal.d.ts +2 -0
- package/dist/types/temporal.d.ts.map +1 -1
- package/package.json +2 -1
- package/src/curator.ts +54 -2
- package/src/db.ts +347 -0
- package/src/distillation.ts +55 -14
- package/src/embedding.ts +28 -3
- package/src/gradient.ts +183 -74
- package/src/index.ts +8 -0
- package/src/ltm.ts +480 -45
- package/src/session-limiter.ts +47 -0
- package/src/temporal.ts +10 -0
package/dist/node/index.js
CHANGED
|
@@ -125,6 +125,7 @@ __export(temporal_exports, {
|
|
|
125
125
|
CHUNK_TERMINATOR: () => CHUNK_TERMINATOR,
|
|
126
126
|
bySession: () => bySession,
|
|
127
127
|
count: () => count,
|
|
128
|
+
hasMessages: () => hasMessages,
|
|
128
129
|
markDistilled: () => markDistilled,
|
|
129
130
|
partsToText: () => partsToText,
|
|
130
131
|
prune: () => prune,
|
|
@@ -754,6 +755,55 @@ var MIGRATIONS = [
|
|
|
754
755
|
WHERE ih.project_id = projects.id
|
|
755
756
|
AND ih.source_id = '__declined__'
|
|
756
757
|
);
|
|
758
|
+
`,
|
|
759
|
+
`
|
|
760
|
+
-- Version 23: Persist volatile session tracking state across restarts.
|
|
761
|
+
-- Previously these were in-memory only, causing duplicate processing,
|
|
762
|
+
-- false compaction detection, and expensive prompt cache busts on restart.
|
|
763
|
+
ALTER TABLE session_state ADD COLUMN last_curated_at INTEGER NOT NULL DEFAULT 0;
|
|
764
|
+
ALTER TABLE session_state ADD COLUMN message_count INTEGER NOT NULL DEFAULT 0;
|
|
765
|
+
ALTER TABLE session_state ADD COLUMN turns_since_curation INTEGER NOT NULL DEFAULT 0;
|
|
766
|
+
ALTER TABLE session_state ADD COLUMN ltm_cache_text TEXT;
|
|
767
|
+
ALTER TABLE session_state ADD COLUMN ltm_cache_tokens INTEGER;
|
|
768
|
+
ALTER TABLE session_state ADD COLUMN ltm_pin_text TEXT;
|
|
769
|
+
ALTER TABLE session_state ADD COLUMN ltm_pin_tokens INTEGER;
|
|
770
|
+
ALTER TABLE session_state ADD COLUMN consecutive_text_only_turns INTEGER NOT NULL DEFAULT 0;
|
|
771
|
+
`,
|
|
772
|
+
`
|
|
773
|
+
-- Version 24: Persist remaining volatile session state across restarts.
|
|
774
|
+
-- Session identity (Tier 1/2/3 session correlation)
|
|
775
|
+
ALTER TABLE session_state ADD COLUMN fingerprint TEXT NOT NULL DEFAULT '';
|
|
776
|
+
ALTER TABLE session_state ADD COLUMN header_session_id TEXT;
|
|
777
|
+
ALTER TABLE session_state ADD COLUMN header_name TEXT;
|
|
778
|
+
-- Cache warming state
|
|
779
|
+
ALTER TABLE session_state ADD COLUMN resolved_conversation_ttl TEXT NOT NULL DEFAULT '5m';
|
|
780
|
+
ALTER TABLE session_state ADD COLUMN warmup_state TEXT;
|
|
781
|
+
-- Gradient calibration state (survives restarts to avoid uncalibrated busts)
|
|
782
|
+
ALTER TABLE session_state ADD COLUMN dynamic_context_cap REAL NOT NULL DEFAULT 0;
|
|
783
|
+
ALTER TABLE session_state ADD COLUMN bust_rate_ema REAL NOT NULL DEFAULT -1;
|
|
784
|
+
ALTER TABLE session_state ADD COLUMN inter_bust_interval_ema REAL NOT NULL DEFAULT -1;
|
|
785
|
+
ALTER TABLE session_state ADD COLUMN last_layer INTEGER NOT NULL DEFAULT 0;
|
|
786
|
+
ALTER TABLE session_state ADD COLUMN last_known_input INTEGER NOT NULL DEFAULT 0;
|
|
787
|
+
ALTER TABLE session_state ADD COLUMN last_turn_at INTEGER NOT NULL DEFAULT 0;
|
|
788
|
+
ALTER TABLE session_state ADD COLUMN last_bust_at INTEGER NOT NULL DEFAULT 0;
|
|
789
|
+
`,
|
|
790
|
+
`
|
|
791
|
+
-- Version 25: Adaptive dedup threshold \u2014 store accept/reject feedback
|
|
792
|
+
-- on embedding-based duplicate pairs for per-project threshold calibration.
|
|
793
|
+
-- Titles stored instead of FK IDs because entries are deleted during dedup;
|
|
794
|
+
-- the similarity float is the actual calibration input.
|
|
795
|
+
CREATE TABLE IF NOT EXISTS dedup_feedback (
|
|
796
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
797
|
+
project_id TEXT,
|
|
798
|
+
entry_a_title TEXT NOT NULL,
|
|
799
|
+
entry_b_title TEXT NOT NULL,
|
|
800
|
+
similarity REAL NOT NULL,
|
|
801
|
+
accepted INTEGER NOT NULL,
|
|
802
|
+
source TEXT NOT NULL DEFAULT 'manual',
|
|
803
|
+
created_at INTEGER NOT NULL
|
|
804
|
+
);
|
|
805
|
+
CREATE INDEX IF NOT EXISTS idx_dedup_feedback_project
|
|
806
|
+
ON dedup_feedback(project_id);
|
|
757
807
|
`
|
|
758
808
|
];
|
|
759
809
|
function dbPath() {
|
|
@@ -890,6 +940,11 @@ function close() {
|
|
|
890
940
|
}
|
|
891
941
|
}
|
|
892
942
|
function ensureProject(path, name) {
|
|
943
|
+
if (!process.env.LORE_DB_PATH && /^\/test\//.test(path)) {
|
|
944
|
+
throw new Error(
|
|
945
|
+
`Refusing to create project with test path "${path}" in the production DB. Set LORE_DB_PATH to a temp path, or run tests via \`bun test\` from the repo root.`
|
|
946
|
+
);
|
|
947
|
+
}
|
|
893
948
|
const existing = db().query("SELECT id, git_remote FROM projects WHERE path = ?").get(path);
|
|
894
949
|
if (existing) {
|
|
895
950
|
if (!existing.git_remote) {
|
|
@@ -1058,6 +1113,153 @@ function loadAllSessionCosts() {
|
|
|
1058
1113
|
}
|
|
1059
1114
|
return result;
|
|
1060
1115
|
}
|
|
1116
|
+
function saveSessionTracking(sessionID, state) {
|
|
1117
|
+
const now = Date.now();
|
|
1118
|
+
db().query(
|
|
1119
|
+
"INSERT OR IGNORE INTO session_state (session_id, force_min_layer, updated_at) VALUES (?, 0, ?)"
|
|
1120
|
+
).run(sessionID, now);
|
|
1121
|
+
const sets = ["updated_at = ?"];
|
|
1122
|
+
const vals = [now];
|
|
1123
|
+
if (state.lastCuratedAt !== void 0) {
|
|
1124
|
+
sets.push("last_curated_at = ?");
|
|
1125
|
+
vals.push(state.lastCuratedAt);
|
|
1126
|
+
}
|
|
1127
|
+
if (state.messageCount !== void 0) {
|
|
1128
|
+
sets.push("message_count = ?");
|
|
1129
|
+
vals.push(state.messageCount);
|
|
1130
|
+
}
|
|
1131
|
+
if (state.turnsSinceCuration !== void 0) {
|
|
1132
|
+
sets.push("turns_since_curation = ?");
|
|
1133
|
+
vals.push(state.turnsSinceCuration);
|
|
1134
|
+
}
|
|
1135
|
+
if (state.consecutiveTextOnlyTurns !== void 0) {
|
|
1136
|
+
sets.push("consecutive_text_only_turns = ?");
|
|
1137
|
+
vals.push(state.consecutiveTextOnlyTurns);
|
|
1138
|
+
}
|
|
1139
|
+
if (state.ltmCacheText !== void 0) {
|
|
1140
|
+
sets.push("ltm_cache_text = ?");
|
|
1141
|
+
vals.push(state.ltmCacheText);
|
|
1142
|
+
}
|
|
1143
|
+
if (state.ltmCacheTokens !== void 0) {
|
|
1144
|
+
sets.push("ltm_cache_tokens = ?");
|
|
1145
|
+
vals.push(state.ltmCacheTokens);
|
|
1146
|
+
}
|
|
1147
|
+
if (state.ltmPinText !== void 0) {
|
|
1148
|
+
sets.push("ltm_pin_text = ?");
|
|
1149
|
+
vals.push(state.ltmPinText);
|
|
1150
|
+
}
|
|
1151
|
+
if (state.ltmPinTokens !== void 0) {
|
|
1152
|
+
sets.push("ltm_pin_tokens = ?");
|
|
1153
|
+
vals.push(state.ltmPinTokens);
|
|
1154
|
+
}
|
|
1155
|
+
if (state.fingerprint !== void 0) {
|
|
1156
|
+
sets.push("fingerprint = ?");
|
|
1157
|
+
vals.push(state.fingerprint);
|
|
1158
|
+
}
|
|
1159
|
+
if (state.headerSessionId !== void 0) {
|
|
1160
|
+
sets.push("header_session_id = ?");
|
|
1161
|
+
vals.push(state.headerSessionId);
|
|
1162
|
+
}
|
|
1163
|
+
if (state.headerName !== void 0) {
|
|
1164
|
+
sets.push("header_name = ?");
|
|
1165
|
+
vals.push(state.headerName);
|
|
1166
|
+
}
|
|
1167
|
+
if (state.resolvedConversationTTL !== void 0) {
|
|
1168
|
+
sets.push("resolved_conversation_ttl = ?");
|
|
1169
|
+
vals.push(state.resolvedConversationTTL);
|
|
1170
|
+
}
|
|
1171
|
+
if (state.warmupState !== void 0) {
|
|
1172
|
+
sets.push("warmup_state = ?");
|
|
1173
|
+
vals.push(state.warmupState);
|
|
1174
|
+
}
|
|
1175
|
+
if (state.dynamicContextCap !== void 0) {
|
|
1176
|
+
sets.push("dynamic_context_cap = ?");
|
|
1177
|
+
vals.push(state.dynamicContextCap);
|
|
1178
|
+
}
|
|
1179
|
+
if (state.bustRateEMA !== void 0) {
|
|
1180
|
+
sets.push("bust_rate_ema = ?");
|
|
1181
|
+
vals.push(state.bustRateEMA);
|
|
1182
|
+
}
|
|
1183
|
+
if (state.interBustIntervalEMA !== void 0) {
|
|
1184
|
+
sets.push("inter_bust_interval_ema = ?");
|
|
1185
|
+
vals.push(state.interBustIntervalEMA);
|
|
1186
|
+
}
|
|
1187
|
+
if (state.lastLayer !== void 0) {
|
|
1188
|
+
sets.push("last_layer = ?");
|
|
1189
|
+
vals.push(state.lastLayer);
|
|
1190
|
+
}
|
|
1191
|
+
if (state.lastKnownInput !== void 0) {
|
|
1192
|
+
sets.push("last_known_input = ?");
|
|
1193
|
+
vals.push(state.lastKnownInput);
|
|
1194
|
+
}
|
|
1195
|
+
if (state.lastTurnAt !== void 0) {
|
|
1196
|
+
sets.push("last_turn_at = ?");
|
|
1197
|
+
vals.push(state.lastTurnAt);
|
|
1198
|
+
}
|
|
1199
|
+
if (state.lastBustAt !== void 0) {
|
|
1200
|
+
sets.push("last_bust_at = ?");
|
|
1201
|
+
vals.push(state.lastBustAt);
|
|
1202
|
+
}
|
|
1203
|
+
db().query(
|
|
1204
|
+
"UPDATE session_state SET " + sets.join(", ") + " WHERE session_id = ?"
|
|
1205
|
+
).run(...vals, sessionID);
|
|
1206
|
+
}
|
|
1207
|
+
function loadSessionTracking(sessionID) {
|
|
1208
|
+
const row = db().query(
|
|
1209
|
+
`SELECT last_curated_at, message_count, turns_since_curation,
|
|
1210
|
+
consecutive_text_only_turns,
|
|
1211
|
+
ltm_cache_text, ltm_cache_tokens, ltm_pin_text, ltm_pin_tokens,
|
|
1212
|
+
fingerprint, header_session_id, header_name,
|
|
1213
|
+
resolved_conversation_ttl, warmup_state,
|
|
1214
|
+
dynamic_context_cap, bust_rate_ema, inter_bust_interval_ema,
|
|
1215
|
+
last_layer, last_known_input, last_turn_at, last_bust_at
|
|
1216
|
+
FROM session_state WHERE session_id = ?`
|
|
1217
|
+
).get(sessionID);
|
|
1218
|
+
if (!row) return null;
|
|
1219
|
+
return {
|
|
1220
|
+
lastCuratedAt: row.last_curated_at,
|
|
1221
|
+
messageCount: row.message_count,
|
|
1222
|
+
turnsSinceCuration: row.turns_since_curation,
|
|
1223
|
+
consecutiveTextOnlyTurns: row.consecutive_text_only_turns,
|
|
1224
|
+
ltmCacheText: row.ltm_cache_text,
|
|
1225
|
+
ltmCacheTokens: row.ltm_cache_tokens,
|
|
1226
|
+
ltmPinText: row.ltm_pin_text,
|
|
1227
|
+
ltmPinTokens: row.ltm_pin_tokens,
|
|
1228
|
+
fingerprint: row.fingerprint,
|
|
1229
|
+
headerSessionId: row.header_session_id,
|
|
1230
|
+
headerName: row.header_name,
|
|
1231
|
+
resolvedConversationTTL: row.resolved_conversation_ttl,
|
|
1232
|
+
warmupState: row.warmup_state,
|
|
1233
|
+
dynamicContextCap: row.dynamic_context_cap,
|
|
1234
|
+
bustRateEMA: row.bust_rate_ema,
|
|
1235
|
+
interBustIntervalEMA: row.inter_bust_interval_ema,
|
|
1236
|
+
lastLayer: row.last_layer,
|
|
1237
|
+
lastKnownInput: row.last_known_input,
|
|
1238
|
+
lastTurnAt: row.last_turn_at,
|
|
1239
|
+
lastBustAt: row.last_bust_at
|
|
1240
|
+
};
|
|
1241
|
+
}
|
|
1242
|
+
function loadHeaderSessionIndex() {
|
|
1243
|
+
const rows = db().query(
|
|
1244
|
+
`SELECT session_id, header_session_id, header_name
|
|
1245
|
+
FROM session_state
|
|
1246
|
+
WHERE header_session_id IS NOT NULL AND header_name IS NOT NULL`
|
|
1247
|
+
).all();
|
|
1248
|
+
return rows.map((row) => ({
|
|
1249
|
+
sessionId: row.session_id,
|
|
1250
|
+
headerSessionId: row.header_session_id,
|
|
1251
|
+
headerName: row.header_name
|
|
1252
|
+
}));
|
|
1253
|
+
}
|
|
1254
|
+
function getKV(key) {
|
|
1255
|
+
const row = db().query("SELECT value FROM kv_meta WHERE key = ?").get(key);
|
|
1256
|
+
return row?.value ?? null;
|
|
1257
|
+
}
|
|
1258
|
+
function setKV(key, value) {
|
|
1259
|
+
db().query(
|
|
1260
|
+
"INSERT INTO kv_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = ?"
|
|
1261
|
+
).run(key, value, value);
|
|
1262
|
+
}
|
|
1061
1263
|
function getMeta(key) {
|
|
1062
1264
|
const row = db().query("SELECT value FROM metadata WHERE key = ?").get(key);
|
|
1063
1265
|
return row?.value ?? null;
|
|
@@ -26770,7 +26972,16 @@ var LocalProvider = class {
|
|
|
26770
26972
|
workerUrl = vendorWorkerUrl;
|
|
26771
26973
|
}
|
|
26772
26974
|
} else {
|
|
26773
|
-
|
|
26975
|
+
const selfUrl = typeof import.meta.url === "string" ? import.meta.url : void 0;
|
|
26976
|
+
if (selfUrl) {
|
|
26977
|
+
workerUrl = new URL(
|
|
26978
|
+
`./embedding-worker${selfUrl.endsWith(".ts") ? ".ts" : ".js"}`,
|
|
26979
|
+
selfUrl
|
|
26980
|
+
);
|
|
26981
|
+
} else {
|
|
26982
|
+
const { pathToFileURL } = await import("node:url");
|
|
26983
|
+
workerUrl = new URL("./embedding-worker.cjs", pathToFileURL(__filename));
|
|
26984
|
+
}
|
|
26774
26985
|
}
|
|
26775
26986
|
const vendor = vendorModelInfo();
|
|
26776
26987
|
const workerInitData = {
|
|
@@ -27045,8 +27256,14 @@ function fromBlob(blob) {
|
|
|
27045
27256
|
const bytes = new Uint8Array(blob);
|
|
27046
27257
|
return new Float32Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / 4);
|
|
27047
27258
|
}
|
|
27048
|
-
function vectorSearch(queryEmbedding, limit = 10) {
|
|
27049
|
-
|
|
27259
|
+
function vectorSearch(queryEmbedding, limit = 10, excludeCategories) {
|
|
27260
|
+
let sql = "SELECT id, embedding FROM knowledge WHERE embedding IS NOT NULL AND confidence > 0.2";
|
|
27261
|
+
const params = [];
|
|
27262
|
+
if (excludeCategories?.length) {
|
|
27263
|
+
sql += ` AND category NOT IN (${excludeCategories.map(() => "?").join(",")})`;
|
|
27264
|
+
params.push(...excludeCategories);
|
|
27265
|
+
}
|
|
27266
|
+
const rows = db().query(sql).all(...params);
|
|
27050
27267
|
const scored = [];
|
|
27051
27268
|
for (const row of rows) {
|
|
27052
27269
|
const vec = fromBlob(row.embedding);
|
|
@@ -27435,6 +27652,12 @@ function count(projectPath, sessionID) {
|
|
|
27435
27652
|
const params = sessionID ? [pid, sessionID] : [pid];
|
|
27436
27653
|
return db().query(query).get(...params).count;
|
|
27437
27654
|
}
|
|
27655
|
+
function hasMessages(projectPath, sessionID) {
|
|
27656
|
+
const pid = ensureProject(projectPath);
|
|
27657
|
+
return !!db().query(
|
|
27658
|
+
"SELECT 1 FROM temporal_messages WHERE project_id = ? AND session_id = ? LIMIT 1"
|
|
27659
|
+
).get(pid, sessionID);
|
|
27660
|
+
}
|
|
27438
27661
|
function undistilledCount(projectPath, sessionID) {
|
|
27439
27662
|
const pid = ensureProject(projectPath);
|
|
27440
27663
|
const query = sessionID ? "SELECT COUNT(*) as count FROM temporal_messages WHERE project_id = ? AND session_id = ? AND distilled = 0" : "SELECT COUNT(*) as count FROM temporal_messages WHERE project_id = ? AND distilled = 0";
|
|
@@ -27493,11 +27716,13 @@ function prune(input) {
|
|
|
27493
27716
|
var ltm_exports = {};
|
|
27494
27717
|
__export(ltm_exports, {
|
|
27495
27718
|
all: () => all2,
|
|
27719
|
+
calibrateDedupThreshold: () => calibrateDedupThreshold,
|
|
27496
27720
|
cascadeRefReplace: () => cascadeRefReplace,
|
|
27497
27721
|
check: () => check2,
|
|
27498
27722
|
cleanDeadRefs: () => cleanDeadRefs,
|
|
27499
27723
|
create: () => create,
|
|
27500
27724
|
crossProject: () => crossProject,
|
|
27725
|
+
dedupPairKey: () => dedupPairKey,
|
|
27501
27726
|
deduplicate: () => deduplicate,
|
|
27502
27727
|
deduplicateGlobal: () => deduplicateGlobal,
|
|
27503
27728
|
extractRefs: () => extractRefs,
|
|
@@ -27505,9 +27730,17 @@ __export(ltm_exports, {
|
|
|
27505
27730
|
forProject: () => forProject,
|
|
27506
27731
|
forSession: () => forSession,
|
|
27507
27732
|
get: () => get,
|
|
27733
|
+
getDedupFeedback: () => getDedupFeedback,
|
|
27734
|
+
getDedupFeedbackCount: () => getDedupFeedbackCount,
|
|
27735
|
+
loadCalibratedThreshold: () => loadCalibratedThreshold,
|
|
27736
|
+
pruneDedupFeedback: () => pruneDedupFeedback,
|
|
27508
27737
|
pruneOversized: () => pruneOversized,
|
|
27738
|
+
recordAutoSignals: () => recordAutoSignals,
|
|
27739
|
+
recordDedupFeedback: () => recordDedupFeedback,
|
|
27740
|
+
recordDedupResultFeedback: () => recordDedupResultFeedback,
|
|
27509
27741
|
remove: () => remove,
|
|
27510
27742
|
resolveRef: () => resolveRef2,
|
|
27743
|
+
saveCalibratedThreshold: () => saveCalibratedThreshold,
|
|
27511
27744
|
search: () => search3,
|
|
27512
27745
|
searchScored: () => searchScored3,
|
|
27513
27746
|
searchScoredOtherProjects: () => searchScoredOtherProjects,
|
|
@@ -28244,18 +28477,29 @@ function scoreEntriesFTS(sessionContext) {
|
|
|
28244
28477
|
return /* @__PURE__ */ new Map();
|
|
28245
28478
|
}
|
|
28246
28479
|
}
|
|
28247
|
-
function forSession(projectPath, sessionID, maxTokens) {
|
|
28480
|
+
async function forSession(projectPath, sessionID, maxTokens, options) {
|
|
28248
28481
|
const pid = ensureProject(projectPath);
|
|
28482
|
+
const categoryFilter = options?.categories;
|
|
28483
|
+
const excludeFilter = options?.excludeCategories;
|
|
28484
|
+
let categoryClause = "";
|
|
28485
|
+
let categoryParams = [];
|
|
28486
|
+
if (categoryFilter?.length) {
|
|
28487
|
+
categoryClause = ` AND category IN (${categoryFilter.map(() => "?").join(",")})`;
|
|
28488
|
+
categoryParams = categoryFilter;
|
|
28489
|
+
} else if (excludeFilter?.length) {
|
|
28490
|
+
categoryClause = ` AND category NOT IN (${excludeFilter.map(() => "?").join(",")})`;
|
|
28491
|
+
categoryParams = excludeFilter;
|
|
28492
|
+
}
|
|
28249
28493
|
const projectEntries = db().query(
|
|
28250
28494
|
`SELECT ${KNOWLEDGE_COLS} FROM knowledge
|
|
28251
|
-
WHERE project_id = ? AND cross_project = 0 AND confidence > 0.2
|
|
28495
|
+
WHERE project_id = ? AND cross_project = 0 AND confidence > 0.2${categoryClause}
|
|
28252
28496
|
ORDER BY confidence DESC, updated_at DESC`
|
|
28253
|
-
).all(pid);
|
|
28497
|
+
).all(pid, ...categoryParams);
|
|
28254
28498
|
const crossEntries = db().query(
|
|
28255
28499
|
`SELECT ${KNOWLEDGE_COLS} FROM knowledge
|
|
28256
|
-
WHERE (project_id IS NULL OR cross_project = 1) AND confidence > 0.2
|
|
28500
|
+
WHERE (project_id IS NULL OR cross_project = 1) AND confidence > 0.2${categoryClause}
|
|
28257
28501
|
ORDER BY confidence DESC, updated_at DESC`
|
|
28258
|
-
).all();
|
|
28502
|
+
).all(...categoryParams);
|
|
28259
28503
|
if (!crossEntries.length && !projectEntries.length) return [];
|
|
28260
28504
|
let sessionContext = "";
|
|
28261
28505
|
if (sessionID) {
|
|
@@ -28276,22 +28520,52 @@ function forSession(projectPath, sessionID, maxTokens) {
|
|
|
28276
28520
|
sessionContext += recentMsgs.map((m) => m.content).join("\n");
|
|
28277
28521
|
}
|
|
28278
28522
|
}
|
|
28523
|
+
if (!sessionContext.trim() && options?.contextHint) {
|
|
28524
|
+
sessionContext = options.contextHint;
|
|
28525
|
+
}
|
|
28279
28526
|
let scoredProject;
|
|
28280
28527
|
let scoredCross;
|
|
28281
|
-
if (sessionContext.trim().length > 20) {
|
|
28528
|
+
if (sessionContext.trim().length > 20 && isAvailable()) {
|
|
28529
|
+
let vectorScores;
|
|
28530
|
+
try {
|
|
28531
|
+
const [contextVec] = await embed([sessionContext], "query");
|
|
28532
|
+
const hits = vectorSearch(contextVec, 50, excludeFilter);
|
|
28533
|
+
vectorScores = new Map(hits.map((h3) => [h3.id, h3.similarity]));
|
|
28534
|
+
} catch (err) {
|
|
28535
|
+
warn("Vector scoring failed, falling back to FTS5:", err);
|
|
28536
|
+
vectorScores = /* @__PURE__ */ new Map();
|
|
28537
|
+
}
|
|
28538
|
+
if (vectorScores.size > 0) {
|
|
28539
|
+
const ftsScores = scoreEntriesFTS(sessionContext);
|
|
28540
|
+
const rawScored = projectEntries.map((entry) => {
|
|
28541
|
+
const vecScore = vectorScores.get(entry.id);
|
|
28542
|
+
const score = vecScore != null ? vecScore * entry.confidence : (ftsScores.get(entry.id) ?? 0) * entry.confidence;
|
|
28543
|
+
return { entry, score };
|
|
28544
|
+
});
|
|
28545
|
+
const matched = rawScored.filter((s) => s.score > 0);
|
|
28546
|
+
const matchedIds = new Set(matched.map((s) => s.entry.id));
|
|
28547
|
+
const safetyNet = projectEntries.filter((e) => !matchedIds.has(e.id)).slice(0, PROJECT_SAFETY_NET).map((e) => ({ entry: e, score: 1e-3 * e.confidence }));
|
|
28548
|
+
scoredProject = [...matched, ...safetyNet];
|
|
28549
|
+
scoredCross = crossEntries.filter((e) => vectorScores.has(e.id) || ftsScores.has(e.id)).map((e) => {
|
|
28550
|
+
const vecScore = vectorScores.get(e.id);
|
|
28551
|
+
const score = vecScore != null ? vecScore * e.confidence : (ftsScores.get(e.id) ?? 0) * e.confidence;
|
|
28552
|
+
return { entry: e, score };
|
|
28553
|
+
});
|
|
28554
|
+
} else {
|
|
28555
|
+
const ftsScores = scoreEntriesFTS(sessionContext);
|
|
28556
|
+
({ scoredProject, scoredCross } = scoreFTS(
|
|
28557
|
+
projectEntries,
|
|
28558
|
+
crossEntries,
|
|
28559
|
+
ftsScores
|
|
28560
|
+
));
|
|
28561
|
+
}
|
|
28562
|
+
} else if (sessionContext.trim().length > 20) {
|
|
28282
28563
|
const ftsScores = scoreEntriesFTS(sessionContext);
|
|
28283
|
-
|
|
28284
|
-
|
|
28285
|
-
|
|
28286
|
-
|
|
28287
|
-
|
|
28288
|
-
const matchedIds = new Set(matched.map((s) => s.entry.id));
|
|
28289
|
-
const safetyNet = projectEntries.filter((e) => !matchedIds.has(e.id)).slice(0, PROJECT_SAFETY_NET).map((e) => ({ entry: e, score: 1e-3 * e.confidence }));
|
|
28290
|
-
scoredProject = [...matched, ...safetyNet];
|
|
28291
|
-
scoredCross = crossEntries.filter((e) => ftsScores.has(e.id)).map((e) => ({
|
|
28292
|
-
entry: e,
|
|
28293
|
-
score: (ftsScores.get(e.id) ?? 0) * e.confidence
|
|
28294
|
-
}));
|
|
28564
|
+
({ scoredProject, scoredCross } = scoreFTS(
|
|
28565
|
+
projectEntries,
|
|
28566
|
+
crossEntries,
|
|
28567
|
+
ftsScores
|
|
28568
|
+
));
|
|
28295
28569
|
} else {
|
|
28296
28570
|
scoredProject = projectEntries.slice(0, NO_CONTEXT_FALLBACK_CAP).map((entry) => ({ entry, score: entry.confidence }));
|
|
28297
28571
|
scoredCross = crossEntries.slice(0, NO_CONTEXT_FALLBACK_CAP).map((entry) => ({ entry, score: entry.confidence }));
|
|
@@ -28337,6 +28611,21 @@ function forSession(projectPath, sessionID, maxTokens) {
|
|
|
28337
28611
|
}
|
|
28338
28612
|
return result;
|
|
28339
28613
|
}
|
|
28614
|
+
function scoreFTS(projectEntries, crossEntries, ftsScores) {
|
|
28615
|
+
const rawScored = projectEntries.map((entry) => ({
|
|
28616
|
+
entry,
|
|
28617
|
+
score: (ftsScores.get(entry.id) ?? 0) * entry.confidence
|
|
28618
|
+
}));
|
|
28619
|
+
const matched = rawScored.filter((s) => s.score > 0);
|
|
28620
|
+
const matchedIds = new Set(matched.map((s) => s.entry.id));
|
|
28621
|
+
const safetyNet = projectEntries.filter((e) => !matchedIds.has(e.id)).slice(0, PROJECT_SAFETY_NET).map((e) => ({ entry: e, score: 1e-3 * e.confidence }));
|
|
28622
|
+
const scoredProject = [...matched, ...safetyNet];
|
|
28623
|
+
const scoredCross = crossEntries.filter((e) => ftsScores.has(e.id)).map((e) => ({
|
|
28624
|
+
entry: e,
|
|
28625
|
+
score: (ftsScores.get(e.id) ?? 0) * e.confidence
|
|
28626
|
+
}));
|
|
28627
|
+
return { scoredProject, scoredCross };
|
|
28628
|
+
}
|
|
28340
28629
|
function all2() {
|
|
28341
28630
|
return db().query(
|
|
28342
28631
|
`SELECT ${KNOWLEDGE_COLS} FROM knowledge WHERE confidence > 0.2 ORDER BY confidence DESC, updated_at DESC`
|
|
@@ -28580,8 +28869,11 @@ function check2(projectPath) {
|
|
|
28580
28869
|
}
|
|
28581
28870
|
return issues;
|
|
28582
28871
|
}
|
|
28583
|
-
function
|
|
28584
|
-
|
|
28872
|
+
function dedupPairKey(idA, idB) {
|
|
28873
|
+
return idA < idB ? `${idA}:${idB}` : `${idB}:${idA}`;
|
|
28874
|
+
}
|
|
28875
|
+
function _dedup(entries, dryRun, embeddingThreshold = EMBEDDING_DEDUP_THRESHOLD) {
|
|
28876
|
+
if (entries.length < 2) return { clusters: [], totalRemoved: 0, pairSimilarities: /* @__PURE__ */ new Map(), entryTitles: /* @__PURE__ */ new Map() };
|
|
28585
28877
|
const embeddingMap = /* @__PURE__ */ new Map();
|
|
28586
28878
|
{
|
|
28587
28879
|
const entryIds = entries.map((e) => e.id);
|
|
@@ -28596,6 +28888,7 @@ function _dedup(entries, dryRun) {
|
|
|
28596
28888
|
}
|
|
28597
28889
|
}
|
|
28598
28890
|
const neighborMap = /* @__PURE__ */ new Map();
|
|
28891
|
+
const pairSimilarities = /* @__PURE__ */ new Map();
|
|
28599
28892
|
for (const entry of entries) {
|
|
28600
28893
|
const neighbors = [];
|
|
28601
28894
|
const entryVec = embeddingMap.get(entry.id);
|
|
@@ -28609,7 +28902,13 @@ function _dedup(entries, dryRun) {
|
|
|
28609
28902
|
const otherVec = embeddingMap.get(other.id);
|
|
28610
28903
|
if (otherVec && entryVec.length === otherVec.length) {
|
|
28611
28904
|
similarity = cosineSimilarity(entryVec, otherVec);
|
|
28612
|
-
embeddingMatch = similarity >=
|
|
28905
|
+
embeddingMatch = similarity >= embeddingThreshold;
|
|
28906
|
+
}
|
|
28907
|
+
}
|
|
28908
|
+
if (similarity > 0) {
|
|
28909
|
+
const pk = dedupPairKey(entry.id, other.id);
|
|
28910
|
+
if (!pairSimilarities.has(pk)) {
|
|
28911
|
+
pairSimilarities.set(pk, similarity);
|
|
28613
28912
|
}
|
|
28614
28913
|
}
|
|
28615
28914
|
if (titleMatch || embeddingMatch) {
|
|
@@ -28661,20 +28960,178 @@ function _dedup(entries, dryRun) {
|
|
|
28661
28960
|
totalRemoved += merged.length;
|
|
28662
28961
|
}
|
|
28663
28962
|
result.sort((a, b) => b.merged.length - a.merged.length);
|
|
28664
|
-
|
|
28963
|
+
const entryTitles = new Map(entries.map((e) => [e.id, e.title]));
|
|
28964
|
+
return { clusters: result, totalRemoved, pairSimilarities, entryTitles };
|
|
28665
28965
|
}
|
|
28666
28966
|
async function deduplicate(projectPath, opts) {
|
|
28967
|
+
const pid = ensureProject(projectPath);
|
|
28968
|
+
const threshold = loadCalibratedThreshold(pid) ?? EMBEDDING_DEDUP_THRESHOLD;
|
|
28667
28969
|
const entries = forProject(projectPath, false);
|
|
28668
|
-
return _dedup(entries, opts?.dryRun ?? true);
|
|
28970
|
+
return _dedup(entries, opts?.dryRun ?? true, threshold);
|
|
28669
28971
|
}
|
|
28670
28972
|
async function deduplicateGlobal(opts) {
|
|
28973
|
+
const threshold = loadCalibratedThreshold(null) ?? EMBEDDING_DEDUP_THRESHOLD;
|
|
28671
28974
|
const entries = db().query(
|
|
28672
28975
|
`SELECT ${KNOWLEDGE_COLS} FROM knowledge
|
|
28673
28976
|
WHERE project_id IS NULL
|
|
28674
28977
|
AND confidence > 0.2
|
|
28675
28978
|
ORDER BY confidence DESC, updated_at DESC`
|
|
28676
28979
|
).all();
|
|
28677
|
-
return _dedup(entries, opts?.dryRun ?? true);
|
|
28980
|
+
return _dedup(entries, opts?.dryRun ?? true, threshold);
|
|
28981
|
+
}
|
|
28982
|
+
var MIN_CALIBRATION_SAMPLES = 20;
|
|
28983
|
+
var DEFAULT_EMBEDDING_DEDUP_THRESHOLD = EMBEDDING_DEDUP_THRESHOLD;
|
|
28984
|
+
var AUTO_SIGNAL_MIN_SIMILARITY = 0.8;
|
|
28985
|
+
var AUTO_SIGNAL_MAX_PAIRS = 50;
|
|
28986
|
+
function recordDedupFeedback(input) {
|
|
28987
|
+
db().query(
|
|
28988
|
+
`INSERT INTO dedup_feedback
|
|
28989
|
+
(project_id, entry_a_title, entry_b_title, similarity, accepted, source, created_at)
|
|
28990
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`
|
|
28991
|
+
).run(
|
|
28992
|
+
input.projectId,
|
|
28993
|
+
input.entryATitle,
|
|
28994
|
+
input.entryBTitle,
|
|
28995
|
+
input.similarity,
|
|
28996
|
+
input.accepted ? 1 : 0,
|
|
28997
|
+
input.source,
|
|
28998
|
+
Date.now()
|
|
28999
|
+
);
|
|
29000
|
+
}
|
|
29001
|
+
function recordDedupResultFeedback(projectId2, result, accepted, source) {
|
|
29002
|
+
for (const cluster of result.clusters) {
|
|
29003
|
+
for (const merged of cluster.merged) {
|
|
29004
|
+
const pk = dedupPairKey(cluster.surviving.id, merged.id);
|
|
29005
|
+
const similarity = result.pairSimilarities.get(pk);
|
|
29006
|
+
if (similarity != null && similarity > 0) {
|
|
29007
|
+
recordDedupFeedback({
|
|
29008
|
+
projectId: projectId2,
|
|
29009
|
+
entryATitle: cluster.surviving.title,
|
|
29010
|
+
entryBTitle: merged.title,
|
|
29011
|
+
similarity,
|
|
29012
|
+
accepted,
|
|
29013
|
+
source
|
|
29014
|
+
});
|
|
29015
|
+
}
|
|
29016
|
+
}
|
|
29017
|
+
}
|
|
29018
|
+
}
|
|
29019
|
+
function recordAutoSignals(projectId2, result) {
|
|
29020
|
+
const mergedPairs = /* @__PURE__ */ new Set();
|
|
29021
|
+
for (const cluster of result.clusters) {
|
|
29022
|
+
for (const merged of cluster.merged) {
|
|
29023
|
+
mergedPairs.add(dedupPairKey(cluster.surviving.id, merged.id));
|
|
29024
|
+
}
|
|
29025
|
+
}
|
|
29026
|
+
const titleMap = new Map(result.entryTitles);
|
|
29027
|
+
for (const cluster of result.clusters) {
|
|
29028
|
+
if (!titleMap.has(cluster.surviving.id)) {
|
|
29029
|
+
titleMap.set(cluster.surviving.id, cluster.surviving.title);
|
|
29030
|
+
}
|
|
29031
|
+
for (const m of cluster.merged) {
|
|
29032
|
+
if (!titleMap.has(m.id)) titleMap.set(m.id, m.title);
|
|
29033
|
+
}
|
|
29034
|
+
}
|
|
29035
|
+
const signals = [];
|
|
29036
|
+
for (const [pk, sim] of result.pairSimilarities) {
|
|
29037
|
+
if (sim < AUTO_SIGNAL_MIN_SIMILARITY) continue;
|
|
29038
|
+
if (mergedPairs.has(pk)) continue;
|
|
29039
|
+
const [idA, idB] = pk.split(":");
|
|
29040
|
+
const titleA = titleMap.get(idA);
|
|
29041
|
+
const titleB = titleMap.get(idB);
|
|
29042
|
+
if (!titleA || !titleB) continue;
|
|
29043
|
+
signals.push({ entryATitle: titleA, entryBTitle: titleB, similarity: sim });
|
|
29044
|
+
}
|
|
29045
|
+
const currentThreshold = loadCalibratedThreshold(projectId2) ?? DEFAULT_EMBEDDING_DEDUP_THRESHOLD;
|
|
29046
|
+
signals.sort((a, b) => Math.abs(a.similarity - currentThreshold) - Math.abs(b.similarity - currentThreshold));
|
|
29047
|
+
const capped = signals.slice(0, AUTO_SIGNAL_MAX_PAIRS);
|
|
29048
|
+
pruneDedupFeedback(projectId2);
|
|
29049
|
+
for (const s of capped) {
|
|
29050
|
+
recordDedupFeedback({
|
|
29051
|
+
projectId: projectId2,
|
|
29052
|
+
entryATitle: s.entryATitle,
|
|
29053
|
+
entryBTitle: s.entryBTitle,
|
|
29054
|
+
similarity: s.similarity,
|
|
29055
|
+
accepted: false,
|
|
29056
|
+
source: "auto_dedup"
|
|
29057
|
+
});
|
|
29058
|
+
}
|
|
29059
|
+
}
|
|
29060
|
+
function getDedupFeedback(projectId2) {
|
|
29061
|
+
const rows = projectId2 !== null ? db().query(
|
|
29062
|
+
"SELECT similarity, accepted, source FROM dedup_feedback WHERE project_id = ? ORDER BY similarity"
|
|
29063
|
+
).all(projectId2) : db().query(
|
|
29064
|
+
"SELECT similarity, accepted, source FROM dedup_feedback WHERE project_id IS NULL ORDER BY similarity"
|
|
29065
|
+
).all();
|
|
29066
|
+
return rows.map((r) => ({ similarity: r.similarity, accepted: r.accepted === 1, source: r.source }));
|
|
29067
|
+
}
|
|
29068
|
+
function getDedupFeedbackCount(projectId2) {
|
|
29069
|
+
const row = projectId2 !== null ? db().query("SELECT COUNT(*) as cnt FROM dedup_feedback WHERE project_id = ?").get(projectId2) : db().query("SELECT COUNT(*) as cnt FROM dedup_feedback WHERE project_id IS NULL").get();
|
|
29070
|
+
return row?.cnt ?? 0;
|
|
29071
|
+
}
|
|
29072
|
+
var MAX_FEEDBACK_ROWS_PER_PROJECT = 500;
|
|
29073
|
+
function pruneDedupFeedback(projectId2) {
|
|
29074
|
+
const count3 = getDedupFeedbackCount(projectId2);
|
|
29075
|
+
if (count3 <= MAX_FEEDBACK_ROWS_PER_PROJECT) return;
|
|
29076
|
+
const excess = count3 - MAX_FEEDBACK_ROWS_PER_PROJECT;
|
|
29077
|
+
if (projectId2 !== null) {
|
|
29078
|
+
db().query(
|
|
29079
|
+
`DELETE FROM dedup_feedback WHERE id IN (
|
|
29080
|
+
SELECT id FROM dedup_feedback WHERE project_id = ?
|
|
29081
|
+
ORDER BY created_at ASC LIMIT ?
|
|
29082
|
+
)`
|
|
29083
|
+
).run(projectId2, excess);
|
|
29084
|
+
} else {
|
|
29085
|
+
db().query(
|
|
29086
|
+
`DELETE FROM dedup_feedback WHERE id IN (
|
|
29087
|
+
SELECT id FROM dedup_feedback WHERE project_id IS NULL
|
|
29088
|
+
ORDER BY created_at ASC LIMIT ?
|
|
29089
|
+
)`
|
|
29090
|
+
).run(excess);
|
|
29091
|
+
}
|
|
29092
|
+
}
|
|
29093
|
+
function calibrateDedupThreshold(projectId2) {
|
|
29094
|
+
const feedback = getDedupFeedback(projectId2);
|
|
29095
|
+
if (feedback.length < MIN_CALIBRATION_SAMPLES) return null;
|
|
29096
|
+
const accepted = feedback.filter((f) => f.accepted);
|
|
29097
|
+
const rejected = feedback.filter((f) => !f.accepted);
|
|
29098
|
+
if (rejected.length === 0) {
|
|
29099
|
+
const minAccepted = Math.min(...accepted.map((f) => f.similarity));
|
|
29100
|
+
return Math.max(0.85, minAccepted - 5e-3);
|
|
29101
|
+
}
|
|
29102
|
+
if (accepted.length === 0) {
|
|
29103
|
+
warn("dedup calibration: all feedback is reject \u2014 keeping default threshold");
|
|
29104
|
+
return null;
|
|
29105
|
+
}
|
|
29106
|
+
const allSims = [...new Set(feedback.map((f) => f.similarity))].sort((a, b) => a - b);
|
|
29107
|
+
let bestThreshold = DEFAULT_EMBEDDING_DEDUP_THRESHOLD;
|
|
29108
|
+
let bestAccuracy = -1;
|
|
29109
|
+
for (let i = 0; i < allSims.length - 1; i++) {
|
|
29110
|
+
const candidate = (allSims[i] + allSims[i + 1]) / 2;
|
|
29111
|
+
const correctAccepted = accepted.filter((f) => f.similarity >= candidate).length;
|
|
29112
|
+
const correctRejected = rejected.filter((f) => f.similarity < candidate).length;
|
|
29113
|
+
const accuracy = (correctAccepted + correctRejected) / feedback.length;
|
|
29114
|
+
if (accuracy > bestAccuracy || accuracy === bestAccuracy && candidate > bestThreshold) {
|
|
29115
|
+
bestAccuracy = accuracy;
|
|
29116
|
+
bestThreshold = candidate;
|
|
29117
|
+
}
|
|
29118
|
+
}
|
|
29119
|
+
return Math.max(0.85, Math.min(0.98, bestThreshold));
|
|
29120
|
+
}
|
|
29121
|
+
function saveCalibratedThreshold(projectId2, threshold, sampleSize) {
|
|
29122
|
+
const key = `dedup_threshold:${projectId2 ?? "global"}`;
|
|
29123
|
+
setKV(key, JSON.stringify({ threshold, sampleSize, calibratedAt: Date.now() }));
|
|
29124
|
+
}
|
|
29125
|
+
function loadCalibratedThreshold(projectId2) {
|
|
29126
|
+
const key = `dedup_threshold:${projectId2 ?? "global"}`;
|
|
29127
|
+
const raw = getKV(key);
|
|
29128
|
+
if (!raw) return null;
|
|
29129
|
+
try {
|
|
29130
|
+
const parsed = JSON.parse(raw);
|
|
29131
|
+
return typeof parsed.threshold === "number" ? parsed.threshold : null;
|
|
29132
|
+
} catch {
|
|
29133
|
+
return null;
|
|
29134
|
+
}
|
|
28678
29135
|
}
|
|
28679
29136
|
|
|
28680
29137
|
// src/data.ts
|
|
@@ -29523,6 +29980,16 @@ function getSessionState(sessionID) {
|
|
|
29523
29980
|
if (!state) {
|
|
29524
29981
|
state = makeSessionState();
|
|
29525
29982
|
state.forceMinLayer = loadForceMinLayer(sessionID);
|
|
29983
|
+
const persisted = loadSessionTracking(sessionID);
|
|
29984
|
+
if (persisted && persisted.lastTurnAt > 0) {
|
|
29985
|
+
state.dynamicContextCap = persisted.dynamicContextCap;
|
|
29986
|
+
state.bustRateEMA = persisted.bustRateEMA;
|
|
29987
|
+
state.interBustIntervalEMA = persisted.interBustIntervalEMA;
|
|
29988
|
+
state.lastLayer = persisted.lastLayer;
|
|
29989
|
+
state.lastKnownInput = persisted.lastKnownInput;
|
|
29990
|
+
state.lastTurnAt = persisted.lastTurnAt;
|
|
29991
|
+
state.lastBustAt = persisted.lastBustAt;
|
|
29992
|
+
}
|
|
29526
29993
|
sessionStates.set(sessionID, state);
|
|
29527
29994
|
}
|
|
29528
29995
|
return state;
|
|
@@ -29628,6 +30095,19 @@ function inspectSessionState(sessionID) {
|
|
|
29628
30095
|
function setLastTurnAtForTest(sessionID, ms) {
|
|
29629
30096
|
getSessionState(sessionID).lastTurnAt = ms;
|
|
29630
30097
|
}
|
|
30098
|
+
function saveGradientState(sessionID) {
|
|
30099
|
+
const state = sessionStates.get(sessionID);
|
|
30100
|
+
if (!state) return;
|
|
30101
|
+
saveSessionTracking(sessionID, {
|
|
30102
|
+
dynamicContextCap: state.dynamicContextCap,
|
|
30103
|
+
bustRateEMA: state.bustRateEMA,
|
|
30104
|
+
interBustIntervalEMA: state.interBustIntervalEMA,
|
|
30105
|
+
lastLayer: state.lastLayer,
|
|
30106
|
+
lastKnownInput: state.lastKnownInput,
|
|
30107
|
+
lastTurnAt: state.lastTurnAt,
|
|
30108
|
+
lastBustAt: state.lastBustAt
|
|
30109
|
+
});
|
|
30110
|
+
}
|
|
29631
30111
|
function loadDistillations(projectPath, sessionID) {
|
|
29632
30112
|
const pid = ensureProject(projectPath);
|
|
29633
30113
|
const query = sessionID ? "SELECT id, observations, generation, token_count, created_at, session_id FROM distillations WHERE project_id = ? AND session_id = ? AND archived = 0 ORDER BY created_at ASC" : "SELECT id, observations, generation, token_count, created_at, session_id FROM distillations WHERE project_id = ? AND archived = 0 ORDER BY created_at ASC";
|
|
@@ -29912,6 +30392,26 @@ function buildPrefixMessages(formatted) {
|
|
|
29912
30392
|
}
|
|
29913
30393
|
];
|
|
29914
30394
|
}
|
|
30395
|
+
var DECISION_RE = /\b(?:decision|decided|chose|chosen|agreed)\b/i;
|
|
30396
|
+
var GOTCHA_RE = /\b(?:gotcha|(?:critical|known|subtle)\s+bug|broken|crash(?:ed|es)?|regression)\b/i;
|
|
30397
|
+
var ARCH_RE = /\b(?:architecture|design.(?:decision|pattern)|system.design)\b/i;
|
|
30398
|
+
function importanceBonus(d) {
|
|
30399
|
+
let bonus = 0;
|
|
30400
|
+
if (DECISION_RE.test(d.observations)) bonus += 0.3;
|
|
30401
|
+
if (GOTCHA_RE.test(d.observations)) bonus += 0.2;
|
|
30402
|
+
if (ARCH_RE.test(d.observations)) bonus += 0.1;
|
|
30403
|
+
if (d.generation >= 1) bonus += 0.2;
|
|
30404
|
+
return Math.min(bonus, 1);
|
|
30405
|
+
}
|
|
30406
|
+
function selectDistillations(all3, limit) {
|
|
30407
|
+
if (all3.length <= limit) return all3;
|
|
30408
|
+
const maxIdx = all3.length - 1;
|
|
30409
|
+
const scored = all3.map((d, i) => ({
|
|
30410
|
+
d,
|
|
30411
|
+
score: (maxIdx > 0 ? i / maxIdx : 1) * 0.7 + importanceBonus(d) * 0.3
|
|
30412
|
+
}));
|
|
30413
|
+
return scored.sort((a, b) => b.score - a.score).slice(0, limit).map((s) => s.d).sort((a, b) => a.created_at - b.created_at);
|
|
30414
|
+
}
|
|
29915
30415
|
function distilledPrefix(distillations) {
|
|
29916
30416
|
if (!distillations.length) return [];
|
|
29917
30417
|
const formatted = formatDistillations(distillations);
|
|
@@ -30029,6 +30529,11 @@ function tryFitStable(input) {
|
|
|
30029
30529
|
}
|
|
30030
30530
|
return result;
|
|
30031
30531
|
}
|
|
30532
|
+
var COMPRESSION_STAGES = [
|
|
30533
|
+
{ strip: "none", rawFrac: null, distFrac: null, distLimit: Infinity, protectedTurns: 0, useStableWindow: true },
|
|
30534
|
+
{ strip: "old-tools", rawFrac: 0.5, distFrac: null, distLimit: Infinity, protectedTurns: 2, useStableWindow: false },
|
|
30535
|
+
{ strip: "all-tools", rawFrac: 0.55, distFrac: 0.15, distLimit: 5, protectedTurns: 0, useStableWindow: false }
|
|
30536
|
+
];
|
|
30032
30537
|
var urgentDistillationMap = /* @__PURE__ */ new Map();
|
|
30033
30538
|
function needsUrgentDistillation(sessionID) {
|
|
30034
30539
|
const v = urgentDistillationMap.get(sessionID) ?? false;
|
|
@@ -30060,7 +30565,7 @@ function transformInner(input) {
|
|
|
30060
30565
|
if (calibrated) return true;
|
|
30061
30566
|
return result.totalTokens * UNCALIBRATED_SAFETY <= maxInput;
|
|
30062
30567
|
}
|
|
30063
|
-
if (calibrated && sessState.lastLayer >= 1 && input.messages.length >= sessState.lastKnownMessageCount) {
|
|
30568
|
+
if (calibrated && sessState.lastLayer >= 1 && sessState.lastLayer <= 3 && input.messages.length >= sessState.lastKnownMessageCount) {
|
|
30064
30569
|
effectiveMinLayer = Math.max(effectiveMinLayer, sessState.lastLayer);
|
|
30065
30570
|
}
|
|
30066
30571
|
const postIdleCompact = sessState.postIdleCompact;
|
|
@@ -30098,7 +30603,8 @@ function transformInner(input) {
|
|
|
30098
30603
|
totalTokens: Math.max(0, messageTokens),
|
|
30099
30604
|
usable,
|
|
30100
30605
|
distilledBudget,
|
|
30101
|
-
rawBudget
|
|
30606
|
+
rawBudget,
|
|
30607
|
+
refreshLtm: false
|
|
30102
30608
|
};
|
|
30103
30609
|
}
|
|
30104
30610
|
const turnStart = currentTurnStart(input.messages);
|
|
@@ -30108,67 +30614,52 @@ function transformInner(input) {
|
|
|
30108
30614
|
const msgs = distilledPrefix(distillations);
|
|
30109
30615
|
return { messages: msgs, tokens: msgs.reduce((sum, m) => sum + estimateMessage(m), 0) };
|
|
30110
30616
|
})();
|
|
30111
|
-
|
|
30112
|
-
const
|
|
30113
|
-
|
|
30114
|
-
|
|
30115
|
-
|
|
30116
|
-
|
|
30117
|
-
|
|
30118
|
-
|
|
30119
|
-
|
|
30120
|
-
|
|
30121
|
-
|
|
30122
|
-
|
|
30123
|
-
|
|
30124
|
-
|
|
30125
|
-
|
|
30126
|
-
|
|
30127
|
-
|
|
30128
|
-
|
|
30129
|
-
|
|
30617
|
+
for (let s = 0; s < COMPRESSION_STAGES.length; s++) {
|
|
30618
|
+
const stageLayer = s + 1;
|
|
30619
|
+
if (effectiveMinLayer > stageLayer) continue;
|
|
30620
|
+
const stage = COMPRESSION_STAGES[s];
|
|
30621
|
+
const stageRawBudget = stage.rawFrac !== null ? Math.floor(usable * stage.rawFrac) : rawBudget;
|
|
30622
|
+
const stageDistBudget = stage.distFrac !== null ? Math.floor(usable * stage.distFrac) : distilledBudget;
|
|
30623
|
+
let stagePrefix = cached2.messages;
|
|
30624
|
+
let stagePrefixTokens = cached2.tokens;
|
|
30625
|
+
if (stage.distLimit !== Infinity && distillations.length > stage.distLimit) {
|
|
30626
|
+
const trimmed = selectDistillations(distillations, stage.distLimit);
|
|
30627
|
+
stagePrefix = distilledPrefix(trimmed);
|
|
30628
|
+
stagePrefixTokens = stagePrefix.reduce((sum, m) => sum + estimateMessage(m), 0);
|
|
30629
|
+
}
|
|
30630
|
+
let result;
|
|
30631
|
+
if (stage.useStableWindow && sid) {
|
|
30632
|
+
result = tryFitStable({
|
|
30633
|
+
messages: dedupMessages,
|
|
30634
|
+
prefix: stagePrefix,
|
|
30635
|
+
prefixTokens: stagePrefixTokens,
|
|
30636
|
+
distilledBudget: stageDistBudget,
|
|
30637
|
+
rawBudget: stageRawBudget,
|
|
30638
|
+
sessionID: sid,
|
|
30639
|
+
sessState
|
|
30640
|
+
});
|
|
30641
|
+
} else {
|
|
30642
|
+
sessState.rawWindowCache = null;
|
|
30643
|
+
result = tryFit({
|
|
30644
|
+
messages: dedupMessages,
|
|
30645
|
+
prefix: stagePrefix,
|
|
30646
|
+
prefixTokens: stagePrefixTokens,
|
|
30647
|
+
distilledBudget: stageDistBudget,
|
|
30648
|
+
rawBudget: stageRawBudget,
|
|
30649
|
+
strip: stage.strip,
|
|
30650
|
+
protectedTurns: stage.protectedTurns
|
|
30651
|
+
});
|
|
30652
|
+
}
|
|
30653
|
+
if (fitsWithSafetyMargin(result)) {
|
|
30654
|
+
if (sid && (s > 0 || cached2.tokens === 0)) {
|
|
30130
30655
|
urgentDistillationMap.set(sid, true);
|
|
30131
30656
|
}
|
|
30132
|
-
return { ...
|
|
30657
|
+
return { ...result, layer: stageLayer, usable, distilledBudget, rawBudget, refreshLtm: false };
|
|
30133
30658
|
}
|
|
30134
30659
|
}
|
|
30135
30660
|
sessState.rawWindowCache = null;
|
|
30136
|
-
if (effectiveMinLayer <= 2) {
|
|
30137
|
-
const layer2 = tryFit({
|
|
30138
|
-
messages: dedupMessages,
|
|
30139
|
-
prefix: cached2.messages,
|
|
30140
|
-
prefixTokens: cached2.tokens,
|
|
30141
|
-
distilledBudget,
|
|
30142
|
-
rawBudget: Math.floor(usable * 0.5),
|
|
30143
|
-
// give raw more room
|
|
30144
|
-
strip: "old-tools",
|
|
30145
|
-
protectedTurns: 2
|
|
30146
|
-
});
|
|
30147
|
-
if (fitsWithSafetyMargin(layer2)) {
|
|
30148
|
-
if (sid) urgentDistillationMap.set(sid, true);
|
|
30149
|
-
return { ...layer2, layer: 2, usable, distilledBudget, rawBudget };
|
|
30150
|
-
}
|
|
30151
|
-
}
|
|
30152
|
-
const trimmedDistillations = distillations.slice(-5);
|
|
30153
|
-
const trimmedPrefix = distilledPrefix(trimmedDistillations);
|
|
30154
|
-
const trimmedPrefixTokens = trimmedPrefix.reduce(
|
|
30155
|
-
(sum, m) => sum + estimateMessage(m),
|
|
30156
|
-
0
|
|
30157
|
-
);
|
|
30158
|
-
const layer3 = tryFit({
|
|
30159
|
-
messages: dedupMessages,
|
|
30160
|
-
prefix: trimmedPrefix,
|
|
30161
|
-
prefixTokens: trimmedPrefixTokens,
|
|
30162
|
-
distilledBudget: Math.floor(usable * 0.15),
|
|
30163
|
-
rawBudget: Math.floor(usable * 0.55),
|
|
30164
|
-
strip: "all-tools"
|
|
30165
|
-
});
|
|
30166
|
-
if (fitsWithSafetyMargin(layer3)) {
|
|
30167
|
-
if (sid) urgentDistillationMap.set(sid, true);
|
|
30168
|
-
return { ...layer3, layer: 3, usable, distilledBudget, rawBudget };
|
|
30169
|
-
}
|
|
30170
30661
|
if (sid) urgentDistillationMap.set(sid, true);
|
|
30171
|
-
const nuclearDistillations = distillations
|
|
30662
|
+
const nuclearDistillations = selectDistillations(distillations, 2);
|
|
30172
30663
|
const nuclearPrefix = distilledPrefix(nuclearDistillations);
|
|
30173
30664
|
const nuclearPrefixTokens = nuclearPrefix.reduce(
|
|
30174
30665
|
(sum, m) => sum + estimateMessage(m),
|
|
@@ -30207,7 +30698,8 @@ function transformInner(input) {
|
|
|
30207
30698
|
totalTokens: nuclearPrefixTokens + nuclearRawTokens,
|
|
30208
30699
|
usable,
|
|
30209
30700
|
distilledBudget,
|
|
30210
|
-
rawBudget
|
|
30701
|
+
rawBudget,
|
|
30702
|
+
refreshLtm: true
|
|
30211
30703
|
};
|
|
30212
30704
|
}
|
|
30213
30705
|
function transform2(input) {
|
|
@@ -30314,6 +30806,185 @@ function isWorkerSession(sessionID) {
|
|
|
30314
30806
|
return workerSessionIDs.has(sessionID);
|
|
30315
30807
|
}
|
|
30316
30808
|
|
|
30809
|
+
// ../../node_modules/.bun/yocto-queue@1.2.2/node_modules/yocto-queue/index.js
|
|
30810
|
+
var Node = class {
|
|
30811
|
+
value;
|
|
30812
|
+
next;
|
|
30813
|
+
constructor(value) {
|
|
30814
|
+
this.value = value;
|
|
30815
|
+
}
|
|
30816
|
+
};
|
|
30817
|
+
var Queue = class {
|
|
30818
|
+
#head;
|
|
30819
|
+
#tail;
|
|
30820
|
+
#size;
|
|
30821
|
+
constructor() {
|
|
30822
|
+
this.clear();
|
|
30823
|
+
}
|
|
30824
|
+
enqueue(value) {
|
|
30825
|
+
const node2 = new Node(value);
|
|
30826
|
+
if (this.#head) {
|
|
30827
|
+
this.#tail.next = node2;
|
|
30828
|
+
this.#tail = node2;
|
|
30829
|
+
} else {
|
|
30830
|
+
this.#head = node2;
|
|
30831
|
+
this.#tail = node2;
|
|
30832
|
+
}
|
|
30833
|
+
this.#size++;
|
|
30834
|
+
}
|
|
30835
|
+
dequeue() {
|
|
30836
|
+
const current2 = this.#head;
|
|
30837
|
+
if (!current2) {
|
|
30838
|
+
return;
|
|
30839
|
+
}
|
|
30840
|
+
this.#head = this.#head.next;
|
|
30841
|
+
this.#size--;
|
|
30842
|
+
if (!this.#head) {
|
|
30843
|
+
this.#tail = void 0;
|
|
30844
|
+
}
|
|
30845
|
+
return current2.value;
|
|
30846
|
+
}
|
|
30847
|
+
peek() {
|
|
30848
|
+
if (!this.#head) {
|
|
30849
|
+
return;
|
|
30850
|
+
}
|
|
30851
|
+
return this.#head.value;
|
|
30852
|
+
}
|
|
30853
|
+
clear() {
|
|
30854
|
+
this.#head = void 0;
|
|
30855
|
+
this.#tail = void 0;
|
|
30856
|
+
this.#size = 0;
|
|
30857
|
+
}
|
|
30858
|
+
get size() {
|
|
30859
|
+
return this.#size;
|
|
30860
|
+
}
|
|
30861
|
+
*[Symbol.iterator]() {
|
|
30862
|
+
let current2 = this.#head;
|
|
30863
|
+
while (current2) {
|
|
30864
|
+
yield current2.value;
|
|
30865
|
+
current2 = current2.next;
|
|
30866
|
+
}
|
|
30867
|
+
}
|
|
30868
|
+
*drain() {
|
|
30869
|
+
while (this.#head) {
|
|
30870
|
+
yield this.dequeue();
|
|
30871
|
+
}
|
|
30872
|
+
}
|
|
30873
|
+
};
|
|
30874
|
+
|
|
30875
|
+
// ../../node_modules/.bun/p-limit@7.3.0/node_modules/p-limit/index.js
|
|
30876
|
+
function pLimit(concurrency) {
|
|
30877
|
+
let rejectOnClear = false;
|
|
30878
|
+
if (typeof concurrency === "object") {
|
|
30879
|
+
({ concurrency, rejectOnClear = false } = concurrency);
|
|
30880
|
+
}
|
|
30881
|
+
validateConcurrency(concurrency);
|
|
30882
|
+
if (typeof rejectOnClear !== "boolean") {
|
|
30883
|
+
throw new TypeError("Expected `rejectOnClear` to be a boolean");
|
|
30884
|
+
}
|
|
30885
|
+
const queue = new Queue();
|
|
30886
|
+
let activeCount = 0;
|
|
30887
|
+
const resumeNext = () => {
|
|
30888
|
+
if (activeCount < concurrency && queue.size > 0) {
|
|
30889
|
+
activeCount++;
|
|
30890
|
+
queue.dequeue().run();
|
|
30891
|
+
}
|
|
30892
|
+
};
|
|
30893
|
+
const next = () => {
|
|
30894
|
+
activeCount--;
|
|
30895
|
+
resumeNext();
|
|
30896
|
+
};
|
|
30897
|
+
const run3 = async (function_, resolve, arguments_) => {
|
|
30898
|
+
const result = (async () => function_(...arguments_))();
|
|
30899
|
+
resolve(result);
|
|
30900
|
+
try {
|
|
30901
|
+
await result;
|
|
30902
|
+
} catch {
|
|
30903
|
+
}
|
|
30904
|
+
next();
|
|
30905
|
+
};
|
|
30906
|
+
const enqueue = (function_, resolve, reject, arguments_) => {
|
|
30907
|
+
const queueItem = { reject };
|
|
30908
|
+
new Promise((internalResolve) => {
|
|
30909
|
+
queueItem.run = internalResolve;
|
|
30910
|
+
queue.enqueue(queueItem);
|
|
30911
|
+
}).then(run3.bind(void 0, function_, resolve, arguments_));
|
|
30912
|
+
if (activeCount < concurrency) {
|
|
30913
|
+
resumeNext();
|
|
30914
|
+
}
|
|
30915
|
+
};
|
|
30916
|
+
const generator = (function_, ...arguments_) => new Promise((resolve, reject) => {
|
|
30917
|
+
enqueue(function_, resolve, reject, arguments_);
|
|
30918
|
+
});
|
|
30919
|
+
Object.defineProperties(generator, {
|
|
30920
|
+
activeCount: {
|
|
30921
|
+
get: () => activeCount
|
|
30922
|
+
},
|
|
30923
|
+
pendingCount: {
|
|
30924
|
+
get: () => queue.size
|
|
30925
|
+
},
|
|
30926
|
+
clearQueue: {
|
|
30927
|
+
value() {
|
|
30928
|
+
if (!rejectOnClear) {
|
|
30929
|
+
queue.clear();
|
|
30930
|
+
return;
|
|
30931
|
+
}
|
|
30932
|
+
const abortError = AbortSignal.abort().reason;
|
|
30933
|
+
while (queue.size > 0) {
|
|
30934
|
+
queue.dequeue().reject(abortError);
|
|
30935
|
+
}
|
|
30936
|
+
}
|
|
30937
|
+
},
|
|
30938
|
+
concurrency: {
|
|
30939
|
+
get: () => concurrency,
|
|
30940
|
+
set(newConcurrency) {
|
|
30941
|
+
validateConcurrency(newConcurrency);
|
|
30942
|
+
concurrency = newConcurrency;
|
|
30943
|
+
queueMicrotask(() => {
|
|
30944
|
+
while (activeCount < concurrency && queue.size > 0) {
|
|
30945
|
+
resumeNext();
|
|
30946
|
+
}
|
|
30947
|
+
});
|
|
30948
|
+
}
|
|
30949
|
+
},
|
|
30950
|
+
map: {
|
|
30951
|
+
async value(iterable, function_) {
|
|
30952
|
+
const promises = Array.from(iterable, (value, index2) => this(function_, value, index2));
|
|
30953
|
+
return Promise.all(promises);
|
|
30954
|
+
}
|
|
30955
|
+
}
|
|
30956
|
+
});
|
|
30957
|
+
return generator;
|
|
30958
|
+
}
|
|
30959
|
+
function validateConcurrency(concurrency) {
|
|
30960
|
+
if (!((Number.isInteger(concurrency) || concurrency === Number.POSITIVE_INFINITY) && concurrency > 0)) {
|
|
30961
|
+
throw new TypeError("Expected `concurrency` to be a number from 1 and up");
|
|
30962
|
+
}
|
|
30963
|
+
}
|
|
30964
|
+
|
|
30965
|
+
// src/session-limiter.ts
|
|
30966
|
+
function createLimiterPool() {
|
|
30967
|
+
const limiters = /* @__PURE__ */ new Map();
|
|
30968
|
+
function get2(key) {
|
|
30969
|
+
let limiter = limiters.get(key);
|
|
30970
|
+
if (!limiter) {
|
|
30971
|
+
limiter = pLimit(1);
|
|
30972
|
+
limiters.set(key, limiter);
|
|
30973
|
+
}
|
|
30974
|
+
return limiter;
|
|
30975
|
+
}
|
|
30976
|
+
function isBusy(key) {
|
|
30977
|
+
const limiter = limiters.get(key);
|
|
30978
|
+
return limiter ? limiter.activeCount + limiter.pendingCount > 0 : false;
|
|
30979
|
+
}
|
|
30980
|
+
function clear() {
|
|
30981
|
+
limiters.clear();
|
|
30982
|
+
}
|
|
30983
|
+
return { get: get2, isBusy, clear };
|
|
30984
|
+
}
|
|
30985
|
+
var distillLimiter = createLimiterPool();
|
|
30986
|
+
var curatorLimiter = createLimiterPool();
|
|
30987
|
+
|
|
30317
30988
|
// src/distillation.ts
|
|
30318
30989
|
function compressionRatio(distilledTokens, sourceTokens) {
|
|
30319
30990
|
if (sourceTokens <= 0) return 0;
|
|
@@ -30558,6 +31229,9 @@ function resetOrphans(projectPath, sessionID) {
|
|
|
30558
31229
|
return orphans.length;
|
|
30559
31230
|
}
|
|
30560
31231
|
async function run(input) {
|
|
31232
|
+
return distillLimiter.get(input.sessionID)(() => runInner(input));
|
|
31233
|
+
}
|
|
31234
|
+
async function runInner(input) {
|
|
30561
31235
|
const orphans = resetOrphans(input.projectPath, input.sessionID);
|
|
30562
31236
|
if (orphans > 0) {
|
|
30563
31237
|
info(
|
|
@@ -30601,7 +31275,7 @@ async function run(input) {
|
|
|
30601
31275
|
}
|
|
30602
31276
|
}
|
|
30603
31277
|
if (!input.skipMeta && gen0Count(input.projectPath, input.sessionID) >= cfg.distillation.metaThreshold) {
|
|
30604
|
-
await
|
|
31278
|
+
await metaDistillInner({
|
|
30605
31279
|
llm: input.llm,
|
|
30606
31280
|
projectPath: input.projectPath,
|
|
30607
31281
|
sessionID: input.sessionID,
|
|
@@ -30651,17 +31325,25 @@ async function distillSegment(input) {
|
|
|
30651
31325
|
);
|
|
30652
31326
|
return null;
|
|
30653
31327
|
}
|
|
30654
|
-
|
|
30655
|
-
|
|
30656
|
-
|
|
30657
|
-
|
|
30658
|
-
|
|
30659
|
-
|
|
30660
|
-
|
|
30661
|
-
|
|
30662
|
-
|
|
30663
|
-
|
|
30664
|
-
|
|
31328
|
+
let distillId;
|
|
31329
|
+
db().exec("BEGIN IMMEDIATE");
|
|
31330
|
+
try {
|
|
31331
|
+
distillId = storeDistillation({
|
|
31332
|
+
projectPath: input.projectPath,
|
|
31333
|
+
sessionID: input.sessionID,
|
|
31334
|
+
observations: result.observations,
|
|
31335
|
+
sourceIDs: input.messages.map((m) => m.id),
|
|
31336
|
+
generation: 0,
|
|
31337
|
+
rCompression: rComp,
|
|
31338
|
+
cNorm,
|
|
31339
|
+
callType: input.callType
|
|
31340
|
+
});
|
|
31341
|
+
markDistilled(input.messages.map((m) => m.id));
|
|
31342
|
+
db().exec("COMMIT");
|
|
31343
|
+
} catch (e) {
|
|
31344
|
+
db().exec("ROLLBACK");
|
|
31345
|
+
throw e;
|
|
31346
|
+
}
|
|
30665
31347
|
info(
|
|
30666
31348
|
`distill segment: ${input.messages.length} msgs, ${sourceTokens}\u2192${distilledTokens} tokens, R=${rComp.toFixed(2)}, C_norm=${cNorm.toFixed(3)}`
|
|
30667
31349
|
);
|
|
@@ -30695,6 +31377,9 @@ async function distillSegment(input) {
|
|
|
30695
31377
|
return result;
|
|
30696
31378
|
}
|
|
30697
31379
|
async function metaDistill(input) {
|
|
31380
|
+
return distillLimiter.get(input.sessionID)(() => metaDistillInner(input));
|
|
31381
|
+
}
|
|
31382
|
+
async function metaDistillInner(input) {
|
|
30698
31383
|
const existing = loadGen0(input.projectPath, input.sessionID);
|
|
30699
31384
|
const priorMeta = latestMeta(input.projectPath, input.sessionID);
|
|
30700
31385
|
if (priorMeta) {
|
|
@@ -30994,11 +31679,27 @@ function applyOps(ops, input) {
|
|
|
30994
31679
|
return { created, updated, deleted };
|
|
30995
31680
|
}
|
|
30996
31681
|
var lastCuratedAt = /* @__PURE__ */ new Map();
|
|
31682
|
+
function getLastCuratedAt(sessionID) {
|
|
31683
|
+
const cached2 = lastCuratedAt.get(sessionID);
|
|
31684
|
+
if (cached2 !== void 0) return cached2;
|
|
31685
|
+
const persisted = loadSessionTracking(sessionID);
|
|
31686
|
+
const ts = persisted?.lastCuratedAt ?? 0;
|
|
31687
|
+
lastCuratedAt.set(sessionID, ts);
|
|
31688
|
+
return ts;
|
|
31689
|
+
}
|
|
30997
31690
|
async function run2(input) {
|
|
30998
31691
|
const cfg = config2();
|
|
30999
31692
|
if (!cfg.curator.enabled) return { created: 0, updated: 0, deleted: 0 };
|
|
31693
|
+
if (curatorLimiter.isBusy(input.sessionID)) {
|
|
31694
|
+
info(`curation skipped: already running for session ${input.sessionID.slice(0, 16)}`);
|
|
31695
|
+
return { created: 0, updated: 0, deleted: 0 };
|
|
31696
|
+
}
|
|
31697
|
+
return curatorLimiter.get(input.sessionID)(() => runInner2(input));
|
|
31698
|
+
}
|
|
31699
|
+
async function runInner2(input) {
|
|
31700
|
+
const cfg = config2();
|
|
31000
31701
|
const all3 = bySession(input.projectPath, input.sessionID);
|
|
31001
|
-
const sessionCuratedAt =
|
|
31702
|
+
const sessionCuratedAt = getLastCuratedAt(input.sessionID);
|
|
31002
31703
|
const recent = all3.filter((m) => m.created_at > sessionCuratedAt);
|
|
31003
31704
|
if (recent.length < 3) return { created: 0, updated: 0, deleted: 0 };
|
|
31004
31705
|
const text4 = recent.map((m) => `[${m.role}] ${m.content}`).join("\n\n");
|
|
@@ -31042,11 +31743,22 @@ async function run2(input) {
|
|
|
31042
31743
|
info(`post-curation dedup: merged ${dupes.totalRemoved} duplicate entries`);
|
|
31043
31744
|
result.deleted += dupes.totalRemoved;
|
|
31044
31745
|
}
|
|
31746
|
+
if (dupes.pairSimilarities.size > 0) {
|
|
31747
|
+
const pid = ensureProject(input.projectPath);
|
|
31748
|
+
recordAutoSignals(pid, dupes);
|
|
31749
|
+
const newThreshold = calibrateDedupThreshold(pid);
|
|
31750
|
+
if (newThreshold !== null) {
|
|
31751
|
+
const count3 = getDedupFeedbackCount(pid);
|
|
31752
|
+
saveCalibratedThreshold(pid, newThreshold, count3);
|
|
31753
|
+
}
|
|
31754
|
+
}
|
|
31045
31755
|
} catch (err) {
|
|
31046
31756
|
warn("post-curation dedup failed (non-fatal):", err);
|
|
31047
31757
|
}
|
|
31048
31758
|
}
|
|
31049
|
-
|
|
31759
|
+
const now = Date.now();
|
|
31760
|
+
lastCuratedAt.set(input.sessionID, now);
|
|
31761
|
+
saveSessionTracking(input.sessionID, { lastCuratedAt: now });
|
|
31050
31762
|
return result;
|
|
31051
31763
|
}
|
|
31052
31764
|
function resetCurationTracker(sessionID) {
|
|
@@ -33169,6 +33881,7 @@ export {
|
|
|
33169
33881
|
ftsQueryRelaxed,
|
|
33170
33882
|
getGitRemote,
|
|
33171
33883
|
getInstanceId,
|
|
33884
|
+
getKV,
|
|
33172
33885
|
getLastImportAt,
|
|
33173
33886
|
getLastTransformEstimate,
|
|
33174
33887
|
getLastTransformedCount,
|
|
@@ -33194,7 +33907,9 @@ export {
|
|
|
33194
33907
|
load,
|
|
33195
33908
|
loadAllSessionCosts,
|
|
33196
33909
|
loadForceMinLayer,
|
|
33910
|
+
loadHeaderSessionIndex,
|
|
33197
33911
|
loadSessionCosts,
|
|
33912
|
+
loadSessionTracking,
|
|
33198
33913
|
log_exports as log,
|
|
33199
33914
|
loreFileExists,
|
|
33200
33915
|
ltm_exports as ltm,
|
|
@@ -33215,10 +33930,13 @@ export {
|
|
|
33215
33930
|
runRecall,
|
|
33216
33931
|
sanitizeSurrogates,
|
|
33217
33932
|
saveForceMinLayer,
|
|
33933
|
+
saveGradientState,
|
|
33218
33934
|
saveSessionCosts,
|
|
33935
|
+
saveSessionTracking,
|
|
33219
33936
|
searchRecall,
|
|
33220
33937
|
serialize,
|
|
33221
33938
|
setForceMinLayer,
|
|
33939
|
+
setKV,
|
|
33222
33940
|
setLastImportAt,
|
|
33223
33941
|
setLastTurnAtForTest,
|
|
33224
33942
|
setLtmTokens,
|