@loreai/core 0.18.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bun/agents-file.d.ts.map +1 -1
- package/dist/bun/config.d.ts.map +1 -1
- package/dist/bun/curator.d.ts.map +1 -1
- package/dist/bun/db.d.ts +86 -1
- package/dist/bun/db.d.ts.map +1 -1
- package/dist/bun/distillation.d.ts +2 -13
- package/dist/bun/distillation.d.ts.map +1 -1
- package/dist/bun/embedding.d.ts +5 -1
- package/dist/bun/embedding.d.ts.map +1 -1
- package/dist/bun/git.d.ts.map +1 -1
- package/dist/bun/gradient.d.ts +13 -1
- package/dist/bun/gradient.d.ts.map +1 -1
- package/dist/bun/hosted.d.ts +36 -0
- package/dist/bun/hosted.d.ts.map +1 -0
- package/dist/bun/index.d.ts +3 -2
- package/dist/bun/index.d.ts.map +1 -1
- package/dist/bun/index.js +1049 -247
- package/dist/bun/index.js.map +4 -4
- package/dist/bun/lat-reader.d.ts.map +1 -1
- package/dist/bun/ltm.d.ts +99 -5
- package/dist/bun/ltm.d.ts.map +1 -1
- package/dist/bun/session-limiter.d.ts +26 -0
- package/dist/bun/session-limiter.d.ts.map +1 -0
- package/dist/bun/temporal.d.ts +2 -0
- package/dist/bun/temporal.d.ts.map +1 -1
- package/dist/node/agents-file.d.ts.map +1 -1
- package/dist/node/config.d.ts.map +1 -1
- package/dist/node/curator.d.ts.map +1 -1
- package/dist/node/db.d.ts +86 -1
- package/dist/node/db.d.ts.map +1 -1
- package/dist/node/distillation.d.ts +2 -13
- package/dist/node/distillation.d.ts.map +1 -1
- package/dist/node/embedding.d.ts +5 -1
- package/dist/node/embedding.d.ts.map +1 -1
- package/dist/node/git.d.ts.map +1 -1
- package/dist/node/gradient.d.ts +13 -1
- package/dist/node/gradient.d.ts.map +1 -1
- package/dist/node/hosted.d.ts +36 -0
- package/dist/node/hosted.d.ts.map +1 -0
- package/dist/node/index.d.ts +3 -2
- package/dist/node/index.d.ts.map +1 -1
- package/dist/node/index.js +1049 -247
- package/dist/node/index.js.map +4 -4
- package/dist/node/lat-reader.d.ts.map +1 -1
- package/dist/node/ltm.d.ts +99 -5
- package/dist/node/ltm.d.ts.map +1 -1
- package/dist/node/session-limiter.d.ts +26 -0
- package/dist/node/session-limiter.d.ts.map +1 -0
- package/dist/node/temporal.d.ts +2 -0
- package/dist/node/temporal.d.ts.map +1 -1
- package/dist/types/agents-file.d.ts.map +1 -1
- package/dist/types/config.d.ts.map +1 -1
- package/dist/types/curator.d.ts.map +1 -1
- package/dist/types/db.d.ts +86 -1
- package/dist/types/db.d.ts.map +1 -1
- package/dist/types/distillation.d.ts +2 -13
- package/dist/types/distillation.d.ts.map +1 -1
- package/dist/types/embedding.d.ts +5 -1
- package/dist/types/embedding.d.ts.map +1 -1
- package/dist/types/git.d.ts.map +1 -1
- package/dist/types/gradient.d.ts +13 -1
- package/dist/types/gradient.d.ts.map +1 -1
- package/dist/types/hosted.d.ts +36 -0
- package/dist/types/hosted.d.ts.map +1 -0
- package/dist/types/index.d.ts +3 -2
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/lat-reader.d.ts.map +1 -1
- package/dist/types/ltm.d.ts +99 -5
- package/dist/types/ltm.d.ts.map +1 -1
- package/dist/types/session-limiter.d.ts +26 -0
- package/dist/types/session-limiter.d.ts.map +1 -0
- package/dist/types/temporal.d.ts +2 -0
- package/dist/types/temporal.d.ts.map +1 -1
- package/package.json +3 -1
- package/src/agents-file.ts +12 -0
- package/src/config.ts +10 -5
- package/src/curator.ts +54 -2
- package/src/db.ts +386 -6
- package/src/distillation.ts +55 -14
- package/src/embedding.ts +71 -8
- package/src/git.ts +4 -0
- package/src/gradient.ts +227 -74
- package/src/hosted.ts +46 -0
- package/src/index.ts +12 -0
- package/src/lat-reader.ts +4 -0
- package/src/ltm.ts +480 -45
- package/src/session-limiter.ts +47 -0
- package/src/temporal.ts +10 -0
package/dist/node/index.js
CHANGED
|
@@ -125,6 +125,7 @@ __export(temporal_exports, {
|
|
|
125
125
|
CHUNK_TERMINATOR: () => CHUNK_TERMINATOR,
|
|
126
126
|
bySession: () => bySession,
|
|
127
127
|
count: () => count,
|
|
128
|
+
hasMessages: () => hasMessages,
|
|
128
129
|
markDistilled: () => markDistilled,
|
|
129
130
|
partsToText: () => partsToText,
|
|
130
131
|
prune: () => prune,
|
|
@@ -167,6 +168,17 @@ import { mkdirSync } from "fs";
|
|
|
167
168
|
|
|
168
169
|
// src/git.ts
|
|
169
170
|
import { execSync } from "child_process";
|
|
171
|
+
|
|
172
|
+
// src/hosted.ts
|
|
173
|
+
var _hostedMode = false;
|
|
174
|
+
function enableHostedMode() {
|
|
175
|
+
_hostedMode = true;
|
|
176
|
+
}
|
|
177
|
+
function isHostedMode() {
|
|
178
|
+
return _hostedMode;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
// src/git.ts
|
|
170
182
|
function normalizeRemoteUrl(url2) {
|
|
171
183
|
let normalized = url2.trim();
|
|
172
184
|
const sshMatch = normalized.match(/^[\w.-]+@([\w.-]+):(.+)$/);
|
|
@@ -191,6 +203,7 @@ function clearGitRemoteCache() {
|
|
|
191
203
|
gitRemoteCache.clear();
|
|
192
204
|
}
|
|
193
205
|
function getGitRemote(path) {
|
|
206
|
+
if (isHostedMode()) return null;
|
|
194
207
|
const cached2 = gitRemoteCache.get(path);
|
|
195
208
|
if (cached2 !== void 0) return cached2;
|
|
196
209
|
try {
|
|
@@ -754,6 +767,55 @@ var MIGRATIONS = [
|
|
|
754
767
|
WHERE ih.project_id = projects.id
|
|
755
768
|
AND ih.source_id = '__declined__'
|
|
756
769
|
);
|
|
770
|
+
`,
|
|
771
|
+
`
|
|
772
|
+
-- Version 23: Persist volatile session tracking state across restarts.
|
|
773
|
+
-- Previously these were in-memory only, causing duplicate processing,
|
|
774
|
+
-- false compaction detection, and expensive prompt cache busts on restart.
|
|
775
|
+
ALTER TABLE session_state ADD COLUMN last_curated_at INTEGER NOT NULL DEFAULT 0;
|
|
776
|
+
ALTER TABLE session_state ADD COLUMN message_count INTEGER NOT NULL DEFAULT 0;
|
|
777
|
+
ALTER TABLE session_state ADD COLUMN turns_since_curation INTEGER NOT NULL DEFAULT 0;
|
|
778
|
+
ALTER TABLE session_state ADD COLUMN ltm_cache_text TEXT;
|
|
779
|
+
ALTER TABLE session_state ADD COLUMN ltm_cache_tokens INTEGER;
|
|
780
|
+
ALTER TABLE session_state ADD COLUMN ltm_pin_text TEXT;
|
|
781
|
+
ALTER TABLE session_state ADD COLUMN ltm_pin_tokens INTEGER;
|
|
782
|
+
ALTER TABLE session_state ADD COLUMN consecutive_text_only_turns INTEGER NOT NULL DEFAULT 0;
|
|
783
|
+
`,
|
|
784
|
+
`
|
|
785
|
+
-- Version 24: Persist remaining volatile session state across restarts.
|
|
786
|
+
-- Session identity (Tier 1/2/3 session correlation)
|
|
787
|
+
ALTER TABLE session_state ADD COLUMN fingerprint TEXT NOT NULL DEFAULT '';
|
|
788
|
+
ALTER TABLE session_state ADD COLUMN header_session_id TEXT;
|
|
789
|
+
ALTER TABLE session_state ADD COLUMN header_name TEXT;
|
|
790
|
+
-- Cache warming state
|
|
791
|
+
ALTER TABLE session_state ADD COLUMN resolved_conversation_ttl TEXT NOT NULL DEFAULT '5m';
|
|
792
|
+
ALTER TABLE session_state ADD COLUMN warmup_state TEXT;
|
|
793
|
+
-- Gradient calibration state (survives restarts to avoid uncalibrated busts)
|
|
794
|
+
ALTER TABLE session_state ADD COLUMN dynamic_context_cap REAL NOT NULL DEFAULT 0;
|
|
795
|
+
ALTER TABLE session_state ADD COLUMN bust_rate_ema REAL NOT NULL DEFAULT -1;
|
|
796
|
+
ALTER TABLE session_state ADD COLUMN inter_bust_interval_ema REAL NOT NULL DEFAULT -1;
|
|
797
|
+
ALTER TABLE session_state ADD COLUMN last_layer INTEGER NOT NULL DEFAULT 0;
|
|
798
|
+
ALTER TABLE session_state ADD COLUMN last_known_input INTEGER NOT NULL DEFAULT 0;
|
|
799
|
+
ALTER TABLE session_state ADD COLUMN last_turn_at INTEGER NOT NULL DEFAULT 0;
|
|
800
|
+
ALTER TABLE session_state ADD COLUMN last_bust_at INTEGER NOT NULL DEFAULT 0;
|
|
801
|
+
`,
|
|
802
|
+
`
|
|
803
|
+
-- Version 25: Adaptive dedup threshold \u2014 store accept/reject feedback
|
|
804
|
+
-- on embedding-based duplicate pairs for per-project threshold calibration.
|
|
805
|
+
-- Titles stored instead of FK IDs because entries are deleted during dedup;
|
|
806
|
+
-- the similarity float is the actual calibration input.
|
|
807
|
+
CREATE TABLE IF NOT EXISTS dedup_feedback (
|
|
808
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
809
|
+
project_id TEXT,
|
|
810
|
+
entry_a_title TEXT NOT NULL,
|
|
811
|
+
entry_b_title TEXT NOT NULL,
|
|
812
|
+
similarity REAL NOT NULL,
|
|
813
|
+
accepted INTEGER NOT NULL,
|
|
814
|
+
source TEXT NOT NULL DEFAULT 'manual',
|
|
815
|
+
created_at INTEGER NOT NULL
|
|
816
|
+
);
|
|
817
|
+
CREATE INDEX IF NOT EXISTS idx_dedup_feedback_project
|
|
818
|
+
ON dedup_feedback(project_id);
|
|
757
819
|
`
|
|
758
820
|
];
|
|
759
821
|
function dbPath() {
|
|
@@ -889,26 +951,31 @@ function close() {
|
|
|
889
951
|
instance = void 0;
|
|
890
952
|
}
|
|
891
953
|
}
|
|
892
|
-
function ensureProject(path, name) {
|
|
954
|
+
function ensureProject(path, name, suppliedGitRemote) {
|
|
955
|
+
if (!process.env.LORE_DB_PATH && /^\/test\//.test(path)) {
|
|
956
|
+
throw new Error(
|
|
957
|
+
`Refusing to create project with test path "${path}" in the production DB. Set LORE_DB_PATH to a temp path, or run tests via \`bun test\` from the repo root.`
|
|
958
|
+
);
|
|
959
|
+
}
|
|
893
960
|
const existing = db().query("SELECT id, git_remote FROM projects WHERE path = ?").get(path);
|
|
894
961
|
if (existing) {
|
|
895
962
|
if (!existing.git_remote) {
|
|
896
|
-
const
|
|
897
|
-
if (
|
|
963
|
+
const resolvedRemote = suppliedGitRemote ?? getGitRemote(path);
|
|
964
|
+
if (resolvedRemote) {
|
|
898
965
|
const conflict = db().query(
|
|
899
966
|
"SELECT id FROM projects WHERE git_remote = ? AND id != ? LIMIT 1"
|
|
900
|
-
).get(
|
|
967
|
+
).get(resolvedRemote, existing.id);
|
|
901
968
|
if (conflict) {
|
|
902
969
|
mergeProjectInternal(conflict.id, existing.id);
|
|
903
970
|
}
|
|
904
|
-
db().query("UPDATE projects SET git_remote = ? WHERE id = ?").run(
|
|
971
|
+
db().query("UPDATE projects SET git_remote = ? WHERE id = ?").run(resolvedRemote, existing.id);
|
|
905
972
|
}
|
|
906
973
|
}
|
|
907
974
|
return existing.id;
|
|
908
975
|
}
|
|
909
976
|
const alias = db().query("SELECT project_id FROM project_path_aliases WHERE path = ?").get(path);
|
|
910
977
|
if (alias) return alias.project_id;
|
|
911
|
-
const gitRemote = getGitRemote(path);
|
|
978
|
+
const gitRemote = suppliedGitRemote ?? getGitRemote(path);
|
|
912
979
|
if (gitRemote) {
|
|
913
980
|
const byRemote = db().query("SELECT id FROM projects WHERE git_remote = ? LIMIT 1").get(gitRemote);
|
|
914
981
|
if (byRemote) {
|
|
@@ -936,6 +1003,20 @@ function projectId(path) {
|
|
|
936
1003
|
const alias = db().query("SELECT project_id FROM project_path_aliases WHERE path = ?").get(path);
|
|
937
1004
|
return alias?.project_id;
|
|
938
1005
|
}
|
|
1006
|
+
function resolveProjectByRemoteOrPath(gitRemote, path) {
|
|
1007
|
+
if (gitRemote) {
|
|
1008
|
+
const row = db().query("SELECT id FROM projects WHERE git_remote = ? LIMIT 1").get(gitRemote);
|
|
1009
|
+
if (row) return row.id;
|
|
1010
|
+
}
|
|
1011
|
+
if (path) {
|
|
1012
|
+
return projectId(path) ?? null;
|
|
1013
|
+
}
|
|
1014
|
+
return null;
|
|
1015
|
+
}
|
|
1016
|
+
function projectPath(id) {
|
|
1017
|
+
const row = db().query("SELECT path FROM projects WHERE id = ?").get(id);
|
|
1018
|
+
return row?.path ?? null;
|
|
1019
|
+
}
|
|
939
1020
|
function projectName(id) {
|
|
940
1021
|
const row = db().query("SELECT name FROM projects WHERE id = ?").get(id);
|
|
941
1022
|
return row?.name ?? null;
|
|
@@ -944,13 +1025,13 @@ function isFirstRun() {
|
|
|
944
1025
|
const row = db().query("SELECT COUNT(*) as count FROM projects").get();
|
|
945
1026
|
return row.count === 0;
|
|
946
1027
|
}
|
|
947
|
-
function getLastImportAt(
|
|
948
|
-
const id = ensureProject(
|
|
1028
|
+
function getLastImportAt(projectPath2) {
|
|
1029
|
+
const id = ensureProject(projectPath2);
|
|
949
1030
|
const row = db().query("SELECT last_import_at FROM projects WHERE id = ?").get(id);
|
|
950
1031
|
return row?.last_import_at ?? null;
|
|
951
1032
|
}
|
|
952
|
-
function setLastImportAt(
|
|
953
|
-
const id = ensureProject(
|
|
1033
|
+
function setLastImportAt(projectPath2, timestamp) {
|
|
1034
|
+
const id = ensureProject(projectPath2);
|
|
954
1035
|
db().query("UPDATE projects SET last_import_at = ? WHERE id = ?").run(timestamp, id);
|
|
955
1036
|
}
|
|
956
1037
|
function loadForceMinLayer(sessionID) {
|
|
@@ -1058,6 +1139,153 @@ function loadAllSessionCosts() {
|
|
|
1058
1139
|
}
|
|
1059
1140
|
return result;
|
|
1060
1141
|
}
|
|
1142
|
+
function saveSessionTracking(sessionID, state) {
|
|
1143
|
+
const now = Date.now();
|
|
1144
|
+
db().query(
|
|
1145
|
+
"INSERT OR IGNORE INTO session_state (session_id, force_min_layer, updated_at) VALUES (?, 0, ?)"
|
|
1146
|
+
).run(sessionID, now);
|
|
1147
|
+
const sets = ["updated_at = ?"];
|
|
1148
|
+
const vals = [now];
|
|
1149
|
+
if (state.lastCuratedAt !== void 0) {
|
|
1150
|
+
sets.push("last_curated_at = ?");
|
|
1151
|
+
vals.push(state.lastCuratedAt);
|
|
1152
|
+
}
|
|
1153
|
+
if (state.messageCount !== void 0) {
|
|
1154
|
+
sets.push("message_count = ?");
|
|
1155
|
+
vals.push(state.messageCount);
|
|
1156
|
+
}
|
|
1157
|
+
if (state.turnsSinceCuration !== void 0) {
|
|
1158
|
+
sets.push("turns_since_curation = ?");
|
|
1159
|
+
vals.push(state.turnsSinceCuration);
|
|
1160
|
+
}
|
|
1161
|
+
if (state.consecutiveTextOnlyTurns !== void 0) {
|
|
1162
|
+
sets.push("consecutive_text_only_turns = ?");
|
|
1163
|
+
vals.push(state.consecutiveTextOnlyTurns);
|
|
1164
|
+
}
|
|
1165
|
+
if (state.ltmCacheText !== void 0) {
|
|
1166
|
+
sets.push("ltm_cache_text = ?");
|
|
1167
|
+
vals.push(state.ltmCacheText);
|
|
1168
|
+
}
|
|
1169
|
+
if (state.ltmCacheTokens !== void 0) {
|
|
1170
|
+
sets.push("ltm_cache_tokens = ?");
|
|
1171
|
+
vals.push(state.ltmCacheTokens);
|
|
1172
|
+
}
|
|
1173
|
+
if (state.ltmPinText !== void 0) {
|
|
1174
|
+
sets.push("ltm_pin_text = ?");
|
|
1175
|
+
vals.push(state.ltmPinText);
|
|
1176
|
+
}
|
|
1177
|
+
if (state.ltmPinTokens !== void 0) {
|
|
1178
|
+
sets.push("ltm_pin_tokens = ?");
|
|
1179
|
+
vals.push(state.ltmPinTokens);
|
|
1180
|
+
}
|
|
1181
|
+
if (state.fingerprint !== void 0) {
|
|
1182
|
+
sets.push("fingerprint = ?");
|
|
1183
|
+
vals.push(state.fingerprint);
|
|
1184
|
+
}
|
|
1185
|
+
if (state.headerSessionId !== void 0) {
|
|
1186
|
+
sets.push("header_session_id = ?");
|
|
1187
|
+
vals.push(state.headerSessionId);
|
|
1188
|
+
}
|
|
1189
|
+
if (state.headerName !== void 0) {
|
|
1190
|
+
sets.push("header_name = ?");
|
|
1191
|
+
vals.push(state.headerName);
|
|
1192
|
+
}
|
|
1193
|
+
if (state.resolvedConversationTTL !== void 0) {
|
|
1194
|
+
sets.push("resolved_conversation_ttl = ?");
|
|
1195
|
+
vals.push(state.resolvedConversationTTL);
|
|
1196
|
+
}
|
|
1197
|
+
if (state.warmupState !== void 0) {
|
|
1198
|
+
sets.push("warmup_state = ?");
|
|
1199
|
+
vals.push(state.warmupState);
|
|
1200
|
+
}
|
|
1201
|
+
if (state.dynamicContextCap !== void 0) {
|
|
1202
|
+
sets.push("dynamic_context_cap = ?");
|
|
1203
|
+
vals.push(state.dynamicContextCap);
|
|
1204
|
+
}
|
|
1205
|
+
if (state.bustRateEMA !== void 0) {
|
|
1206
|
+
sets.push("bust_rate_ema = ?");
|
|
1207
|
+
vals.push(state.bustRateEMA);
|
|
1208
|
+
}
|
|
1209
|
+
if (state.interBustIntervalEMA !== void 0) {
|
|
1210
|
+
sets.push("inter_bust_interval_ema = ?");
|
|
1211
|
+
vals.push(state.interBustIntervalEMA);
|
|
1212
|
+
}
|
|
1213
|
+
if (state.lastLayer !== void 0) {
|
|
1214
|
+
sets.push("last_layer = ?");
|
|
1215
|
+
vals.push(state.lastLayer);
|
|
1216
|
+
}
|
|
1217
|
+
if (state.lastKnownInput !== void 0) {
|
|
1218
|
+
sets.push("last_known_input = ?");
|
|
1219
|
+
vals.push(state.lastKnownInput);
|
|
1220
|
+
}
|
|
1221
|
+
if (state.lastTurnAt !== void 0) {
|
|
1222
|
+
sets.push("last_turn_at = ?");
|
|
1223
|
+
vals.push(state.lastTurnAt);
|
|
1224
|
+
}
|
|
1225
|
+
if (state.lastBustAt !== void 0) {
|
|
1226
|
+
sets.push("last_bust_at = ?");
|
|
1227
|
+
vals.push(state.lastBustAt);
|
|
1228
|
+
}
|
|
1229
|
+
db().query(
|
|
1230
|
+
"UPDATE session_state SET " + sets.join(", ") + " WHERE session_id = ?"
|
|
1231
|
+
).run(...vals, sessionID);
|
|
1232
|
+
}
|
|
1233
|
+
function loadSessionTracking(sessionID) {
|
|
1234
|
+
const row = db().query(
|
|
1235
|
+
`SELECT last_curated_at, message_count, turns_since_curation,
|
|
1236
|
+
consecutive_text_only_turns,
|
|
1237
|
+
ltm_cache_text, ltm_cache_tokens, ltm_pin_text, ltm_pin_tokens,
|
|
1238
|
+
fingerprint, header_session_id, header_name,
|
|
1239
|
+
resolved_conversation_ttl, warmup_state,
|
|
1240
|
+
dynamic_context_cap, bust_rate_ema, inter_bust_interval_ema,
|
|
1241
|
+
last_layer, last_known_input, last_turn_at, last_bust_at
|
|
1242
|
+
FROM session_state WHERE session_id = ?`
|
|
1243
|
+
).get(sessionID);
|
|
1244
|
+
if (!row) return null;
|
|
1245
|
+
return {
|
|
1246
|
+
lastCuratedAt: row.last_curated_at,
|
|
1247
|
+
messageCount: row.message_count,
|
|
1248
|
+
turnsSinceCuration: row.turns_since_curation,
|
|
1249
|
+
consecutiveTextOnlyTurns: row.consecutive_text_only_turns,
|
|
1250
|
+
ltmCacheText: row.ltm_cache_text,
|
|
1251
|
+
ltmCacheTokens: row.ltm_cache_tokens,
|
|
1252
|
+
ltmPinText: row.ltm_pin_text,
|
|
1253
|
+
ltmPinTokens: row.ltm_pin_tokens,
|
|
1254
|
+
fingerprint: row.fingerprint,
|
|
1255
|
+
headerSessionId: row.header_session_id,
|
|
1256
|
+
headerName: row.header_name,
|
|
1257
|
+
resolvedConversationTTL: row.resolved_conversation_ttl,
|
|
1258
|
+
warmupState: row.warmup_state,
|
|
1259
|
+
dynamicContextCap: row.dynamic_context_cap,
|
|
1260
|
+
bustRateEMA: row.bust_rate_ema,
|
|
1261
|
+
interBustIntervalEMA: row.inter_bust_interval_ema,
|
|
1262
|
+
lastLayer: row.last_layer,
|
|
1263
|
+
lastKnownInput: row.last_known_input,
|
|
1264
|
+
lastTurnAt: row.last_turn_at,
|
|
1265
|
+
lastBustAt: row.last_bust_at
|
|
1266
|
+
};
|
|
1267
|
+
}
|
|
1268
|
+
function loadHeaderSessionIndex() {
|
|
1269
|
+
const rows = db().query(
|
|
1270
|
+
`SELECT session_id, header_session_id, header_name
|
|
1271
|
+
FROM session_state
|
|
1272
|
+
WHERE header_session_id IS NOT NULL AND header_name IS NOT NULL`
|
|
1273
|
+
).all();
|
|
1274
|
+
return rows.map((row) => ({
|
|
1275
|
+
sessionId: row.session_id,
|
|
1276
|
+
headerSessionId: row.header_session_id,
|
|
1277
|
+
headerName: row.header_name
|
|
1278
|
+
}));
|
|
1279
|
+
}
|
|
1280
|
+
function getKV(key) {
|
|
1281
|
+
const row = db().query("SELECT value FROM kv_meta WHERE key = ?").get(key);
|
|
1282
|
+
return row?.value ?? null;
|
|
1283
|
+
}
|
|
1284
|
+
function setKV(key, value) {
|
|
1285
|
+
db().query(
|
|
1286
|
+
"INSERT INTO kv_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = ?"
|
|
1287
|
+
).run(key, value, value);
|
|
1288
|
+
}
|
|
1061
1289
|
function getMeta(key) {
|
|
1062
1290
|
const row = db().query("SELECT value FROM metadata WHERE key = ?").get(key);
|
|
1063
1291
|
return row?.value ?? null;
|
|
@@ -26589,11 +26817,13 @@ function config2() {
|
|
|
26589
26817
|
return current;
|
|
26590
26818
|
}
|
|
26591
26819
|
async function load(directory) {
|
|
26592
|
-
|
|
26593
|
-
|
|
26594
|
-
|
|
26595
|
-
|
|
26596
|
-
|
|
26820
|
+
if (!isHostedMode()) {
|
|
26821
|
+
const path = join5(directory, ".lore.json");
|
|
26822
|
+
if (existsSync2(path)) {
|
|
26823
|
+
const raw = JSON.parse(readFileSync(path, "utf8"));
|
|
26824
|
+
current = LoreConfig.parse(raw);
|
|
26825
|
+
return current;
|
|
26826
|
+
}
|
|
26597
26827
|
}
|
|
26598
26828
|
current = LoreConfig.parse({});
|
|
26599
26829
|
return current;
|
|
@@ -26633,6 +26863,14 @@ function vendorRegistration() {
|
|
|
26633
26863
|
|
|
26634
26864
|
// src/embedding.ts
|
|
26635
26865
|
var EMBED_TIMEOUT_MS = 1e4;
|
|
26866
|
+
var LOCAL_MAX_CHARS = 4096 * 4;
|
|
26867
|
+
function safeLocalTruncate(text4) {
|
|
26868
|
+
if (text4.length <= LOCAL_MAX_CHARS) return text4;
|
|
26869
|
+
let end = LOCAL_MAX_CHARS;
|
|
26870
|
+
const code2 = text4.charCodeAt(end - 1);
|
|
26871
|
+
if (code2 >= 55296 && code2 <= 56319) end--;
|
|
26872
|
+
return text4.slice(0, end);
|
|
26873
|
+
}
|
|
26636
26874
|
var VOYAGE_API_URL = "https://api.voyageai.com/v1/embeddings";
|
|
26637
26875
|
var VoyageProvider = class {
|
|
26638
26876
|
maxBatchSize = 128;
|
|
@@ -26770,7 +27008,16 @@ var LocalProvider = class {
|
|
|
26770
27008
|
workerUrl = vendorWorkerUrl;
|
|
26771
27009
|
}
|
|
26772
27010
|
} else {
|
|
26773
|
-
|
|
27011
|
+
const selfUrl = typeof import.meta.url === "string" ? import.meta.url : void 0;
|
|
27012
|
+
if (selfUrl) {
|
|
27013
|
+
workerUrl = new URL(
|
|
27014
|
+
`./embedding-worker${selfUrl.endsWith(".ts") ? ".ts" : ".js"}`,
|
|
27015
|
+
selfUrl
|
|
27016
|
+
);
|
|
27017
|
+
} else {
|
|
27018
|
+
const { pathToFileURL } = await import("node:url");
|
|
27019
|
+
workerUrl = new URL("./embedding-worker.cjs", pathToFileURL(__filename));
|
|
27020
|
+
}
|
|
26774
27021
|
}
|
|
26775
27022
|
const vendor = vendorModelInfo();
|
|
26776
27023
|
const workerInitData = {
|
|
@@ -26806,8 +27053,9 @@ var LocalProvider = class {
|
|
|
26806
27053
|
localProviderKnownBroken = true;
|
|
26807
27054
|
if (!localProviderErrorLogged) {
|
|
26808
27055
|
localProviderErrorLogged = true;
|
|
26809
|
-
|
|
26810
|
-
`local embedding provider failed to init: ${msg.error}. Set VOYAGE_API_KEY/OPENAI_API_KEY for automatic remote fallback
|
|
27056
|
+
error(
|
|
27057
|
+
`local embedding provider failed to init: ${msg.error}. Set VOYAGE_API_KEY/OPENAI_API_KEY for automatic remote fallback.`,
|
|
27058
|
+
new Error(`embedding worker init failed: ${msg.error}`)
|
|
26811
27059
|
);
|
|
26812
27060
|
}
|
|
26813
27061
|
for (const [, p2] of this.pendingRequests) {
|
|
@@ -26822,6 +27070,7 @@ var LocalProvider = class {
|
|
|
26822
27070
|
this.worker.on("error", (err) => {
|
|
26823
27071
|
this.workerInitError = err.message;
|
|
26824
27072
|
this.workerReady = false;
|
|
27073
|
+
error("embedding worker crashed:", err);
|
|
26825
27074
|
for (const [, p2] of this.pendingRequests) {
|
|
26826
27075
|
p2.reject(new LocalProviderUnavailableError(err));
|
|
26827
27076
|
}
|
|
@@ -26831,6 +27080,10 @@ var LocalProvider = class {
|
|
|
26831
27080
|
this.worker.on("exit", (code2) => {
|
|
26832
27081
|
if (code2 !== 0 && !this.workerInitError) {
|
|
26833
27082
|
this.workerInitError = `embedding worker exited with code ${code2}`;
|
|
27083
|
+
error(
|
|
27084
|
+
this.workerInitError,
|
|
27085
|
+
new Error(this.workerInitError)
|
|
27086
|
+
);
|
|
26834
27087
|
}
|
|
26835
27088
|
this.workerReady = false;
|
|
26836
27089
|
for (const [, p2] of this.pendingRequests) {
|
|
@@ -26861,8 +27114,9 @@ var LocalProvider = class {
|
|
|
26861
27114
|
}
|
|
26862
27115
|
async embed(texts, inputType) {
|
|
26863
27116
|
await this.ensureWorker();
|
|
27117
|
+
const truncated = texts.map(safeLocalTruncate);
|
|
26864
27118
|
const prefix = inputType === "document" ? "search_document: " : "search_query: ";
|
|
26865
|
-
const prefixed =
|
|
27119
|
+
const prefixed = truncated.map((t2) => prefix + t2);
|
|
26866
27120
|
const id = this.nextRequestId++;
|
|
26867
27121
|
const priority = inputType === "query" && texts.length === 1 ? "high" : "normal";
|
|
26868
27122
|
return new Promise((resolve, reject) => {
|
|
@@ -27045,8 +27299,14 @@ function fromBlob(blob) {
|
|
|
27045
27299
|
const bytes = new Uint8Array(blob);
|
|
27046
27300
|
return new Float32Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / 4);
|
|
27047
27301
|
}
|
|
27048
|
-
function vectorSearch(queryEmbedding, limit = 10) {
|
|
27049
|
-
|
|
27302
|
+
function vectorSearch(queryEmbedding, limit = 10, excludeCategories) {
|
|
27303
|
+
let sql = "SELECT id, embedding FROM knowledge WHERE embedding IS NOT NULL AND confidence > 0.2";
|
|
27304
|
+
const params = [];
|
|
27305
|
+
if (excludeCategories?.length) {
|
|
27306
|
+
sql += ` AND category NOT IN (${excludeCategories.map(() => "?").join(",")})`;
|
|
27307
|
+
params.push(...excludeCategories);
|
|
27308
|
+
}
|
|
27309
|
+
const rows = db().query(sql).all(...params);
|
|
27050
27310
|
const scored = [];
|
|
27051
27311
|
for (const row of rows) {
|
|
27052
27312
|
const vec = fromBlob(row.embedding);
|
|
@@ -27084,27 +27344,30 @@ function vectorSearchAllDistillations(queryEmbedding, projectId2, limit = 20) {
|
|
|
27084
27344
|
return scored.slice(0, limit);
|
|
27085
27345
|
}
|
|
27086
27346
|
function embedKnowledgeEntry(id, title, content3) {
|
|
27347
|
+
if (!isAvailable()) return;
|
|
27087
27348
|
const text4 = `${title}
|
|
27088
27349
|
${content3}`;
|
|
27089
27350
|
embed([text4], "document").then(([vec]) => {
|
|
27090
27351
|
db().query("UPDATE knowledge SET embedding = ? WHERE id = ?").run(toBlob(vec), id);
|
|
27091
27352
|
}).catch((err) => {
|
|
27092
|
-
|
|
27353
|
+
error("embedding failed for knowledge entry", id, ":", err);
|
|
27093
27354
|
});
|
|
27094
27355
|
}
|
|
27095
27356
|
function embedDistillation(id, observations) {
|
|
27357
|
+
if (!isAvailable()) return;
|
|
27096
27358
|
embed([observations], "document").then(([vec]) => {
|
|
27097
27359
|
db().query("UPDATE distillations SET embedding = ? WHERE id = ?").run(toBlob(vec), id);
|
|
27098
27360
|
}).catch((err) => {
|
|
27099
|
-
|
|
27361
|
+
error("embedding failed for distillation", id, ":", err);
|
|
27100
27362
|
});
|
|
27101
27363
|
}
|
|
27102
27364
|
function embedTemporalMessage(id, content3) {
|
|
27365
|
+
if (!isAvailable()) return;
|
|
27103
27366
|
if (content3.length < 50) return;
|
|
27104
27367
|
embed([content3], "document").then(([vec]) => {
|
|
27105
27368
|
db().query("UPDATE temporal_messages SET embedding = ? WHERE id = ?").run(toBlob(vec), id);
|
|
27106
27369
|
}).catch((err) => {
|
|
27107
|
-
|
|
27370
|
+
error("embedding failed for temporal message", id, ":", err);
|
|
27108
27371
|
});
|
|
27109
27372
|
}
|
|
27110
27373
|
function vectorSearchTemporal(queryEmbedding, projectId2, limit = 10, sessionId) {
|
|
@@ -27224,6 +27487,7 @@ ${r.content}` }));
|
|
|
27224
27487
|
}
|
|
27225
27488
|
} catch (err) {
|
|
27226
27489
|
error(`embedding backfill batch failed (${batch.length} items):`, err);
|
|
27490
|
+
if (err instanceof LocalProviderUnavailableError) break;
|
|
27227
27491
|
}
|
|
27228
27492
|
}
|
|
27229
27493
|
if (embedded > 0) {
|
|
@@ -27257,6 +27521,7 @@ async function backfillDistillationEmbeddings() {
|
|
|
27257
27521
|
}
|
|
27258
27522
|
} catch (err) {
|
|
27259
27523
|
error(`distillation embedding backfill batch failed (${batch.length} items):`, err);
|
|
27524
|
+
if (err instanceof LocalProviderUnavailableError) break;
|
|
27260
27525
|
}
|
|
27261
27526
|
if (embedded >= nextProgressAt) {
|
|
27262
27527
|
info(`embedding distillations: ${embedded}/${rows.length}\u2026`);
|
|
@@ -27346,14 +27611,14 @@ function store(input) {
|
|
|
27346
27611
|
embedTemporalMessage(input.info.id, content3);
|
|
27347
27612
|
}
|
|
27348
27613
|
}
|
|
27349
|
-
function undistilled(
|
|
27350
|
-
const pid = ensureProject(
|
|
27614
|
+
function undistilled(projectPath2, sessionID) {
|
|
27615
|
+
const pid = ensureProject(projectPath2);
|
|
27351
27616
|
const query = sessionID ? "SELECT * FROM temporal_messages WHERE project_id = ? AND session_id = ? AND distilled = 0 ORDER BY created_at ASC" : "SELECT * FROM temporal_messages WHERE project_id = ? AND distilled = 0 ORDER BY created_at ASC";
|
|
27352
27617
|
const params = sessionID ? [pid, sessionID] : [pid];
|
|
27353
27618
|
return db().query(query).all(...params);
|
|
27354
27619
|
}
|
|
27355
|
-
function bySession(
|
|
27356
|
-
const pid = ensureProject(
|
|
27620
|
+
function bySession(projectPath2, sessionID) {
|
|
27621
|
+
const pid = ensureProject(projectPath2);
|
|
27357
27622
|
return db().query(
|
|
27358
27623
|
"SELECT * FROM temporal_messages WHERE project_id = ? AND session_id = ? ORDER BY created_at ASC"
|
|
27359
27624
|
).all(pid, sessionID);
|
|
@@ -27429,20 +27694,26 @@ function temporalCnorm(timestamps, now = Date.now()) {
|
|
|
27429
27694
|
const maxVariance = (n - 1) / (n * n);
|
|
27430
27695
|
return maxVariance === 0 ? 0 : variance / maxVariance;
|
|
27431
27696
|
}
|
|
27432
|
-
function count(
|
|
27433
|
-
const pid = ensureProject(
|
|
27697
|
+
function count(projectPath2, sessionID) {
|
|
27698
|
+
const pid = ensureProject(projectPath2);
|
|
27434
27699
|
const query = sessionID ? "SELECT COUNT(*) as count FROM temporal_messages WHERE project_id = ? AND session_id = ?" : "SELECT COUNT(*) as count FROM temporal_messages WHERE project_id = ?";
|
|
27435
27700
|
const params = sessionID ? [pid, sessionID] : [pid];
|
|
27436
27701
|
return db().query(query).get(...params).count;
|
|
27437
27702
|
}
|
|
27438
|
-
function
|
|
27439
|
-
const pid = ensureProject(
|
|
27703
|
+
function hasMessages(projectPath2, sessionID) {
|
|
27704
|
+
const pid = ensureProject(projectPath2);
|
|
27705
|
+
return !!db().query(
|
|
27706
|
+
"SELECT 1 FROM temporal_messages WHERE project_id = ? AND session_id = ? LIMIT 1"
|
|
27707
|
+
).get(pid, sessionID);
|
|
27708
|
+
}
|
|
27709
|
+
function undistilledCount(projectPath2, sessionID) {
|
|
27710
|
+
const pid = ensureProject(projectPath2);
|
|
27440
27711
|
const query = sessionID ? "SELECT COUNT(*) as count FROM temporal_messages WHERE project_id = ? AND session_id = ? AND distilled = 0" : "SELECT COUNT(*) as count FROM temporal_messages WHERE project_id = ? AND distilled = 0";
|
|
27441
27712
|
const params = sessionID ? [pid, sessionID] : [pid];
|
|
27442
27713
|
return db().query(query).get(...params).count;
|
|
27443
27714
|
}
|
|
27444
|
-
function undistilledTokens(
|
|
27445
|
-
const pid = ensureProject(
|
|
27715
|
+
function undistilledTokens(projectPath2, sessionID) {
|
|
27716
|
+
const pid = ensureProject(projectPath2);
|
|
27446
27717
|
const query = sessionID ? "SELECT COALESCE(SUM(tokens), 0) as total FROM temporal_messages WHERE project_id = ? AND session_id = ? AND distilled = 0" : "SELECT COALESCE(SUM(tokens), 0) as total FROM temporal_messages WHERE project_id = ? AND distilled = 0";
|
|
27447
27718
|
const params = sessionID ? [pid, sessionID] : [pid];
|
|
27448
27719
|
return db().query(query).get(...params).total;
|
|
@@ -27493,11 +27764,13 @@ function prune(input) {
|
|
|
27493
27764
|
var ltm_exports = {};
|
|
27494
27765
|
__export(ltm_exports, {
|
|
27495
27766
|
all: () => all2,
|
|
27767
|
+
calibrateDedupThreshold: () => calibrateDedupThreshold,
|
|
27496
27768
|
cascadeRefReplace: () => cascadeRefReplace,
|
|
27497
27769
|
check: () => check2,
|
|
27498
27770
|
cleanDeadRefs: () => cleanDeadRefs,
|
|
27499
27771
|
create: () => create,
|
|
27500
27772
|
crossProject: () => crossProject,
|
|
27773
|
+
dedupPairKey: () => dedupPairKey,
|
|
27501
27774
|
deduplicate: () => deduplicate,
|
|
27502
27775
|
deduplicateGlobal: () => deduplicateGlobal,
|
|
27503
27776
|
extractRefs: () => extractRefs,
|
|
@@ -27505,9 +27778,17 @@ __export(ltm_exports, {
|
|
|
27505
27778
|
forProject: () => forProject,
|
|
27506
27779
|
forSession: () => forSession,
|
|
27507
27780
|
get: () => get,
|
|
27781
|
+
getDedupFeedback: () => getDedupFeedback,
|
|
27782
|
+
getDedupFeedbackCount: () => getDedupFeedbackCount,
|
|
27783
|
+
loadCalibratedThreshold: () => loadCalibratedThreshold,
|
|
27784
|
+
pruneDedupFeedback: () => pruneDedupFeedback,
|
|
27508
27785
|
pruneOversized: () => pruneOversized,
|
|
27786
|
+
recordAutoSignals: () => recordAutoSignals,
|
|
27787
|
+
recordDedupFeedback: () => recordDedupFeedback,
|
|
27788
|
+
recordDedupResultFeedback: () => recordDedupResultFeedback,
|
|
27509
27789
|
remove: () => remove,
|
|
27510
27790
|
resolveRef: () => resolveRef2,
|
|
27791
|
+
saveCalibratedThreshold: () => saveCalibratedThreshold,
|
|
27511
27792
|
search: () => search3,
|
|
27512
27793
|
searchScored: () => searchScored3,
|
|
27513
27794
|
searchScoredOtherProjects: () => searchScoredOtherProjects,
|
|
@@ -27953,14 +28234,16 @@ function listMarkdownFiles(dir) {
|
|
|
27953
28234
|
function contentHash(content3) {
|
|
27954
28235
|
return sha256(content3);
|
|
27955
28236
|
}
|
|
27956
|
-
function hasLatDir(
|
|
27957
|
-
|
|
28237
|
+
function hasLatDir(projectPath2) {
|
|
28238
|
+
if (isHostedMode()) return false;
|
|
28239
|
+
const latDir = join6(projectPath2, "lat.md");
|
|
27958
28240
|
return existsSync3(latDir) && statSync2(latDir).isDirectory();
|
|
27959
28241
|
}
|
|
27960
|
-
function refresh(
|
|
27961
|
-
|
|
28242
|
+
function refresh(projectPath2) {
|
|
28243
|
+
if (isHostedMode()) return 0;
|
|
28244
|
+
const latDir = join6(projectPath2, "lat.md");
|
|
27962
28245
|
if (!existsSync3(latDir) || !statSync2(latDir).isDirectory()) return 0;
|
|
27963
|
-
const pid = ensureProject(
|
|
28246
|
+
const pid = ensureProject(projectPath2);
|
|
27964
28247
|
const files = listMarkdownFiles(latDir);
|
|
27965
28248
|
let upserted = 0;
|
|
27966
28249
|
const seenFiles = /* @__PURE__ */ new Set();
|
|
@@ -27975,7 +28258,7 @@ function refresh(projectPath) {
|
|
|
27975
28258
|
} catch {
|
|
27976
28259
|
continue;
|
|
27977
28260
|
}
|
|
27978
|
-
const fileRel = relative(
|
|
28261
|
+
const fileRel = relative(projectPath2, filePath);
|
|
27979
28262
|
seenFiles.add(fileRel);
|
|
27980
28263
|
const hash2 = contentHash(content3);
|
|
27981
28264
|
const existing = db().query("SELECT content_hash FROM lat_sections WHERE project_id = ? AND file = ? LIMIT 1").get(pid, fileRel.replace(/\.md$/, ""));
|
|
@@ -27983,7 +28266,7 @@ function refresh(projectPath) {
|
|
|
27983
28266
|
continue;
|
|
27984
28267
|
}
|
|
27985
28268
|
db().query("DELETE FROM lat_sections WHERE project_id = ? AND file = ?").run(pid, fileRel.replace(/\.md$/, ""));
|
|
27986
|
-
const sections = parseSections(filePath, content3,
|
|
28269
|
+
const sections = parseSections(filePath, content3, projectPath2);
|
|
27987
28270
|
const now = Date.now();
|
|
27988
28271
|
for (const section of sections) {
|
|
27989
28272
|
upsertStmt.run(
|
|
@@ -28033,9 +28316,9 @@ function searchScored2(input) {
|
|
|
28033
28316
|
return [];
|
|
28034
28317
|
}
|
|
28035
28318
|
}
|
|
28036
|
-
function scoreForSession(
|
|
28037
|
-
if (!hasLatDir(
|
|
28038
|
-
const pid = ensureProject(
|
|
28319
|
+
function scoreForSession(projectPath2, sessionContext, maxTokens) {
|
|
28320
|
+
if (!hasLatDir(projectPath2)) return [];
|
|
28321
|
+
const pid = ensureProject(projectPath2);
|
|
28039
28322
|
const terms = extractTopTerms(sessionContext);
|
|
28040
28323
|
if (!terms.length) return [];
|
|
28041
28324
|
const q = terms.map((t2) => `${t2}*`).join(" OR ");
|
|
@@ -28067,8 +28350,8 @@ function scoreForSession(projectPath, sessionContext, maxTokens) {
|
|
|
28067
28350
|
}
|
|
28068
28351
|
return packed;
|
|
28069
28352
|
}
|
|
28070
|
-
function count2(
|
|
28071
|
-
const pid = ensureProject(
|
|
28353
|
+
function count2(projectPath2) {
|
|
28354
|
+
const pid = ensureProject(projectPath2);
|
|
28072
28355
|
const row = db().query("SELECT COUNT(*) as cnt FROM lat_sections WHERE project_id = ?").get(pid);
|
|
28073
28356
|
return row.cnt;
|
|
28074
28357
|
}
|
|
@@ -28195,8 +28478,8 @@ function findFuzzyDuplicate(input) {
|
|
|
28195
28478
|
}
|
|
28196
28479
|
return null;
|
|
28197
28480
|
}
|
|
28198
|
-
function forProject(
|
|
28199
|
-
const pid = ensureProject(
|
|
28481
|
+
function forProject(projectPath2, includeCross = true) {
|
|
28482
|
+
const pid = ensureProject(projectPath2);
|
|
28200
28483
|
if (includeCross) {
|
|
28201
28484
|
return db().query(
|
|
28202
28485
|
`SELECT ${KNOWLEDGE_COLS} FROM knowledge
|
|
@@ -28244,18 +28527,29 @@ function scoreEntriesFTS(sessionContext) {
|
|
|
28244
28527
|
return /* @__PURE__ */ new Map();
|
|
28245
28528
|
}
|
|
28246
28529
|
}
|
|
28247
|
-
function forSession(
|
|
28248
|
-
const pid = ensureProject(
|
|
28530
|
+
async function forSession(projectPath2, sessionID, maxTokens, options) {
|
|
28531
|
+
const pid = ensureProject(projectPath2);
|
|
28532
|
+
const categoryFilter = options?.categories;
|
|
28533
|
+
const excludeFilter = options?.excludeCategories;
|
|
28534
|
+
let categoryClause = "";
|
|
28535
|
+
let categoryParams = [];
|
|
28536
|
+
if (categoryFilter?.length) {
|
|
28537
|
+
categoryClause = ` AND category IN (${categoryFilter.map(() => "?").join(",")})`;
|
|
28538
|
+
categoryParams = categoryFilter;
|
|
28539
|
+
} else if (excludeFilter?.length) {
|
|
28540
|
+
categoryClause = ` AND category NOT IN (${excludeFilter.map(() => "?").join(",")})`;
|
|
28541
|
+
categoryParams = excludeFilter;
|
|
28542
|
+
}
|
|
28249
28543
|
const projectEntries = db().query(
|
|
28250
28544
|
`SELECT ${KNOWLEDGE_COLS} FROM knowledge
|
|
28251
|
-
WHERE project_id = ? AND cross_project = 0 AND confidence > 0.2
|
|
28545
|
+
WHERE project_id = ? AND cross_project = 0 AND confidence > 0.2${categoryClause}
|
|
28252
28546
|
ORDER BY confidence DESC, updated_at DESC`
|
|
28253
|
-
).all(pid);
|
|
28547
|
+
).all(pid, ...categoryParams);
|
|
28254
28548
|
const crossEntries = db().query(
|
|
28255
28549
|
`SELECT ${KNOWLEDGE_COLS} FROM knowledge
|
|
28256
|
-
WHERE (project_id IS NULL OR cross_project = 1) AND confidence > 0.2
|
|
28550
|
+
WHERE (project_id IS NULL OR cross_project = 1) AND confidence > 0.2${categoryClause}
|
|
28257
28551
|
ORDER BY confidence DESC, updated_at DESC`
|
|
28258
|
-
).all();
|
|
28552
|
+
).all(...categoryParams);
|
|
28259
28553
|
if (!crossEntries.length && !projectEntries.length) return [];
|
|
28260
28554
|
let sessionContext = "";
|
|
28261
28555
|
if (sessionID) {
|
|
@@ -28276,22 +28570,52 @@ function forSession(projectPath, sessionID, maxTokens) {
|
|
|
28276
28570
|
sessionContext += recentMsgs.map((m) => m.content).join("\n");
|
|
28277
28571
|
}
|
|
28278
28572
|
}
|
|
28573
|
+
if (!sessionContext.trim() && options?.contextHint) {
|
|
28574
|
+
sessionContext = options.contextHint;
|
|
28575
|
+
}
|
|
28279
28576
|
let scoredProject;
|
|
28280
28577
|
let scoredCross;
|
|
28281
|
-
if (sessionContext.trim().length > 20) {
|
|
28578
|
+
if (sessionContext.trim().length > 20 && isAvailable()) {
|
|
28579
|
+
let vectorScores;
|
|
28580
|
+
try {
|
|
28581
|
+
const [contextVec] = await embed([sessionContext], "query");
|
|
28582
|
+
const hits = vectorSearch(contextVec, 50, excludeFilter);
|
|
28583
|
+
vectorScores = new Map(hits.map((h3) => [h3.id, h3.similarity]));
|
|
28584
|
+
} catch (err) {
|
|
28585
|
+
warn("Vector scoring failed, falling back to FTS5:", err);
|
|
28586
|
+
vectorScores = /* @__PURE__ */ new Map();
|
|
28587
|
+
}
|
|
28588
|
+
if (vectorScores.size > 0) {
|
|
28589
|
+
const ftsScores = scoreEntriesFTS(sessionContext);
|
|
28590
|
+
const rawScored = projectEntries.map((entry) => {
|
|
28591
|
+
const vecScore = vectorScores.get(entry.id);
|
|
28592
|
+
const score = vecScore != null ? vecScore * entry.confidence : (ftsScores.get(entry.id) ?? 0) * entry.confidence;
|
|
28593
|
+
return { entry, score };
|
|
28594
|
+
});
|
|
28595
|
+
const matched = rawScored.filter((s) => s.score > 0);
|
|
28596
|
+
const matchedIds = new Set(matched.map((s) => s.entry.id));
|
|
28597
|
+
const safetyNet = projectEntries.filter((e) => !matchedIds.has(e.id)).slice(0, PROJECT_SAFETY_NET).map((e) => ({ entry: e, score: 1e-3 * e.confidence }));
|
|
28598
|
+
scoredProject = [...matched, ...safetyNet];
|
|
28599
|
+
scoredCross = crossEntries.filter((e) => vectorScores.has(e.id) || ftsScores.has(e.id)).map((e) => {
|
|
28600
|
+
const vecScore = vectorScores.get(e.id);
|
|
28601
|
+
const score = vecScore != null ? vecScore * e.confidence : (ftsScores.get(e.id) ?? 0) * e.confidence;
|
|
28602
|
+
return { entry: e, score };
|
|
28603
|
+
});
|
|
28604
|
+
} else {
|
|
28605
|
+
const ftsScores = scoreEntriesFTS(sessionContext);
|
|
28606
|
+
({ scoredProject, scoredCross } = scoreFTS(
|
|
28607
|
+
projectEntries,
|
|
28608
|
+
crossEntries,
|
|
28609
|
+
ftsScores
|
|
28610
|
+
));
|
|
28611
|
+
}
|
|
28612
|
+
} else if (sessionContext.trim().length > 20) {
|
|
28282
28613
|
const ftsScores = scoreEntriesFTS(sessionContext);
|
|
28283
|
-
|
|
28284
|
-
|
|
28285
|
-
|
|
28286
|
-
|
|
28287
|
-
|
|
28288
|
-
const matchedIds = new Set(matched.map((s) => s.entry.id));
|
|
28289
|
-
const safetyNet = projectEntries.filter((e) => !matchedIds.has(e.id)).slice(0, PROJECT_SAFETY_NET).map((e) => ({ entry: e, score: 1e-3 * e.confidence }));
|
|
28290
|
-
scoredProject = [...matched, ...safetyNet];
|
|
28291
|
-
scoredCross = crossEntries.filter((e) => ftsScores.has(e.id)).map((e) => ({
|
|
28292
|
-
entry: e,
|
|
28293
|
-
score: (ftsScores.get(e.id) ?? 0) * e.confidence
|
|
28294
|
-
}));
|
|
28614
|
+
({ scoredProject, scoredCross } = scoreFTS(
|
|
28615
|
+
projectEntries,
|
|
28616
|
+
crossEntries,
|
|
28617
|
+
ftsScores
|
|
28618
|
+
));
|
|
28295
28619
|
} else {
|
|
28296
28620
|
scoredProject = projectEntries.slice(0, NO_CONTEXT_FALLBACK_CAP).map((entry) => ({ entry, score: entry.confidence }));
|
|
28297
28621
|
scoredCross = crossEntries.slice(0, NO_CONTEXT_FALLBACK_CAP).map((entry) => ({ entry, score: entry.confidence }));
|
|
@@ -28308,9 +28632,9 @@ function forSession(projectPath, sessionID, maxTokens) {
|
|
|
28308
28632
|
result.push(entry);
|
|
28309
28633
|
used += cost;
|
|
28310
28634
|
}
|
|
28311
|
-
if (hasLatDir(
|
|
28635
|
+
if (hasLatDir(projectPath2) && used < maxTokens) {
|
|
28312
28636
|
const latSections = scoreForSession(
|
|
28313
|
-
|
|
28637
|
+
projectPath2,
|
|
28314
28638
|
sessionContext,
|
|
28315
28639
|
maxTokens - used
|
|
28316
28640
|
);
|
|
@@ -28337,6 +28661,21 @@ function forSession(projectPath, sessionID, maxTokens) {
|
|
|
28337
28661
|
}
|
|
28338
28662
|
return result;
|
|
28339
28663
|
}
|
|
28664
|
+
function scoreFTS(projectEntries, crossEntries, ftsScores) {
|
|
28665
|
+
const rawScored = projectEntries.map((entry) => ({
|
|
28666
|
+
entry,
|
|
28667
|
+
score: (ftsScores.get(entry.id) ?? 0) * entry.confidence
|
|
28668
|
+
}));
|
|
28669
|
+
const matched = rawScored.filter((s) => s.score > 0);
|
|
28670
|
+
const matchedIds = new Set(matched.map((s) => s.entry.id));
|
|
28671
|
+
const safetyNet = projectEntries.filter((e) => !matchedIds.has(e.id)).slice(0, PROJECT_SAFETY_NET).map((e) => ({ entry: e, score: 1e-3 * e.confidence }));
|
|
28672
|
+
const scoredProject = [...matched, ...safetyNet];
|
|
28673
|
+
const scoredCross = crossEntries.filter((e) => ftsScores.has(e.id)).map((e) => ({
|
|
28674
|
+
entry: e,
|
|
28675
|
+
score: (ftsScores.get(e.id) ?? 0) * e.confidence
|
|
28676
|
+
}));
|
|
28677
|
+
return { scoredProject, scoredCross };
|
|
28678
|
+
}
|
|
28340
28679
|
function all2() {
|
|
28341
28680
|
return db().query(
|
|
28342
28681
|
`SELECT ${KNOWLEDGE_COLS} FROM knowledge WHERE confidence > 0.2 ORDER BY confidence DESC, updated_at DESC`
|
|
@@ -28518,8 +28857,8 @@ function cleanDeadRefs() {
|
|
|
28518
28857
|
}
|
|
28519
28858
|
return cleaned;
|
|
28520
28859
|
}
|
|
28521
|
-
function check2(
|
|
28522
|
-
const entries = forProject(
|
|
28860
|
+
function check2(projectPath2) {
|
|
28861
|
+
const entries = forProject(projectPath2, false);
|
|
28523
28862
|
const issues = [];
|
|
28524
28863
|
for (const entry of entries) {
|
|
28525
28864
|
if (entry.content.length > 1200) {
|
|
@@ -28580,8 +28919,11 @@ function check2(projectPath) {
|
|
|
28580
28919
|
}
|
|
28581
28920
|
return issues;
|
|
28582
28921
|
}
|
|
28583
|
-
function
|
|
28584
|
-
|
|
28922
|
+
function dedupPairKey(idA, idB) {
|
|
28923
|
+
return idA < idB ? `${idA}:${idB}` : `${idB}:${idA}`;
|
|
28924
|
+
}
|
|
28925
|
+
function _dedup(entries, dryRun, embeddingThreshold = EMBEDDING_DEDUP_THRESHOLD) {
|
|
28926
|
+
if (entries.length < 2) return { clusters: [], totalRemoved: 0, pairSimilarities: /* @__PURE__ */ new Map(), entryTitles: /* @__PURE__ */ new Map() };
|
|
28585
28927
|
const embeddingMap = /* @__PURE__ */ new Map();
|
|
28586
28928
|
{
|
|
28587
28929
|
const entryIds = entries.map((e) => e.id);
|
|
@@ -28596,6 +28938,7 @@ function _dedup(entries, dryRun) {
|
|
|
28596
28938
|
}
|
|
28597
28939
|
}
|
|
28598
28940
|
const neighborMap = /* @__PURE__ */ new Map();
|
|
28941
|
+
const pairSimilarities = /* @__PURE__ */ new Map();
|
|
28599
28942
|
for (const entry of entries) {
|
|
28600
28943
|
const neighbors = [];
|
|
28601
28944
|
const entryVec = embeddingMap.get(entry.id);
|
|
@@ -28609,7 +28952,13 @@ function _dedup(entries, dryRun) {
|
|
|
28609
28952
|
const otherVec = embeddingMap.get(other.id);
|
|
28610
28953
|
if (otherVec && entryVec.length === otherVec.length) {
|
|
28611
28954
|
similarity = cosineSimilarity(entryVec, otherVec);
|
|
28612
|
-
embeddingMatch = similarity >=
|
|
28955
|
+
embeddingMatch = similarity >= embeddingThreshold;
|
|
28956
|
+
}
|
|
28957
|
+
}
|
|
28958
|
+
if (similarity > 0) {
|
|
28959
|
+
const pk = dedupPairKey(entry.id, other.id);
|
|
28960
|
+
if (!pairSimilarities.has(pk)) {
|
|
28961
|
+
pairSimilarities.set(pk, similarity);
|
|
28613
28962
|
}
|
|
28614
28963
|
}
|
|
28615
28964
|
if (titleMatch || embeddingMatch) {
|
|
@@ -28661,20 +29010,178 @@ function _dedup(entries, dryRun) {
|
|
|
28661
29010
|
totalRemoved += merged.length;
|
|
28662
29011
|
}
|
|
28663
29012
|
result.sort((a, b) => b.merged.length - a.merged.length);
|
|
28664
|
-
|
|
29013
|
+
const entryTitles = new Map(entries.map((e) => [e.id, e.title]));
|
|
29014
|
+
return { clusters: result, totalRemoved, pairSimilarities, entryTitles };
|
|
28665
29015
|
}
|
|
28666
|
-
async function deduplicate(
|
|
28667
|
-
const
|
|
28668
|
-
|
|
29016
|
+
async function deduplicate(projectPath2, opts) {
|
|
29017
|
+
const pid = ensureProject(projectPath2);
|
|
29018
|
+
const threshold = loadCalibratedThreshold(pid) ?? EMBEDDING_DEDUP_THRESHOLD;
|
|
29019
|
+
const entries = forProject(projectPath2, false);
|
|
29020
|
+
return _dedup(entries, opts?.dryRun ?? true, threshold);
|
|
28669
29021
|
}
|
|
28670
29022
|
async function deduplicateGlobal(opts) {
|
|
29023
|
+
const threshold = loadCalibratedThreshold(null) ?? EMBEDDING_DEDUP_THRESHOLD;
|
|
28671
29024
|
const entries = db().query(
|
|
28672
29025
|
`SELECT ${KNOWLEDGE_COLS} FROM knowledge
|
|
28673
29026
|
WHERE project_id IS NULL
|
|
28674
29027
|
AND confidence > 0.2
|
|
28675
29028
|
ORDER BY confidence DESC, updated_at DESC`
|
|
28676
29029
|
).all();
|
|
28677
|
-
return _dedup(entries, opts?.dryRun ?? true);
|
|
29030
|
+
return _dedup(entries, opts?.dryRun ?? true, threshold);
|
|
29031
|
+
}
|
|
29032
|
+
var MIN_CALIBRATION_SAMPLES = 20;
|
|
29033
|
+
var DEFAULT_EMBEDDING_DEDUP_THRESHOLD = EMBEDDING_DEDUP_THRESHOLD;
|
|
29034
|
+
var AUTO_SIGNAL_MIN_SIMILARITY = 0.8;
|
|
29035
|
+
var AUTO_SIGNAL_MAX_PAIRS = 50;
|
|
29036
|
+
function recordDedupFeedback(input) {
|
|
29037
|
+
db().query(
|
|
29038
|
+
`INSERT INTO dedup_feedback
|
|
29039
|
+
(project_id, entry_a_title, entry_b_title, similarity, accepted, source, created_at)
|
|
29040
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`
|
|
29041
|
+
).run(
|
|
29042
|
+
input.projectId,
|
|
29043
|
+
input.entryATitle,
|
|
29044
|
+
input.entryBTitle,
|
|
29045
|
+
input.similarity,
|
|
29046
|
+
input.accepted ? 1 : 0,
|
|
29047
|
+
input.source,
|
|
29048
|
+
Date.now()
|
|
29049
|
+
);
|
|
29050
|
+
}
|
|
29051
|
+
function recordDedupResultFeedback(projectId2, result, accepted, source) {
|
|
29052
|
+
for (const cluster of result.clusters) {
|
|
29053
|
+
for (const merged of cluster.merged) {
|
|
29054
|
+
const pk = dedupPairKey(cluster.surviving.id, merged.id);
|
|
29055
|
+
const similarity = result.pairSimilarities.get(pk);
|
|
29056
|
+
if (similarity != null && similarity > 0) {
|
|
29057
|
+
recordDedupFeedback({
|
|
29058
|
+
projectId: projectId2,
|
|
29059
|
+
entryATitle: cluster.surviving.title,
|
|
29060
|
+
entryBTitle: merged.title,
|
|
29061
|
+
similarity,
|
|
29062
|
+
accepted,
|
|
29063
|
+
source
|
|
29064
|
+
});
|
|
29065
|
+
}
|
|
29066
|
+
}
|
|
29067
|
+
}
|
|
29068
|
+
}
|
|
29069
|
+
function recordAutoSignals(projectId2, result) {
|
|
29070
|
+
const mergedPairs = /* @__PURE__ */ new Set();
|
|
29071
|
+
for (const cluster of result.clusters) {
|
|
29072
|
+
for (const merged of cluster.merged) {
|
|
29073
|
+
mergedPairs.add(dedupPairKey(cluster.surviving.id, merged.id));
|
|
29074
|
+
}
|
|
29075
|
+
}
|
|
29076
|
+
const titleMap = new Map(result.entryTitles);
|
|
29077
|
+
for (const cluster of result.clusters) {
|
|
29078
|
+
if (!titleMap.has(cluster.surviving.id)) {
|
|
29079
|
+
titleMap.set(cluster.surviving.id, cluster.surviving.title);
|
|
29080
|
+
}
|
|
29081
|
+
for (const m of cluster.merged) {
|
|
29082
|
+
if (!titleMap.has(m.id)) titleMap.set(m.id, m.title);
|
|
29083
|
+
}
|
|
29084
|
+
}
|
|
29085
|
+
const signals = [];
|
|
29086
|
+
for (const [pk, sim] of result.pairSimilarities) {
|
|
29087
|
+
if (sim < AUTO_SIGNAL_MIN_SIMILARITY) continue;
|
|
29088
|
+
if (mergedPairs.has(pk)) continue;
|
|
29089
|
+
const [idA, idB] = pk.split(":");
|
|
29090
|
+
const titleA = titleMap.get(idA);
|
|
29091
|
+
const titleB = titleMap.get(idB);
|
|
29092
|
+
if (!titleA || !titleB) continue;
|
|
29093
|
+
signals.push({ entryATitle: titleA, entryBTitle: titleB, similarity: sim });
|
|
29094
|
+
}
|
|
29095
|
+
const currentThreshold = loadCalibratedThreshold(projectId2) ?? DEFAULT_EMBEDDING_DEDUP_THRESHOLD;
|
|
29096
|
+
signals.sort((a, b) => Math.abs(a.similarity - currentThreshold) - Math.abs(b.similarity - currentThreshold));
|
|
29097
|
+
const capped = signals.slice(0, AUTO_SIGNAL_MAX_PAIRS);
|
|
29098
|
+
pruneDedupFeedback(projectId2);
|
|
29099
|
+
for (const s of capped) {
|
|
29100
|
+
recordDedupFeedback({
|
|
29101
|
+
projectId: projectId2,
|
|
29102
|
+
entryATitle: s.entryATitle,
|
|
29103
|
+
entryBTitle: s.entryBTitle,
|
|
29104
|
+
similarity: s.similarity,
|
|
29105
|
+
accepted: false,
|
|
29106
|
+
source: "auto_dedup"
|
|
29107
|
+
});
|
|
29108
|
+
}
|
|
29109
|
+
}
|
|
29110
|
+
function getDedupFeedback(projectId2) {
|
|
29111
|
+
const rows = projectId2 !== null ? db().query(
|
|
29112
|
+
"SELECT similarity, accepted, source FROM dedup_feedback WHERE project_id = ? ORDER BY similarity"
|
|
29113
|
+
).all(projectId2) : db().query(
|
|
29114
|
+
"SELECT similarity, accepted, source FROM dedup_feedback WHERE project_id IS NULL ORDER BY similarity"
|
|
29115
|
+
).all();
|
|
29116
|
+
return rows.map((r) => ({ similarity: r.similarity, accepted: r.accepted === 1, source: r.source }));
|
|
29117
|
+
}
|
|
29118
|
+
function getDedupFeedbackCount(projectId2) {
|
|
29119
|
+
const row = projectId2 !== null ? db().query("SELECT COUNT(*) as cnt FROM dedup_feedback WHERE project_id = ?").get(projectId2) : db().query("SELECT COUNT(*) as cnt FROM dedup_feedback WHERE project_id IS NULL").get();
|
|
29120
|
+
return row?.cnt ?? 0;
|
|
29121
|
+
}
|
|
29122
|
+
var MAX_FEEDBACK_ROWS_PER_PROJECT = 500;
|
|
29123
|
+
function pruneDedupFeedback(projectId2) {
|
|
29124
|
+
const count3 = getDedupFeedbackCount(projectId2);
|
|
29125
|
+
if (count3 <= MAX_FEEDBACK_ROWS_PER_PROJECT) return;
|
|
29126
|
+
const excess = count3 - MAX_FEEDBACK_ROWS_PER_PROJECT;
|
|
29127
|
+
if (projectId2 !== null) {
|
|
29128
|
+
db().query(
|
|
29129
|
+
`DELETE FROM dedup_feedback WHERE id IN (
|
|
29130
|
+
SELECT id FROM dedup_feedback WHERE project_id = ?
|
|
29131
|
+
ORDER BY created_at ASC LIMIT ?
|
|
29132
|
+
)`
|
|
29133
|
+
).run(projectId2, excess);
|
|
29134
|
+
} else {
|
|
29135
|
+
db().query(
|
|
29136
|
+
`DELETE FROM dedup_feedback WHERE id IN (
|
|
29137
|
+
SELECT id FROM dedup_feedback WHERE project_id IS NULL
|
|
29138
|
+
ORDER BY created_at ASC LIMIT ?
|
|
29139
|
+
)`
|
|
29140
|
+
).run(excess);
|
|
29141
|
+
}
|
|
29142
|
+
}
|
|
29143
|
+
function calibrateDedupThreshold(projectId2) {
|
|
29144
|
+
const feedback = getDedupFeedback(projectId2);
|
|
29145
|
+
if (feedback.length < MIN_CALIBRATION_SAMPLES) return null;
|
|
29146
|
+
const accepted = feedback.filter((f) => f.accepted);
|
|
29147
|
+
const rejected = feedback.filter((f) => !f.accepted);
|
|
29148
|
+
if (rejected.length === 0) {
|
|
29149
|
+
const minAccepted = Math.min(...accepted.map((f) => f.similarity));
|
|
29150
|
+
return Math.max(0.85, minAccepted - 5e-3);
|
|
29151
|
+
}
|
|
29152
|
+
if (accepted.length === 0) {
|
|
29153
|
+
warn("dedup calibration: all feedback is reject \u2014 keeping default threshold");
|
|
29154
|
+
return null;
|
|
29155
|
+
}
|
|
29156
|
+
const allSims = [...new Set(feedback.map((f) => f.similarity))].sort((a, b) => a - b);
|
|
29157
|
+
let bestThreshold = DEFAULT_EMBEDDING_DEDUP_THRESHOLD;
|
|
29158
|
+
let bestAccuracy = -1;
|
|
29159
|
+
for (let i = 0; i < allSims.length - 1; i++) {
|
|
29160
|
+
const candidate = (allSims[i] + allSims[i + 1]) / 2;
|
|
29161
|
+
const correctAccepted = accepted.filter((f) => f.similarity >= candidate).length;
|
|
29162
|
+
const correctRejected = rejected.filter((f) => f.similarity < candidate).length;
|
|
29163
|
+
const accuracy = (correctAccepted + correctRejected) / feedback.length;
|
|
29164
|
+
if (accuracy > bestAccuracy || accuracy === bestAccuracy && candidate > bestThreshold) {
|
|
29165
|
+
bestAccuracy = accuracy;
|
|
29166
|
+
bestThreshold = candidate;
|
|
29167
|
+
}
|
|
29168
|
+
}
|
|
29169
|
+
return Math.max(0.85, Math.min(0.98, bestThreshold));
|
|
29170
|
+
}
|
|
29171
|
+
function saveCalibratedThreshold(projectId2, threshold, sampleSize) {
|
|
29172
|
+
const key = `dedup_threshold:${projectId2 ?? "global"}`;
|
|
29173
|
+
setKV(key, JSON.stringify({ threshold, sampleSize, calibratedAt: Date.now() }));
|
|
29174
|
+
}
|
|
29175
|
+
function loadCalibratedThreshold(projectId2) {
|
|
29176
|
+
const key = `dedup_threshold:${projectId2 ?? "global"}`;
|
|
29177
|
+
const raw = getKV(key);
|
|
29178
|
+
if (!raw) return null;
|
|
29179
|
+
try {
|
|
29180
|
+
const parsed = JSON.parse(raw);
|
|
29181
|
+
return typeof parsed.threshold === "number" ? parsed.threshold : null;
|
|
29182
|
+
} catch {
|
|
29183
|
+
return null;
|
|
29184
|
+
}
|
|
28678
29185
|
}
|
|
28679
29186
|
|
|
28680
29187
|
// src/data.ts
|
|
@@ -28734,8 +29241,8 @@ function setCache(fp, entry) {
|
|
|
28734
29241
|
"INSERT INTO kv_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = ?"
|
|
28735
29242
|
).run(key, value, value);
|
|
28736
29243
|
}
|
|
28737
|
-
function clearLoreFileCache(
|
|
28738
|
-
db().query("DELETE FROM kv_meta WHERE key = ?").run(CACHE_PREFIX + join7(
|
|
29244
|
+
function clearLoreFileCache(projectPath2) {
|
|
29245
|
+
db().query("DELETE FROM kv_meta WHERE key = ?").run(CACHE_PREFIX + join7(projectPath2, LORE_FILE));
|
|
28739
29246
|
}
|
|
28740
29247
|
function splitFile(fileContent) {
|
|
28741
29248
|
const spans = [];
|
|
@@ -28811,8 +29318,8 @@ function hashSection(section) {
|
|
|
28811
29318
|
}
|
|
28812
29319
|
return (h3 >>> 0).toString(16).padStart(8, "0");
|
|
28813
29320
|
}
|
|
28814
|
-
function buildSection(
|
|
28815
|
-
const entries = forProject(
|
|
29321
|
+
function buildSection(projectPath2) {
|
|
29322
|
+
const entries = forProject(projectPath2, false);
|
|
28816
29323
|
if (!entries.length) {
|
|
28817
29324
|
return "\n";
|
|
28818
29325
|
}
|
|
@@ -28844,6 +29351,7 @@ function buildSection(projectPath) {
|
|
|
28844
29351
|
return out.join("\n");
|
|
28845
29352
|
}
|
|
28846
29353
|
function exportToFile(input) {
|
|
29354
|
+
if (isHostedMode()) return;
|
|
28847
29355
|
exportLoreFile(input.projectPath);
|
|
28848
29356
|
const pointerBody = "\n## Long-term Knowledge\n\nFor long-term knowledge entries managed by [lore](https://github.com/BYK/loreai) (gotchas, patterns, decisions, architecture), see [`.lore.md`](.lore.md) in the project root.\n";
|
|
28849
29357
|
const newSection = LORE_SECTION_START + pointerBody + LORE_SECTION_END + "\n";
|
|
@@ -28861,6 +29369,7 @@ function exportToFile(input) {
|
|
|
28861
29369
|
writeFileSync(input.filePath, result, "utf8");
|
|
28862
29370
|
}
|
|
28863
29371
|
function shouldImport(input) {
|
|
29372
|
+
if (isHostedMode()) return false;
|
|
28864
29373
|
if (!existsSync4(input.filePath)) return false;
|
|
28865
29374
|
const fileContent = readFileSync3(input.filePath, "utf8");
|
|
28866
29375
|
const { section } = splitFile(fileContent);
|
|
@@ -28870,7 +29379,7 @@ function shouldImport(input) {
|
|
|
28870
29379
|
const expected = buildSection(input.projectPath);
|
|
28871
29380
|
return hashSection(section) !== hashSection(expected);
|
|
28872
29381
|
}
|
|
28873
|
-
function _importEntries(entries,
|
|
29382
|
+
function _importEntries(entries, projectPath2) {
|
|
28874
29383
|
const seenIds = /* @__PURE__ */ new Set();
|
|
28875
29384
|
for (const entry of entries) {
|
|
28876
29385
|
if (entry.id !== null) {
|
|
@@ -28882,7 +29391,7 @@ function _importEntries(entries, projectPath) {
|
|
|
28882
29391
|
update(entry.id, { content: entry.content });
|
|
28883
29392
|
}
|
|
28884
29393
|
} else {
|
|
28885
|
-
const pid = ensureProject(
|
|
29394
|
+
const pid = ensureProject(projectPath2);
|
|
28886
29395
|
const fuzzyMatch = findFuzzyDuplicate({ title: entry.title, projectId: pid });
|
|
28887
29396
|
if (fuzzyMatch) {
|
|
28888
29397
|
if (fuzzyMatch.title !== entry.title || get(fuzzyMatch.id)?.content !== entry.content) {
|
|
@@ -28890,7 +29399,7 @@ function _importEntries(entries, projectPath) {
|
|
|
28890
29399
|
}
|
|
28891
29400
|
} else {
|
|
28892
29401
|
create({
|
|
28893
|
-
projectPath,
|
|
29402
|
+
projectPath: projectPath2,
|
|
28894
29403
|
category: entry.category,
|
|
28895
29404
|
title: entry.title,
|
|
28896
29405
|
content: entry.content,
|
|
@@ -28901,13 +29410,13 @@ function _importEntries(entries, projectPath) {
|
|
|
28901
29410
|
}
|
|
28902
29411
|
}
|
|
28903
29412
|
} else {
|
|
28904
|
-
const existing = forProject(
|
|
29413
|
+
const existing = forProject(projectPath2, false);
|
|
28905
29414
|
const titleMatch = existing.find(
|
|
28906
29415
|
(e) => e.title.toLowerCase() === entry.title.toLowerCase()
|
|
28907
29416
|
);
|
|
28908
29417
|
if (!titleMatch) {
|
|
28909
29418
|
create({
|
|
28910
|
-
projectPath,
|
|
29419
|
+
projectPath: projectPath2,
|
|
28911
29420
|
category: entry.category,
|
|
28912
29421
|
title: entry.title,
|
|
28913
29422
|
content: entry.content,
|
|
@@ -28919,6 +29428,7 @@ function _importEntries(entries, projectPath) {
|
|
|
28919
29428
|
}
|
|
28920
29429
|
}
|
|
28921
29430
|
function importFromFile(input) {
|
|
29431
|
+
if (isHostedMode()) return;
|
|
28922
29432
|
if (!existsSync4(input.filePath)) return;
|
|
28923
29433
|
const fileContent = readFileSync3(input.filePath, "utf8");
|
|
28924
29434
|
const { section } = splitFile(fileContent);
|
|
@@ -28927,14 +29437,16 @@ function importFromFile(input) {
|
|
|
28927
29437
|
if (!fileEntries.length) return;
|
|
28928
29438
|
_importEntries(fileEntries, input.projectPath);
|
|
28929
29439
|
}
|
|
28930
|
-
function loreFileExists(
|
|
28931
|
-
|
|
29440
|
+
function loreFileExists(projectPath2) {
|
|
29441
|
+
if (isHostedMode()) return false;
|
|
29442
|
+
return existsSync4(join7(projectPath2, LORE_FILE));
|
|
28932
29443
|
}
|
|
28933
|
-
function exportLoreFile(
|
|
28934
|
-
|
|
29444
|
+
function exportLoreFile(projectPath2) {
|
|
29445
|
+
if (isHostedMode()) return;
|
|
29446
|
+
const sectionBody = buildSection(projectPath2);
|
|
28935
29447
|
const content3 = LORE_FILE_HEADER + "\n" + sectionBody;
|
|
28936
29448
|
const contentHash2 = hashSection(content3);
|
|
28937
|
-
const fp = join7(
|
|
29449
|
+
const fp = join7(projectPath2, LORE_FILE);
|
|
28938
29450
|
const cached2 = getCache(fp);
|
|
28939
29451
|
if (cached2 && cached2.hash === contentHash2) {
|
|
28940
29452
|
return;
|
|
@@ -28943,8 +29455,9 @@ function exportLoreFile(projectPath) {
|
|
|
28943
29455
|
const { mtimeMs } = statSync3(fp);
|
|
28944
29456
|
setCache(fp, { mtimeMs, hash: contentHash2 });
|
|
28945
29457
|
}
|
|
28946
|
-
function shouldImportLoreFile(
|
|
28947
|
-
|
|
29458
|
+
function shouldImportLoreFile(projectPath2) {
|
|
29459
|
+
if (isHostedMode()) return false;
|
|
29460
|
+
const fp = join7(projectPath2, LORE_FILE);
|
|
28948
29461
|
if (!existsSync4(fp)) return false;
|
|
28949
29462
|
const { mtimeMs } = statSync3(fp);
|
|
28950
29463
|
const cached2 = getCache(fp);
|
|
@@ -28953,7 +29466,7 @@ function shouldImportLoreFile(projectPath) {
|
|
|
28953
29466
|
}
|
|
28954
29467
|
const fileContent = readFileSync3(fp, "utf8");
|
|
28955
29468
|
const fileHash = hashSection(fileContent);
|
|
28956
|
-
const expected = LORE_FILE_HEADER + "\n" + buildSection(
|
|
29469
|
+
const expected = LORE_FILE_HEADER + "\n" + buildSection(projectPath2);
|
|
28957
29470
|
const expectedHash = hashSection(expected);
|
|
28958
29471
|
if (fileHash === expectedHash) {
|
|
28959
29472
|
setCache(fp, { mtimeMs, hash: fileHash });
|
|
@@ -28961,13 +29474,14 @@ function shouldImportLoreFile(projectPath) {
|
|
|
28961
29474
|
}
|
|
28962
29475
|
return true;
|
|
28963
29476
|
}
|
|
28964
|
-
function importLoreFile(
|
|
28965
|
-
|
|
29477
|
+
function importLoreFile(projectPath2) {
|
|
29478
|
+
if (isHostedMode()) return;
|
|
29479
|
+
const fp = join7(projectPath2, LORE_FILE);
|
|
28966
29480
|
if (!existsSync4(fp)) return;
|
|
28967
29481
|
const fileContent = readFileSync3(fp, "utf8");
|
|
28968
29482
|
const fileEntries = parseEntriesFromSection(fileContent);
|
|
28969
29483
|
if (!fileEntries.length) return;
|
|
28970
|
-
_importEntries(fileEntries,
|
|
29484
|
+
_importEntries(fileEntries, projectPath2);
|
|
28971
29485
|
try {
|
|
28972
29486
|
const { mtimeMs } = statSync3(fp);
|
|
28973
29487
|
setCache(fp, { mtimeMs, hash: hashSection(fileContent) });
|
|
@@ -28986,8 +29500,8 @@ function listProjects() {
|
|
|
28986
29500
|
FROM projects p ORDER BY p.created_at DESC`
|
|
28987
29501
|
).all();
|
|
28988
29502
|
}
|
|
28989
|
-
function listSessions(
|
|
28990
|
-
const pid = ensureProject(
|
|
29503
|
+
function listSessions(projectPath2, limit = 50) {
|
|
29504
|
+
const pid = ensureProject(projectPath2);
|
|
28991
29505
|
return db().query(
|
|
28992
29506
|
`SELECT
|
|
28993
29507
|
session_id,
|
|
@@ -29006,8 +29520,8 @@ function listSessions(projectPath, limit = 50) {
|
|
|
29006
29520
|
LIMIT ?`
|
|
29007
29521
|
).all(pid, limit);
|
|
29008
29522
|
}
|
|
29009
|
-
function listDistillations(
|
|
29010
|
-
const pid = ensureProject(
|
|
29523
|
+
function listDistillations(projectPath2, opts) {
|
|
29524
|
+
const pid = ensureProject(projectPath2);
|
|
29011
29525
|
const limit = opts?.limit ?? 50;
|
|
29012
29526
|
if (opts?.sessionId) {
|
|
29013
29527
|
return db().query(
|
|
@@ -29056,8 +29570,8 @@ function globalStats() {
|
|
|
29056
29570
|
}
|
|
29057
29571
|
return { ...row, db_size_bytes };
|
|
29058
29572
|
}
|
|
29059
|
-
function countForProject(
|
|
29060
|
-
const pid = projectId(
|
|
29573
|
+
function countForProject(projectPath2) {
|
|
29574
|
+
const pid = projectId(projectPath2);
|
|
29061
29575
|
if (!pid)
|
|
29062
29576
|
return { knowledge: 0, messages: 0, distillations: 0, sessions: 0 };
|
|
29063
29577
|
const row = db().query(
|
|
@@ -29069,8 +29583,8 @@ function countForProject(projectPath) {
|
|
|
29069
29583
|
).get(pid, pid, pid, pid);
|
|
29070
29584
|
return row;
|
|
29071
29585
|
}
|
|
29072
|
-
function clearProject(
|
|
29073
|
-
const pid = ensureProject(
|
|
29586
|
+
function clearProject(projectPath2) {
|
|
29587
|
+
const pid = ensureProject(projectPath2);
|
|
29074
29588
|
const database = db();
|
|
29075
29589
|
const counts = {
|
|
29076
29590
|
knowledge: database.query(
|
|
@@ -29101,9 +29615,9 @@ function clearProject(projectPath) {
|
|
|
29101
29615
|
database.exec("ROLLBACK");
|
|
29102
29616
|
throw e;
|
|
29103
29617
|
}
|
|
29104
|
-
if (existsSync5(
|
|
29618
|
+
if (existsSync5(projectPath2)) {
|
|
29105
29619
|
try {
|
|
29106
|
-
exportLoreFile(
|
|
29620
|
+
exportLoreFile(projectPath2);
|
|
29107
29621
|
} catch {
|
|
29108
29622
|
}
|
|
29109
29623
|
}
|
|
@@ -29166,30 +29680,30 @@ function renameProject(projectId2, newName) {
|
|
|
29166
29680
|
const result = db().query("UPDATE projects SET name = ? WHERE id = ?").run(newName.trim(), projectId2);
|
|
29167
29681
|
return result.changes > 0;
|
|
29168
29682
|
}
|
|
29169
|
-
function clearKnowledge(
|
|
29170
|
-
const pid = ensureProject(
|
|
29683
|
+
function clearKnowledge(projectPath2) {
|
|
29684
|
+
const pid = ensureProject(projectPath2);
|
|
29171
29685
|
const count3 = db().query(
|
|
29172
29686
|
"SELECT COUNT(*) as c FROM knowledge WHERE project_id = ?"
|
|
29173
29687
|
).get(pid).c;
|
|
29174
29688
|
db().query("DELETE FROM knowledge WHERE project_id = ?").run(pid);
|
|
29175
|
-
if (existsSync5(
|
|
29689
|
+
if (existsSync5(projectPath2)) {
|
|
29176
29690
|
try {
|
|
29177
|
-
exportLoreFile(
|
|
29691
|
+
exportLoreFile(projectPath2);
|
|
29178
29692
|
} catch {
|
|
29179
29693
|
}
|
|
29180
29694
|
}
|
|
29181
29695
|
return count3;
|
|
29182
29696
|
}
|
|
29183
|
-
function clearTemporal(
|
|
29184
|
-
const pid = ensureProject(
|
|
29697
|
+
function clearTemporal(projectPath2) {
|
|
29698
|
+
const pid = ensureProject(projectPath2);
|
|
29185
29699
|
const count3 = db().query(
|
|
29186
29700
|
"SELECT COUNT(*) as c FROM temporal_messages WHERE project_id = ?"
|
|
29187
29701
|
).get(pid).c;
|
|
29188
29702
|
db().query("DELETE FROM temporal_messages WHERE project_id = ?").run(pid);
|
|
29189
29703
|
return count3;
|
|
29190
29704
|
}
|
|
29191
|
-
function clearDistillations(
|
|
29192
|
-
const pid = ensureProject(
|
|
29705
|
+
function clearDistillations(projectPath2) {
|
|
29706
|
+
const pid = ensureProject(projectPath2);
|
|
29193
29707
|
const count3 = db().query(
|
|
29194
29708
|
"SELECT COUNT(*) as c FROM distillations WHERE project_id = ?"
|
|
29195
29709
|
).get(pid).c;
|
|
@@ -29208,8 +29722,8 @@ function deleteDistillation(id) {
|
|
|
29208
29722
|
db().query("DELETE FROM distillations WHERE id = ?").run(id);
|
|
29209
29723
|
return true;
|
|
29210
29724
|
}
|
|
29211
|
-
function deleteSession(
|
|
29212
|
-
const pid = ensureProject(
|
|
29725
|
+
function deleteSession(projectPath2, sessionId) {
|
|
29726
|
+
const pid = ensureProject(projectPath2);
|
|
29213
29727
|
const database = db();
|
|
29214
29728
|
const msgCount = database.query(
|
|
29215
29729
|
"SELECT COUNT(*) as c FROM temporal_messages WHERE project_id = ? AND session_id = ?"
|
|
@@ -29453,9 +29967,22 @@ function setMaxContextTokens(tokens) {
|
|
|
29453
29967
|
function getMaxContextTokens() {
|
|
29454
29968
|
return maxContextTokensCeiling;
|
|
29455
29969
|
}
|
|
29456
|
-
function updateBustRate(cacheWrite, cacheRead, sessionID) {
|
|
29970
|
+
function updateBustRate(cacheWrite, cacheRead, sessionID, lastLayer) {
|
|
29457
29971
|
if (!sessionID) return;
|
|
29458
29972
|
const state = getSessionState(sessionID);
|
|
29973
|
+
if (lastLayer === 4) {
|
|
29974
|
+
state.consecutiveLayer4++;
|
|
29975
|
+
if (state.consecutiveLayer4 >= 5 && state.dynamicContextCap > 0 && maxContextTokensCeiling > 0) {
|
|
29976
|
+
state.dynamicContextCap = Math.min(
|
|
29977
|
+
maxContextTokensCeiling,
|
|
29978
|
+
Math.floor(state.dynamicContextCap * 1.1)
|
|
29979
|
+
);
|
|
29980
|
+
}
|
|
29981
|
+
return;
|
|
29982
|
+
}
|
|
29983
|
+
if (lastLayer !== void 0) {
|
|
29984
|
+
state.consecutiveLayer4 = 0;
|
|
29985
|
+
}
|
|
29459
29986
|
const total = cacheWrite + cacheRead;
|
|
29460
29987
|
if (total === 0) return;
|
|
29461
29988
|
const bustRatio = cacheWrite / total;
|
|
@@ -29510,6 +30037,7 @@ function makeSessionState() {
|
|
|
29510
30037
|
cameOutOfIdle: false,
|
|
29511
30038
|
postIdleCompact: false,
|
|
29512
30039
|
consecutiveHighLayer: 0,
|
|
30040
|
+
consecutiveLayer4: 0,
|
|
29513
30041
|
bustRateEMA: -1,
|
|
29514
30042
|
interBustIntervalEMA: -1,
|
|
29515
30043
|
lastBustAt: 0,
|
|
@@ -29523,6 +30051,16 @@ function getSessionState(sessionID) {
|
|
|
29523
30051
|
if (!state) {
|
|
29524
30052
|
state = makeSessionState();
|
|
29525
30053
|
state.forceMinLayer = loadForceMinLayer(sessionID);
|
|
30054
|
+
const persisted = loadSessionTracking(sessionID);
|
|
30055
|
+
if (persisted && persisted.lastTurnAt > 0) {
|
|
30056
|
+
state.dynamicContextCap = persisted.dynamicContextCap;
|
|
30057
|
+
state.bustRateEMA = persisted.bustRateEMA;
|
|
30058
|
+
state.interBustIntervalEMA = persisted.interBustIntervalEMA;
|
|
30059
|
+
state.lastLayer = persisted.lastLayer;
|
|
30060
|
+
state.lastKnownInput = persisted.lastKnownInput;
|
|
30061
|
+
state.lastTurnAt = persisted.lastTurnAt;
|
|
30062
|
+
state.lastBustAt = persisted.lastBustAt;
|
|
30063
|
+
}
|
|
29526
30064
|
sessionStates.set(sessionID, state);
|
|
29527
30065
|
}
|
|
29528
30066
|
return state;
|
|
@@ -29602,6 +30140,11 @@ function getLastTransformedCount(sessionID) {
|
|
|
29602
30140
|
function getLastTransformEstimate(sessionID) {
|
|
29603
30141
|
return sessionStates.get(sessionID)?.lastTransformEstimate ?? 0;
|
|
29604
30142
|
}
|
|
30143
|
+
function getLastLayer(sessionID) {
|
|
30144
|
+
if (sessionID) return sessionStates.get(sessionID)?.lastLayer ?? 0;
|
|
30145
|
+
const first = sessionStates.values().next().value;
|
|
30146
|
+
return first?.lastLayer ?? 0;
|
|
30147
|
+
}
|
|
29605
30148
|
function setForceMinLayer(layer, sessionID) {
|
|
29606
30149
|
if (sessionID) {
|
|
29607
30150
|
getSessionState(sessionID).forceMinLayer = layer;
|
|
@@ -29622,19 +30165,35 @@ function inspectSessionState(sessionID) {
|
|
|
29622
30165
|
cameOutOfIdle: state.cameOutOfIdle,
|
|
29623
30166
|
postIdleCompact: state.postIdleCompact,
|
|
29624
30167
|
lastTurnAt: state.lastTurnAt,
|
|
29625
|
-
distillationSnapshot: state.distillationSnapshot
|
|
30168
|
+
distillationSnapshot: state.distillationSnapshot,
|
|
30169
|
+
bustRateEMA: state.bustRateEMA,
|
|
30170
|
+
dynamicContextCap: state.dynamicContextCap,
|
|
30171
|
+
consecutiveLayer4: state.consecutiveLayer4
|
|
29626
30172
|
};
|
|
29627
30173
|
}
|
|
29628
30174
|
function setLastTurnAtForTest(sessionID, ms) {
|
|
29629
30175
|
getSessionState(sessionID).lastTurnAt = ms;
|
|
29630
30176
|
}
|
|
29631
|
-
function
|
|
29632
|
-
const
|
|
30177
|
+
function saveGradientState(sessionID) {
|
|
30178
|
+
const state = sessionStates.get(sessionID);
|
|
30179
|
+
if (!state) return;
|
|
30180
|
+
saveSessionTracking(sessionID, {
|
|
30181
|
+
dynamicContextCap: state.dynamicContextCap,
|
|
30182
|
+
bustRateEMA: state.bustRateEMA,
|
|
30183
|
+
interBustIntervalEMA: state.interBustIntervalEMA,
|
|
30184
|
+
lastLayer: state.lastLayer,
|
|
30185
|
+
lastKnownInput: state.lastKnownInput,
|
|
30186
|
+
lastTurnAt: state.lastTurnAt,
|
|
30187
|
+
lastBustAt: state.lastBustAt
|
|
30188
|
+
});
|
|
30189
|
+
}
|
|
30190
|
+
function loadDistillations(projectPath2, sessionID) {
|
|
30191
|
+
const pid = ensureProject(projectPath2);
|
|
29633
30192
|
const query = sessionID ? "SELECT id, observations, generation, token_count, created_at, session_id FROM distillations WHERE project_id = ? AND session_id = ? AND archived = 0 ORDER BY created_at ASC" : "SELECT id, observations, generation, token_count, created_at, session_id FROM distillations WHERE project_id = ? AND archived = 0 ORDER BY created_at ASC";
|
|
29634
30193
|
const params = sessionID ? [pid, sessionID] : [pid];
|
|
29635
30194
|
return db().query(query).all(...params);
|
|
29636
30195
|
}
|
|
29637
|
-
function loadDistillationsCached(
|
|
30196
|
+
function loadDistillationsCached(projectPath2, sessionID, messages, sessState) {
|
|
29638
30197
|
let lastUserMsgId = null;
|
|
29639
30198
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
29640
30199
|
if (messages[i].info.role === "user") {
|
|
@@ -29646,7 +30205,7 @@ function loadDistillationsCached(projectPath, sessionID, messages, sessState) {
|
|
|
29646
30205
|
if (snapshot && snapshot.lastUserMsgId === lastUserMsgId) {
|
|
29647
30206
|
return snapshot.rows;
|
|
29648
30207
|
}
|
|
29649
|
-
const rows = loadDistillations(
|
|
30208
|
+
const rows = loadDistillations(projectPath2, sessionID);
|
|
29650
30209
|
sessState.distillationSnapshot = { rows, lastUserMsgId };
|
|
29651
30210
|
info(
|
|
29652
30211
|
`distillation refresh: ${rows.length} rows (user msg ${lastUserMsgId?.substring(0, 16) ?? "none"})`
|
|
@@ -29912,6 +30471,26 @@ function buildPrefixMessages(formatted) {
|
|
|
29912
30471
|
}
|
|
29913
30472
|
];
|
|
29914
30473
|
}
|
|
30474
|
+
var DECISION_RE = /\b(?:decision|decided|chose|chosen|agreed)\b/i;
|
|
30475
|
+
var GOTCHA_RE = /\b(?:gotcha|(?:critical|known|subtle)\s+bug|broken|crash(?:ed|es)?|regression)\b/i;
|
|
30476
|
+
var ARCH_RE = /\b(?:architecture|design.(?:decision|pattern)|system.design)\b/i;
|
|
30477
|
+
function importanceBonus(d) {
|
|
30478
|
+
let bonus = 0;
|
|
30479
|
+
if (DECISION_RE.test(d.observations)) bonus += 0.3;
|
|
30480
|
+
if (GOTCHA_RE.test(d.observations)) bonus += 0.2;
|
|
30481
|
+
if (ARCH_RE.test(d.observations)) bonus += 0.1;
|
|
30482
|
+
if (d.generation >= 1) bonus += 0.2;
|
|
30483
|
+
return Math.min(bonus, 1);
|
|
30484
|
+
}
|
|
30485
|
+
function selectDistillations(all3, limit) {
|
|
30486
|
+
if (all3.length <= limit) return all3;
|
|
30487
|
+
const maxIdx = all3.length - 1;
|
|
30488
|
+
const scored = all3.map((d, i) => ({
|
|
30489
|
+
d,
|
|
30490
|
+
score: (maxIdx > 0 ? i / maxIdx : 1) * 0.7 + importanceBonus(d) * 0.3
|
|
30491
|
+
}));
|
|
30492
|
+
return scored.sort((a, b) => b.score - a.score).slice(0, limit).map((s) => s.d).sort((a, b) => a.created_at - b.created_at);
|
|
30493
|
+
}
|
|
29915
30494
|
function distilledPrefix(distillations) {
|
|
29916
30495
|
if (!distillations.length) return [];
|
|
29917
30496
|
const formatted = formatDistillations(distillations);
|
|
@@ -30029,6 +30608,11 @@ function tryFitStable(input) {
|
|
|
30029
30608
|
}
|
|
30030
30609
|
return result;
|
|
30031
30610
|
}
|
|
30611
|
+
var COMPRESSION_STAGES = [
|
|
30612
|
+
{ strip: "none", rawFrac: null, distFrac: null, distLimit: Infinity, protectedTurns: 0, useStableWindow: true },
|
|
30613
|
+
{ strip: "old-tools", rawFrac: 0.5, distFrac: null, distLimit: Infinity, protectedTurns: 2, useStableWindow: false },
|
|
30614
|
+
{ strip: "all-tools", rawFrac: 0.55, distFrac: 0.15, distLimit: 5, protectedTurns: 0, useStableWindow: false }
|
|
30615
|
+
];
|
|
30032
30616
|
var urgentDistillationMap = /* @__PURE__ */ new Map();
|
|
30033
30617
|
function needsUrgentDistillation(sessionID) {
|
|
30034
30618
|
const v = urgentDistillationMap.get(sessionID) ?? false;
|
|
@@ -30060,7 +30644,7 @@ function transformInner(input) {
|
|
|
30060
30644
|
if (calibrated) return true;
|
|
30061
30645
|
return result.totalTokens * UNCALIBRATED_SAFETY <= maxInput;
|
|
30062
30646
|
}
|
|
30063
|
-
if (calibrated && sessState.lastLayer >= 1 && input.messages.length >= sessState.lastKnownMessageCount) {
|
|
30647
|
+
if (calibrated && sessState.lastLayer >= 1 && sessState.lastLayer <= 3 && input.messages.length >= sessState.lastKnownMessageCount) {
|
|
30064
30648
|
effectiveMinLayer = Math.max(effectiveMinLayer, sessState.lastLayer);
|
|
30065
30649
|
}
|
|
30066
30650
|
const postIdleCompact = sessState.postIdleCompact;
|
|
@@ -30098,7 +30682,8 @@ function transformInner(input) {
|
|
|
30098
30682
|
totalTokens: Math.max(0, messageTokens),
|
|
30099
30683
|
usable,
|
|
30100
30684
|
distilledBudget,
|
|
30101
|
-
rawBudget
|
|
30685
|
+
rawBudget,
|
|
30686
|
+
refreshLtm: false
|
|
30102
30687
|
};
|
|
30103
30688
|
}
|
|
30104
30689
|
const turnStart = currentTurnStart(input.messages);
|
|
@@ -30108,67 +30693,52 @@ function transformInner(input) {
|
|
|
30108
30693
|
const msgs = distilledPrefix(distillations);
|
|
30109
30694
|
return { messages: msgs, tokens: msgs.reduce((sum, m) => sum + estimateMessage(m), 0) };
|
|
30110
30695
|
})();
|
|
30111
|
-
|
|
30112
|
-
const
|
|
30113
|
-
|
|
30114
|
-
|
|
30115
|
-
|
|
30116
|
-
|
|
30117
|
-
|
|
30118
|
-
|
|
30119
|
-
|
|
30120
|
-
|
|
30121
|
-
|
|
30122
|
-
|
|
30123
|
-
|
|
30124
|
-
|
|
30125
|
-
|
|
30126
|
-
|
|
30127
|
-
|
|
30128
|
-
|
|
30129
|
-
|
|
30696
|
+
for (let s = 0; s < COMPRESSION_STAGES.length; s++) {
|
|
30697
|
+
const stageLayer = s + 1;
|
|
30698
|
+
if (effectiveMinLayer > stageLayer) continue;
|
|
30699
|
+
const stage = COMPRESSION_STAGES[s];
|
|
30700
|
+
const stageRawBudget = stage.rawFrac !== null ? Math.floor(usable * stage.rawFrac) : rawBudget;
|
|
30701
|
+
const stageDistBudget = stage.distFrac !== null ? Math.floor(usable * stage.distFrac) : distilledBudget;
|
|
30702
|
+
let stagePrefix = cached2.messages;
|
|
30703
|
+
let stagePrefixTokens = cached2.tokens;
|
|
30704
|
+
if (stage.distLimit !== Infinity && distillations.length > stage.distLimit) {
|
|
30705
|
+
const trimmed = selectDistillations(distillations, stage.distLimit);
|
|
30706
|
+
stagePrefix = distilledPrefix(trimmed);
|
|
30707
|
+
stagePrefixTokens = stagePrefix.reduce((sum, m) => sum + estimateMessage(m), 0);
|
|
30708
|
+
}
|
|
30709
|
+
let result;
|
|
30710
|
+
if (stage.useStableWindow && sid) {
|
|
30711
|
+
result = tryFitStable({
|
|
30712
|
+
messages: dedupMessages,
|
|
30713
|
+
prefix: stagePrefix,
|
|
30714
|
+
prefixTokens: stagePrefixTokens,
|
|
30715
|
+
distilledBudget: stageDistBudget,
|
|
30716
|
+
rawBudget: stageRawBudget,
|
|
30717
|
+
sessionID: sid,
|
|
30718
|
+
sessState
|
|
30719
|
+
});
|
|
30720
|
+
} else {
|
|
30721
|
+
sessState.rawWindowCache = null;
|
|
30722
|
+
result = tryFit({
|
|
30723
|
+
messages: dedupMessages,
|
|
30724
|
+
prefix: stagePrefix,
|
|
30725
|
+
prefixTokens: stagePrefixTokens,
|
|
30726
|
+
distilledBudget: stageDistBudget,
|
|
30727
|
+
rawBudget: stageRawBudget,
|
|
30728
|
+
strip: stage.strip,
|
|
30729
|
+
protectedTurns: stage.protectedTurns
|
|
30730
|
+
});
|
|
30731
|
+
}
|
|
30732
|
+
if (fitsWithSafetyMargin(result)) {
|
|
30733
|
+
if (sid && (s > 0 || cached2.tokens === 0)) {
|
|
30130
30734
|
urgentDistillationMap.set(sid, true);
|
|
30131
30735
|
}
|
|
30132
|
-
return { ...
|
|
30736
|
+
return { ...result, layer: stageLayer, usable, distilledBudget, rawBudget, refreshLtm: false };
|
|
30133
30737
|
}
|
|
30134
30738
|
}
|
|
30135
30739
|
sessState.rawWindowCache = null;
|
|
30136
|
-
if (effectiveMinLayer <= 2) {
|
|
30137
|
-
const layer2 = tryFit({
|
|
30138
|
-
messages: dedupMessages,
|
|
30139
|
-
prefix: cached2.messages,
|
|
30140
|
-
prefixTokens: cached2.tokens,
|
|
30141
|
-
distilledBudget,
|
|
30142
|
-
rawBudget: Math.floor(usable * 0.5),
|
|
30143
|
-
// give raw more room
|
|
30144
|
-
strip: "old-tools",
|
|
30145
|
-
protectedTurns: 2
|
|
30146
|
-
});
|
|
30147
|
-
if (fitsWithSafetyMargin(layer2)) {
|
|
30148
|
-
if (sid) urgentDistillationMap.set(sid, true);
|
|
30149
|
-
return { ...layer2, layer: 2, usable, distilledBudget, rawBudget };
|
|
30150
|
-
}
|
|
30151
|
-
}
|
|
30152
|
-
const trimmedDistillations = distillations.slice(-5);
|
|
30153
|
-
const trimmedPrefix = distilledPrefix(trimmedDistillations);
|
|
30154
|
-
const trimmedPrefixTokens = trimmedPrefix.reduce(
|
|
30155
|
-
(sum, m) => sum + estimateMessage(m),
|
|
30156
|
-
0
|
|
30157
|
-
);
|
|
30158
|
-
const layer3 = tryFit({
|
|
30159
|
-
messages: dedupMessages,
|
|
30160
|
-
prefix: trimmedPrefix,
|
|
30161
|
-
prefixTokens: trimmedPrefixTokens,
|
|
30162
|
-
distilledBudget: Math.floor(usable * 0.15),
|
|
30163
|
-
rawBudget: Math.floor(usable * 0.55),
|
|
30164
|
-
strip: "all-tools"
|
|
30165
|
-
});
|
|
30166
|
-
if (fitsWithSafetyMargin(layer3)) {
|
|
30167
|
-
if (sid) urgentDistillationMap.set(sid, true);
|
|
30168
|
-
return { ...layer3, layer: 3, usable, distilledBudget, rawBudget };
|
|
30169
|
-
}
|
|
30170
30740
|
if (sid) urgentDistillationMap.set(sid, true);
|
|
30171
|
-
const nuclearDistillations = distillations
|
|
30741
|
+
const nuclearDistillations = selectDistillations(distillations, 2);
|
|
30172
30742
|
const nuclearPrefix = distilledPrefix(nuclearDistillations);
|
|
30173
30743
|
const nuclearPrefixTokens = nuclearPrefix.reduce(
|
|
30174
30744
|
(sum, m) => sum + estimateMessage(m),
|
|
@@ -30207,7 +30777,8 @@ function transformInner(input) {
|
|
|
30207
30777
|
totalTokens: nuclearPrefixTokens + nuclearRawTokens,
|
|
30208
30778
|
usable,
|
|
30209
30779
|
distilledBudget,
|
|
30210
|
-
rawBudget
|
|
30780
|
+
rawBudget,
|
|
30781
|
+
refreshLtm: true
|
|
30211
30782
|
};
|
|
30212
30783
|
}
|
|
30213
30784
|
function transform2(input) {
|
|
@@ -30314,6 +30885,185 @@ function isWorkerSession(sessionID) {
|
|
|
30314
30885
|
return workerSessionIDs.has(sessionID);
|
|
30315
30886
|
}
|
|
30316
30887
|
|
|
30888
|
+
// ../../node_modules/.bun/yocto-queue@1.2.2/node_modules/yocto-queue/index.js
|
|
30889
|
+
var Node = class {
|
|
30890
|
+
value;
|
|
30891
|
+
next;
|
|
30892
|
+
constructor(value) {
|
|
30893
|
+
this.value = value;
|
|
30894
|
+
}
|
|
30895
|
+
};
|
|
30896
|
+
var Queue = class {
|
|
30897
|
+
#head;
|
|
30898
|
+
#tail;
|
|
30899
|
+
#size;
|
|
30900
|
+
constructor() {
|
|
30901
|
+
this.clear();
|
|
30902
|
+
}
|
|
30903
|
+
enqueue(value) {
|
|
30904
|
+
const node2 = new Node(value);
|
|
30905
|
+
if (this.#head) {
|
|
30906
|
+
this.#tail.next = node2;
|
|
30907
|
+
this.#tail = node2;
|
|
30908
|
+
} else {
|
|
30909
|
+
this.#head = node2;
|
|
30910
|
+
this.#tail = node2;
|
|
30911
|
+
}
|
|
30912
|
+
this.#size++;
|
|
30913
|
+
}
|
|
30914
|
+
dequeue() {
|
|
30915
|
+
const current2 = this.#head;
|
|
30916
|
+
if (!current2) {
|
|
30917
|
+
return;
|
|
30918
|
+
}
|
|
30919
|
+
this.#head = this.#head.next;
|
|
30920
|
+
this.#size--;
|
|
30921
|
+
if (!this.#head) {
|
|
30922
|
+
this.#tail = void 0;
|
|
30923
|
+
}
|
|
30924
|
+
return current2.value;
|
|
30925
|
+
}
|
|
30926
|
+
peek() {
|
|
30927
|
+
if (!this.#head) {
|
|
30928
|
+
return;
|
|
30929
|
+
}
|
|
30930
|
+
return this.#head.value;
|
|
30931
|
+
}
|
|
30932
|
+
clear() {
|
|
30933
|
+
this.#head = void 0;
|
|
30934
|
+
this.#tail = void 0;
|
|
30935
|
+
this.#size = 0;
|
|
30936
|
+
}
|
|
30937
|
+
get size() {
|
|
30938
|
+
return this.#size;
|
|
30939
|
+
}
|
|
30940
|
+
*[Symbol.iterator]() {
|
|
30941
|
+
let current2 = this.#head;
|
|
30942
|
+
while (current2) {
|
|
30943
|
+
yield current2.value;
|
|
30944
|
+
current2 = current2.next;
|
|
30945
|
+
}
|
|
30946
|
+
}
|
|
30947
|
+
*drain() {
|
|
30948
|
+
while (this.#head) {
|
|
30949
|
+
yield this.dequeue();
|
|
30950
|
+
}
|
|
30951
|
+
}
|
|
30952
|
+
};
|
|
30953
|
+
|
|
30954
|
+
// ../../node_modules/.bun/p-limit@7.3.0/node_modules/p-limit/index.js
|
|
30955
|
+
function pLimit(concurrency) {
|
|
30956
|
+
let rejectOnClear = false;
|
|
30957
|
+
if (typeof concurrency === "object") {
|
|
30958
|
+
({ concurrency, rejectOnClear = false } = concurrency);
|
|
30959
|
+
}
|
|
30960
|
+
validateConcurrency(concurrency);
|
|
30961
|
+
if (typeof rejectOnClear !== "boolean") {
|
|
30962
|
+
throw new TypeError("Expected `rejectOnClear` to be a boolean");
|
|
30963
|
+
}
|
|
30964
|
+
const queue = new Queue();
|
|
30965
|
+
let activeCount = 0;
|
|
30966
|
+
const resumeNext = () => {
|
|
30967
|
+
if (activeCount < concurrency && queue.size > 0) {
|
|
30968
|
+
activeCount++;
|
|
30969
|
+
queue.dequeue().run();
|
|
30970
|
+
}
|
|
30971
|
+
};
|
|
30972
|
+
const next = () => {
|
|
30973
|
+
activeCount--;
|
|
30974
|
+
resumeNext();
|
|
30975
|
+
};
|
|
30976
|
+
const run3 = async (function_, resolve, arguments_) => {
|
|
30977
|
+
const result = (async () => function_(...arguments_))();
|
|
30978
|
+
resolve(result);
|
|
30979
|
+
try {
|
|
30980
|
+
await result;
|
|
30981
|
+
} catch {
|
|
30982
|
+
}
|
|
30983
|
+
next();
|
|
30984
|
+
};
|
|
30985
|
+
const enqueue = (function_, resolve, reject, arguments_) => {
|
|
30986
|
+
const queueItem = { reject };
|
|
30987
|
+
new Promise((internalResolve) => {
|
|
30988
|
+
queueItem.run = internalResolve;
|
|
30989
|
+
queue.enqueue(queueItem);
|
|
30990
|
+
}).then(run3.bind(void 0, function_, resolve, arguments_));
|
|
30991
|
+
if (activeCount < concurrency) {
|
|
30992
|
+
resumeNext();
|
|
30993
|
+
}
|
|
30994
|
+
};
|
|
30995
|
+
const generator = (function_, ...arguments_) => new Promise((resolve, reject) => {
|
|
30996
|
+
enqueue(function_, resolve, reject, arguments_);
|
|
30997
|
+
});
|
|
30998
|
+
Object.defineProperties(generator, {
|
|
30999
|
+
activeCount: {
|
|
31000
|
+
get: () => activeCount
|
|
31001
|
+
},
|
|
31002
|
+
pendingCount: {
|
|
31003
|
+
get: () => queue.size
|
|
31004
|
+
},
|
|
31005
|
+
clearQueue: {
|
|
31006
|
+
value() {
|
|
31007
|
+
if (!rejectOnClear) {
|
|
31008
|
+
queue.clear();
|
|
31009
|
+
return;
|
|
31010
|
+
}
|
|
31011
|
+
const abortError = AbortSignal.abort().reason;
|
|
31012
|
+
while (queue.size > 0) {
|
|
31013
|
+
queue.dequeue().reject(abortError);
|
|
31014
|
+
}
|
|
31015
|
+
}
|
|
31016
|
+
},
|
|
31017
|
+
concurrency: {
|
|
31018
|
+
get: () => concurrency,
|
|
31019
|
+
set(newConcurrency) {
|
|
31020
|
+
validateConcurrency(newConcurrency);
|
|
31021
|
+
concurrency = newConcurrency;
|
|
31022
|
+
queueMicrotask(() => {
|
|
31023
|
+
while (activeCount < concurrency && queue.size > 0) {
|
|
31024
|
+
resumeNext();
|
|
31025
|
+
}
|
|
31026
|
+
});
|
|
31027
|
+
}
|
|
31028
|
+
},
|
|
31029
|
+
map: {
|
|
31030
|
+
async value(iterable, function_) {
|
|
31031
|
+
const promises = Array.from(iterable, (value, index2) => this(function_, value, index2));
|
|
31032
|
+
return Promise.all(promises);
|
|
31033
|
+
}
|
|
31034
|
+
}
|
|
31035
|
+
});
|
|
31036
|
+
return generator;
|
|
31037
|
+
}
|
|
31038
|
+
function validateConcurrency(concurrency) {
|
|
31039
|
+
if (!((Number.isInteger(concurrency) || concurrency === Number.POSITIVE_INFINITY) && concurrency > 0)) {
|
|
31040
|
+
throw new TypeError("Expected `concurrency` to be a number from 1 and up");
|
|
31041
|
+
}
|
|
31042
|
+
}
|
|
31043
|
+
|
|
31044
|
+
// src/session-limiter.ts
|
|
31045
|
+
function createLimiterPool() {
|
|
31046
|
+
const limiters = /* @__PURE__ */ new Map();
|
|
31047
|
+
function get2(key) {
|
|
31048
|
+
let limiter = limiters.get(key);
|
|
31049
|
+
if (!limiter) {
|
|
31050
|
+
limiter = pLimit(1);
|
|
31051
|
+
limiters.set(key, limiter);
|
|
31052
|
+
}
|
|
31053
|
+
return limiter;
|
|
31054
|
+
}
|
|
31055
|
+
function isBusy(key) {
|
|
31056
|
+
const limiter = limiters.get(key);
|
|
31057
|
+
return limiter ? limiter.activeCount + limiter.pendingCount > 0 : false;
|
|
31058
|
+
}
|
|
31059
|
+
function clear() {
|
|
31060
|
+
limiters.clear();
|
|
31061
|
+
}
|
|
31062
|
+
return { get: get2, isBusy, clear };
|
|
31063
|
+
}
|
|
31064
|
+
var distillLimiter = createLimiterPool();
|
|
31065
|
+
var curatorLimiter = createLimiterPool();
|
|
31066
|
+
|
|
30317
31067
|
// src/distillation.ts
|
|
30318
31068
|
function compressionRatio(distilledTokens, sourceTokens) {
|
|
30319
31069
|
if (sourceTokens <= 0) return 0;
|
|
@@ -30439,18 +31189,18 @@ function parseDistillationResult(text4) {
|
|
|
30439
31189
|
if (!observations) return null;
|
|
30440
31190
|
return { observations };
|
|
30441
31191
|
}
|
|
30442
|
-
function latestObservations(
|
|
30443
|
-
const pid = ensureProject(
|
|
31192
|
+
function latestObservations(projectPath2, sessionID) {
|
|
31193
|
+
const pid = ensureProject(projectPath2);
|
|
30444
31194
|
const row = db().query(
|
|
30445
31195
|
"SELECT observations FROM distillations WHERE project_id = ? AND session_id = ? ORDER BY created_at DESC LIMIT 1"
|
|
30446
31196
|
).get(pid, sessionID);
|
|
30447
31197
|
return row?.observations || void 0;
|
|
30448
31198
|
}
|
|
30449
|
-
function latestMetaObservations(
|
|
30450
|
-
return latestMeta(
|
|
31199
|
+
function latestMetaObservations(projectPath2, sessionID) {
|
|
31200
|
+
return latestMeta(projectPath2, sessionID)?.observations;
|
|
30451
31201
|
}
|
|
30452
|
-
function latestMeta(
|
|
30453
|
-
const pid = ensureProject(
|
|
31202
|
+
function latestMeta(projectPath2, sessionID) {
|
|
31203
|
+
const pid = ensureProject(projectPath2);
|
|
30454
31204
|
const row = db().query(
|
|
30455
31205
|
`SELECT observations, generation FROM distillations
|
|
30456
31206
|
WHERE project_id = ? AND session_id = ? AND generation > 0
|
|
@@ -30468,8 +31218,8 @@ function parseSourceIds(raw) {
|
|
|
30468
31218
|
return [];
|
|
30469
31219
|
}
|
|
30470
31220
|
}
|
|
30471
|
-
function loadForSession(
|
|
30472
|
-
const pid = ensureProject(
|
|
31221
|
+
function loadForSession(projectPath2, sessionID, includeArchived = false) {
|
|
31222
|
+
const pid = ensureProject(projectPath2);
|
|
30473
31223
|
const sql = includeArchived ? "SELECT id, project_id, session_id, observations, source_ids, generation, token_count, created_at, r_compression, c_norm FROM distillations WHERE project_id = ? AND session_id = ? ORDER BY created_at ASC" : "SELECT id, project_id, session_id, observations, source_ids, generation, token_count, created_at, r_compression, c_norm FROM distillations WHERE project_id = ? AND session_id = ? AND archived = 0 ORDER BY created_at ASC";
|
|
30474
31224
|
const rows = db().query(sql).all(pid, sessionID);
|
|
30475
31225
|
return rows.map((r) => ({
|
|
@@ -30504,14 +31254,14 @@ function storeDistillation(input) {
|
|
|
30504
31254
|
);
|
|
30505
31255
|
return id;
|
|
30506
31256
|
}
|
|
30507
|
-
function gen0Count(
|
|
30508
|
-
const pid = ensureProject(
|
|
31257
|
+
function gen0Count(projectPath2, sessionID) {
|
|
31258
|
+
const pid = ensureProject(projectPath2);
|
|
30509
31259
|
return db().query(
|
|
30510
31260
|
"SELECT COUNT(*) as count FROM distillations WHERE project_id = ? AND session_id = ? AND generation = 0 AND archived = 0"
|
|
30511
31261
|
).get(pid, sessionID).count;
|
|
30512
31262
|
}
|
|
30513
|
-
function loadGen0(
|
|
30514
|
-
const pid = ensureProject(
|
|
31263
|
+
function loadGen0(projectPath2, sessionID) {
|
|
31264
|
+
const pid = ensureProject(projectPath2);
|
|
30515
31265
|
const rows = db().query(
|
|
30516
31266
|
"SELECT id, project_id, session_id, observations, source_ids, generation, token_count, created_at, r_compression, c_norm FROM distillations WHERE project_id = ? AND session_id = ? AND generation = 0 AND archived = 0 ORDER BY created_at ASC"
|
|
30517
31267
|
).all(pid, sessionID);
|
|
@@ -30527,8 +31277,8 @@ function archiveDistillations(ids) {
|
|
|
30527
31277
|
`UPDATE distillations SET archived = 1 WHERE id IN (${placeholders})`
|
|
30528
31278
|
).run(...ids);
|
|
30529
31279
|
}
|
|
30530
|
-
function resetOrphans(
|
|
30531
|
-
const pid = ensureProject(
|
|
31280
|
+
function resetOrphans(projectPath2, sessionID) {
|
|
31281
|
+
const pid = ensureProject(projectPath2);
|
|
30532
31282
|
const rows = db().query(
|
|
30533
31283
|
"SELECT source_ids FROM distillations WHERE project_id = ? AND session_id = ?"
|
|
30534
31284
|
).all(pid, sessionID);
|
|
@@ -30558,6 +31308,9 @@ function resetOrphans(projectPath, sessionID) {
|
|
|
30558
31308
|
return orphans.length;
|
|
30559
31309
|
}
|
|
30560
31310
|
async function run(input) {
|
|
31311
|
+
return distillLimiter.get(input.sessionID)(() => runInner(input));
|
|
31312
|
+
}
|
|
31313
|
+
async function runInner(input) {
|
|
30561
31314
|
const orphans = resetOrphans(input.projectPath, input.sessionID);
|
|
30562
31315
|
if (orphans > 0) {
|
|
30563
31316
|
info(
|
|
@@ -30601,7 +31354,7 @@ async function run(input) {
|
|
|
30601
31354
|
}
|
|
30602
31355
|
}
|
|
30603
31356
|
if (!input.skipMeta && gen0Count(input.projectPath, input.sessionID) >= cfg.distillation.metaThreshold) {
|
|
30604
|
-
await
|
|
31357
|
+
await metaDistillInner({
|
|
30605
31358
|
llm: input.llm,
|
|
30606
31359
|
projectPath: input.projectPath,
|
|
30607
31360
|
sessionID: input.sessionID,
|
|
@@ -30651,17 +31404,25 @@ async function distillSegment(input) {
|
|
|
30651
31404
|
);
|
|
30652
31405
|
return null;
|
|
30653
31406
|
}
|
|
30654
|
-
|
|
30655
|
-
|
|
30656
|
-
|
|
30657
|
-
|
|
30658
|
-
|
|
30659
|
-
|
|
30660
|
-
|
|
30661
|
-
|
|
30662
|
-
|
|
30663
|
-
|
|
30664
|
-
|
|
31407
|
+
let distillId;
|
|
31408
|
+
db().exec("BEGIN IMMEDIATE");
|
|
31409
|
+
try {
|
|
31410
|
+
distillId = storeDistillation({
|
|
31411
|
+
projectPath: input.projectPath,
|
|
31412
|
+
sessionID: input.sessionID,
|
|
31413
|
+
observations: result.observations,
|
|
31414
|
+
sourceIDs: input.messages.map((m) => m.id),
|
|
31415
|
+
generation: 0,
|
|
31416
|
+
rCompression: rComp,
|
|
31417
|
+
cNorm,
|
|
31418
|
+
callType: input.callType
|
|
31419
|
+
});
|
|
31420
|
+
markDistilled(input.messages.map((m) => m.id));
|
|
31421
|
+
db().exec("COMMIT");
|
|
31422
|
+
} catch (e) {
|
|
31423
|
+
db().exec("ROLLBACK");
|
|
31424
|
+
throw e;
|
|
31425
|
+
}
|
|
30665
31426
|
info(
|
|
30666
31427
|
`distill segment: ${input.messages.length} msgs, ${sourceTokens}\u2192${distilledTokens} tokens, R=${rComp.toFixed(2)}, C_norm=${cNorm.toFixed(3)}`
|
|
30667
31428
|
);
|
|
@@ -30695,6 +31456,9 @@ async function distillSegment(input) {
|
|
|
30695
31456
|
return result;
|
|
30696
31457
|
}
|
|
30697
31458
|
async function metaDistill(input) {
|
|
31459
|
+
return distillLimiter.get(input.sessionID)(() => metaDistillInner(input));
|
|
31460
|
+
}
|
|
31461
|
+
async function metaDistillInner(input) {
|
|
30698
31462
|
const existing = loadGen0(input.projectPath, input.sessionID);
|
|
30699
31463
|
const priorMeta = latestMeta(input.projectPath, input.sessionID);
|
|
30700
31464
|
if (priorMeta) {
|
|
@@ -30994,11 +31758,27 @@ function applyOps(ops, input) {
|
|
|
30994
31758
|
return { created, updated, deleted };
|
|
30995
31759
|
}
|
|
30996
31760
|
var lastCuratedAt = /* @__PURE__ */ new Map();
|
|
31761
|
+
function getLastCuratedAt(sessionID) {
|
|
31762
|
+
const cached2 = lastCuratedAt.get(sessionID);
|
|
31763
|
+
if (cached2 !== void 0) return cached2;
|
|
31764
|
+
const persisted = loadSessionTracking(sessionID);
|
|
31765
|
+
const ts = persisted?.lastCuratedAt ?? 0;
|
|
31766
|
+
lastCuratedAt.set(sessionID, ts);
|
|
31767
|
+
return ts;
|
|
31768
|
+
}
|
|
30997
31769
|
async function run2(input) {
|
|
30998
31770
|
const cfg = config2();
|
|
30999
31771
|
if (!cfg.curator.enabled) return { created: 0, updated: 0, deleted: 0 };
|
|
31772
|
+
if (curatorLimiter.isBusy(input.sessionID)) {
|
|
31773
|
+
info(`curation skipped: already running for session ${input.sessionID.slice(0, 16)}`);
|
|
31774
|
+
return { created: 0, updated: 0, deleted: 0 };
|
|
31775
|
+
}
|
|
31776
|
+
return curatorLimiter.get(input.sessionID)(() => runInner2(input));
|
|
31777
|
+
}
|
|
31778
|
+
async function runInner2(input) {
|
|
31779
|
+
const cfg = config2();
|
|
31000
31780
|
const all3 = bySession(input.projectPath, input.sessionID);
|
|
31001
|
-
const sessionCuratedAt =
|
|
31781
|
+
const sessionCuratedAt = getLastCuratedAt(input.sessionID);
|
|
31002
31782
|
const recent = all3.filter((m) => m.created_at > sessionCuratedAt);
|
|
31003
31783
|
if (recent.length < 3) return { created: 0, updated: 0, deleted: 0 };
|
|
31004
31784
|
const text4 = recent.map((m) => `[${m.role}] ${m.content}`).join("\n\n");
|
|
@@ -31042,11 +31822,22 @@ async function run2(input) {
|
|
|
31042
31822
|
info(`post-curation dedup: merged ${dupes.totalRemoved} duplicate entries`);
|
|
31043
31823
|
result.deleted += dupes.totalRemoved;
|
|
31044
31824
|
}
|
|
31825
|
+
if (dupes.pairSimilarities.size > 0) {
|
|
31826
|
+
const pid = ensureProject(input.projectPath);
|
|
31827
|
+
recordAutoSignals(pid, dupes);
|
|
31828
|
+
const newThreshold = calibrateDedupThreshold(pid);
|
|
31829
|
+
if (newThreshold !== null) {
|
|
31830
|
+
const count3 = getDedupFeedbackCount(pid);
|
|
31831
|
+
saveCalibratedThreshold(pid, newThreshold, count3);
|
|
31832
|
+
}
|
|
31833
|
+
}
|
|
31045
31834
|
} catch (err) {
|
|
31046
31835
|
warn("post-curation dedup failed (non-fatal):", err);
|
|
31047
31836
|
}
|
|
31048
31837
|
}
|
|
31049
|
-
|
|
31838
|
+
const now = Date.now();
|
|
31839
|
+
lastCuratedAt.set(input.sessionID, now);
|
|
31840
|
+
saveSessionTracking(input.sessionID, { lastCuratedAt: now });
|
|
31050
31841
|
return result;
|
|
31051
31842
|
}
|
|
31052
31843
|
function resetCurationTracker(sessionID) {
|
|
@@ -31119,11 +31910,11 @@ function clearProviders() {
|
|
|
31119
31910
|
}
|
|
31120
31911
|
|
|
31121
31912
|
// src/import/detect.ts
|
|
31122
|
-
function detectAll(
|
|
31913
|
+
function detectAll(projectPath2) {
|
|
31123
31914
|
const results = [];
|
|
31124
31915
|
for (const provider of getProviders()) {
|
|
31125
31916
|
try {
|
|
31126
|
-
const sessions = provider.detect(
|
|
31917
|
+
const sessions = provider.detect(projectPath2);
|
|
31127
31918
|
if (sessions.length > 0) {
|
|
31128
31919
|
results.push({
|
|
31129
31920
|
agentName: provider.name,
|
|
@@ -31212,8 +32003,8 @@ async function extractKnowledge(input) {
|
|
|
31212
32003
|
}
|
|
31213
32004
|
|
|
31214
32005
|
// src/import/history.ts
|
|
31215
|
-
function isImported(
|
|
31216
|
-
const projectId2 = ensureProject(
|
|
32006
|
+
function isImported(projectPath2, agentName, sourceId, sourceHash) {
|
|
32007
|
+
const projectId2 = ensureProject(projectPath2);
|
|
31217
32008
|
const row = db().query(
|
|
31218
32009
|
`SELECT * FROM import_history
|
|
31219
32010
|
WHERE project_id = ? AND agent_name = ? AND source_id = ?`
|
|
@@ -31222,8 +32013,8 @@ function isImported(projectPath, agentName, sourceId, sourceHash) {
|
|
|
31222
32013
|
if (row.source_hash !== sourceHash) return null;
|
|
31223
32014
|
return row;
|
|
31224
32015
|
}
|
|
31225
|
-
function recordImport(
|
|
31226
|
-
const projectId2 = ensureProject(
|
|
32016
|
+
function recordImport(projectPath2, agentName, sourceId, sourceHash, stats) {
|
|
32017
|
+
const projectId2 = ensureProject(projectPath2);
|
|
31227
32018
|
db().query(
|
|
31228
32019
|
`INSERT OR REPLACE INTO import_history
|
|
31229
32020
|
(id, project_id, agent_name, source_id, source_hash, entries_created, entries_updated, imported_at)
|
|
@@ -31239,8 +32030,8 @@ function recordImport(projectPath, agentName, sourceId, sourceHash, stats) {
|
|
|
31239
32030
|
Date.now()
|
|
31240
32031
|
);
|
|
31241
32032
|
}
|
|
31242
|
-
function listImports(
|
|
31243
|
-
const projectId2 = ensureProject(
|
|
32033
|
+
function listImports(projectPath2) {
|
|
32034
|
+
const projectId2 = ensureProject(projectPath2);
|
|
31244
32035
|
return db().query(
|
|
31245
32036
|
`SELECT * FROM import_history
|
|
31246
32037
|
WHERE project_id = ? AND source_id != '__declined__'
|
|
@@ -31258,8 +32049,8 @@ import { homedir as homedir2 } from "os";
|
|
|
31258
32049
|
var CLAUDE_DIR = join8(homedir2(), ".claude", "projects");
|
|
31259
32050
|
var MAX_TOOL_OUTPUT_CHARS = 500;
|
|
31260
32051
|
var DEFAULT_MAX_TOKENS = 12288;
|
|
31261
|
-
function manglePath(
|
|
31262
|
-
return
|
|
32052
|
+
function manglePath(projectPath2) {
|
|
32053
|
+
return projectPath2.replace(/\//g, "-");
|
|
31263
32054
|
}
|
|
31264
32055
|
function estimateTokens4(text4) {
|
|
31265
32056
|
return Math.ceil(text4.length / 3);
|
|
@@ -31373,8 +32164,8 @@ function getSessionMetadata(filePath) {
|
|
|
31373
32164
|
var claudeCodeProvider = {
|
|
31374
32165
|
name: "claude-code",
|
|
31375
32166
|
displayName: "Claude Code",
|
|
31376
|
-
detect(
|
|
31377
|
-
const mangled = manglePath(
|
|
32167
|
+
detect(projectPath2) {
|
|
32168
|
+
const mangled = manglePath(projectPath2);
|
|
31378
32169
|
const dir = join8(CLAUDE_DIR, mangled);
|
|
31379
32170
|
let entries;
|
|
31380
32171
|
try {
|
|
@@ -31572,7 +32363,7 @@ function getSessionMeta(filePath) {
|
|
|
31572
32363
|
var codexProvider = {
|
|
31573
32364
|
name: "codex",
|
|
31574
32365
|
displayName: "Codex",
|
|
31575
|
-
detect(
|
|
32366
|
+
detect(projectPath2) {
|
|
31576
32367
|
const sessions = [];
|
|
31577
32368
|
const allFiles = [
|
|
31578
32369
|
...findJsonlFiles(SESSIONS_DIR),
|
|
@@ -31581,7 +32372,7 @@ var codexProvider = {
|
|
|
31581
32372
|
for (const filePath of allFiles) {
|
|
31582
32373
|
const meta3 = getSessionMeta(filePath);
|
|
31583
32374
|
if (!meta3) continue;
|
|
31584
|
-
if (meta3.cwd !==
|
|
32375
|
+
if (meta3.cwd !== projectPath2) continue;
|
|
31585
32376
|
if (meta3.messageCount < 3) continue;
|
|
31586
32377
|
const ts = new Date(meta3.timestamp).getTime();
|
|
31587
32378
|
const estimatedTokens = Math.ceil(meta3.fileSize / 5);
|
|
@@ -31713,14 +32504,14 @@ function partsToConversationText(parts) {
|
|
|
31713
32504
|
var opencodeProvider = {
|
|
31714
32505
|
name: "opencode",
|
|
31715
32506
|
displayName: "OpenCode",
|
|
31716
|
-
detect(
|
|
32507
|
+
detect(projectPath2) {
|
|
31717
32508
|
const database = openDB();
|
|
31718
32509
|
if (!database) return [];
|
|
31719
32510
|
try {
|
|
31720
32511
|
if (!tableExists(database, "project") || !tableExists(database, "session") || !tableExists(database, "message")) {
|
|
31721
32512
|
return [];
|
|
31722
32513
|
}
|
|
31723
|
-
const project = database.query("SELECT id FROM project WHERE worktree = ?").get(
|
|
32514
|
+
const project = database.query("SELECT id FROM project WHERE worktree = ?").get(projectPath2);
|
|
31724
32515
|
if (!project) return [];
|
|
31725
32516
|
const sessions = database.query(
|
|
31726
32517
|
`SELECT s.id, s.title, s.time_created, s.time_updated,
|
|
@@ -31885,7 +32676,7 @@ function findGlobalStorageDirs() {
|
|
|
31885
32676
|
}
|
|
31886
32677
|
return dirs;
|
|
31887
32678
|
}
|
|
31888
|
-
function loadTaskHistory(storageDir,
|
|
32679
|
+
function loadTaskHistory(storageDir, projectPath2) {
|
|
31889
32680
|
const paths = [
|
|
31890
32681
|
join11(storageDir, "state", "taskHistory.json"),
|
|
31891
32682
|
join11(storageDir, "taskHistory.json")
|
|
@@ -31897,7 +32688,7 @@ function loadTaskHistory(storageDir, projectPath) {
|
|
|
31897
32688
|
const items = JSON.parse(raw);
|
|
31898
32689
|
if (!Array.isArray(items)) continue;
|
|
31899
32690
|
return items.filter(
|
|
31900
|
-
(item) => item.cwdOnTaskInitialization ===
|
|
32691
|
+
(item) => item.cwdOnTaskInitialization === projectPath2
|
|
31901
32692
|
);
|
|
31902
32693
|
} catch {
|
|
31903
32694
|
continue;
|
|
@@ -31950,11 +32741,11 @@ function messageToText(msg) {
|
|
|
31950
32741
|
var clineProvider = {
|
|
31951
32742
|
name: "cline",
|
|
31952
32743
|
displayName: "Cline",
|
|
31953
|
-
detect(
|
|
32744
|
+
detect(projectPath2) {
|
|
31954
32745
|
const sessions = [];
|
|
31955
32746
|
const storageDirs = findGlobalStorageDirs();
|
|
31956
32747
|
for (const storageDir of storageDirs) {
|
|
31957
|
-
const tasks = loadTaskHistory(storageDir,
|
|
32748
|
+
const tasks = loadTaskHistory(storageDir, projectPath2);
|
|
31958
32749
|
for (const task of tasks) {
|
|
31959
32750
|
const taskDir = join11(storageDir, "tasks", task.id);
|
|
31960
32751
|
if (!existsSync8(taskDir)) continue;
|
|
@@ -32102,11 +32893,11 @@ function historyItemToText(item) {
|
|
|
32102
32893
|
var continueProvider = {
|
|
32103
32894
|
name: "continue",
|
|
32104
32895
|
displayName: "Continue",
|
|
32105
|
-
detect(
|
|
32896
|
+
detect(projectPath2) {
|
|
32106
32897
|
const sessions = [];
|
|
32107
32898
|
const index2 = loadSessionIndex();
|
|
32108
32899
|
for (const meta3 of index2) {
|
|
32109
|
-
if (meta3.workspaceDirectory !==
|
|
32900
|
+
if (meta3.workspaceDirectory !== projectPath2) continue;
|
|
32110
32901
|
const session = loadSession(meta3.sessionId);
|
|
32111
32902
|
if (!session || !session.history || session.history.length < 3) continue;
|
|
32112
32903
|
const ts = new Date(meta3.dateCreated).getTime();
|
|
@@ -32138,7 +32929,7 @@ var continueProvider = {
|
|
|
32138
32929
|
if (existingIds.has(sessionId)) continue;
|
|
32139
32930
|
const session = loadSession(sessionId);
|
|
32140
32931
|
if (!session) continue;
|
|
32141
|
-
if (session.workspaceDirectory !==
|
|
32932
|
+
if (session.workspaceDirectory !== projectPath2) continue;
|
|
32142
32933
|
if (!session.history || session.history.length < 3) continue;
|
|
32143
32934
|
const dateStr = session.title ? truncate5(session.title, 60) : sessionId.slice(0, 8);
|
|
32144
32935
|
sessions.push({
|
|
@@ -32287,8 +33078,8 @@ function getSessionMeta2(filePath) {
|
|
|
32287
33078
|
var piProvider = {
|
|
32288
33079
|
name: "pi",
|
|
32289
33080
|
displayName: "Pi",
|
|
32290
|
-
detect(
|
|
32291
|
-
const encoded = encodeCwd(
|
|
33081
|
+
detect(projectPath2) {
|
|
33082
|
+
const encoded = encodeCwd(projectPath2);
|
|
32292
33083
|
const dir = join13(PI_DIR, encoded);
|
|
32293
33084
|
let entries;
|
|
32294
33085
|
try {
|
|
@@ -32426,8 +33217,8 @@ function parseAiderHistory(content3) {
|
|
|
32426
33217
|
var aiderProvider = {
|
|
32427
33218
|
name: "aider",
|
|
32428
33219
|
displayName: "Aider",
|
|
32429
|
-
detect(
|
|
32430
|
-
const filePath = join14(
|
|
33220
|
+
detect(projectPath2) {
|
|
33221
|
+
const filePath = join14(projectPath2, HISTORY_FILE);
|
|
32431
33222
|
if (!existsSync11(filePath)) return [];
|
|
32432
33223
|
let stat;
|
|
32433
33224
|
try {
|
|
@@ -32456,7 +33247,7 @@ var aiderProvider = {
|
|
|
32456
33247
|
}
|
|
32457
33248
|
];
|
|
32458
33249
|
},
|
|
32459
|
-
readChunks(
|
|
33250
|
+
readChunks(projectPath2, sessionIds, maxTokens = DEFAULT_MAX_TOKENS7) {
|
|
32460
33251
|
const chunks = [];
|
|
32461
33252
|
for (const filePath of sessionIds) {
|
|
32462
33253
|
let content3;
|
|
@@ -32751,7 +33542,7 @@ async function searchRecall(input) {
|
|
|
32751
33542
|
const {
|
|
32752
33543
|
query,
|
|
32753
33544
|
scope = "all",
|
|
32754
|
-
projectPath,
|
|
33545
|
+
projectPath: projectPath2,
|
|
32755
33546
|
sessionID,
|
|
32756
33547
|
knowledgeEnabled = true,
|
|
32757
33548
|
llm,
|
|
@@ -32778,7 +33569,7 @@ async function searchRecall(input) {
|
|
|
32778
33569
|
if (knowledgeEnabled && scope !== "session") {
|
|
32779
33570
|
try {
|
|
32780
33571
|
knowledgeResults.push(
|
|
32781
|
-
...searchScored3({ query: q, projectPath, limit })
|
|
33572
|
+
...searchScored3({ query: q, projectPath: projectPath2, limit })
|
|
32782
33573
|
);
|
|
32783
33574
|
} catch (err) {
|
|
32784
33575
|
error("recall: knowledge search failed:", err);
|
|
@@ -32789,7 +33580,7 @@ async function searchRecall(input) {
|
|
|
32789
33580
|
try {
|
|
32790
33581
|
distillationResults.push(
|
|
32791
33582
|
...searchDistillationsScored({
|
|
32792
|
-
projectPath,
|
|
33583
|
+
projectPath: projectPath2,
|
|
32793
33584
|
query: q,
|
|
32794
33585
|
sessionID: scope === "session" ? sessionID : void 0,
|
|
32795
33586
|
limit
|
|
@@ -32804,7 +33595,7 @@ async function searchRecall(input) {
|
|
|
32804
33595
|
try {
|
|
32805
33596
|
temporalResults.push(
|
|
32806
33597
|
...searchScored({
|
|
32807
|
-
projectPath,
|
|
33598
|
+
projectPath: projectPath2,
|
|
32808
33599
|
query: q,
|
|
32809
33600
|
sessionID: scope === "session" ? sessionID : void 0,
|
|
32810
33601
|
limit
|
|
@@ -32898,7 +33689,7 @@ async function searchRecall(input) {
|
|
|
32898
33689
|
}
|
|
32899
33690
|
}
|
|
32900
33691
|
if (scope !== "knowledge") {
|
|
32901
|
-
const pid = ensureProject(
|
|
33692
|
+
const pid = ensureProject(projectPath2);
|
|
32902
33693
|
const temporalVectorHits = vectorSearchTemporal(
|
|
32903
33694
|
queryVec,
|
|
32904
33695
|
pid,
|
|
@@ -32926,11 +33717,11 @@ async function searchRecall(input) {
|
|
|
32926
33717
|
info("recall: vector search failed:", err);
|
|
32927
33718
|
}
|
|
32928
33719
|
}
|
|
32929
|
-
if (scope !== "session" && hasLatDir(
|
|
33720
|
+
if (scope !== "session" && hasLatDir(projectPath2)) {
|
|
32930
33721
|
try {
|
|
32931
33722
|
const latResults = searchScored2({
|
|
32932
33723
|
query,
|
|
32933
|
-
projectPath,
|
|
33724
|
+
projectPath: projectPath2,
|
|
32934
33725
|
limit
|
|
32935
33726
|
});
|
|
32936
33727
|
if (latResults.length) {
|
|
@@ -32950,7 +33741,7 @@ async function searchRecall(input) {
|
|
|
32950
33741
|
try {
|
|
32951
33742
|
const crossProjectResults = searchScoredOtherProjects({
|
|
32952
33743
|
query,
|
|
32953
|
-
excludeProjectPath:
|
|
33744
|
+
excludeProjectPath: projectPath2,
|
|
32954
33745
|
limit
|
|
32955
33746
|
});
|
|
32956
33747
|
if (crossProjectResults.length) {
|
|
@@ -33156,6 +33947,7 @@ export {
|
|
|
33156
33947
|
distillationUser,
|
|
33157
33948
|
embedding_exports as embedding,
|
|
33158
33949
|
embedding_vendor_exports as embeddingVendor,
|
|
33950
|
+
enableHostedMode,
|
|
33159
33951
|
ensureProject,
|
|
33160
33952
|
exactTermMatchRank,
|
|
33161
33953
|
expandQuery,
|
|
@@ -33169,7 +33961,9 @@ export {
|
|
|
33169
33961
|
ftsQueryRelaxed,
|
|
33170
33962
|
getGitRemote,
|
|
33171
33963
|
getInstanceId,
|
|
33964
|
+
getKV,
|
|
33172
33965
|
getLastImportAt,
|
|
33966
|
+
getLastLayer,
|
|
33173
33967
|
getLastTransformEstimate,
|
|
33174
33968
|
getLastTransformedCount,
|
|
33175
33969
|
getLastTurnAt,
|
|
@@ -33184,6 +33978,7 @@ export {
|
|
|
33184
33978
|
inspectSessionState,
|
|
33185
33979
|
instruction_detect_exports as instructionDetect,
|
|
33186
33980
|
isFirstRun,
|
|
33981
|
+
isHostedMode,
|
|
33187
33982
|
isReasoningPart,
|
|
33188
33983
|
isTextPart,
|
|
33189
33984
|
isToolPart,
|
|
@@ -33194,7 +33989,9 @@ export {
|
|
|
33194
33989
|
load,
|
|
33195
33990
|
loadAllSessionCosts,
|
|
33196
33991
|
loadForceMinLayer,
|
|
33992
|
+
loadHeaderSessionIndex,
|
|
33197
33993
|
loadSessionCosts,
|
|
33994
|
+
loadSessionTracking,
|
|
33198
33995
|
log_exports as log,
|
|
33199
33996
|
loreFileExists,
|
|
33200
33997
|
ltm_exports as ltm,
|
|
@@ -33207,18 +34004,23 @@ export {
|
|
|
33207
34004
|
pattern_extract_exports as patternExtract,
|
|
33208
34005
|
projectId,
|
|
33209
34006
|
projectName,
|
|
34007
|
+
projectPath,
|
|
33210
34008
|
recallById,
|
|
33211
34009
|
reciprocalRankFusion,
|
|
33212
34010
|
recursiveUser,
|
|
33213
34011
|
renderMarkdown,
|
|
34012
|
+
resolveProjectByRemoteOrPath,
|
|
33214
34013
|
root2 as root,
|
|
33215
34014
|
runRecall,
|
|
33216
34015
|
sanitizeSurrogates,
|
|
33217
34016
|
saveForceMinLayer,
|
|
34017
|
+
saveGradientState,
|
|
33218
34018
|
saveSessionCosts,
|
|
34019
|
+
saveSessionTracking,
|
|
33219
34020
|
searchRecall,
|
|
33220
34021
|
serialize,
|
|
33221
34022
|
setForceMinLayer,
|
|
34023
|
+
setKV,
|
|
33222
34024
|
setLastImportAt,
|
|
33223
34025
|
setLastTurnAtForTest,
|
|
33224
34026
|
setLtmTokens,
|