@loreai/core 0.18.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bun/agents-file.d.ts.map +1 -1
- package/dist/bun/config.d.ts.map +1 -1
- package/dist/bun/curator.d.ts.map +1 -1
- package/dist/bun/db.d.ts +86 -1
- package/dist/bun/db.d.ts.map +1 -1
- package/dist/bun/distillation.d.ts +2 -13
- package/dist/bun/distillation.d.ts.map +1 -1
- package/dist/bun/embedding.d.ts +5 -1
- package/dist/bun/embedding.d.ts.map +1 -1
- package/dist/bun/git.d.ts.map +1 -1
- package/dist/bun/gradient.d.ts +13 -1
- package/dist/bun/gradient.d.ts.map +1 -1
- package/dist/bun/hosted.d.ts +36 -0
- package/dist/bun/hosted.d.ts.map +1 -0
- package/dist/bun/index.d.ts +3 -2
- package/dist/bun/index.d.ts.map +1 -1
- package/dist/bun/index.js +1049 -247
- package/dist/bun/index.js.map +4 -4
- package/dist/bun/lat-reader.d.ts.map +1 -1
- package/dist/bun/ltm.d.ts +99 -5
- package/dist/bun/ltm.d.ts.map +1 -1
- package/dist/bun/session-limiter.d.ts +26 -0
- package/dist/bun/session-limiter.d.ts.map +1 -0
- package/dist/bun/temporal.d.ts +2 -0
- package/dist/bun/temporal.d.ts.map +1 -1
- package/dist/node/agents-file.d.ts.map +1 -1
- package/dist/node/config.d.ts.map +1 -1
- package/dist/node/curator.d.ts.map +1 -1
- package/dist/node/db.d.ts +86 -1
- package/dist/node/db.d.ts.map +1 -1
- package/dist/node/distillation.d.ts +2 -13
- package/dist/node/distillation.d.ts.map +1 -1
- package/dist/node/embedding.d.ts +5 -1
- package/dist/node/embedding.d.ts.map +1 -1
- package/dist/node/git.d.ts.map +1 -1
- package/dist/node/gradient.d.ts +13 -1
- package/dist/node/gradient.d.ts.map +1 -1
- package/dist/node/hosted.d.ts +36 -0
- package/dist/node/hosted.d.ts.map +1 -0
- package/dist/node/index.d.ts +3 -2
- package/dist/node/index.d.ts.map +1 -1
- package/dist/node/index.js +1049 -247
- package/dist/node/index.js.map +4 -4
- package/dist/node/lat-reader.d.ts.map +1 -1
- package/dist/node/ltm.d.ts +99 -5
- package/dist/node/ltm.d.ts.map +1 -1
- package/dist/node/session-limiter.d.ts +26 -0
- package/dist/node/session-limiter.d.ts.map +1 -0
- package/dist/node/temporal.d.ts +2 -0
- package/dist/node/temporal.d.ts.map +1 -1
- package/dist/types/agents-file.d.ts.map +1 -1
- package/dist/types/config.d.ts.map +1 -1
- package/dist/types/curator.d.ts.map +1 -1
- package/dist/types/db.d.ts +86 -1
- package/dist/types/db.d.ts.map +1 -1
- package/dist/types/distillation.d.ts +2 -13
- package/dist/types/distillation.d.ts.map +1 -1
- package/dist/types/embedding.d.ts +5 -1
- package/dist/types/embedding.d.ts.map +1 -1
- package/dist/types/git.d.ts.map +1 -1
- package/dist/types/gradient.d.ts +13 -1
- package/dist/types/gradient.d.ts.map +1 -1
- package/dist/types/hosted.d.ts +36 -0
- package/dist/types/hosted.d.ts.map +1 -0
- package/dist/types/index.d.ts +3 -2
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/lat-reader.d.ts.map +1 -1
- package/dist/types/ltm.d.ts +99 -5
- package/dist/types/ltm.d.ts.map +1 -1
- package/dist/types/session-limiter.d.ts +26 -0
- package/dist/types/session-limiter.d.ts.map +1 -0
- package/dist/types/temporal.d.ts +2 -0
- package/dist/types/temporal.d.ts.map +1 -1
- package/package.json +3 -1
- package/src/agents-file.ts +12 -0
- package/src/config.ts +10 -5
- package/src/curator.ts +54 -2
- package/src/db.ts +386 -6
- package/src/distillation.ts +55 -14
- package/src/embedding.ts +71 -8
- package/src/git.ts +4 -0
- package/src/gradient.ts +227 -74
- package/src/hosted.ts +46 -0
- package/src/index.ts +12 -0
- package/src/lat-reader.ts +4 -0
- package/src/ltm.ts +480 -45
- package/src/session-limiter.ts +47 -0
- package/src/temporal.ts +10 -0
package/dist/bun/index.js
CHANGED
|
@@ -125,6 +125,7 @@ __export(temporal_exports, {
|
|
|
125
125
|
CHUNK_TERMINATOR: () => CHUNK_TERMINATOR,
|
|
126
126
|
bySession: () => bySession,
|
|
127
127
|
count: () => count,
|
|
128
|
+
hasMessages: () => hasMessages,
|
|
128
129
|
markDistilled: () => markDistilled,
|
|
129
130
|
partsToText: () => partsToText,
|
|
130
131
|
prune: () => prune,
|
|
@@ -150,6 +151,17 @@ import { mkdirSync } from "fs";
|
|
|
150
151
|
|
|
151
152
|
// src/git.ts
|
|
152
153
|
import { execSync } from "child_process";
|
|
154
|
+
|
|
155
|
+
// src/hosted.ts
|
|
156
|
+
var _hostedMode = false;
|
|
157
|
+
function enableHostedMode() {
|
|
158
|
+
_hostedMode = true;
|
|
159
|
+
}
|
|
160
|
+
function isHostedMode() {
|
|
161
|
+
return _hostedMode;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
// src/git.ts
|
|
153
165
|
function normalizeRemoteUrl(url2) {
|
|
154
166
|
let normalized = url2.trim();
|
|
155
167
|
const sshMatch = normalized.match(/^[\w.-]+@([\w.-]+):(.+)$/);
|
|
@@ -174,6 +186,7 @@ function clearGitRemoteCache() {
|
|
|
174
186
|
gitRemoteCache.clear();
|
|
175
187
|
}
|
|
176
188
|
function getGitRemote(path) {
|
|
189
|
+
if (isHostedMode()) return null;
|
|
177
190
|
const cached2 = gitRemoteCache.get(path);
|
|
178
191
|
if (cached2 !== void 0) return cached2;
|
|
179
192
|
try {
|
|
@@ -737,6 +750,55 @@ var MIGRATIONS = [
|
|
|
737
750
|
WHERE ih.project_id = projects.id
|
|
738
751
|
AND ih.source_id = '__declined__'
|
|
739
752
|
);
|
|
753
|
+
`,
|
|
754
|
+
`
|
|
755
|
+
-- Version 23: Persist volatile session tracking state across restarts.
|
|
756
|
+
-- Previously these were in-memory only, causing duplicate processing,
|
|
757
|
+
-- false compaction detection, and expensive prompt cache busts on restart.
|
|
758
|
+
ALTER TABLE session_state ADD COLUMN last_curated_at INTEGER NOT NULL DEFAULT 0;
|
|
759
|
+
ALTER TABLE session_state ADD COLUMN message_count INTEGER NOT NULL DEFAULT 0;
|
|
760
|
+
ALTER TABLE session_state ADD COLUMN turns_since_curation INTEGER NOT NULL DEFAULT 0;
|
|
761
|
+
ALTER TABLE session_state ADD COLUMN ltm_cache_text TEXT;
|
|
762
|
+
ALTER TABLE session_state ADD COLUMN ltm_cache_tokens INTEGER;
|
|
763
|
+
ALTER TABLE session_state ADD COLUMN ltm_pin_text TEXT;
|
|
764
|
+
ALTER TABLE session_state ADD COLUMN ltm_pin_tokens INTEGER;
|
|
765
|
+
ALTER TABLE session_state ADD COLUMN consecutive_text_only_turns INTEGER NOT NULL DEFAULT 0;
|
|
766
|
+
`,
|
|
767
|
+
`
|
|
768
|
+
-- Version 24: Persist remaining volatile session state across restarts.
|
|
769
|
+
-- Session identity (Tier 1/2/3 session correlation)
|
|
770
|
+
ALTER TABLE session_state ADD COLUMN fingerprint TEXT NOT NULL DEFAULT '';
|
|
771
|
+
ALTER TABLE session_state ADD COLUMN header_session_id TEXT;
|
|
772
|
+
ALTER TABLE session_state ADD COLUMN header_name TEXT;
|
|
773
|
+
-- Cache warming state
|
|
774
|
+
ALTER TABLE session_state ADD COLUMN resolved_conversation_ttl TEXT NOT NULL DEFAULT '5m';
|
|
775
|
+
ALTER TABLE session_state ADD COLUMN warmup_state TEXT;
|
|
776
|
+
-- Gradient calibration state (survives restarts to avoid uncalibrated busts)
|
|
777
|
+
ALTER TABLE session_state ADD COLUMN dynamic_context_cap REAL NOT NULL DEFAULT 0;
|
|
778
|
+
ALTER TABLE session_state ADD COLUMN bust_rate_ema REAL NOT NULL DEFAULT -1;
|
|
779
|
+
ALTER TABLE session_state ADD COLUMN inter_bust_interval_ema REAL NOT NULL DEFAULT -1;
|
|
780
|
+
ALTER TABLE session_state ADD COLUMN last_layer INTEGER NOT NULL DEFAULT 0;
|
|
781
|
+
ALTER TABLE session_state ADD COLUMN last_known_input INTEGER NOT NULL DEFAULT 0;
|
|
782
|
+
ALTER TABLE session_state ADD COLUMN last_turn_at INTEGER NOT NULL DEFAULT 0;
|
|
783
|
+
ALTER TABLE session_state ADD COLUMN last_bust_at INTEGER NOT NULL DEFAULT 0;
|
|
784
|
+
`,
|
|
785
|
+
`
|
|
786
|
+
-- Version 25: Adaptive dedup threshold \u2014 store accept/reject feedback
|
|
787
|
+
-- on embedding-based duplicate pairs for per-project threshold calibration.
|
|
788
|
+
-- Titles stored instead of FK IDs because entries are deleted during dedup;
|
|
789
|
+
-- the similarity float is the actual calibration input.
|
|
790
|
+
CREATE TABLE IF NOT EXISTS dedup_feedback (
|
|
791
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
792
|
+
project_id TEXT,
|
|
793
|
+
entry_a_title TEXT NOT NULL,
|
|
794
|
+
entry_b_title TEXT NOT NULL,
|
|
795
|
+
similarity REAL NOT NULL,
|
|
796
|
+
accepted INTEGER NOT NULL,
|
|
797
|
+
source TEXT NOT NULL DEFAULT 'manual',
|
|
798
|
+
created_at INTEGER NOT NULL
|
|
799
|
+
);
|
|
800
|
+
CREATE INDEX IF NOT EXISTS idx_dedup_feedback_project
|
|
801
|
+
ON dedup_feedback(project_id);
|
|
740
802
|
`
|
|
741
803
|
];
|
|
742
804
|
function dbPath() {
|
|
@@ -872,26 +934,31 @@ function close() {
|
|
|
872
934
|
instance = void 0;
|
|
873
935
|
}
|
|
874
936
|
}
|
|
875
|
-
function ensureProject(path, name) {
|
|
937
|
+
function ensureProject(path, name, suppliedGitRemote) {
|
|
938
|
+
if (!process.env.LORE_DB_PATH && /^\/test\//.test(path)) {
|
|
939
|
+
throw new Error(
|
|
940
|
+
`Refusing to create project with test path "${path}" in the production DB. Set LORE_DB_PATH to a temp path, or run tests via \`bun test\` from the repo root.`
|
|
941
|
+
);
|
|
942
|
+
}
|
|
876
943
|
const existing = db().query("SELECT id, git_remote FROM projects WHERE path = ?").get(path);
|
|
877
944
|
if (existing) {
|
|
878
945
|
if (!existing.git_remote) {
|
|
879
|
-
const
|
|
880
|
-
if (
|
|
946
|
+
const resolvedRemote = suppliedGitRemote ?? getGitRemote(path);
|
|
947
|
+
if (resolvedRemote) {
|
|
881
948
|
const conflict = db().query(
|
|
882
949
|
"SELECT id FROM projects WHERE git_remote = ? AND id != ? LIMIT 1"
|
|
883
|
-
).get(
|
|
950
|
+
).get(resolvedRemote, existing.id);
|
|
884
951
|
if (conflict) {
|
|
885
952
|
mergeProjectInternal(conflict.id, existing.id);
|
|
886
953
|
}
|
|
887
|
-
db().query("UPDATE projects SET git_remote = ? WHERE id = ?").run(
|
|
954
|
+
db().query("UPDATE projects SET git_remote = ? WHERE id = ?").run(resolvedRemote, existing.id);
|
|
888
955
|
}
|
|
889
956
|
}
|
|
890
957
|
return existing.id;
|
|
891
958
|
}
|
|
892
959
|
const alias = db().query("SELECT project_id FROM project_path_aliases WHERE path = ?").get(path);
|
|
893
960
|
if (alias) return alias.project_id;
|
|
894
|
-
const gitRemote = getGitRemote(path);
|
|
961
|
+
const gitRemote = suppliedGitRemote ?? getGitRemote(path);
|
|
895
962
|
if (gitRemote) {
|
|
896
963
|
const byRemote = db().query("SELECT id FROM projects WHERE git_remote = ? LIMIT 1").get(gitRemote);
|
|
897
964
|
if (byRemote) {
|
|
@@ -919,6 +986,20 @@ function projectId(path) {
|
|
|
919
986
|
const alias = db().query("SELECT project_id FROM project_path_aliases WHERE path = ?").get(path);
|
|
920
987
|
return alias?.project_id;
|
|
921
988
|
}
|
|
989
|
+
function resolveProjectByRemoteOrPath(gitRemote, path) {
|
|
990
|
+
if (gitRemote) {
|
|
991
|
+
const row = db().query("SELECT id FROM projects WHERE git_remote = ? LIMIT 1").get(gitRemote);
|
|
992
|
+
if (row) return row.id;
|
|
993
|
+
}
|
|
994
|
+
if (path) {
|
|
995
|
+
return projectId(path) ?? null;
|
|
996
|
+
}
|
|
997
|
+
return null;
|
|
998
|
+
}
|
|
999
|
+
function projectPath(id) {
|
|
1000
|
+
const row = db().query("SELECT path FROM projects WHERE id = ?").get(id);
|
|
1001
|
+
return row?.path ?? null;
|
|
1002
|
+
}
|
|
922
1003
|
function projectName(id) {
|
|
923
1004
|
const row = db().query("SELECT name FROM projects WHERE id = ?").get(id);
|
|
924
1005
|
return row?.name ?? null;
|
|
@@ -927,13 +1008,13 @@ function isFirstRun() {
|
|
|
927
1008
|
const row = db().query("SELECT COUNT(*) as count FROM projects").get();
|
|
928
1009
|
return row.count === 0;
|
|
929
1010
|
}
|
|
930
|
-
function getLastImportAt(
|
|
931
|
-
const id = ensureProject(
|
|
1011
|
+
function getLastImportAt(projectPath2) {
|
|
1012
|
+
const id = ensureProject(projectPath2);
|
|
932
1013
|
const row = db().query("SELECT last_import_at FROM projects WHERE id = ?").get(id);
|
|
933
1014
|
return row?.last_import_at ?? null;
|
|
934
1015
|
}
|
|
935
|
-
function setLastImportAt(
|
|
936
|
-
const id = ensureProject(
|
|
1016
|
+
function setLastImportAt(projectPath2, timestamp) {
|
|
1017
|
+
const id = ensureProject(projectPath2);
|
|
937
1018
|
db().query("UPDATE projects SET last_import_at = ? WHERE id = ?").run(timestamp, id);
|
|
938
1019
|
}
|
|
939
1020
|
function loadForceMinLayer(sessionID) {
|
|
@@ -1041,6 +1122,153 @@ function loadAllSessionCosts() {
|
|
|
1041
1122
|
}
|
|
1042
1123
|
return result;
|
|
1043
1124
|
}
|
|
1125
|
+
function saveSessionTracking(sessionID, state) {
|
|
1126
|
+
const now = Date.now();
|
|
1127
|
+
db().query(
|
|
1128
|
+
"INSERT OR IGNORE INTO session_state (session_id, force_min_layer, updated_at) VALUES (?, 0, ?)"
|
|
1129
|
+
).run(sessionID, now);
|
|
1130
|
+
const sets = ["updated_at = ?"];
|
|
1131
|
+
const vals = [now];
|
|
1132
|
+
if (state.lastCuratedAt !== void 0) {
|
|
1133
|
+
sets.push("last_curated_at = ?");
|
|
1134
|
+
vals.push(state.lastCuratedAt);
|
|
1135
|
+
}
|
|
1136
|
+
if (state.messageCount !== void 0) {
|
|
1137
|
+
sets.push("message_count = ?");
|
|
1138
|
+
vals.push(state.messageCount);
|
|
1139
|
+
}
|
|
1140
|
+
if (state.turnsSinceCuration !== void 0) {
|
|
1141
|
+
sets.push("turns_since_curation = ?");
|
|
1142
|
+
vals.push(state.turnsSinceCuration);
|
|
1143
|
+
}
|
|
1144
|
+
if (state.consecutiveTextOnlyTurns !== void 0) {
|
|
1145
|
+
sets.push("consecutive_text_only_turns = ?");
|
|
1146
|
+
vals.push(state.consecutiveTextOnlyTurns);
|
|
1147
|
+
}
|
|
1148
|
+
if (state.ltmCacheText !== void 0) {
|
|
1149
|
+
sets.push("ltm_cache_text = ?");
|
|
1150
|
+
vals.push(state.ltmCacheText);
|
|
1151
|
+
}
|
|
1152
|
+
if (state.ltmCacheTokens !== void 0) {
|
|
1153
|
+
sets.push("ltm_cache_tokens = ?");
|
|
1154
|
+
vals.push(state.ltmCacheTokens);
|
|
1155
|
+
}
|
|
1156
|
+
if (state.ltmPinText !== void 0) {
|
|
1157
|
+
sets.push("ltm_pin_text = ?");
|
|
1158
|
+
vals.push(state.ltmPinText);
|
|
1159
|
+
}
|
|
1160
|
+
if (state.ltmPinTokens !== void 0) {
|
|
1161
|
+
sets.push("ltm_pin_tokens = ?");
|
|
1162
|
+
vals.push(state.ltmPinTokens);
|
|
1163
|
+
}
|
|
1164
|
+
if (state.fingerprint !== void 0) {
|
|
1165
|
+
sets.push("fingerprint = ?");
|
|
1166
|
+
vals.push(state.fingerprint);
|
|
1167
|
+
}
|
|
1168
|
+
if (state.headerSessionId !== void 0) {
|
|
1169
|
+
sets.push("header_session_id = ?");
|
|
1170
|
+
vals.push(state.headerSessionId);
|
|
1171
|
+
}
|
|
1172
|
+
if (state.headerName !== void 0) {
|
|
1173
|
+
sets.push("header_name = ?");
|
|
1174
|
+
vals.push(state.headerName);
|
|
1175
|
+
}
|
|
1176
|
+
if (state.resolvedConversationTTL !== void 0) {
|
|
1177
|
+
sets.push("resolved_conversation_ttl = ?");
|
|
1178
|
+
vals.push(state.resolvedConversationTTL);
|
|
1179
|
+
}
|
|
1180
|
+
if (state.warmupState !== void 0) {
|
|
1181
|
+
sets.push("warmup_state = ?");
|
|
1182
|
+
vals.push(state.warmupState);
|
|
1183
|
+
}
|
|
1184
|
+
if (state.dynamicContextCap !== void 0) {
|
|
1185
|
+
sets.push("dynamic_context_cap = ?");
|
|
1186
|
+
vals.push(state.dynamicContextCap);
|
|
1187
|
+
}
|
|
1188
|
+
if (state.bustRateEMA !== void 0) {
|
|
1189
|
+
sets.push("bust_rate_ema = ?");
|
|
1190
|
+
vals.push(state.bustRateEMA);
|
|
1191
|
+
}
|
|
1192
|
+
if (state.interBustIntervalEMA !== void 0) {
|
|
1193
|
+
sets.push("inter_bust_interval_ema = ?");
|
|
1194
|
+
vals.push(state.interBustIntervalEMA);
|
|
1195
|
+
}
|
|
1196
|
+
if (state.lastLayer !== void 0) {
|
|
1197
|
+
sets.push("last_layer = ?");
|
|
1198
|
+
vals.push(state.lastLayer);
|
|
1199
|
+
}
|
|
1200
|
+
if (state.lastKnownInput !== void 0) {
|
|
1201
|
+
sets.push("last_known_input = ?");
|
|
1202
|
+
vals.push(state.lastKnownInput);
|
|
1203
|
+
}
|
|
1204
|
+
if (state.lastTurnAt !== void 0) {
|
|
1205
|
+
sets.push("last_turn_at = ?");
|
|
1206
|
+
vals.push(state.lastTurnAt);
|
|
1207
|
+
}
|
|
1208
|
+
if (state.lastBustAt !== void 0) {
|
|
1209
|
+
sets.push("last_bust_at = ?");
|
|
1210
|
+
vals.push(state.lastBustAt);
|
|
1211
|
+
}
|
|
1212
|
+
db().query(
|
|
1213
|
+
"UPDATE session_state SET " + sets.join(", ") + " WHERE session_id = ?"
|
|
1214
|
+
).run(...vals, sessionID);
|
|
1215
|
+
}
|
|
1216
|
+
function loadSessionTracking(sessionID) {
|
|
1217
|
+
const row = db().query(
|
|
1218
|
+
`SELECT last_curated_at, message_count, turns_since_curation,
|
|
1219
|
+
consecutive_text_only_turns,
|
|
1220
|
+
ltm_cache_text, ltm_cache_tokens, ltm_pin_text, ltm_pin_tokens,
|
|
1221
|
+
fingerprint, header_session_id, header_name,
|
|
1222
|
+
resolved_conversation_ttl, warmup_state,
|
|
1223
|
+
dynamic_context_cap, bust_rate_ema, inter_bust_interval_ema,
|
|
1224
|
+
last_layer, last_known_input, last_turn_at, last_bust_at
|
|
1225
|
+
FROM session_state WHERE session_id = ?`
|
|
1226
|
+
).get(sessionID);
|
|
1227
|
+
if (!row) return null;
|
|
1228
|
+
return {
|
|
1229
|
+
lastCuratedAt: row.last_curated_at,
|
|
1230
|
+
messageCount: row.message_count,
|
|
1231
|
+
turnsSinceCuration: row.turns_since_curation,
|
|
1232
|
+
consecutiveTextOnlyTurns: row.consecutive_text_only_turns,
|
|
1233
|
+
ltmCacheText: row.ltm_cache_text,
|
|
1234
|
+
ltmCacheTokens: row.ltm_cache_tokens,
|
|
1235
|
+
ltmPinText: row.ltm_pin_text,
|
|
1236
|
+
ltmPinTokens: row.ltm_pin_tokens,
|
|
1237
|
+
fingerprint: row.fingerprint,
|
|
1238
|
+
headerSessionId: row.header_session_id,
|
|
1239
|
+
headerName: row.header_name,
|
|
1240
|
+
resolvedConversationTTL: row.resolved_conversation_ttl,
|
|
1241
|
+
warmupState: row.warmup_state,
|
|
1242
|
+
dynamicContextCap: row.dynamic_context_cap,
|
|
1243
|
+
bustRateEMA: row.bust_rate_ema,
|
|
1244
|
+
interBustIntervalEMA: row.inter_bust_interval_ema,
|
|
1245
|
+
lastLayer: row.last_layer,
|
|
1246
|
+
lastKnownInput: row.last_known_input,
|
|
1247
|
+
lastTurnAt: row.last_turn_at,
|
|
1248
|
+
lastBustAt: row.last_bust_at
|
|
1249
|
+
};
|
|
1250
|
+
}
|
|
1251
|
+
function loadHeaderSessionIndex() {
|
|
1252
|
+
const rows = db().query(
|
|
1253
|
+
`SELECT session_id, header_session_id, header_name
|
|
1254
|
+
FROM session_state
|
|
1255
|
+
WHERE header_session_id IS NOT NULL AND header_name IS NOT NULL`
|
|
1256
|
+
).all();
|
|
1257
|
+
return rows.map((row) => ({
|
|
1258
|
+
sessionId: row.session_id,
|
|
1259
|
+
headerSessionId: row.header_session_id,
|
|
1260
|
+
headerName: row.header_name
|
|
1261
|
+
}));
|
|
1262
|
+
}
|
|
1263
|
+
function getKV(key) {
|
|
1264
|
+
const row = db().query("SELECT value FROM kv_meta WHERE key = ?").get(key);
|
|
1265
|
+
return row?.value ?? null;
|
|
1266
|
+
}
|
|
1267
|
+
function setKV(key, value) {
|
|
1268
|
+
db().query(
|
|
1269
|
+
"INSERT INTO kv_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = ?"
|
|
1270
|
+
).run(key, value, value);
|
|
1271
|
+
}
|
|
1044
1272
|
function getMeta(key) {
|
|
1045
1273
|
const row = db().query("SELECT value FROM metadata WHERE key = ?").get(key);
|
|
1046
1274
|
return row?.value ?? null;
|
|
@@ -26572,11 +26800,13 @@ function config2() {
|
|
|
26572
26800
|
return current;
|
|
26573
26801
|
}
|
|
26574
26802
|
async function load(directory) {
|
|
26575
|
-
|
|
26576
|
-
|
|
26577
|
-
|
|
26578
|
-
|
|
26579
|
-
|
|
26803
|
+
if (!isHostedMode()) {
|
|
26804
|
+
const path = join5(directory, ".lore.json");
|
|
26805
|
+
if (existsSync2(path)) {
|
|
26806
|
+
const raw = JSON.parse(readFileSync(path, "utf8"));
|
|
26807
|
+
current = LoreConfig.parse(raw);
|
|
26808
|
+
return current;
|
|
26809
|
+
}
|
|
26580
26810
|
}
|
|
26581
26811
|
current = LoreConfig.parse({});
|
|
26582
26812
|
return current;
|
|
@@ -26616,6 +26846,14 @@ function vendorRegistration() {
|
|
|
26616
26846
|
|
|
26617
26847
|
// src/embedding.ts
|
|
26618
26848
|
var EMBED_TIMEOUT_MS = 1e4;
|
|
26849
|
+
var LOCAL_MAX_CHARS = 4096 * 4;
|
|
26850
|
+
function safeLocalTruncate(text4) {
|
|
26851
|
+
if (text4.length <= LOCAL_MAX_CHARS) return text4;
|
|
26852
|
+
let end = LOCAL_MAX_CHARS;
|
|
26853
|
+
const code2 = text4.charCodeAt(end - 1);
|
|
26854
|
+
if (code2 >= 55296 && code2 <= 56319) end--;
|
|
26855
|
+
return text4.slice(0, end);
|
|
26856
|
+
}
|
|
26619
26857
|
var VOYAGE_API_URL = "https://api.voyageai.com/v1/embeddings";
|
|
26620
26858
|
var VoyageProvider = class {
|
|
26621
26859
|
maxBatchSize = 128;
|
|
@@ -26753,7 +26991,16 @@ var LocalProvider = class {
|
|
|
26753
26991
|
workerUrl = vendorWorkerUrl;
|
|
26754
26992
|
}
|
|
26755
26993
|
} else {
|
|
26756
|
-
|
|
26994
|
+
const selfUrl = typeof import.meta.url === "string" ? import.meta.url : void 0;
|
|
26995
|
+
if (selfUrl) {
|
|
26996
|
+
workerUrl = new URL(
|
|
26997
|
+
`./embedding-worker${selfUrl.endsWith(".ts") ? ".ts" : ".js"}`,
|
|
26998
|
+
selfUrl
|
|
26999
|
+
);
|
|
27000
|
+
} else {
|
|
27001
|
+
const { pathToFileURL } = await import("node:url");
|
|
27002
|
+
workerUrl = new URL("./embedding-worker.cjs", pathToFileURL(__filename));
|
|
27003
|
+
}
|
|
26757
27004
|
}
|
|
26758
27005
|
const vendor = vendorModelInfo();
|
|
26759
27006
|
const workerInitData = {
|
|
@@ -26789,8 +27036,9 @@ var LocalProvider = class {
|
|
|
26789
27036
|
localProviderKnownBroken = true;
|
|
26790
27037
|
if (!localProviderErrorLogged) {
|
|
26791
27038
|
localProviderErrorLogged = true;
|
|
26792
|
-
|
|
26793
|
-
`local embedding provider failed to init: ${msg.error}. Set VOYAGE_API_KEY/OPENAI_API_KEY for automatic remote fallback
|
|
27039
|
+
error(
|
|
27040
|
+
`local embedding provider failed to init: ${msg.error}. Set VOYAGE_API_KEY/OPENAI_API_KEY for automatic remote fallback.`,
|
|
27041
|
+
new Error(`embedding worker init failed: ${msg.error}`)
|
|
26794
27042
|
);
|
|
26795
27043
|
}
|
|
26796
27044
|
for (const [, p2] of this.pendingRequests) {
|
|
@@ -26805,6 +27053,7 @@ var LocalProvider = class {
|
|
|
26805
27053
|
this.worker.on("error", (err) => {
|
|
26806
27054
|
this.workerInitError = err.message;
|
|
26807
27055
|
this.workerReady = false;
|
|
27056
|
+
error("embedding worker crashed:", err);
|
|
26808
27057
|
for (const [, p2] of this.pendingRequests) {
|
|
26809
27058
|
p2.reject(new LocalProviderUnavailableError(err));
|
|
26810
27059
|
}
|
|
@@ -26814,6 +27063,10 @@ var LocalProvider = class {
|
|
|
26814
27063
|
this.worker.on("exit", (code2) => {
|
|
26815
27064
|
if (code2 !== 0 && !this.workerInitError) {
|
|
26816
27065
|
this.workerInitError = `embedding worker exited with code ${code2}`;
|
|
27066
|
+
error(
|
|
27067
|
+
this.workerInitError,
|
|
27068
|
+
new Error(this.workerInitError)
|
|
27069
|
+
);
|
|
26817
27070
|
}
|
|
26818
27071
|
this.workerReady = false;
|
|
26819
27072
|
for (const [, p2] of this.pendingRequests) {
|
|
@@ -26844,8 +27097,9 @@ var LocalProvider = class {
|
|
|
26844
27097
|
}
|
|
26845
27098
|
async embed(texts, inputType) {
|
|
26846
27099
|
await this.ensureWorker();
|
|
27100
|
+
const truncated = texts.map(safeLocalTruncate);
|
|
26847
27101
|
const prefix = inputType === "document" ? "search_document: " : "search_query: ";
|
|
26848
|
-
const prefixed =
|
|
27102
|
+
const prefixed = truncated.map((t2) => prefix + t2);
|
|
26849
27103
|
const id = this.nextRequestId++;
|
|
26850
27104
|
const priority = inputType === "query" && texts.length === 1 ? "high" : "normal";
|
|
26851
27105
|
return new Promise((resolve, reject) => {
|
|
@@ -27028,8 +27282,14 @@ function fromBlob(blob) {
|
|
|
27028
27282
|
const bytes = new Uint8Array(blob);
|
|
27029
27283
|
return new Float32Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / 4);
|
|
27030
27284
|
}
|
|
27031
|
-
function vectorSearch(queryEmbedding, limit = 10) {
|
|
27032
|
-
|
|
27285
|
+
function vectorSearch(queryEmbedding, limit = 10, excludeCategories) {
|
|
27286
|
+
let sql = "SELECT id, embedding FROM knowledge WHERE embedding IS NOT NULL AND confidence > 0.2";
|
|
27287
|
+
const params = [];
|
|
27288
|
+
if (excludeCategories?.length) {
|
|
27289
|
+
sql += ` AND category NOT IN (${excludeCategories.map(() => "?").join(",")})`;
|
|
27290
|
+
params.push(...excludeCategories);
|
|
27291
|
+
}
|
|
27292
|
+
const rows = db().query(sql).all(...params);
|
|
27033
27293
|
const scored = [];
|
|
27034
27294
|
for (const row of rows) {
|
|
27035
27295
|
const vec = fromBlob(row.embedding);
|
|
@@ -27067,27 +27327,30 @@ function vectorSearchAllDistillations(queryEmbedding, projectId2, limit = 20) {
|
|
|
27067
27327
|
return scored.slice(0, limit);
|
|
27068
27328
|
}
|
|
27069
27329
|
function embedKnowledgeEntry(id, title, content3) {
|
|
27330
|
+
if (!isAvailable()) return;
|
|
27070
27331
|
const text4 = `${title}
|
|
27071
27332
|
${content3}`;
|
|
27072
27333
|
embed([text4], "document").then(([vec]) => {
|
|
27073
27334
|
db().query("UPDATE knowledge SET embedding = ? WHERE id = ?").run(toBlob(vec), id);
|
|
27074
27335
|
}).catch((err) => {
|
|
27075
|
-
|
|
27336
|
+
error("embedding failed for knowledge entry", id, ":", err);
|
|
27076
27337
|
});
|
|
27077
27338
|
}
|
|
27078
27339
|
function embedDistillation(id, observations) {
|
|
27340
|
+
if (!isAvailable()) return;
|
|
27079
27341
|
embed([observations], "document").then(([vec]) => {
|
|
27080
27342
|
db().query("UPDATE distillations SET embedding = ? WHERE id = ?").run(toBlob(vec), id);
|
|
27081
27343
|
}).catch((err) => {
|
|
27082
|
-
|
|
27344
|
+
error("embedding failed for distillation", id, ":", err);
|
|
27083
27345
|
});
|
|
27084
27346
|
}
|
|
27085
27347
|
function embedTemporalMessage(id, content3) {
|
|
27348
|
+
if (!isAvailable()) return;
|
|
27086
27349
|
if (content3.length < 50) return;
|
|
27087
27350
|
embed([content3], "document").then(([vec]) => {
|
|
27088
27351
|
db().query("UPDATE temporal_messages SET embedding = ? WHERE id = ?").run(toBlob(vec), id);
|
|
27089
27352
|
}).catch((err) => {
|
|
27090
|
-
|
|
27353
|
+
error("embedding failed for temporal message", id, ":", err);
|
|
27091
27354
|
});
|
|
27092
27355
|
}
|
|
27093
27356
|
function vectorSearchTemporal(queryEmbedding, projectId2, limit = 10, sessionId) {
|
|
@@ -27207,6 +27470,7 @@ ${r.content}` }));
|
|
|
27207
27470
|
}
|
|
27208
27471
|
} catch (err) {
|
|
27209
27472
|
error(`embedding backfill batch failed (${batch.length} items):`, err);
|
|
27473
|
+
if (err instanceof LocalProviderUnavailableError) break;
|
|
27210
27474
|
}
|
|
27211
27475
|
}
|
|
27212
27476
|
if (embedded > 0) {
|
|
@@ -27240,6 +27504,7 @@ async function backfillDistillationEmbeddings() {
|
|
|
27240
27504
|
}
|
|
27241
27505
|
} catch (err) {
|
|
27242
27506
|
error(`distillation embedding backfill batch failed (${batch.length} items):`, err);
|
|
27507
|
+
if (err instanceof LocalProviderUnavailableError) break;
|
|
27243
27508
|
}
|
|
27244
27509
|
if (embedded >= nextProgressAt) {
|
|
27245
27510
|
info(`embedding distillations: ${embedded}/${rows.length}\u2026`);
|
|
@@ -27329,14 +27594,14 @@ function store(input) {
|
|
|
27329
27594
|
embedTemporalMessage(input.info.id, content3);
|
|
27330
27595
|
}
|
|
27331
27596
|
}
|
|
27332
|
-
function undistilled(
|
|
27333
|
-
const pid = ensureProject(
|
|
27597
|
+
function undistilled(projectPath2, sessionID) {
|
|
27598
|
+
const pid = ensureProject(projectPath2);
|
|
27334
27599
|
const query = sessionID ? "SELECT * FROM temporal_messages WHERE project_id = ? AND session_id = ? AND distilled = 0 ORDER BY created_at ASC" : "SELECT * FROM temporal_messages WHERE project_id = ? AND distilled = 0 ORDER BY created_at ASC";
|
|
27335
27600
|
const params = sessionID ? [pid, sessionID] : [pid];
|
|
27336
27601
|
return db().query(query).all(...params);
|
|
27337
27602
|
}
|
|
27338
|
-
function bySession(
|
|
27339
|
-
const pid = ensureProject(
|
|
27603
|
+
function bySession(projectPath2, sessionID) {
|
|
27604
|
+
const pid = ensureProject(projectPath2);
|
|
27340
27605
|
return db().query(
|
|
27341
27606
|
"SELECT * FROM temporal_messages WHERE project_id = ? AND session_id = ? ORDER BY created_at ASC"
|
|
27342
27607
|
).all(pid, sessionID);
|
|
@@ -27412,20 +27677,26 @@ function temporalCnorm(timestamps, now = Date.now()) {
|
|
|
27412
27677
|
const maxVariance = (n - 1) / (n * n);
|
|
27413
27678
|
return maxVariance === 0 ? 0 : variance / maxVariance;
|
|
27414
27679
|
}
|
|
27415
|
-
function count(
|
|
27416
|
-
const pid = ensureProject(
|
|
27680
|
+
function count(projectPath2, sessionID) {
|
|
27681
|
+
const pid = ensureProject(projectPath2);
|
|
27417
27682
|
const query = sessionID ? "SELECT COUNT(*) as count FROM temporal_messages WHERE project_id = ? AND session_id = ?" : "SELECT COUNT(*) as count FROM temporal_messages WHERE project_id = ?";
|
|
27418
27683
|
const params = sessionID ? [pid, sessionID] : [pid];
|
|
27419
27684
|
return db().query(query).get(...params).count;
|
|
27420
27685
|
}
|
|
27421
|
-
function
|
|
27422
|
-
const pid = ensureProject(
|
|
27686
|
+
function hasMessages(projectPath2, sessionID) {
|
|
27687
|
+
const pid = ensureProject(projectPath2);
|
|
27688
|
+
return !!db().query(
|
|
27689
|
+
"SELECT 1 FROM temporal_messages WHERE project_id = ? AND session_id = ? LIMIT 1"
|
|
27690
|
+
).get(pid, sessionID);
|
|
27691
|
+
}
|
|
27692
|
+
function undistilledCount(projectPath2, sessionID) {
|
|
27693
|
+
const pid = ensureProject(projectPath2);
|
|
27423
27694
|
const query = sessionID ? "SELECT COUNT(*) as count FROM temporal_messages WHERE project_id = ? AND session_id = ? AND distilled = 0" : "SELECT COUNT(*) as count FROM temporal_messages WHERE project_id = ? AND distilled = 0";
|
|
27424
27695
|
const params = sessionID ? [pid, sessionID] : [pid];
|
|
27425
27696
|
return db().query(query).get(...params).count;
|
|
27426
27697
|
}
|
|
27427
|
-
function undistilledTokens(
|
|
27428
|
-
const pid = ensureProject(
|
|
27698
|
+
function undistilledTokens(projectPath2, sessionID) {
|
|
27699
|
+
const pid = ensureProject(projectPath2);
|
|
27429
27700
|
const query = sessionID ? "SELECT COALESCE(SUM(tokens), 0) as total FROM temporal_messages WHERE project_id = ? AND session_id = ? AND distilled = 0" : "SELECT COALESCE(SUM(tokens), 0) as total FROM temporal_messages WHERE project_id = ? AND distilled = 0";
|
|
27430
27701
|
const params = sessionID ? [pid, sessionID] : [pid];
|
|
27431
27702
|
return db().query(query).get(...params).total;
|
|
@@ -27476,11 +27747,13 @@ function prune(input) {
|
|
|
27476
27747
|
var ltm_exports = {};
|
|
27477
27748
|
__export(ltm_exports, {
|
|
27478
27749
|
all: () => all2,
|
|
27750
|
+
calibrateDedupThreshold: () => calibrateDedupThreshold,
|
|
27479
27751
|
cascadeRefReplace: () => cascadeRefReplace,
|
|
27480
27752
|
check: () => check2,
|
|
27481
27753
|
cleanDeadRefs: () => cleanDeadRefs,
|
|
27482
27754
|
create: () => create,
|
|
27483
27755
|
crossProject: () => crossProject,
|
|
27756
|
+
dedupPairKey: () => dedupPairKey,
|
|
27484
27757
|
deduplicate: () => deduplicate,
|
|
27485
27758
|
deduplicateGlobal: () => deduplicateGlobal,
|
|
27486
27759
|
extractRefs: () => extractRefs,
|
|
@@ -27488,9 +27761,17 @@ __export(ltm_exports, {
|
|
|
27488
27761
|
forProject: () => forProject,
|
|
27489
27762
|
forSession: () => forSession,
|
|
27490
27763
|
get: () => get,
|
|
27764
|
+
getDedupFeedback: () => getDedupFeedback,
|
|
27765
|
+
getDedupFeedbackCount: () => getDedupFeedbackCount,
|
|
27766
|
+
loadCalibratedThreshold: () => loadCalibratedThreshold,
|
|
27767
|
+
pruneDedupFeedback: () => pruneDedupFeedback,
|
|
27491
27768
|
pruneOversized: () => pruneOversized,
|
|
27769
|
+
recordAutoSignals: () => recordAutoSignals,
|
|
27770
|
+
recordDedupFeedback: () => recordDedupFeedback,
|
|
27771
|
+
recordDedupResultFeedback: () => recordDedupResultFeedback,
|
|
27492
27772
|
remove: () => remove,
|
|
27493
27773
|
resolveRef: () => resolveRef2,
|
|
27774
|
+
saveCalibratedThreshold: () => saveCalibratedThreshold,
|
|
27494
27775
|
search: () => search3,
|
|
27495
27776
|
searchScored: () => searchScored3,
|
|
27496
27777
|
searchScoredOtherProjects: () => searchScoredOtherProjects,
|
|
@@ -27936,14 +28217,16 @@ function listMarkdownFiles(dir) {
|
|
|
27936
28217
|
function contentHash(content3) {
|
|
27937
28218
|
return sha256(content3);
|
|
27938
28219
|
}
|
|
27939
|
-
function hasLatDir(
|
|
27940
|
-
|
|
28220
|
+
function hasLatDir(projectPath2) {
|
|
28221
|
+
if (isHostedMode()) return false;
|
|
28222
|
+
const latDir = join6(projectPath2, "lat.md");
|
|
27941
28223
|
return existsSync3(latDir) && statSync2(latDir).isDirectory();
|
|
27942
28224
|
}
|
|
27943
|
-
function refresh(
|
|
27944
|
-
|
|
28225
|
+
function refresh(projectPath2) {
|
|
28226
|
+
if (isHostedMode()) return 0;
|
|
28227
|
+
const latDir = join6(projectPath2, "lat.md");
|
|
27945
28228
|
if (!existsSync3(latDir) || !statSync2(latDir).isDirectory()) return 0;
|
|
27946
|
-
const pid = ensureProject(
|
|
28229
|
+
const pid = ensureProject(projectPath2);
|
|
27947
28230
|
const files = listMarkdownFiles(latDir);
|
|
27948
28231
|
let upserted = 0;
|
|
27949
28232
|
const seenFiles = /* @__PURE__ */ new Set();
|
|
@@ -27958,7 +28241,7 @@ function refresh(projectPath) {
|
|
|
27958
28241
|
} catch {
|
|
27959
28242
|
continue;
|
|
27960
28243
|
}
|
|
27961
|
-
const fileRel = relative(
|
|
28244
|
+
const fileRel = relative(projectPath2, filePath);
|
|
27962
28245
|
seenFiles.add(fileRel);
|
|
27963
28246
|
const hash2 = contentHash(content3);
|
|
27964
28247
|
const existing = db().query("SELECT content_hash FROM lat_sections WHERE project_id = ? AND file = ? LIMIT 1").get(pid, fileRel.replace(/\.md$/, ""));
|
|
@@ -27966,7 +28249,7 @@ function refresh(projectPath) {
|
|
|
27966
28249
|
continue;
|
|
27967
28250
|
}
|
|
27968
28251
|
db().query("DELETE FROM lat_sections WHERE project_id = ? AND file = ?").run(pid, fileRel.replace(/\.md$/, ""));
|
|
27969
|
-
const sections = parseSections(filePath, content3,
|
|
28252
|
+
const sections = parseSections(filePath, content3, projectPath2);
|
|
27970
28253
|
const now = Date.now();
|
|
27971
28254
|
for (const section of sections) {
|
|
27972
28255
|
upsertStmt.run(
|
|
@@ -28016,9 +28299,9 @@ function searchScored2(input) {
|
|
|
28016
28299
|
return [];
|
|
28017
28300
|
}
|
|
28018
28301
|
}
|
|
28019
|
-
function scoreForSession(
|
|
28020
|
-
if (!hasLatDir(
|
|
28021
|
-
const pid = ensureProject(
|
|
28302
|
+
function scoreForSession(projectPath2, sessionContext, maxTokens) {
|
|
28303
|
+
if (!hasLatDir(projectPath2)) return [];
|
|
28304
|
+
const pid = ensureProject(projectPath2);
|
|
28022
28305
|
const terms = extractTopTerms(sessionContext);
|
|
28023
28306
|
if (!terms.length) return [];
|
|
28024
28307
|
const q = terms.map((t2) => `${t2}*`).join(" OR ");
|
|
@@ -28050,8 +28333,8 @@ function scoreForSession(projectPath, sessionContext, maxTokens) {
|
|
|
28050
28333
|
}
|
|
28051
28334
|
return packed;
|
|
28052
28335
|
}
|
|
28053
|
-
function count2(
|
|
28054
|
-
const pid = ensureProject(
|
|
28336
|
+
function count2(projectPath2) {
|
|
28337
|
+
const pid = ensureProject(projectPath2);
|
|
28055
28338
|
const row = db().query("SELECT COUNT(*) as cnt FROM lat_sections WHERE project_id = ?").get(pid);
|
|
28056
28339
|
return row.cnt;
|
|
28057
28340
|
}
|
|
@@ -28178,8 +28461,8 @@ function findFuzzyDuplicate(input) {
|
|
|
28178
28461
|
}
|
|
28179
28462
|
return null;
|
|
28180
28463
|
}
|
|
28181
|
-
function forProject(
|
|
28182
|
-
const pid = ensureProject(
|
|
28464
|
+
function forProject(projectPath2, includeCross = true) {
|
|
28465
|
+
const pid = ensureProject(projectPath2);
|
|
28183
28466
|
if (includeCross) {
|
|
28184
28467
|
return db().query(
|
|
28185
28468
|
`SELECT ${KNOWLEDGE_COLS} FROM knowledge
|
|
@@ -28227,18 +28510,29 @@ function scoreEntriesFTS(sessionContext) {
|
|
|
28227
28510
|
return /* @__PURE__ */ new Map();
|
|
28228
28511
|
}
|
|
28229
28512
|
}
|
|
28230
|
-
function forSession(
|
|
28231
|
-
const pid = ensureProject(
|
|
28513
|
+
async function forSession(projectPath2, sessionID, maxTokens, options) {
|
|
28514
|
+
const pid = ensureProject(projectPath2);
|
|
28515
|
+
const categoryFilter = options?.categories;
|
|
28516
|
+
const excludeFilter = options?.excludeCategories;
|
|
28517
|
+
let categoryClause = "";
|
|
28518
|
+
let categoryParams = [];
|
|
28519
|
+
if (categoryFilter?.length) {
|
|
28520
|
+
categoryClause = ` AND category IN (${categoryFilter.map(() => "?").join(",")})`;
|
|
28521
|
+
categoryParams = categoryFilter;
|
|
28522
|
+
} else if (excludeFilter?.length) {
|
|
28523
|
+
categoryClause = ` AND category NOT IN (${excludeFilter.map(() => "?").join(",")})`;
|
|
28524
|
+
categoryParams = excludeFilter;
|
|
28525
|
+
}
|
|
28232
28526
|
const projectEntries = db().query(
|
|
28233
28527
|
`SELECT ${KNOWLEDGE_COLS} FROM knowledge
|
|
28234
|
-
WHERE project_id = ? AND cross_project = 0 AND confidence > 0.2
|
|
28528
|
+
WHERE project_id = ? AND cross_project = 0 AND confidence > 0.2${categoryClause}
|
|
28235
28529
|
ORDER BY confidence DESC, updated_at DESC`
|
|
28236
|
-
).all(pid);
|
|
28530
|
+
).all(pid, ...categoryParams);
|
|
28237
28531
|
const crossEntries = db().query(
|
|
28238
28532
|
`SELECT ${KNOWLEDGE_COLS} FROM knowledge
|
|
28239
|
-
WHERE (project_id IS NULL OR cross_project = 1) AND confidence > 0.2
|
|
28533
|
+
WHERE (project_id IS NULL OR cross_project = 1) AND confidence > 0.2${categoryClause}
|
|
28240
28534
|
ORDER BY confidence DESC, updated_at DESC`
|
|
28241
|
-
).all();
|
|
28535
|
+
).all(...categoryParams);
|
|
28242
28536
|
if (!crossEntries.length && !projectEntries.length) return [];
|
|
28243
28537
|
let sessionContext = "";
|
|
28244
28538
|
if (sessionID) {
|
|
@@ -28259,22 +28553,52 @@ function forSession(projectPath, sessionID, maxTokens) {
|
|
|
28259
28553
|
sessionContext += recentMsgs.map((m) => m.content).join("\n");
|
|
28260
28554
|
}
|
|
28261
28555
|
}
|
|
28556
|
+
if (!sessionContext.trim() && options?.contextHint) {
|
|
28557
|
+
sessionContext = options.contextHint;
|
|
28558
|
+
}
|
|
28262
28559
|
let scoredProject;
|
|
28263
28560
|
let scoredCross;
|
|
28264
|
-
if (sessionContext.trim().length > 20) {
|
|
28561
|
+
if (sessionContext.trim().length > 20 && isAvailable()) {
|
|
28562
|
+
let vectorScores;
|
|
28563
|
+
try {
|
|
28564
|
+
const [contextVec] = await embed([sessionContext], "query");
|
|
28565
|
+
const hits = vectorSearch(contextVec, 50, excludeFilter);
|
|
28566
|
+
vectorScores = new Map(hits.map((h3) => [h3.id, h3.similarity]));
|
|
28567
|
+
} catch (err) {
|
|
28568
|
+
warn("Vector scoring failed, falling back to FTS5:", err);
|
|
28569
|
+
vectorScores = /* @__PURE__ */ new Map();
|
|
28570
|
+
}
|
|
28571
|
+
if (vectorScores.size > 0) {
|
|
28572
|
+
const ftsScores = scoreEntriesFTS(sessionContext);
|
|
28573
|
+
const rawScored = projectEntries.map((entry) => {
|
|
28574
|
+
const vecScore = vectorScores.get(entry.id);
|
|
28575
|
+
const score = vecScore != null ? vecScore * entry.confidence : (ftsScores.get(entry.id) ?? 0) * entry.confidence;
|
|
28576
|
+
return { entry, score };
|
|
28577
|
+
});
|
|
28578
|
+
const matched = rawScored.filter((s) => s.score > 0);
|
|
28579
|
+
const matchedIds = new Set(matched.map((s) => s.entry.id));
|
|
28580
|
+
const safetyNet = projectEntries.filter((e) => !matchedIds.has(e.id)).slice(0, PROJECT_SAFETY_NET).map((e) => ({ entry: e, score: 1e-3 * e.confidence }));
|
|
28581
|
+
scoredProject = [...matched, ...safetyNet];
|
|
28582
|
+
scoredCross = crossEntries.filter((e) => vectorScores.has(e.id) || ftsScores.has(e.id)).map((e) => {
|
|
28583
|
+
const vecScore = vectorScores.get(e.id);
|
|
28584
|
+
const score = vecScore != null ? vecScore * e.confidence : (ftsScores.get(e.id) ?? 0) * e.confidence;
|
|
28585
|
+
return { entry: e, score };
|
|
28586
|
+
});
|
|
28587
|
+
} else {
|
|
28588
|
+
const ftsScores = scoreEntriesFTS(sessionContext);
|
|
28589
|
+
({ scoredProject, scoredCross } = scoreFTS(
|
|
28590
|
+
projectEntries,
|
|
28591
|
+
crossEntries,
|
|
28592
|
+
ftsScores
|
|
28593
|
+
));
|
|
28594
|
+
}
|
|
28595
|
+
} else if (sessionContext.trim().length > 20) {
|
|
28265
28596
|
const ftsScores = scoreEntriesFTS(sessionContext);
|
|
28266
|
-
|
|
28267
|
-
|
|
28268
|
-
|
|
28269
|
-
|
|
28270
|
-
|
|
28271
|
-
const matchedIds = new Set(matched.map((s) => s.entry.id));
|
|
28272
|
-
const safetyNet = projectEntries.filter((e) => !matchedIds.has(e.id)).slice(0, PROJECT_SAFETY_NET).map((e) => ({ entry: e, score: 1e-3 * e.confidence }));
|
|
28273
|
-
scoredProject = [...matched, ...safetyNet];
|
|
28274
|
-
scoredCross = crossEntries.filter((e) => ftsScores.has(e.id)).map((e) => ({
|
|
28275
|
-
entry: e,
|
|
28276
|
-
score: (ftsScores.get(e.id) ?? 0) * e.confidence
|
|
28277
|
-
}));
|
|
28597
|
+
({ scoredProject, scoredCross } = scoreFTS(
|
|
28598
|
+
projectEntries,
|
|
28599
|
+
crossEntries,
|
|
28600
|
+
ftsScores
|
|
28601
|
+
));
|
|
28278
28602
|
} else {
|
|
28279
28603
|
scoredProject = projectEntries.slice(0, NO_CONTEXT_FALLBACK_CAP).map((entry) => ({ entry, score: entry.confidence }));
|
|
28280
28604
|
scoredCross = crossEntries.slice(0, NO_CONTEXT_FALLBACK_CAP).map((entry) => ({ entry, score: entry.confidence }));
|
|
@@ -28291,9 +28615,9 @@ function forSession(projectPath, sessionID, maxTokens) {
|
|
|
28291
28615
|
result.push(entry);
|
|
28292
28616
|
used += cost;
|
|
28293
28617
|
}
|
|
28294
|
-
if (hasLatDir(
|
|
28618
|
+
if (hasLatDir(projectPath2) && used < maxTokens) {
|
|
28295
28619
|
const latSections = scoreForSession(
|
|
28296
|
-
|
|
28620
|
+
projectPath2,
|
|
28297
28621
|
sessionContext,
|
|
28298
28622
|
maxTokens - used
|
|
28299
28623
|
);
|
|
@@ -28320,6 +28644,21 @@ function forSession(projectPath, sessionID, maxTokens) {
|
|
|
28320
28644
|
}
|
|
28321
28645
|
return result;
|
|
28322
28646
|
}
|
|
28647
|
+
function scoreFTS(projectEntries, crossEntries, ftsScores) {
|
|
28648
|
+
const rawScored = projectEntries.map((entry) => ({
|
|
28649
|
+
entry,
|
|
28650
|
+
score: (ftsScores.get(entry.id) ?? 0) * entry.confidence
|
|
28651
|
+
}));
|
|
28652
|
+
const matched = rawScored.filter((s) => s.score > 0);
|
|
28653
|
+
const matchedIds = new Set(matched.map((s) => s.entry.id));
|
|
28654
|
+
const safetyNet = projectEntries.filter((e) => !matchedIds.has(e.id)).slice(0, PROJECT_SAFETY_NET).map((e) => ({ entry: e, score: 1e-3 * e.confidence }));
|
|
28655
|
+
const scoredProject = [...matched, ...safetyNet];
|
|
28656
|
+
const scoredCross = crossEntries.filter((e) => ftsScores.has(e.id)).map((e) => ({
|
|
28657
|
+
entry: e,
|
|
28658
|
+
score: (ftsScores.get(e.id) ?? 0) * e.confidence
|
|
28659
|
+
}));
|
|
28660
|
+
return { scoredProject, scoredCross };
|
|
28661
|
+
}
|
|
28323
28662
|
function all2() {
|
|
28324
28663
|
return db().query(
|
|
28325
28664
|
`SELECT ${KNOWLEDGE_COLS} FROM knowledge WHERE confidence > 0.2 ORDER BY confidence DESC, updated_at DESC`
|
|
@@ -28501,8 +28840,8 @@ function cleanDeadRefs() {
|
|
|
28501
28840
|
}
|
|
28502
28841
|
return cleaned;
|
|
28503
28842
|
}
|
|
28504
|
-
function check2(
|
|
28505
|
-
const entries = forProject(
|
|
28843
|
+
function check2(projectPath2) {
|
|
28844
|
+
const entries = forProject(projectPath2, false);
|
|
28506
28845
|
const issues = [];
|
|
28507
28846
|
for (const entry of entries) {
|
|
28508
28847
|
if (entry.content.length > 1200) {
|
|
@@ -28563,8 +28902,11 @@ function check2(projectPath) {
|
|
|
28563
28902
|
}
|
|
28564
28903
|
return issues;
|
|
28565
28904
|
}
|
|
28566
|
-
function
|
|
28567
|
-
|
|
28905
|
+
function dedupPairKey(idA, idB) {
|
|
28906
|
+
return idA < idB ? `${idA}:${idB}` : `${idB}:${idA}`;
|
|
28907
|
+
}
|
|
28908
|
+
function _dedup(entries, dryRun, embeddingThreshold = EMBEDDING_DEDUP_THRESHOLD) {
|
|
28909
|
+
if (entries.length < 2) return { clusters: [], totalRemoved: 0, pairSimilarities: /* @__PURE__ */ new Map(), entryTitles: /* @__PURE__ */ new Map() };
|
|
28568
28910
|
const embeddingMap = /* @__PURE__ */ new Map();
|
|
28569
28911
|
{
|
|
28570
28912
|
const entryIds = entries.map((e) => e.id);
|
|
@@ -28579,6 +28921,7 @@ function _dedup(entries, dryRun) {
|
|
|
28579
28921
|
}
|
|
28580
28922
|
}
|
|
28581
28923
|
const neighborMap = /* @__PURE__ */ new Map();
|
|
28924
|
+
const pairSimilarities = /* @__PURE__ */ new Map();
|
|
28582
28925
|
for (const entry of entries) {
|
|
28583
28926
|
const neighbors = [];
|
|
28584
28927
|
const entryVec = embeddingMap.get(entry.id);
|
|
@@ -28592,7 +28935,13 @@ function _dedup(entries, dryRun) {
|
|
|
28592
28935
|
const otherVec = embeddingMap.get(other.id);
|
|
28593
28936
|
if (otherVec && entryVec.length === otherVec.length) {
|
|
28594
28937
|
similarity = cosineSimilarity(entryVec, otherVec);
|
|
28595
|
-
embeddingMatch = similarity >=
|
|
28938
|
+
embeddingMatch = similarity >= embeddingThreshold;
|
|
28939
|
+
}
|
|
28940
|
+
}
|
|
28941
|
+
if (similarity > 0) {
|
|
28942
|
+
const pk = dedupPairKey(entry.id, other.id);
|
|
28943
|
+
if (!pairSimilarities.has(pk)) {
|
|
28944
|
+
pairSimilarities.set(pk, similarity);
|
|
28596
28945
|
}
|
|
28597
28946
|
}
|
|
28598
28947
|
if (titleMatch || embeddingMatch) {
|
|
@@ -28644,20 +28993,178 @@ function _dedup(entries, dryRun) {
|
|
|
28644
28993
|
totalRemoved += merged.length;
|
|
28645
28994
|
}
|
|
28646
28995
|
result.sort((a, b) => b.merged.length - a.merged.length);
|
|
28647
|
-
|
|
28996
|
+
const entryTitles = new Map(entries.map((e) => [e.id, e.title]));
|
|
28997
|
+
return { clusters: result, totalRemoved, pairSimilarities, entryTitles };
|
|
28648
28998
|
}
|
|
28649
|
-
async function deduplicate(
|
|
28650
|
-
const
|
|
28651
|
-
|
|
28999
|
+
async function deduplicate(projectPath2, opts) {
|
|
29000
|
+
const pid = ensureProject(projectPath2);
|
|
29001
|
+
const threshold = loadCalibratedThreshold(pid) ?? EMBEDDING_DEDUP_THRESHOLD;
|
|
29002
|
+
const entries = forProject(projectPath2, false);
|
|
29003
|
+
return _dedup(entries, opts?.dryRun ?? true, threshold);
|
|
28652
29004
|
}
|
|
28653
29005
|
async function deduplicateGlobal(opts) {
|
|
29006
|
+
const threshold = loadCalibratedThreshold(null) ?? EMBEDDING_DEDUP_THRESHOLD;
|
|
28654
29007
|
const entries = db().query(
|
|
28655
29008
|
`SELECT ${KNOWLEDGE_COLS} FROM knowledge
|
|
28656
29009
|
WHERE project_id IS NULL
|
|
28657
29010
|
AND confidence > 0.2
|
|
28658
29011
|
ORDER BY confidence DESC, updated_at DESC`
|
|
28659
29012
|
).all();
|
|
28660
|
-
return _dedup(entries, opts?.dryRun ?? true);
|
|
29013
|
+
return _dedup(entries, opts?.dryRun ?? true, threshold);
|
|
29014
|
+
}
|
|
29015
|
+
var MIN_CALIBRATION_SAMPLES = 20;
|
|
29016
|
+
var DEFAULT_EMBEDDING_DEDUP_THRESHOLD = EMBEDDING_DEDUP_THRESHOLD;
|
|
29017
|
+
var AUTO_SIGNAL_MIN_SIMILARITY = 0.8;
|
|
29018
|
+
var AUTO_SIGNAL_MAX_PAIRS = 50;
|
|
29019
|
+
function recordDedupFeedback(input) {
|
|
29020
|
+
db().query(
|
|
29021
|
+
`INSERT INTO dedup_feedback
|
|
29022
|
+
(project_id, entry_a_title, entry_b_title, similarity, accepted, source, created_at)
|
|
29023
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`
|
|
29024
|
+
).run(
|
|
29025
|
+
input.projectId,
|
|
29026
|
+
input.entryATitle,
|
|
29027
|
+
input.entryBTitle,
|
|
29028
|
+
input.similarity,
|
|
29029
|
+
input.accepted ? 1 : 0,
|
|
29030
|
+
input.source,
|
|
29031
|
+
Date.now()
|
|
29032
|
+
);
|
|
29033
|
+
}
|
|
29034
|
+
function recordDedupResultFeedback(projectId2, result, accepted, source) {
|
|
29035
|
+
for (const cluster of result.clusters) {
|
|
29036
|
+
for (const merged of cluster.merged) {
|
|
29037
|
+
const pk = dedupPairKey(cluster.surviving.id, merged.id);
|
|
29038
|
+
const similarity = result.pairSimilarities.get(pk);
|
|
29039
|
+
if (similarity != null && similarity > 0) {
|
|
29040
|
+
recordDedupFeedback({
|
|
29041
|
+
projectId: projectId2,
|
|
29042
|
+
entryATitle: cluster.surviving.title,
|
|
29043
|
+
entryBTitle: merged.title,
|
|
29044
|
+
similarity,
|
|
29045
|
+
accepted,
|
|
29046
|
+
source
|
|
29047
|
+
});
|
|
29048
|
+
}
|
|
29049
|
+
}
|
|
29050
|
+
}
|
|
29051
|
+
}
|
|
29052
|
+
function recordAutoSignals(projectId2, result) {
|
|
29053
|
+
const mergedPairs = /* @__PURE__ */ new Set();
|
|
29054
|
+
for (const cluster of result.clusters) {
|
|
29055
|
+
for (const merged of cluster.merged) {
|
|
29056
|
+
mergedPairs.add(dedupPairKey(cluster.surviving.id, merged.id));
|
|
29057
|
+
}
|
|
29058
|
+
}
|
|
29059
|
+
const titleMap = new Map(result.entryTitles);
|
|
29060
|
+
for (const cluster of result.clusters) {
|
|
29061
|
+
if (!titleMap.has(cluster.surviving.id)) {
|
|
29062
|
+
titleMap.set(cluster.surviving.id, cluster.surviving.title);
|
|
29063
|
+
}
|
|
29064
|
+
for (const m of cluster.merged) {
|
|
29065
|
+
if (!titleMap.has(m.id)) titleMap.set(m.id, m.title);
|
|
29066
|
+
}
|
|
29067
|
+
}
|
|
29068
|
+
const signals = [];
|
|
29069
|
+
for (const [pk, sim] of result.pairSimilarities) {
|
|
29070
|
+
if (sim < AUTO_SIGNAL_MIN_SIMILARITY) continue;
|
|
29071
|
+
if (mergedPairs.has(pk)) continue;
|
|
29072
|
+
const [idA, idB] = pk.split(":");
|
|
29073
|
+
const titleA = titleMap.get(idA);
|
|
29074
|
+
const titleB = titleMap.get(idB);
|
|
29075
|
+
if (!titleA || !titleB) continue;
|
|
29076
|
+
signals.push({ entryATitle: titleA, entryBTitle: titleB, similarity: sim });
|
|
29077
|
+
}
|
|
29078
|
+
const currentThreshold = loadCalibratedThreshold(projectId2) ?? DEFAULT_EMBEDDING_DEDUP_THRESHOLD;
|
|
29079
|
+
signals.sort((a, b) => Math.abs(a.similarity - currentThreshold) - Math.abs(b.similarity - currentThreshold));
|
|
29080
|
+
const capped = signals.slice(0, AUTO_SIGNAL_MAX_PAIRS);
|
|
29081
|
+
pruneDedupFeedback(projectId2);
|
|
29082
|
+
for (const s of capped) {
|
|
29083
|
+
recordDedupFeedback({
|
|
29084
|
+
projectId: projectId2,
|
|
29085
|
+
entryATitle: s.entryATitle,
|
|
29086
|
+
entryBTitle: s.entryBTitle,
|
|
29087
|
+
similarity: s.similarity,
|
|
29088
|
+
accepted: false,
|
|
29089
|
+
source: "auto_dedup"
|
|
29090
|
+
});
|
|
29091
|
+
}
|
|
29092
|
+
}
|
|
29093
|
+
function getDedupFeedback(projectId2) {
|
|
29094
|
+
const rows = projectId2 !== null ? db().query(
|
|
29095
|
+
"SELECT similarity, accepted, source FROM dedup_feedback WHERE project_id = ? ORDER BY similarity"
|
|
29096
|
+
).all(projectId2) : db().query(
|
|
29097
|
+
"SELECT similarity, accepted, source FROM dedup_feedback WHERE project_id IS NULL ORDER BY similarity"
|
|
29098
|
+
).all();
|
|
29099
|
+
return rows.map((r) => ({ similarity: r.similarity, accepted: r.accepted === 1, source: r.source }));
|
|
29100
|
+
}
|
|
29101
|
+
function getDedupFeedbackCount(projectId2) {
|
|
29102
|
+
const row = projectId2 !== null ? db().query("SELECT COUNT(*) as cnt FROM dedup_feedback WHERE project_id = ?").get(projectId2) : db().query("SELECT COUNT(*) as cnt FROM dedup_feedback WHERE project_id IS NULL").get();
|
|
29103
|
+
return row?.cnt ?? 0;
|
|
29104
|
+
}
|
|
29105
|
+
var MAX_FEEDBACK_ROWS_PER_PROJECT = 500;
|
|
29106
|
+
function pruneDedupFeedback(projectId2) {
|
|
29107
|
+
const count3 = getDedupFeedbackCount(projectId2);
|
|
29108
|
+
if (count3 <= MAX_FEEDBACK_ROWS_PER_PROJECT) return;
|
|
29109
|
+
const excess = count3 - MAX_FEEDBACK_ROWS_PER_PROJECT;
|
|
29110
|
+
if (projectId2 !== null) {
|
|
29111
|
+
db().query(
|
|
29112
|
+
`DELETE FROM dedup_feedback WHERE id IN (
|
|
29113
|
+
SELECT id FROM dedup_feedback WHERE project_id = ?
|
|
29114
|
+
ORDER BY created_at ASC LIMIT ?
|
|
29115
|
+
)`
|
|
29116
|
+
).run(projectId2, excess);
|
|
29117
|
+
} else {
|
|
29118
|
+
db().query(
|
|
29119
|
+
`DELETE FROM dedup_feedback WHERE id IN (
|
|
29120
|
+
SELECT id FROM dedup_feedback WHERE project_id IS NULL
|
|
29121
|
+
ORDER BY created_at ASC LIMIT ?
|
|
29122
|
+
)`
|
|
29123
|
+
).run(excess);
|
|
29124
|
+
}
|
|
29125
|
+
}
|
|
29126
|
+
function calibrateDedupThreshold(projectId2) {
|
|
29127
|
+
const feedback = getDedupFeedback(projectId2);
|
|
29128
|
+
if (feedback.length < MIN_CALIBRATION_SAMPLES) return null;
|
|
29129
|
+
const accepted = feedback.filter((f) => f.accepted);
|
|
29130
|
+
const rejected = feedback.filter((f) => !f.accepted);
|
|
29131
|
+
if (rejected.length === 0) {
|
|
29132
|
+
const minAccepted = Math.min(...accepted.map((f) => f.similarity));
|
|
29133
|
+
return Math.max(0.85, minAccepted - 5e-3);
|
|
29134
|
+
}
|
|
29135
|
+
if (accepted.length === 0) {
|
|
29136
|
+
warn("dedup calibration: all feedback is reject \u2014 keeping default threshold");
|
|
29137
|
+
return null;
|
|
29138
|
+
}
|
|
29139
|
+
const allSims = [...new Set(feedback.map((f) => f.similarity))].sort((a, b) => a - b);
|
|
29140
|
+
let bestThreshold = DEFAULT_EMBEDDING_DEDUP_THRESHOLD;
|
|
29141
|
+
let bestAccuracy = -1;
|
|
29142
|
+
for (let i = 0; i < allSims.length - 1; i++) {
|
|
29143
|
+
const candidate = (allSims[i] + allSims[i + 1]) / 2;
|
|
29144
|
+
const correctAccepted = accepted.filter((f) => f.similarity >= candidate).length;
|
|
29145
|
+
const correctRejected = rejected.filter((f) => f.similarity < candidate).length;
|
|
29146
|
+
const accuracy = (correctAccepted + correctRejected) / feedback.length;
|
|
29147
|
+
if (accuracy > bestAccuracy || accuracy === bestAccuracy && candidate > bestThreshold) {
|
|
29148
|
+
bestAccuracy = accuracy;
|
|
29149
|
+
bestThreshold = candidate;
|
|
29150
|
+
}
|
|
29151
|
+
}
|
|
29152
|
+
return Math.max(0.85, Math.min(0.98, bestThreshold));
|
|
29153
|
+
}
|
|
29154
|
+
function saveCalibratedThreshold(projectId2, threshold, sampleSize) {
|
|
29155
|
+
const key = `dedup_threshold:${projectId2 ?? "global"}`;
|
|
29156
|
+
setKV(key, JSON.stringify({ threshold, sampleSize, calibratedAt: Date.now() }));
|
|
29157
|
+
}
|
|
29158
|
+
function loadCalibratedThreshold(projectId2) {
|
|
29159
|
+
const key = `dedup_threshold:${projectId2 ?? "global"}`;
|
|
29160
|
+
const raw = getKV(key);
|
|
29161
|
+
if (!raw) return null;
|
|
29162
|
+
try {
|
|
29163
|
+
const parsed = JSON.parse(raw);
|
|
29164
|
+
return typeof parsed.threshold === "number" ? parsed.threshold : null;
|
|
29165
|
+
} catch {
|
|
29166
|
+
return null;
|
|
29167
|
+
}
|
|
28661
29168
|
}
|
|
28662
29169
|
|
|
28663
29170
|
// src/data.ts
|
|
@@ -28717,8 +29224,8 @@ function setCache(fp, entry) {
|
|
|
28717
29224
|
"INSERT INTO kv_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = ?"
|
|
28718
29225
|
).run(key, value, value);
|
|
28719
29226
|
}
|
|
28720
|
-
function clearLoreFileCache(
|
|
28721
|
-
db().query("DELETE FROM kv_meta WHERE key = ?").run(CACHE_PREFIX + join7(
|
|
29227
|
+
function clearLoreFileCache(projectPath2) {
|
|
29228
|
+
db().query("DELETE FROM kv_meta WHERE key = ?").run(CACHE_PREFIX + join7(projectPath2, LORE_FILE));
|
|
28722
29229
|
}
|
|
28723
29230
|
function splitFile(fileContent) {
|
|
28724
29231
|
const spans = [];
|
|
@@ -28794,8 +29301,8 @@ function hashSection(section) {
|
|
|
28794
29301
|
}
|
|
28795
29302
|
return (h3 >>> 0).toString(16).padStart(8, "0");
|
|
28796
29303
|
}
|
|
28797
|
-
function buildSection(
|
|
28798
|
-
const entries = forProject(
|
|
29304
|
+
function buildSection(projectPath2) {
|
|
29305
|
+
const entries = forProject(projectPath2, false);
|
|
28799
29306
|
if (!entries.length) {
|
|
28800
29307
|
return "\n";
|
|
28801
29308
|
}
|
|
@@ -28827,6 +29334,7 @@ function buildSection(projectPath) {
|
|
|
28827
29334
|
return out.join("\n");
|
|
28828
29335
|
}
|
|
28829
29336
|
function exportToFile(input) {
|
|
29337
|
+
if (isHostedMode()) return;
|
|
28830
29338
|
exportLoreFile(input.projectPath);
|
|
28831
29339
|
const pointerBody = "\n## Long-term Knowledge\n\nFor long-term knowledge entries managed by [lore](https://github.com/BYK/loreai) (gotchas, patterns, decisions, architecture), see [`.lore.md`](.lore.md) in the project root.\n";
|
|
28832
29340
|
const newSection = LORE_SECTION_START + pointerBody + LORE_SECTION_END + "\n";
|
|
@@ -28844,6 +29352,7 @@ function exportToFile(input) {
|
|
|
28844
29352
|
writeFileSync(input.filePath, result, "utf8");
|
|
28845
29353
|
}
|
|
28846
29354
|
function shouldImport(input) {
|
|
29355
|
+
if (isHostedMode()) return false;
|
|
28847
29356
|
if (!existsSync4(input.filePath)) return false;
|
|
28848
29357
|
const fileContent = readFileSync3(input.filePath, "utf8");
|
|
28849
29358
|
const { section } = splitFile(fileContent);
|
|
@@ -28853,7 +29362,7 @@ function shouldImport(input) {
|
|
|
28853
29362
|
const expected = buildSection(input.projectPath);
|
|
28854
29363
|
return hashSection(section) !== hashSection(expected);
|
|
28855
29364
|
}
|
|
28856
|
-
function _importEntries(entries,
|
|
29365
|
+
function _importEntries(entries, projectPath2) {
|
|
28857
29366
|
const seenIds = /* @__PURE__ */ new Set();
|
|
28858
29367
|
for (const entry of entries) {
|
|
28859
29368
|
if (entry.id !== null) {
|
|
@@ -28865,7 +29374,7 @@ function _importEntries(entries, projectPath) {
|
|
|
28865
29374
|
update(entry.id, { content: entry.content });
|
|
28866
29375
|
}
|
|
28867
29376
|
} else {
|
|
28868
|
-
const pid = ensureProject(
|
|
29377
|
+
const pid = ensureProject(projectPath2);
|
|
28869
29378
|
const fuzzyMatch = findFuzzyDuplicate({ title: entry.title, projectId: pid });
|
|
28870
29379
|
if (fuzzyMatch) {
|
|
28871
29380
|
if (fuzzyMatch.title !== entry.title || get(fuzzyMatch.id)?.content !== entry.content) {
|
|
@@ -28873,7 +29382,7 @@ function _importEntries(entries, projectPath) {
|
|
|
28873
29382
|
}
|
|
28874
29383
|
} else {
|
|
28875
29384
|
create({
|
|
28876
|
-
projectPath,
|
|
29385
|
+
projectPath: projectPath2,
|
|
28877
29386
|
category: entry.category,
|
|
28878
29387
|
title: entry.title,
|
|
28879
29388
|
content: entry.content,
|
|
@@ -28884,13 +29393,13 @@ function _importEntries(entries, projectPath) {
|
|
|
28884
29393
|
}
|
|
28885
29394
|
}
|
|
28886
29395
|
} else {
|
|
28887
|
-
const existing = forProject(
|
|
29396
|
+
const existing = forProject(projectPath2, false);
|
|
28888
29397
|
const titleMatch = existing.find(
|
|
28889
29398
|
(e) => e.title.toLowerCase() === entry.title.toLowerCase()
|
|
28890
29399
|
);
|
|
28891
29400
|
if (!titleMatch) {
|
|
28892
29401
|
create({
|
|
28893
|
-
projectPath,
|
|
29402
|
+
projectPath: projectPath2,
|
|
28894
29403
|
category: entry.category,
|
|
28895
29404
|
title: entry.title,
|
|
28896
29405
|
content: entry.content,
|
|
@@ -28902,6 +29411,7 @@ function _importEntries(entries, projectPath) {
|
|
|
28902
29411
|
}
|
|
28903
29412
|
}
|
|
28904
29413
|
function importFromFile(input) {
|
|
29414
|
+
if (isHostedMode()) return;
|
|
28905
29415
|
if (!existsSync4(input.filePath)) return;
|
|
28906
29416
|
const fileContent = readFileSync3(input.filePath, "utf8");
|
|
28907
29417
|
const { section } = splitFile(fileContent);
|
|
@@ -28910,14 +29420,16 @@ function importFromFile(input) {
|
|
|
28910
29420
|
if (!fileEntries.length) return;
|
|
28911
29421
|
_importEntries(fileEntries, input.projectPath);
|
|
28912
29422
|
}
|
|
28913
|
-
function loreFileExists(
|
|
28914
|
-
|
|
29423
|
+
function loreFileExists(projectPath2) {
|
|
29424
|
+
if (isHostedMode()) return false;
|
|
29425
|
+
return existsSync4(join7(projectPath2, LORE_FILE));
|
|
28915
29426
|
}
|
|
28916
|
-
function exportLoreFile(
|
|
28917
|
-
|
|
29427
|
+
function exportLoreFile(projectPath2) {
|
|
29428
|
+
if (isHostedMode()) return;
|
|
29429
|
+
const sectionBody = buildSection(projectPath2);
|
|
28918
29430
|
const content3 = LORE_FILE_HEADER + "\n" + sectionBody;
|
|
28919
29431
|
const contentHash2 = hashSection(content3);
|
|
28920
|
-
const fp = join7(
|
|
29432
|
+
const fp = join7(projectPath2, LORE_FILE);
|
|
28921
29433
|
const cached2 = getCache(fp);
|
|
28922
29434
|
if (cached2 && cached2.hash === contentHash2) {
|
|
28923
29435
|
return;
|
|
@@ -28926,8 +29438,9 @@ function exportLoreFile(projectPath) {
|
|
|
28926
29438
|
const { mtimeMs } = statSync3(fp);
|
|
28927
29439
|
setCache(fp, { mtimeMs, hash: contentHash2 });
|
|
28928
29440
|
}
|
|
28929
|
-
function shouldImportLoreFile(
|
|
28930
|
-
|
|
29441
|
+
function shouldImportLoreFile(projectPath2) {
|
|
29442
|
+
if (isHostedMode()) return false;
|
|
29443
|
+
const fp = join7(projectPath2, LORE_FILE);
|
|
28931
29444
|
if (!existsSync4(fp)) return false;
|
|
28932
29445
|
const { mtimeMs } = statSync3(fp);
|
|
28933
29446
|
const cached2 = getCache(fp);
|
|
@@ -28936,7 +29449,7 @@ function shouldImportLoreFile(projectPath) {
|
|
|
28936
29449
|
}
|
|
28937
29450
|
const fileContent = readFileSync3(fp, "utf8");
|
|
28938
29451
|
const fileHash = hashSection(fileContent);
|
|
28939
|
-
const expected = LORE_FILE_HEADER + "\n" + buildSection(
|
|
29452
|
+
const expected = LORE_FILE_HEADER + "\n" + buildSection(projectPath2);
|
|
28940
29453
|
const expectedHash = hashSection(expected);
|
|
28941
29454
|
if (fileHash === expectedHash) {
|
|
28942
29455
|
setCache(fp, { mtimeMs, hash: fileHash });
|
|
@@ -28944,13 +29457,14 @@ function shouldImportLoreFile(projectPath) {
|
|
|
28944
29457
|
}
|
|
28945
29458
|
return true;
|
|
28946
29459
|
}
|
|
28947
|
-
function importLoreFile(
|
|
28948
|
-
|
|
29460
|
+
function importLoreFile(projectPath2) {
|
|
29461
|
+
if (isHostedMode()) return;
|
|
29462
|
+
const fp = join7(projectPath2, LORE_FILE);
|
|
28949
29463
|
if (!existsSync4(fp)) return;
|
|
28950
29464
|
const fileContent = readFileSync3(fp, "utf8");
|
|
28951
29465
|
const fileEntries = parseEntriesFromSection(fileContent);
|
|
28952
29466
|
if (!fileEntries.length) return;
|
|
28953
|
-
_importEntries(fileEntries,
|
|
29467
|
+
_importEntries(fileEntries, projectPath2);
|
|
28954
29468
|
try {
|
|
28955
29469
|
const { mtimeMs } = statSync3(fp);
|
|
28956
29470
|
setCache(fp, { mtimeMs, hash: hashSection(fileContent) });
|
|
@@ -28969,8 +29483,8 @@ function listProjects() {
|
|
|
28969
29483
|
FROM projects p ORDER BY p.created_at DESC`
|
|
28970
29484
|
).all();
|
|
28971
29485
|
}
|
|
28972
|
-
function listSessions(
|
|
28973
|
-
const pid = ensureProject(
|
|
29486
|
+
function listSessions(projectPath2, limit = 50) {
|
|
29487
|
+
const pid = ensureProject(projectPath2);
|
|
28974
29488
|
return db().query(
|
|
28975
29489
|
`SELECT
|
|
28976
29490
|
session_id,
|
|
@@ -28989,8 +29503,8 @@ function listSessions(projectPath, limit = 50) {
|
|
|
28989
29503
|
LIMIT ?`
|
|
28990
29504
|
).all(pid, limit);
|
|
28991
29505
|
}
|
|
28992
|
-
function listDistillations(
|
|
28993
|
-
const pid = ensureProject(
|
|
29506
|
+
function listDistillations(projectPath2, opts) {
|
|
29507
|
+
const pid = ensureProject(projectPath2);
|
|
28994
29508
|
const limit = opts?.limit ?? 50;
|
|
28995
29509
|
if (opts?.sessionId) {
|
|
28996
29510
|
return db().query(
|
|
@@ -29039,8 +29553,8 @@ function globalStats() {
|
|
|
29039
29553
|
}
|
|
29040
29554
|
return { ...row, db_size_bytes };
|
|
29041
29555
|
}
|
|
29042
|
-
function countForProject(
|
|
29043
|
-
const pid = projectId(
|
|
29556
|
+
function countForProject(projectPath2) {
|
|
29557
|
+
const pid = projectId(projectPath2);
|
|
29044
29558
|
if (!pid)
|
|
29045
29559
|
return { knowledge: 0, messages: 0, distillations: 0, sessions: 0 };
|
|
29046
29560
|
const row = db().query(
|
|
@@ -29052,8 +29566,8 @@ function countForProject(projectPath) {
|
|
|
29052
29566
|
).get(pid, pid, pid, pid);
|
|
29053
29567
|
return row;
|
|
29054
29568
|
}
|
|
29055
|
-
function clearProject(
|
|
29056
|
-
const pid = ensureProject(
|
|
29569
|
+
function clearProject(projectPath2) {
|
|
29570
|
+
const pid = ensureProject(projectPath2);
|
|
29057
29571
|
const database = db();
|
|
29058
29572
|
const counts = {
|
|
29059
29573
|
knowledge: database.query(
|
|
@@ -29084,9 +29598,9 @@ function clearProject(projectPath) {
|
|
|
29084
29598
|
database.exec("ROLLBACK");
|
|
29085
29599
|
throw e;
|
|
29086
29600
|
}
|
|
29087
|
-
if (existsSync5(
|
|
29601
|
+
if (existsSync5(projectPath2)) {
|
|
29088
29602
|
try {
|
|
29089
|
-
exportLoreFile(
|
|
29603
|
+
exportLoreFile(projectPath2);
|
|
29090
29604
|
} catch {
|
|
29091
29605
|
}
|
|
29092
29606
|
}
|
|
@@ -29149,30 +29663,30 @@ function renameProject(projectId2, newName) {
|
|
|
29149
29663
|
const result = db().query("UPDATE projects SET name = ? WHERE id = ?").run(newName.trim(), projectId2);
|
|
29150
29664
|
return result.changes > 0;
|
|
29151
29665
|
}
|
|
29152
|
-
function clearKnowledge(
|
|
29153
|
-
const pid = ensureProject(
|
|
29666
|
+
function clearKnowledge(projectPath2) {
|
|
29667
|
+
const pid = ensureProject(projectPath2);
|
|
29154
29668
|
const count3 = db().query(
|
|
29155
29669
|
"SELECT COUNT(*) as c FROM knowledge WHERE project_id = ?"
|
|
29156
29670
|
).get(pid).c;
|
|
29157
29671
|
db().query("DELETE FROM knowledge WHERE project_id = ?").run(pid);
|
|
29158
|
-
if (existsSync5(
|
|
29672
|
+
if (existsSync5(projectPath2)) {
|
|
29159
29673
|
try {
|
|
29160
|
-
exportLoreFile(
|
|
29674
|
+
exportLoreFile(projectPath2);
|
|
29161
29675
|
} catch {
|
|
29162
29676
|
}
|
|
29163
29677
|
}
|
|
29164
29678
|
return count3;
|
|
29165
29679
|
}
|
|
29166
|
-
function clearTemporal(
|
|
29167
|
-
const pid = ensureProject(
|
|
29680
|
+
function clearTemporal(projectPath2) {
|
|
29681
|
+
const pid = ensureProject(projectPath2);
|
|
29168
29682
|
const count3 = db().query(
|
|
29169
29683
|
"SELECT COUNT(*) as c FROM temporal_messages WHERE project_id = ?"
|
|
29170
29684
|
).get(pid).c;
|
|
29171
29685
|
db().query("DELETE FROM temporal_messages WHERE project_id = ?").run(pid);
|
|
29172
29686
|
return count3;
|
|
29173
29687
|
}
|
|
29174
|
-
function clearDistillations(
|
|
29175
|
-
const pid = ensureProject(
|
|
29688
|
+
function clearDistillations(projectPath2) {
|
|
29689
|
+
const pid = ensureProject(projectPath2);
|
|
29176
29690
|
const count3 = db().query(
|
|
29177
29691
|
"SELECT COUNT(*) as c FROM distillations WHERE project_id = ?"
|
|
29178
29692
|
).get(pid).c;
|
|
@@ -29191,8 +29705,8 @@ function deleteDistillation(id) {
|
|
|
29191
29705
|
db().query("DELETE FROM distillations WHERE id = ?").run(id);
|
|
29192
29706
|
return true;
|
|
29193
29707
|
}
|
|
29194
|
-
function deleteSession(
|
|
29195
|
-
const pid = ensureProject(
|
|
29708
|
+
function deleteSession(projectPath2, sessionId) {
|
|
29709
|
+
const pid = ensureProject(projectPath2);
|
|
29196
29710
|
const database = db();
|
|
29197
29711
|
const msgCount = database.query(
|
|
29198
29712
|
"SELECT COUNT(*) as c FROM temporal_messages WHERE project_id = ? AND session_id = ?"
|
|
@@ -29436,9 +29950,22 @@ function setMaxContextTokens(tokens) {
|
|
|
29436
29950
|
function getMaxContextTokens() {
|
|
29437
29951
|
return maxContextTokensCeiling;
|
|
29438
29952
|
}
|
|
29439
|
-
function updateBustRate(cacheWrite, cacheRead, sessionID) {
|
|
29953
|
+
function updateBustRate(cacheWrite, cacheRead, sessionID, lastLayer) {
|
|
29440
29954
|
if (!sessionID) return;
|
|
29441
29955
|
const state = getSessionState(sessionID);
|
|
29956
|
+
if (lastLayer === 4) {
|
|
29957
|
+
state.consecutiveLayer4++;
|
|
29958
|
+
if (state.consecutiveLayer4 >= 5 && state.dynamicContextCap > 0 && maxContextTokensCeiling > 0) {
|
|
29959
|
+
state.dynamicContextCap = Math.min(
|
|
29960
|
+
maxContextTokensCeiling,
|
|
29961
|
+
Math.floor(state.dynamicContextCap * 1.1)
|
|
29962
|
+
);
|
|
29963
|
+
}
|
|
29964
|
+
return;
|
|
29965
|
+
}
|
|
29966
|
+
if (lastLayer !== void 0) {
|
|
29967
|
+
state.consecutiveLayer4 = 0;
|
|
29968
|
+
}
|
|
29442
29969
|
const total = cacheWrite + cacheRead;
|
|
29443
29970
|
if (total === 0) return;
|
|
29444
29971
|
const bustRatio = cacheWrite / total;
|
|
@@ -29493,6 +30020,7 @@ function makeSessionState() {
|
|
|
29493
30020
|
cameOutOfIdle: false,
|
|
29494
30021
|
postIdleCompact: false,
|
|
29495
30022
|
consecutiveHighLayer: 0,
|
|
30023
|
+
consecutiveLayer4: 0,
|
|
29496
30024
|
bustRateEMA: -1,
|
|
29497
30025
|
interBustIntervalEMA: -1,
|
|
29498
30026
|
lastBustAt: 0,
|
|
@@ -29506,6 +30034,16 @@ function getSessionState(sessionID) {
|
|
|
29506
30034
|
if (!state) {
|
|
29507
30035
|
state = makeSessionState();
|
|
29508
30036
|
state.forceMinLayer = loadForceMinLayer(sessionID);
|
|
30037
|
+
const persisted = loadSessionTracking(sessionID);
|
|
30038
|
+
if (persisted && persisted.lastTurnAt > 0) {
|
|
30039
|
+
state.dynamicContextCap = persisted.dynamicContextCap;
|
|
30040
|
+
state.bustRateEMA = persisted.bustRateEMA;
|
|
30041
|
+
state.interBustIntervalEMA = persisted.interBustIntervalEMA;
|
|
30042
|
+
state.lastLayer = persisted.lastLayer;
|
|
30043
|
+
state.lastKnownInput = persisted.lastKnownInput;
|
|
30044
|
+
state.lastTurnAt = persisted.lastTurnAt;
|
|
30045
|
+
state.lastBustAt = persisted.lastBustAt;
|
|
30046
|
+
}
|
|
29509
30047
|
sessionStates.set(sessionID, state);
|
|
29510
30048
|
}
|
|
29511
30049
|
return state;
|
|
@@ -29585,6 +30123,11 @@ function getLastTransformedCount(sessionID) {
|
|
|
29585
30123
|
function getLastTransformEstimate(sessionID) {
|
|
29586
30124
|
return sessionStates.get(sessionID)?.lastTransformEstimate ?? 0;
|
|
29587
30125
|
}
|
|
30126
|
+
function getLastLayer(sessionID) {
|
|
30127
|
+
if (sessionID) return sessionStates.get(sessionID)?.lastLayer ?? 0;
|
|
30128
|
+
const first = sessionStates.values().next().value;
|
|
30129
|
+
return first?.lastLayer ?? 0;
|
|
30130
|
+
}
|
|
29588
30131
|
function setForceMinLayer(layer, sessionID) {
|
|
29589
30132
|
if (sessionID) {
|
|
29590
30133
|
getSessionState(sessionID).forceMinLayer = layer;
|
|
@@ -29605,19 +30148,35 @@ function inspectSessionState(sessionID) {
|
|
|
29605
30148
|
cameOutOfIdle: state.cameOutOfIdle,
|
|
29606
30149
|
postIdleCompact: state.postIdleCompact,
|
|
29607
30150
|
lastTurnAt: state.lastTurnAt,
|
|
29608
|
-
distillationSnapshot: state.distillationSnapshot
|
|
30151
|
+
distillationSnapshot: state.distillationSnapshot,
|
|
30152
|
+
bustRateEMA: state.bustRateEMA,
|
|
30153
|
+
dynamicContextCap: state.dynamicContextCap,
|
|
30154
|
+
consecutiveLayer4: state.consecutiveLayer4
|
|
29609
30155
|
};
|
|
29610
30156
|
}
|
|
29611
30157
|
function setLastTurnAtForTest(sessionID, ms) {
|
|
29612
30158
|
getSessionState(sessionID).lastTurnAt = ms;
|
|
29613
30159
|
}
|
|
29614
|
-
function
|
|
29615
|
-
const
|
|
30160
|
+
function saveGradientState(sessionID) {
|
|
30161
|
+
const state = sessionStates.get(sessionID);
|
|
30162
|
+
if (!state) return;
|
|
30163
|
+
saveSessionTracking(sessionID, {
|
|
30164
|
+
dynamicContextCap: state.dynamicContextCap,
|
|
30165
|
+
bustRateEMA: state.bustRateEMA,
|
|
30166
|
+
interBustIntervalEMA: state.interBustIntervalEMA,
|
|
30167
|
+
lastLayer: state.lastLayer,
|
|
30168
|
+
lastKnownInput: state.lastKnownInput,
|
|
30169
|
+
lastTurnAt: state.lastTurnAt,
|
|
30170
|
+
lastBustAt: state.lastBustAt
|
|
30171
|
+
});
|
|
30172
|
+
}
|
|
30173
|
+
function loadDistillations(projectPath2, sessionID) {
|
|
30174
|
+
const pid = ensureProject(projectPath2);
|
|
29616
30175
|
const query = sessionID ? "SELECT id, observations, generation, token_count, created_at, session_id FROM distillations WHERE project_id = ? AND session_id = ? AND archived = 0 ORDER BY created_at ASC" : "SELECT id, observations, generation, token_count, created_at, session_id FROM distillations WHERE project_id = ? AND archived = 0 ORDER BY created_at ASC";
|
|
29617
30176
|
const params = sessionID ? [pid, sessionID] : [pid];
|
|
29618
30177
|
return db().query(query).all(...params);
|
|
29619
30178
|
}
|
|
29620
|
-
function loadDistillationsCached(
|
|
30179
|
+
function loadDistillationsCached(projectPath2, sessionID, messages, sessState) {
|
|
29621
30180
|
let lastUserMsgId = null;
|
|
29622
30181
|
for (let i = messages.length - 1; i >= 0; i--) {
|
|
29623
30182
|
if (messages[i].info.role === "user") {
|
|
@@ -29629,7 +30188,7 @@ function loadDistillationsCached(projectPath, sessionID, messages, sessState) {
|
|
|
29629
30188
|
if (snapshot && snapshot.lastUserMsgId === lastUserMsgId) {
|
|
29630
30189
|
return snapshot.rows;
|
|
29631
30190
|
}
|
|
29632
|
-
const rows = loadDistillations(
|
|
30191
|
+
const rows = loadDistillations(projectPath2, sessionID);
|
|
29633
30192
|
sessState.distillationSnapshot = { rows, lastUserMsgId };
|
|
29634
30193
|
info(
|
|
29635
30194
|
`distillation refresh: ${rows.length} rows (user msg ${lastUserMsgId?.substring(0, 16) ?? "none"})`
|
|
@@ -29895,6 +30454,26 @@ function buildPrefixMessages(formatted) {
|
|
|
29895
30454
|
}
|
|
29896
30455
|
];
|
|
29897
30456
|
}
|
|
30457
|
+
var DECISION_RE = /\b(?:decision|decided|chose|chosen|agreed)\b/i;
|
|
30458
|
+
var GOTCHA_RE = /\b(?:gotcha|(?:critical|known|subtle)\s+bug|broken|crash(?:ed|es)?|regression)\b/i;
|
|
30459
|
+
var ARCH_RE = /\b(?:architecture|design.(?:decision|pattern)|system.design)\b/i;
|
|
30460
|
+
function importanceBonus(d) {
|
|
30461
|
+
let bonus = 0;
|
|
30462
|
+
if (DECISION_RE.test(d.observations)) bonus += 0.3;
|
|
30463
|
+
if (GOTCHA_RE.test(d.observations)) bonus += 0.2;
|
|
30464
|
+
if (ARCH_RE.test(d.observations)) bonus += 0.1;
|
|
30465
|
+
if (d.generation >= 1) bonus += 0.2;
|
|
30466
|
+
return Math.min(bonus, 1);
|
|
30467
|
+
}
|
|
30468
|
+
function selectDistillations(all3, limit) {
|
|
30469
|
+
if (all3.length <= limit) return all3;
|
|
30470
|
+
const maxIdx = all3.length - 1;
|
|
30471
|
+
const scored = all3.map((d, i) => ({
|
|
30472
|
+
d,
|
|
30473
|
+
score: (maxIdx > 0 ? i / maxIdx : 1) * 0.7 + importanceBonus(d) * 0.3
|
|
30474
|
+
}));
|
|
30475
|
+
return scored.sort((a, b) => b.score - a.score).slice(0, limit).map((s) => s.d).sort((a, b) => a.created_at - b.created_at);
|
|
30476
|
+
}
|
|
29898
30477
|
function distilledPrefix(distillations) {
|
|
29899
30478
|
if (!distillations.length) return [];
|
|
29900
30479
|
const formatted = formatDistillations(distillations);
|
|
@@ -30012,6 +30591,11 @@ function tryFitStable(input) {
|
|
|
30012
30591
|
}
|
|
30013
30592
|
return result;
|
|
30014
30593
|
}
|
|
30594
|
+
var COMPRESSION_STAGES = [
|
|
30595
|
+
{ strip: "none", rawFrac: null, distFrac: null, distLimit: Infinity, protectedTurns: 0, useStableWindow: true },
|
|
30596
|
+
{ strip: "old-tools", rawFrac: 0.5, distFrac: null, distLimit: Infinity, protectedTurns: 2, useStableWindow: false },
|
|
30597
|
+
{ strip: "all-tools", rawFrac: 0.55, distFrac: 0.15, distLimit: 5, protectedTurns: 0, useStableWindow: false }
|
|
30598
|
+
];
|
|
30015
30599
|
var urgentDistillationMap = /* @__PURE__ */ new Map();
|
|
30016
30600
|
function needsUrgentDistillation(sessionID) {
|
|
30017
30601
|
const v = urgentDistillationMap.get(sessionID) ?? false;
|
|
@@ -30043,7 +30627,7 @@ function transformInner(input) {
|
|
|
30043
30627
|
if (calibrated) return true;
|
|
30044
30628
|
return result.totalTokens * UNCALIBRATED_SAFETY <= maxInput;
|
|
30045
30629
|
}
|
|
30046
|
-
if (calibrated && sessState.lastLayer >= 1 && input.messages.length >= sessState.lastKnownMessageCount) {
|
|
30630
|
+
if (calibrated && sessState.lastLayer >= 1 && sessState.lastLayer <= 3 && input.messages.length >= sessState.lastKnownMessageCount) {
|
|
30047
30631
|
effectiveMinLayer = Math.max(effectiveMinLayer, sessState.lastLayer);
|
|
30048
30632
|
}
|
|
30049
30633
|
const postIdleCompact = sessState.postIdleCompact;
|
|
@@ -30081,7 +30665,8 @@ function transformInner(input) {
|
|
|
30081
30665
|
totalTokens: Math.max(0, messageTokens),
|
|
30082
30666
|
usable,
|
|
30083
30667
|
distilledBudget,
|
|
30084
|
-
rawBudget
|
|
30668
|
+
rawBudget,
|
|
30669
|
+
refreshLtm: false
|
|
30085
30670
|
};
|
|
30086
30671
|
}
|
|
30087
30672
|
const turnStart = currentTurnStart(input.messages);
|
|
@@ -30091,67 +30676,52 @@ function transformInner(input) {
|
|
|
30091
30676
|
const msgs = distilledPrefix(distillations);
|
|
30092
30677
|
return { messages: msgs, tokens: msgs.reduce((sum, m) => sum + estimateMessage(m), 0) };
|
|
30093
30678
|
})();
|
|
30094
|
-
|
|
30095
|
-
const
|
|
30096
|
-
|
|
30097
|
-
|
|
30098
|
-
|
|
30099
|
-
|
|
30100
|
-
|
|
30101
|
-
|
|
30102
|
-
|
|
30103
|
-
|
|
30104
|
-
|
|
30105
|
-
|
|
30106
|
-
|
|
30107
|
-
|
|
30108
|
-
|
|
30109
|
-
|
|
30110
|
-
|
|
30111
|
-
|
|
30112
|
-
|
|
30679
|
+
for (let s = 0; s < COMPRESSION_STAGES.length; s++) {
|
|
30680
|
+
const stageLayer = s + 1;
|
|
30681
|
+
if (effectiveMinLayer > stageLayer) continue;
|
|
30682
|
+
const stage = COMPRESSION_STAGES[s];
|
|
30683
|
+
const stageRawBudget = stage.rawFrac !== null ? Math.floor(usable * stage.rawFrac) : rawBudget;
|
|
30684
|
+
const stageDistBudget = stage.distFrac !== null ? Math.floor(usable * stage.distFrac) : distilledBudget;
|
|
30685
|
+
let stagePrefix = cached2.messages;
|
|
30686
|
+
let stagePrefixTokens = cached2.tokens;
|
|
30687
|
+
if (stage.distLimit !== Infinity && distillations.length > stage.distLimit) {
|
|
30688
|
+
const trimmed = selectDistillations(distillations, stage.distLimit);
|
|
30689
|
+
stagePrefix = distilledPrefix(trimmed);
|
|
30690
|
+
stagePrefixTokens = stagePrefix.reduce((sum, m) => sum + estimateMessage(m), 0);
|
|
30691
|
+
}
|
|
30692
|
+
let result;
|
|
30693
|
+
if (stage.useStableWindow && sid) {
|
|
30694
|
+
result = tryFitStable({
|
|
30695
|
+
messages: dedupMessages,
|
|
30696
|
+
prefix: stagePrefix,
|
|
30697
|
+
prefixTokens: stagePrefixTokens,
|
|
30698
|
+
distilledBudget: stageDistBudget,
|
|
30699
|
+
rawBudget: stageRawBudget,
|
|
30700
|
+
sessionID: sid,
|
|
30701
|
+
sessState
|
|
30702
|
+
});
|
|
30703
|
+
} else {
|
|
30704
|
+
sessState.rawWindowCache = null;
|
|
30705
|
+
result = tryFit({
|
|
30706
|
+
messages: dedupMessages,
|
|
30707
|
+
prefix: stagePrefix,
|
|
30708
|
+
prefixTokens: stagePrefixTokens,
|
|
30709
|
+
distilledBudget: stageDistBudget,
|
|
30710
|
+
rawBudget: stageRawBudget,
|
|
30711
|
+
strip: stage.strip,
|
|
30712
|
+
protectedTurns: stage.protectedTurns
|
|
30713
|
+
});
|
|
30714
|
+
}
|
|
30715
|
+
if (fitsWithSafetyMargin(result)) {
|
|
30716
|
+
if (sid && (s > 0 || cached2.tokens === 0)) {
|
|
30113
30717
|
urgentDistillationMap.set(sid, true);
|
|
30114
30718
|
}
|
|
30115
|
-
return { ...
|
|
30719
|
+
return { ...result, layer: stageLayer, usable, distilledBudget, rawBudget, refreshLtm: false };
|
|
30116
30720
|
}
|
|
30117
30721
|
}
|
|
30118
30722
|
sessState.rawWindowCache = null;
|
|
30119
|
-
if (effectiveMinLayer <= 2) {
|
|
30120
|
-
const layer2 = tryFit({
|
|
30121
|
-
messages: dedupMessages,
|
|
30122
|
-
prefix: cached2.messages,
|
|
30123
|
-
prefixTokens: cached2.tokens,
|
|
30124
|
-
distilledBudget,
|
|
30125
|
-
rawBudget: Math.floor(usable * 0.5),
|
|
30126
|
-
// give raw more room
|
|
30127
|
-
strip: "old-tools",
|
|
30128
|
-
protectedTurns: 2
|
|
30129
|
-
});
|
|
30130
|
-
if (fitsWithSafetyMargin(layer2)) {
|
|
30131
|
-
if (sid) urgentDistillationMap.set(sid, true);
|
|
30132
|
-
return { ...layer2, layer: 2, usable, distilledBudget, rawBudget };
|
|
30133
|
-
}
|
|
30134
|
-
}
|
|
30135
|
-
const trimmedDistillations = distillations.slice(-5);
|
|
30136
|
-
const trimmedPrefix = distilledPrefix(trimmedDistillations);
|
|
30137
|
-
const trimmedPrefixTokens = trimmedPrefix.reduce(
|
|
30138
|
-
(sum, m) => sum + estimateMessage(m),
|
|
30139
|
-
0
|
|
30140
|
-
);
|
|
30141
|
-
const layer3 = tryFit({
|
|
30142
|
-
messages: dedupMessages,
|
|
30143
|
-
prefix: trimmedPrefix,
|
|
30144
|
-
prefixTokens: trimmedPrefixTokens,
|
|
30145
|
-
distilledBudget: Math.floor(usable * 0.15),
|
|
30146
|
-
rawBudget: Math.floor(usable * 0.55),
|
|
30147
|
-
strip: "all-tools"
|
|
30148
|
-
});
|
|
30149
|
-
if (fitsWithSafetyMargin(layer3)) {
|
|
30150
|
-
if (sid) urgentDistillationMap.set(sid, true);
|
|
30151
|
-
return { ...layer3, layer: 3, usable, distilledBudget, rawBudget };
|
|
30152
|
-
}
|
|
30153
30723
|
if (sid) urgentDistillationMap.set(sid, true);
|
|
30154
|
-
const nuclearDistillations = distillations
|
|
30724
|
+
const nuclearDistillations = selectDistillations(distillations, 2);
|
|
30155
30725
|
const nuclearPrefix = distilledPrefix(nuclearDistillations);
|
|
30156
30726
|
const nuclearPrefixTokens = nuclearPrefix.reduce(
|
|
30157
30727
|
(sum, m) => sum + estimateMessage(m),
|
|
@@ -30190,7 +30760,8 @@ function transformInner(input) {
|
|
|
30190
30760
|
totalTokens: nuclearPrefixTokens + nuclearRawTokens,
|
|
30191
30761
|
usable,
|
|
30192
30762
|
distilledBudget,
|
|
30193
|
-
rawBudget
|
|
30763
|
+
rawBudget,
|
|
30764
|
+
refreshLtm: true
|
|
30194
30765
|
};
|
|
30195
30766
|
}
|
|
30196
30767
|
function transform2(input) {
|
|
@@ -30297,6 +30868,185 @@ function isWorkerSession(sessionID) {
|
|
|
30297
30868
|
return workerSessionIDs.has(sessionID);
|
|
30298
30869
|
}
|
|
30299
30870
|
|
|
30871
|
+
// ../../node_modules/.bun/yocto-queue@1.2.2/node_modules/yocto-queue/index.js
|
|
30872
|
+
var Node = class {
|
|
30873
|
+
value;
|
|
30874
|
+
next;
|
|
30875
|
+
constructor(value) {
|
|
30876
|
+
this.value = value;
|
|
30877
|
+
}
|
|
30878
|
+
};
|
|
30879
|
+
var Queue = class {
|
|
30880
|
+
#head;
|
|
30881
|
+
#tail;
|
|
30882
|
+
#size;
|
|
30883
|
+
constructor() {
|
|
30884
|
+
this.clear();
|
|
30885
|
+
}
|
|
30886
|
+
enqueue(value) {
|
|
30887
|
+
const node2 = new Node(value);
|
|
30888
|
+
if (this.#head) {
|
|
30889
|
+
this.#tail.next = node2;
|
|
30890
|
+
this.#tail = node2;
|
|
30891
|
+
} else {
|
|
30892
|
+
this.#head = node2;
|
|
30893
|
+
this.#tail = node2;
|
|
30894
|
+
}
|
|
30895
|
+
this.#size++;
|
|
30896
|
+
}
|
|
30897
|
+
dequeue() {
|
|
30898
|
+
const current2 = this.#head;
|
|
30899
|
+
if (!current2) {
|
|
30900
|
+
return;
|
|
30901
|
+
}
|
|
30902
|
+
this.#head = this.#head.next;
|
|
30903
|
+
this.#size--;
|
|
30904
|
+
if (!this.#head) {
|
|
30905
|
+
this.#tail = void 0;
|
|
30906
|
+
}
|
|
30907
|
+
return current2.value;
|
|
30908
|
+
}
|
|
30909
|
+
peek() {
|
|
30910
|
+
if (!this.#head) {
|
|
30911
|
+
return;
|
|
30912
|
+
}
|
|
30913
|
+
return this.#head.value;
|
|
30914
|
+
}
|
|
30915
|
+
clear() {
|
|
30916
|
+
this.#head = void 0;
|
|
30917
|
+
this.#tail = void 0;
|
|
30918
|
+
this.#size = 0;
|
|
30919
|
+
}
|
|
30920
|
+
get size() {
|
|
30921
|
+
return this.#size;
|
|
30922
|
+
}
|
|
30923
|
+
*[Symbol.iterator]() {
|
|
30924
|
+
let current2 = this.#head;
|
|
30925
|
+
while (current2) {
|
|
30926
|
+
yield current2.value;
|
|
30927
|
+
current2 = current2.next;
|
|
30928
|
+
}
|
|
30929
|
+
}
|
|
30930
|
+
*drain() {
|
|
30931
|
+
while (this.#head) {
|
|
30932
|
+
yield this.dequeue();
|
|
30933
|
+
}
|
|
30934
|
+
}
|
|
30935
|
+
};
|
|
30936
|
+
|
|
30937
|
+
// ../../node_modules/.bun/p-limit@7.3.0/node_modules/p-limit/index.js
|
|
30938
|
+
function pLimit(concurrency) {
|
|
30939
|
+
let rejectOnClear = false;
|
|
30940
|
+
if (typeof concurrency === "object") {
|
|
30941
|
+
({ concurrency, rejectOnClear = false } = concurrency);
|
|
30942
|
+
}
|
|
30943
|
+
validateConcurrency(concurrency);
|
|
30944
|
+
if (typeof rejectOnClear !== "boolean") {
|
|
30945
|
+
throw new TypeError("Expected `rejectOnClear` to be a boolean");
|
|
30946
|
+
}
|
|
30947
|
+
const queue = new Queue();
|
|
30948
|
+
let activeCount = 0;
|
|
30949
|
+
const resumeNext = () => {
|
|
30950
|
+
if (activeCount < concurrency && queue.size > 0) {
|
|
30951
|
+
activeCount++;
|
|
30952
|
+
queue.dequeue().run();
|
|
30953
|
+
}
|
|
30954
|
+
};
|
|
30955
|
+
const next = () => {
|
|
30956
|
+
activeCount--;
|
|
30957
|
+
resumeNext();
|
|
30958
|
+
};
|
|
30959
|
+
const run3 = async (function_, resolve, arguments_) => {
|
|
30960
|
+
const result = (async () => function_(...arguments_))();
|
|
30961
|
+
resolve(result);
|
|
30962
|
+
try {
|
|
30963
|
+
await result;
|
|
30964
|
+
} catch {
|
|
30965
|
+
}
|
|
30966
|
+
next();
|
|
30967
|
+
};
|
|
30968
|
+
const enqueue = (function_, resolve, reject, arguments_) => {
|
|
30969
|
+
const queueItem = { reject };
|
|
30970
|
+
new Promise((internalResolve) => {
|
|
30971
|
+
queueItem.run = internalResolve;
|
|
30972
|
+
queue.enqueue(queueItem);
|
|
30973
|
+
}).then(run3.bind(void 0, function_, resolve, arguments_));
|
|
30974
|
+
if (activeCount < concurrency) {
|
|
30975
|
+
resumeNext();
|
|
30976
|
+
}
|
|
30977
|
+
};
|
|
30978
|
+
const generator = (function_, ...arguments_) => new Promise((resolve, reject) => {
|
|
30979
|
+
enqueue(function_, resolve, reject, arguments_);
|
|
30980
|
+
});
|
|
30981
|
+
Object.defineProperties(generator, {
|
|
30982
|
+
activeCount: {
|
|
30983
|
+
get: () => activeCount
|
|
30984
|
+
},
|
|
30985
|
+
pendingCount: {
|
|
30986
|
+
get: () => queue.size
|
|
30987
|
+
},
|
|
30988
|
+
clearQueue: {
|
|
30989
|
+
value() {
|
|
30990
|
+
if (!rejectOnClear) {
|
|
30991
|
+
queue.clear();
|
|
30992
|
+
return;
|
|
30993
|
+
}
|
|
30994
|
+
const abortError = AbortSignal.abort().reason;
|
|
30995
|
+
while (queue.size > 0) {
|
|
30996
|
+
queue.dequeue().reject(abortError);
|
|
30997
|
+
}
|
|
30998
|
+
}
|
|
30999
|
+
},
|
|
31000
|
+
concurrency: {
|
|
31001
|
+
get: () => concurrency,
|
|
31002
|
+
set(newConcurrency) {
|
|
31003
|
+
validateConcurrency(newConcurrency);
|
|
31004
|
+
concurrency = newConcurrency;
|
|
31005
|
+
queueMicrotask(() => {
|
|
31006
|
+
while (activeCount < concurrency && queue.size > 0) {
|
|
31007
|
+
resumeNext();
|
|
31008
|
+
}
|
|
31009
|
+
});
|
|
31010
|
+
}
|
|
31011
|
+
},
|
|
31012
|
+
map: {
|
|
31013
|
+
async value(iterable, function_) {
|
|
31014
|
+
const promises = Array.from(iterable, (value, index2) => this(function_, value, index2));
|
|
31015
|
+
return Promise.all(promises);
|
|
31016
|
+
}
|
|
31017
|
+
}
|
|
31018
|
+
});
|
|
31019
|
+
return generator;
|
|
31020
|
+
}
|
|
31021
|
+
function validateConcurrency(concurrency) {
|
|
31022
|
+
if (!((Number.isInteger(concurrency) || concurrency === Number.POSITIVE_INFINITY) && concurrency > 0)) {
|
|
31023
|
+
throw new TypeError("Expected `concurrency` to be a number from 1 and up");
|
|
31024
|
+
}
|
|
31025
|
+
}
|
|
31026
|
+
|
|
31027
|
+
// src/session-limiter.ts
|
|
31028
|
+
function createLimiterPool() {
|
|
31029
|
+
const limiters = /* @__PURE__ */ new Map();
|
|
31030
|
+
function get2(key) {
|
|
31031
|
+
let limiter = limiters.get(key);
|
|
31032
|
+
if (!limiter) {
|
|
31033
|
+
limiter = pLimit(1);
|
|
31034
|
+
limiters.set(key, limiter);
|
|
31035
|
+
}
|
|
31036
|
+
return limiter;
|
|
31037
|
+
}
|
|
31038
|
+
function isBusy(key) {
|
|
31039
|
+
const limiter = limiters.get(key);
|
|
31040
|
+
return limiter ? limiter.activeCount + limiter.pendingCount > 0 : false;
|
|
31041
|
+
}
|
|
31042
|
+
function clear() {
|
|
31043
|
+
limiters.clear();
|
|
31044
|
+
}
|
|
31045
|
+
return { get: get2, isBusy, clear };
|
|
31046
|
+
}
|
|
31047
|
+
var distillLimiter = createLimiterPool();
|
|
31048
|
+
var curatorLimiter = createLimiterPool();
|
|
31049
|
+
|
|
30300
31050
|
// src/distillation.ts
|
|
30301
31051
|
function compressionRatio(distilledTokens, sourceTokens) {
|
|
30302
31052
|
if (sourceTokens <= 0) return 0;
|
|
@@ -30422,18 +31172,18 @@ function parseDistillationResult(text4) {
|
|
|
30422
31172
|
if (!observations) return null;
|
|
30423
31173
|
return { observations };
|
|
30424
31174
|
}
|
|
30425
|
-
function latestObservations(
|
|
30426
|
-
const pid = ensureProject(
|
|
31175
|
+
function latestObservations(projectPath2, sessionID) {
|
|
31176
|
+
const pid = ensureProject(projectPath2);
|
|
30427
31177
|
const row = db().query(
|
|
30428
31178
|
"SELECT observations FROM distillations WHERE project_id = ? AND session_id = ? ORDER BY created_at DESC LIMIT 1"
|
|
30429
31179
|
).get(pid, sessionID);
|
|
30430
31180
|
return row?.observations || void 0;
|
|
30431
31181
|
}
|
|
30432
|
-
function latestMetaObservations(
|
|
30433
|
-
return latestMeta(
|
|
31182
|
+
function latestMetaObservations(projectPath2, sessionID) {
|
|
31183
|
+
return latestMeta(projectPath2, sessionID)?.observations;
|
|
30434
31184
|
}
|
|
30435
|
-
function latestMeta(
|
|
30436
|
-
const pid = ensureProject(
|
|
31185
|
+
function latestMeta(projectPath2, sessionID) {
|
|
31186
|
+
const pid = ensureProject(projectPath2);
|
|
30437
31187
|
const row = db().query(
|
|
30438
31188
|
`SELECT observations, generation FROM distillations
|
|
30439
31189
|
WHERE project_id = ? AND session_id = ? AND generation > 0
|
|
@@ -30451,8 +31201,8 @@ function parseSourceIds(raw) {
|
|
|
30451
31201
|
return [];
|
|
30452
31202
|
}
|
|
30453
31203
|
}
|
|
30454
|
-
function loadForSession(
|
|
30455
|
-
const pid = ensureProject(
|
|
31204
|
+
function loadForSession(projectPath2, sessionID, includeArchived = false) {
|
|
31205
|
+
const pid = ensureProject(projectPath2);
|
|
30456
31206
|
const sql = includeArchived ? "SELECT id, project_id, session_id, observations, source_ids, generation, token_count, created_at, r_compression, c_norm FROM distillations WHERE project_id = ? AND session_id = ? ORDER BY created_at ASC" : "SELECT id, project_id, session_id, observations, source_ids, generation, token_count, created_at, r_compression, c_norm FROM distillations WHERE project_id = ? AND session_id = ? AND archived = 0 ORDER BY created_at ASC";
|
|
30457
31207
|
const rows = db().query(sql).all(pid, sessionID);
|
|
30458
31208
|
return rows.map((r) => ({
|
|
@@ -30487,14 +31237,14 @@ function storeDistillation(input) {
|
|
|
30487
31237
|
);
|
|
30488
31238
|
return id;
|
|
30489
31239
|
}
|
|
30490
|
-
function gen0Count(
|
|
30491
|
-
const pid = ensureProject(
|
|
31240
|
+
function gen0Count(projectPath2, sessionID) {
|
|
31241
|
+
const pid = ensureProject(projectPath2);
|
|
30492
31242
|
return db().query(
|
|
30493
31243
|
"SELECT COUNT(*) as count FROM distillations WHERE project_id = ? AND session_id = ? AND generation = 0 AND archived = 0"
|
|
30494
31244
|
).get(pid, sessionID).count;
|
|
30495
31245
|
}
|
|
30496
|
-
function loadGen0(
|
|
30497
|
-
const pid = ensureProject(
|
|
31246
|
+
function loadGen0(projectPath2, sessionID) {
|
|
31247
|
+
const pid = ensureProject(projectPath2);
|
|
30498
31248
|
const rows = db().query(
|
|
30499
31249
|
"SELECT id, project_id, session_id, observations, source_ids, generation, token_count, created_at, r_compression, c_norm FROM distillations WHERE project_id = ? AND session_id = ? AND generation = 0 AND archived = 0 ORDER BY created_at ASC"
|
|
30500
31250
|
).all(pid, sessionID);
|
|
@@ -30510,8 +31260,8 @@ function archiveDistillations(ids) {
|
|
|
30510
31260
|
`UPDATE distillations SET archived = 1 WHERE id IN (${placeholders})`
|
|
30511
31261
|
).run(...ids);
|
|
30512
31262
|
}
|
|
30513
|
-
function resetOrphans(
|
|
30514
|
-
const pid = ensureProject(
|
|
31263
|
+
function resetOrphans(projectPath2, sessionID) {
|
|
31264
|
+
const pid = ensureProject(projectPath2);
|
|
30515
31265
|
const rows = db().query(
|
|
30516
31266
|
"SELECT source_ids FROM distillations WHERE project_id = ? AND session_id = ?"
|
|
30517
31267
|
).all(pid, sessionID);
|
|
@@ -30541,6 +31291,9 @@ function resetOrphans(projectPath, sessionID) {
|
|
|
30541
31291
|
return orphans.length;
|
|
30542
31292
|
}
|
|
30543
31293
|
async function run(input) {
|
|
31294
|
+
return distillLimiter.get(input.sessionID)(() => runInner(input));
|
|
31295
|
+
}
|
|
31296
|
+
async function runInner(input) {
|
|
30544
31297
|
const orphans = resetOrphans(input.projectPath, input.sessionID);
|
|
30545
31298
|
if (orphans > 0) {
|
|
30546
31299
|
info(
|
|
@@ -30584,7 +31337,7 @@ async function run(input) {
|
|
|
30584
31337
|
}
|
|
30585
31338
|
}
|
|
30586
31339
|
if (!input.skipMeta && gen0Count(input.projectPath, input.sessionID) >= cfg.distillation.metaThreshold) {
|
|
30587
|
-
await
|
|
31340
|
+
await metaDistillInner({
|
|
30588
31341
|
llm: input.llm,
|
|
30589
31342
|
projectPath: input.projectPath,
|
|
30590
31343
|
sessionID: input.sessionID,
|
|
@@ -30634,17 +31387,25 @@ async function distillSegment(input) {
|
|
|
30634
31387
|
);
|
|
30635
31388
|
return null;
|
|
30636
31389
|
}
|
|
30637
|
-
|
|
30638
|
-
|
|
30639
|
-
|
|
30640
|
-
|
|
30641
|
-
|
|
30642
|
-
|
|
30643
|
-
|
|
30644
|
-
|
|
30645
|
-
|
|
30646
|
-
|
|
30647
|
-
|
|
31390
|
+
let distillId;
|
|
31391
|
+
db().exec("BEGIN IMMEDIATE");
|
|
31392
|
+
try {
|
|
31393
|
+
distillId = storeDistillation({
|
|
31394
|
+
projectPath: input.projectPath,
|
|
31395
|
+
sessionID: input.sessionID,
|
|
31396
|
+
observations: result.observations,
|
|
31397
|
+
sourceIDs: input.messages.map((m) => m.id),
|
|
31398
|
+
generation: 0,
|
|
31399
|
+
rCompression: rComp,
|
|
31400
|
+
cNorm,
|
|
31401
|
+
callType: input.callType
|
|
31402
|
+
});
|
|
31403
|
+
markDistilled(input.messages.map((m) => m.id));
|
|
31404
|
+
db().exec("COMMIT");
|
|
31405
|
+
} catch (e) {
|
|
31406
|
+
db().exec("ROLLBACK");
|
|
31407
|
+
throw e;
|
|
31408
|
+
}
|
|
30648
31409
|
info(
|
|
30649
31410
|
`distill segment: ${input.messages.length} msgs, ${sourceTokens}\u2192${distilledTokens} tokens, R=${rComp.toFixed(2)}, C_norm=${cNorm.toFixed(3)}`
|
|
30650
31411
|
);
|
|
@@ -30678,6 +31439,9 @@ async function distillSegment(input) {
|
|
|
30678
31439
|
return result;
|
|
30679
31440
|
}
|
|
30680
31441
|
async function metaDistill(input) {
|
|
31442
|
+
return distillLimiter.get(input.sessionID)(() => metaDistillInner(input));
|
|
31443
|
+
}
|
|
31444
|
+
async function metaDistillInner(input) {
|
|
30681
31445
|
const existing = loadGen0(input.projectPath, input.sessionID);
|
|
30682
31446
|
const priorMeta = latestMeta(input.projectPath, input.sessionID);
|
|
30683
31447
|
if (priorMeta) {
|
|
@@ -30977,11 +31741,27 @@ function applyOps(ops, input) {
|
|
|
30977
31741
|
return { created, updated, deleted };
|
|
30978
31742
|
}
|
|
30979
31743
|
var lastCuratedAt = /* @__PURE__ */ new Map();
|
|
31744
|
+
function getLastCuratedAt(sessionID) {
|
|
31745
|
+
const cached2 = lastCuratedAt.get(sessionID);
|
|
31746
|
+
if (cached2 !== void 0) return cached2;
|
|
31747
|
+
const persisted = loadSessionTracking(sessionID);
|
|
31748
|
+
const ts = persisted?.lastCuratedAt ?? 0;
|
|
31749
|
+
lastCuratedAt.set(sessionID, ts);
|
|
31750
|
+
return ts;
|
|
31751
|
+
}
|
|
30980
31752
|
async function run2(input) {
|
|
30981
31753
|
const cfg = config2();
|
|
30982
31754
|
if (!cfg.curator.enabled) return { created: 0, updated: 0, deleted: 0 };
|
|
31755
|
+
if (curatorLimiter.isBusy(input.sessionID)) {
|
|
31756
|
+
info(`curation skipped: already running for session ${input.sessionID.slice(0, 16)}`);
|
|
31757
|
+
return { created: 0, updated: 0, deleted: 0 };
|
|
31758
|
+
}
|
|
31759
|
+
return curatorLimiter.get(input.sessionID)(() => runInner2(input));
|
|
31760
|
+
}
|
|
31761
|
+
async function runInner2(input) {
|
|
31762
|
+
const cfg = config2();
|
|
30983
31763
|
const all3 = bySession(input.projectPath, input.sessionID);
|
|
30984
|
-
const sessionCuratedAt =
|
|
31764
|
+
const sessionCuratedAt = getLastCuratedAt(input.sessionID);
|
|
30985
31765
|
const recent = all3.filter((m) => m.created_at > sessionCuratedAt);
|
|
30986
31766
|
if (recent.length < 3) return { created: 0, updated: 0, deleted: 0 };
|
|
30987
31767
|
const text4 = recent.map((m) => `[${m.role}] ${m.content}`).join("\n\n");
|
|
@@ -31025,11 +31805,22 @@ async function run2(input) {
|
|
|
31025
31805
|
info(`post-curation dedup: merged ${dupes.totalRemoved} duplicate entries`);
|
|
31026
31806
|
result.deleted += dupes.totalRemoved;
|
|
31027
31807
|
}
|
|
31808
|
+
if (dupes.pairSimilarities.size > 0) {
|
|
31809
|
+
const pid = ensureProject(input.projectPath);
|
|
31810
|
+
recordAutoSignals(pid, dupes);
|
|
31811
|
+
const newThreshold = calibrateDedupThreshold(pid);
|
|
31812
|
+
if (newThreshold !== null) {
|
|
31813
|
+
const count3 = getDedupFeedbackCount(pid);
|
|
31814
|
+
saveCalibratedThreshold(pid, newThreshold, count3);
|
|
31815
|
+
}
|
|
31816
|
+
}
|
|
31028
31817
|
} catch (err) {
|
|
31029
31818
|
warn("post-curation dedup failed (non-fatal):", err);
|
|
31030
31819
|
}
|
|
31031
31820
|
}
|
|
31032
|
-
|
|
31821
|
+
const now = Date.now();
|
|
31822
|
+
lastCuratedAt.set(input.sessionID, now);
|
|
31823
|
+
saveSessionTracking(input.sessionID, { lastCuratedAt: now });
|
|
31033
31824
|
return result;
|
|
31034
31825
|
}
|
|
31035
31826
|
function resetCurationTracker(sessionID) {
|
|
@@ -31102,11 +31893,11 @@ function clearProviders() {
|
|
|
31102
31893
|
}
|
|
31103
31894
|
|
|
31104
31895
|
// src/import/detect.ts
|
|
31105
|
-
function detectAll(
|
|
31896
|
+
function detectAll(projectPath2) {
|
|
31106
31897
|
const results = [];
|
|
31107
31898
|
for (const provider of getProviders()) {
|
|
31108
31899
|
try {
|
|
31109
|
-
const sessions = provider.detect(
|
|
31900
|
+
const sessions = provider.detect(projectPath2);
|
|
31110
31901
|
if (sessions.length > 0) {
|
|
31111
31902
|
results.push({
|
|
31112
31903
|
agentName: provider.name,
|
|
@@ -31195,8 +31986,8 @@ async function extractKnowledge(input) {
|
|
|
31195
31986
|
}
|
|
31196
31987
|
|
|
31197
31988
|
// src/import/history.ts
|
|
31198
|
-
function isImported(
|
|
31199
|
-
const projectId2 = ensureProject(
|
|
31989
|
+
function isImported(projectPath2, agentName, sourceId, sourceHash) {
|
|
31990
|
+
const projectId2 = ensureProject(projectPath2);
|
|
31200
31991
|
const row = db().query(
|
|
31201
31992
|
`SELECT * FROM import_history
|
|
31202
31993
|
WHERE project_id = ? AND agent_name = ? AND source_id = ?`
|
|
@@ -31205,8 +31996,8 @@ function isImported(projectPath, agentName, sourceId, sourceHash) {
|
|
|
31205
31996
|
if (row.source_hash !== sourceHash) return null;
|
|
31206
31997
|
return row;
|
|
31207
31998
|
}
|
|
31208
|
-
function recordImport(
|
|
31209
|
-
const projectId2 = ensureProject(
|
|
31999
|
+
function recordImport(projectPath2, agentName, sourceId, sourceHash, stats) {
|
|
32000
|
+
const projectId2 = ensureProject(projectPath2);
|
|
31210
32001
|
db().query(
|
|
31211
32002
|
`INSERT OR REPLACE INTO import_history
|
|
31212
32003
|
(id, project_id, agent_name, source_id, source_hash, entries_created, entries_updated, imported_at)
|
|
@@ -31222,8 +32013,8 @@ function recordImport(projectPath, agentName, sourceId, sourceHash, stats) {
|
|
|
31222
32013
|
Date.now()
|
|
31223
32014
|
);
|
|
31224
32015
|
}
|
|
31225
|
-
function listImports(
|
|
31226
|
-
const projectId2 = ensureProject(
|
|
32016
|
+
function listImports(projectPath2) {
|
|
32017
|
+
const projectId2 = ensureProject(projectPath2);
|
|
31227
32018
|
return db().query(
|
|
31228
32019
|
`SELECT * FROM import_history
|
|
31229
32020
|
WHERE project_id = ? AND source_id != '__declined__'
|
|
@@ -31241,8 +32032,8 @@ import { homedir as homedir2 } from "os";
|
|
|
31241
32032
|
var CLAUDE_DIR = join8(homedir2(), ".claude", "projects");
|
|
31242
32033
|
var MAX_TOOL_OUTPUT_CHARS = 500;
|
|
31243
32034
|
var DEFAULT_MAX_TOKENS = 12288;
|
|
31244
|
-
function manglePath(
|
|
31245
|
-
return
|
|
32035
|
+
function manglePath(projectPath2) {
|
|
32036
|
+
return projectPath2.replace(/\//g, "-");
|
|
31246
32037
|
}
|
|
31247
32038
|
function estimateTokens4(text4) {
|
|
31248
32039
|
return Math.ceil(text4.length / 3);
|
|
@@ -31356,8 +32147,8 @@ function getSessionMetadata(filePath) {
|
|
|
31356
32147
|
var claudeCodeProvider = {
|
|
31357
32148
|
name: "claude-code",
|
|
31358
32149
|
displayName: "Claude Code",
|
|
31359
|
-
detect(
|
|
31360
|
-
const mangled = manglePath(
|
|
32150
|
+
detect(projectPath2) {
|
|
32151
|
+
const mangled = manglePath(projectPath2);
|
|
31361
32152
|
const dir = join8(CLAUDE_DIR, mangled);
|
|
31362
32153
|
let entries;
|
|
31363
32154
|
try {
|
|
@@ -31555,7 +32346,7 @@ function getSessionMeta(filePath) {
|
|
|
31555
32346
|
var codexProvider = {
|
|
31556
32347
|
name: "codex",
|
|
31557
32348
|
displayName: "Codex",
|
|
31558
|
-
detect(
|
|
32349
|
+
detect(projectPath2) {
|
|
31559
32350
|
const sessions = [];
|
|
31560
32351
|
const allFiles = [
|
|
31561
32352
|
...findJsonlFiles(SESSIONS_DIR),
|
|
@@ -31564,7 +32355,7 @@ var codexProvider = {
|
|
|
31564
32355
|
for (const filePath of allFiles) {
|
|
31565
32356
|
const meta3 = getSessionMeta(filePath);
|
|
31566
32357
|
if (!meta3) continue;
|
|
31567
|
-
if (meta3.cwd !==
|
|
32358
|
+
if (meta3.cwd !== projectPath2) continue;
|
|
31568
32359
|
if (meta3.messageCount < 3) continue;
|
|
31569
32360
|
const ts = new Date(meta3.timestamp).getTime();
|
|
31570
32361
|
const estimatedTokens = Math.ceil(meta3.fileSize / 5);
|
|
@@ -31696,14 +32487,14 @@ function partsToConversationText(parts) {
|
|
|
31696
32487
|
var opencodeProvider = {
|
|
31697
32488
|
name: "opencode",
|
|
31698
32489
|
displayName: "OpenCode",
|
|
31699
|
-
detect(
|
|
32490
|
+
detect(projectPath2) {
|
|
31700
32491
|
const database = openDB();
|
|
31701
32492
|
if (!database) return [];
|
|
31702
32493
|
try {
|
|
31703
32494
|
if (!tableExists(database, "project") || !tableExists(database, "session") || !tableExists(database, "message")) {
|
|
31704
32495
|
return [];
|
|
31705
32496
|
}
|
|
31706
|
-
const project = database.query("SELECT id FROM project WHERE worktree = ?").get(
|
|
32497
|
+
const project = database.query("SELECT id FROM project WHERE worktree = ?").get(projectPath2);
|
|
31707
32498
|
if (!project) return [];
|
|
31708
32499
|
const sessions = database.query(
|
|
31709
32500
|
`SELECT s.id, s.title, s.time_created, s.time_updated,
|
|
@@ -31868,7 +32659,7 @@ function findGlobalStorageDirs() {
|
|
|
31868
32659
|
}
|
|
31869
32660
|
return dirs;
|
|
31870
32661
|
}
|
|
31871
|
-
function loadTaskHistory(storageDir,
|
|
32662
|
+
function loadTaskHistory(storageDir, projectPath2) {
|
|
31872
32663
|
const paths = [
|
|
31873
32664
|
join11(storageDir, "state", "taskHistory.json"),
|
|
31874
32665
|
join11(storageDir, "taskHistory.json")
|
|
@@ -31880,7 +32671,7 @@ function loadTaskHistory(storageDir, projectPath) {
|
|
|
31880
32671
|
const items = JSON.parse(raw);
|
|
31881
32672
|
if (!Array.isArray(items)) continue;
|
|
31882
32673
|
return items.filter(
|
|
31883
|
-
(item) => item.cwdOnTaskInitialization ===
|
|
32674
|
+
(item) => item.cwdOnTaskInitialization === projectPath2
|
|
31884
32675
|
);
|
|
31885
32676
|
} catch {
|
|
31886
32677
|
continue;
|
|
@@ -31933,11 +32724,11 @@ function messageToText(msg) {
|
|
|
31933
32724
|
var clineProvider = {
|
|
31934
32725
|
name: "cline",
|
|
31935
32726
|
displayName: "Cline",
|
|
31936
|
-
detect(
|
|
32727
|
+
detect(projectPath2) {
|
|
31937
32728
|
const sessions = [];
|
|
31938
32729
|
const storageDirs = findGlobalStorageDirs();
|
|
31939
32730
|
for (const storageDir of storageDirs) {
|
|
31940
|
-
const tasks = loadTaskHistory(storageDir,
|
|
32731
|
+
const tasks = loadTaskHistory(storageDir, projectPath2);
|
|
31941
32732
|
for (const task of tasks) {
|
|
31942
32733
|
const taskDir = join11(storageDir, "tasks", task.id);
|
|
31943
32734
|
if (!existsSync8(taskDir)) continue;
|
|
@@ -32085,11 +32876,11 @@ function historyItemToText(item) {
|
|
|
32085
32876
|
var continueProvider = {
|
|
32086
32877
|
name: "continue",
|
|
32087
32878
|
displayName: "Continue",
|
|
32088
|
-
detect(
|
|
32879
|
+
detect(projectPath2) {
|
|
32089
32880
|
const sessions = [];
|
|
32090
32881
|
const index2 = loadSessionIndex();
|
|
32091
32882
|
for (const meta3 of index2) {
|
|
32092
|
-
if (meta3.workspaceDirectory !==
|
|
32883
|
+
if (meta3.workspaceDirectory !== projectPath2) continue;
|
|
32093
32884
|
const session = loadSession(meta3.sessionId);
|
|
32094
32885
|
if (!session || !session.history || session.history.length < 3) continue;
|
|
32095
32886
|
const ts = new Date(meta3.dateCreated).getTime();
|
|
@@ -32121,7 +32912,7 @@ var continueProvider = {
|
|
|
32121
32912
|
if (existingIds.has(sessionId)) continue;
|
|
32122
32913
|
const session = loadSession(sessionId);
|
|
32123
32914
|
if (!session) continue;
|
|
32124
|
-
if (session.workspaceDirectory !==
|
|
32915
|
+
if (session.workspaceDirectory !== projectPath2) continue;
|
|
32125
32916
|
if (!session.history || session.history.length < 3) continue;
|
|
32126
32917
|
const dateStr = session.title ? truncate5(session.title, 60) : sessionId.slice(0, 8);
|
|
32127
32918
|
sessions.push({
|
|
@@ -32270,8 +33061,8 @@ function getSessionMeta2(filePath) {
|
|
|
32270
33061
|
var piProvider = {
|
|
32271
33062
|
name: "pi",
|
|
32272
33063
|
displayName: "Pi",
|
|
32273
|
-
detect(
|
|
32274
|
-
const encoded = encodeCwd(
|
|
33064
|
+
detect(projectPath2) {
|
|
33065
|
+
const encoded = encodeCwd(projectPath2);
|
|
32275
33066
|
const dir = join13(PI_DIR, encoded);
|
|
32276
33067
|
let entries;
|
|
32277
33068
|
try {
|
|
@@ -32409,8 +33200,8 @@ function parseAiderHistory(content3) {
|
|
|
32409
33200
|
var aiderProvider = {
|
|
32410
33201
|
name: "aider",
|
|
32411
33202
|
displayName: "Aider",
|
|
32412
|
-
detect(
|
|
32413
|
-
const filePath = join14(
|
|
33203
|
+
detect(projectPath2) {
|
|
33204
|
+
const filePath = join14(projectPath2, HISTORY_FILE);
|
|
32414
33205
|
if (!existsSync11(filePath)) return [];
|
|
32415
33206
|
let stat;
|
|
32416
33207
|
try {
|
|
@@ -32439,7 +33230,7 @@ var aiderProvider = {
|
|
|
32439
33230
|
}
|
|
32440
33231
|
];
|
|
32441
33232
|
},
|
|
32442
|
-
readChunks(
|
|
33233
|
+
readChunks(projectPath2, sessionIds, maxTokens = DEFAULT_MAX_TOKENS7) {
|
|
32443
33234
|
const chunks = [];
|
|
32444
33235
|
for (const filePath of sessionIds) {
|
|
32445
33236
|
let content3;
|
|
@@ -32734,7 +33525,7 @@ async function searchRecall(input) {
|
|
|
32734
33525
|
const {
|
|
32735
33526
|
query,
|
|
32736
33527
|
scope = "all",
|
|
32737
|
-
projectPath,
|
|
33528
|
+
projectPath: projectPath2,
|
|
32738
33529
|
sessionID,
|
|
32739
33530
|
knowledgeEnabled = true,
|
|
32740
33531
|
llm,
|
|
@@ -32761,7 +33552,7 @@ async function searchRecall(input) {
|
|
|
32761
33552
|
if (knowledgeEnabled && scope !== "session") {
|
|
32762
33553
|
try {
|
|
32763
33554
|
knowledgeResults.push(
|
|
32764
|
-
...searchScored3({ query: q, projectPath, limit })
|
|
33555
|
+
...searchScored3({ query: q, projectPath: projectPath2, limit })
|
|
32765
33556
|
);
|
|
32766
33557
|
} catch (err) {
|
|
32767
33558
|
error("recall: knowledge search failed:", err);
|
|
@@ -32772,7 +33563,7 @@ async function searchRecall(input) {
|
|
|
32772
33563
|
try {
|
|
32773
33564
|
distillationResults.push(
|
|
32774
33565
|
...searchDistillationsScored({
|
|
32775
|
-
projectPath,
|
|
33566
|
+
projectPath: projectPath2,
|
|
32776
33567
|
query: q,
|
|
32777
33568
|
sessionID: scope === "session" ? sessionID : void 0,
|
|
32778
33569
|
limit
|
|
@@ -32787,7 +33578,7 @@ async function searchRecall(input) {
|
|
|
32787
33578
|
try {
|
|
32788
33579
|
temporalResults.push(
|
|
32789
33580
|
...searchScored({
|
|
32790
|
-
projectPath,
|
|
33581
|
+
projectPath: projectPath2,
|
|
32791
33582
|
query: q,
|
|
32792
33583
|
sessionID: scope === "session" ? sessionID : void 0,
|
|
32793
33584
|
limit
|
|
@@ -32881,7 +33672,7 @@ async function searchRecall(input) {
|
|
|
32881
33672
|
}
|
|
32882
33673
|
}
|
|
32883
33674
|
if (scope !== "knowledge") {
|
|
32884
|
-
const pid = ensureProject(
|
|
33675
|
+
const pid = ensureProject(projectPath2);
|
|
32885
33676
|
const temporalVectorHits = vectorSearchTemporal(
|
|
32886
33677
|
queryVec,
|
|
32887
33678
|
pid,
|
|
@@ -32909,11 +33700,11 @@ async function searchRecall(input) {
|
|
|
32909
33700
|
info("recall: vector search failed:", err);
|
|
32910
33701
|
}
|
|
32911
33702
|
}
|
|
32912
|
-
if (scope !== "session" && hasLatDir(
|
|
33703
|
+
if (scope !== "session" && hasLatDir(projectPath2)) {
|
|
32913
33704
|
try {
|
|
32914
33705
|
const latResults = searchScored2({
|
|
32915
33706
|
query,
|
|
32916
|
-
projectPath,
|
|
33707
|
+
projectPath: projectPath2,
|
|
32917
33708
|
limit
|
|
32918
33709
|
});
|
|
32919
33710
|
if (latResults.length) {
|
|
@@ -32933,7 +33724,7 @@ async function searchRecall(input) {
|
|
|
32933
33724
|
try {
|
|
32934
33725
|
const crossProjectResults = searchScoredOtherProjects({
|
|
32935
33726
|
query,
|
|
32936
|
-
excludeProjectPath:
|
|
33727
|
+
excludeProjectPath: projectPath2,
|
|
32937
33728
|
limit
|
|
32938
33729
|
});
|
|
32939
33730
|
if (crossProjectResults.length) {
|
|
@@ -33139,6 +33930,7 @@ export {
|
|
|
33139
33930
|
distillationUser,
|
|
33140
33931
|
embedding_exports as embedding,
|
|
33141
33932
|
embedding_vendor_exports as embeddingVendor,
|
|
33933
|
+
enableHostedMode,
|
|
33142
33934
|
ensureProject,
|
|
33143
33935
|
exactTermMatchRank,
|
|
33144
33936
|
expandQuery,
|
|
@@ -33152,7 +33944,9 @@ export {
|
|
|
33152
33944
|
ftsQueryRelaxed,
|
|
33153
33945
|
getGitRemote,
|
|
33154
33946
|
getInstanceId,
|
|
33947
|
+
getKV,
|
|
33155
33948
|
getLastImportAt,
|
|
33949
|
+
getLastLayer,
|
|
33156
33950
|
getLastTransformEstimate,
|
|
33157
33951
|
getLastTransformedCount,
|
|
33158
33952
|
getLastTurnAt,
|
|
@@ -33167,6 +33961,7 @@ export {
|
|
|
33167
33961
|
inspectSessionState,
|
|
33168
33962
|
instruction_detect_exports as instructionDetect,
|
|
33169
33963
|
isFirstRun,
|
|
33964
|
+
isHostedMode,
|
|
33170
33965
|
isReasoningPart,
|
|
33171
33966
|
isTextPart,
|
|
33172
33967
|
isToolPart,
|
|
@@ -33177,7 +33972,9 @@ export {
|
|
|
33177
33972
|
load,
|
|
33178
33973
|
loadAllSessionCosts,
|
|
33179
33974
|
loadForceMinLayer,
|
|
33975
|
+
loadHeaderSessionIndex,
|
|
33180
33976
|
loadSessionCosts,
|
|
33977
|
+
loadSessionTracking,
|
|
33181
33978
|
log_exports as log,
|
|
33182
33979
|
loreFileExists,
|
|
33183
33980
|
ltm_exports as ltm,
|
|
@@ -33190,18 +33987,23 @@ export {
|
|
|
33190
33987
|
pattern_extract_exports as patternExtract,
|
|
33191
33988
|
projectId,
|
|
33192
33989
|
projectName,
|
|
33990
|
+
projectPath,
|
|
33193
33991
|
recallById,
|
|
33194
33992
|
reciprocalRankFusion,
|
|
33195
33993
|
recursiveUser,
|
|
33196
33994
|
renderMarkdown,
|
|
33995
|
+
resolveProjectByRemoteOrPath,
|
|
33197
33996
|
root2 as root,
|
|
33198
33997
|
runRecall,
|
|
33199
33998
|
sanitizeSurrogates,
|
|
33200
33999
|
saveForceMinLayer,
|
|
34000
|
+
saveGradientState,
|
|
33201
34001
|
saveSessionCosts,
|
|
34002
|
+
saveSessionTracking,
|
|
33202
34003
|
searchRecall,
|
|
33203
34004
|
serialize,
|
|
33204
34005
|
setForceMinLayer,
|
|
34006
|
+
setKV,
|
|
33205
34007
|
setLastImportAt,
|
|
33206
34008
|
setLastTurnAtForTest,
|
|
33207
34009
|
setLtmTokens,
|