@loreai/core 0.15.0 → 0.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. package/README.md +11 -0
  2. package/dist/bun/agents-file.d.ts +13 -1
  3. package/dist/bun/agents-file.d.ts.map +1 -1
  4. package/dist/bun/config.d.ts +20 -1
  5. package/dist/bun/config.d.ts.map +1 -1
  6. package/dist/bun/data.d.ts +174 -0
  7. package/dist/bun/data.d.ts.map +1 -0
  8. package/dist/bun/db.d.ts +65 -0
  9. package/dist/bun/db.d.ts.map +1 -1
  10. package/dist/bun/distillation.d.ts +49 -6
  11. package/dist/bun/distillation.d.ts.map +1 -1
  12. package/dist/bun/embedding-vendor.d.ts +66 -0
  13. package/dist/bun/embedding-vendor.d.ts.map +1 -0
  14. package/dist/bun/embedding-worker-types.d.ts +66 -0
  15. package/dist/bun/embedding-worker-types.d.ts.map +1 -0
  16. package/dist/bun/embedding-worker.d.ts +16 -0
  17. package/dist/bun/embedding-worker.d.ts.map +1 -0
  18. package/dist/bun/embedding-worker.js +100 -0
  19. package/dist/bun/embedding-worker.js.map +7 -0
  20. package/dist/bun/embedding.d.ts +91 -8
  21. package/dist/bun/embedding.d.ts.map +1 -1
  22. package/dist/bun/git.d.ts +47 -0
  23. package/dist/bun/git.d.ts.map +1 -0
  24. package/dist/bun/gradient.d.ts +19 -1
  25. package/dist/bun/gradient.d.ts.map +1 -1
  26. package/dist/bun/index.d.ts +9 -6
  27. package/dist/bun/index.d.ts.map +1 -1
  28. package/dist/bun/index.js +13205 -11259
  29. package/dist/bun/index.js.map +4 -4
  30. package/dist/bun/lat-reader.d.ts +1 -1
  31. package/dist/bun/lat-reader.d.ts.map +1 -1
  32. package/dist/bun/ltm.d.ts.map +1 -1
  33. package/dist/bun/markdown.d.ts +11 -0
  34. package/dist/bun/markdown.d.ts.map +1 -1
  35. package/dist/bun/prompt.d.ts +1 -1
  36. package/dist/bun/prompt.d.ts.map +1 -1
  37. package/dist/bun/recall.d.ts +53 -0
  38. package/dist/bun/recall.d.ts.map +1 -1
  39. package/dist/bun/search.d.ts +29 -0
  40. package/dist/bun/search.d.ts.map +1 -1
  41. package/dist/bun/temporal.d.ts +2 -0
  42. package/dist/bun/temporal.d.ts.map +1 -1
  43. package/dist/bun/types.d.ts +15 -0
  44. package/dist/bun/types.d.ts.map +1 -1
  45. package/dist/bun/worker-model.d.ts +15 -80
  46. package/dist/bun/worker-model.d.ts.map +1 -1
  47. package/dist/node/agents-file.d.ts +13 -1
  48. package/dist/node/agents-file.d.ts.map +1 -1
  49. package/dist/node/config.d.ts +20 -1
  50. package/dist/node/config.d.ts.map +1 -1
  51. package/dist/node/data.d.ts +174 -0
  52. package/dist/node/data.d.ts.map +1 -0
  53. package/dist/node/db.d.ts +65 -0
  54. package/dist/node/db.d.ts.map +1 -1
  55. package/dist/node/distillation.d.ts +49 -6
  56. package/dist/node/distillation.d.ts.map +1 -1
  57. package/dist/node/embedding-vendor.d.ts +66 -0
  58. package/dist/node/embedding-vendor.d.ts.map +1 -0
  59. package/dist/node/embedding-worker-types.d.ts +66 -0
  60. package/dist/node/embedding-worker-types.d.ts.map +1 -0
  61. package/dist/node/embedding-worker.d.ts +16 -0
  62. package/dist/node/embedding-worker.d.ts.map +1 -0
  63. package/dist/node/embedding-worker.js +100 -0
  64. package/dist/node/embedding-worker.js.map +7 -0
  65. package/dist/node/embedding.d.ts +91 -8
  66. package/dist/node/embedding.d.ts.map +1 -1
  67. package/dist/node/git.d.ts +47 -0
  68. package/dist/node/git.d.ts.map +1 -0
  69. package/dist/node/gradient.d.ts +19 -1
  70. package/dist/node/gradient.d.ts.map +1 -1
  71. package/dist/node/index.d.ts +9 -6
  72. package/dist/node/index.d.ts.map +1 -1
  73. package/dist/node/index.js +13205 -11259
  74. package/dist/node/index.js.map +4 -4
  75. package/dist/node/lat-reader.d.ts +1 -1
  76. package/dist/node/lat-reader.d.ts.map +1 -1
  77. package/dist/node/ltm.d.ts.map +1 -1
  78. package/dist/node/markdown.d.ts +11 -0
  79. package/dist/node/markdown.d.ts.map +1 -1
  80. package/dist/node/prompt.d.ts +1 -1
  81. package/dist/node/prompt.d.ts.map +1 -1
  82. package/dist/node/recall.d.ts +53 -0
  83. package/dist/node/recall.d.ts.map +1 -1
  84. package/dist/node/search.d.ts +29 -0
  85. package/dist/node/search.d.ts.map +1 -1
  86. package/dist/node/temporal.d.ts +2 -0
  87. package/dist/node/temporal.d.ts.map +1 -1
  88. package/dist/node/types.d.ts +15 -0
  89. package/dist/node/types.d.ts.map +1 -1
  90. package/dist/node/worker-model.d.ts +15 -80
  91. package/dist/node/worker-model.d.ts.map +1 -1
  92. package/dist/types/agents-file.d.ts +13 -1
  93. package/dist/types/agents-file.d.ts.map +1 -1
  94. package/dist/types/config.d.ts +20 -1
  95. package/dist/types/config.d.ts.map +1 -1
  96. package/dist/types/data.d.ts +174 -0
  97. package/dist/types/data.d.ts.map +1 -0
  98. package/dist/types/db.d.ts +65 -0
  99. package/dist/types/db.d.ts.map +1 -1
  100. package/dist/types/distillation.d.ts +49 -6
  101. package/dist/types/distillation.d.ts.map +1 -1
  102. package/dist/types/embedding-vendor.d.ts +66 -0
  103. package/dist/types/embedding-vendor.d.ts.map +1 -0
  104. package/dist/types/embedding-worker-types.d.ts +66 -0
  105. package/dist/types/embedding-worker-types.d.ts.map +1 -0
  106. package/dist/types/embedding-worker.d.ts +16 -0
  107. package/dist/types/embedding-worker.d.ts.map +1 -0
  108. package/dist/types/embedding.d.ts +91 -8
  109. package/dist/types/embedding.d.ts.map +1 -1
  110. package/dist/types/git.d.ts +47 -0
  111. package/dist/types/git.d.ts.map +1 -0
  112. package/dist/types/gradient.d.ts +19 -1
  113. package/dist/types/gradient.d.ts.map +1 -1
  114. package/dist/types/index.d.ts +9 -6
  115. package/dist/types/index.d.ts.map +1 -1
  116. package/dist/types/lat-reader.d.ts +1 -1
  117. package/dist/types/lat-reader.d.ts.map +1 -1
  118. package/dist/types/ltm.d.ts.map +1 -1
  119. package/dist/types/markdown.d.ts +11 -0
  120. package/dist/types/markdown.d.ts.map +1 -1
  121. package/dist/types/prompt.d.ts +1 -1
  122. package/dist/types/prompt.d.ts.map +1 -1
  123. package/dist/types/recall.d.ts +53 -0
  124. package/dist/types/recall.d.ts.map +1 -1
  125. package/dist/types/search.d.ts +29 -0
  126. package/dist/types/search.d.ts.map +1 -1
  127. package/dist/types/temporal.d.ts +2 -0
  128. package/dist/types/temporal.d.ts.map +1 -1
  129. package/dist/types/types.d.ts +15 -0
  130. package/dist/types/types.d.ts.map +1 -1
  131. package/dist/types/worker-model.d.ts +15 -80
  132. package/dist/types/worker-model.d.ts.map +1 -1
  133. package/package.json +5 -2
  134. package/src/agents-file.ts +87 -4
  135. package/src/config.ts +68 -5
  136. package/src/curator.ts +2 -2
  137. package/src/data.ts +768 -0
  138. package/src/db.ts +386 -7
  139. package/src/distillation.ts +178 -35
  140. package/src/embedding-vendor.ts +102 -0
  141. package/src/embedding-worker-types.ts +82 -0
  142. package/src/embedding-worker.ts +185 -0
  143. package/src/embedding.ts +607 -61
  144. package/src/git.ts +144 -0
  145. package/src/gradient.ts +174 -17
  146. package/src/index.ts +20 -0
  147. package/src/lat-reader.ts +5 -11
  148. package/src/ltm.ts +17 -44
  149. package/src/markdown.ts +15 -0
  150. package/src/prompt.ts +1 -2
  151. package/src/recall.ts +401 -70
  152. package/src/search.ts +71 -1
  153. package/src/temporal.ts +42 -35
  154. package/src/types.ts +15 -0
  155. package/src/worker-model.ts +17 -363
package/src/git.ts ADDED
@@ -0,0 +1,144 @@
1
+ /**
2
+ * git.ts — Git repository identification utilities.
3
+ *
4
+ * Extracts and normalizes git remote URLs to identify projects by their
5
+ * repository identity rather than filesystem path. This enables:
6
+ * - Worktree awareness: main checkout and worktrees share one project
7
+ * - Clone deduplication: same repo cloned to different paths is one project
8
+ * - Fork awareness: prefers `upstream` remote to unify forks with their source
9
+ *
10
+ * Remote URL normalization strips protocol, auth, and `.git` suffix to produce
11
+ * a stable canonical identifier (e.g. "github.com/user/repo") regardless of
12
+ * how the remote was configured (SSH, HTTPS, git://).
13
+ */
14
+
15
+ import { execSync } from "child_process";
16
+
17
+ // ---------------------------------------------------------------------------
18
+ // URL normalization
19
+ // ---------------------------------------------------------------------------
20
+
21
+ /**
22
+ * Normalize a git remote URL to a canonical form for comparison.
23
+ *
24
+ * Strips protocol, auth, `.git` suffix, and normalizes SSH ↔ HTTPS
25
+ * to produce a stable identifier regardless of how the remote was
26
+ * configured.
27
+ *
28
+ * Examples:
29
+ * git@github.com:user/repo.git → github.com/user/repo
30
+ * https://github.com/user/repo.git → github.com/user/repo
31
+ * ssh://git@github.com/user/repo → github.com/user/repo
32
+ * git://github.com/user/repo.git → github.com/user/repo
33
+ * https://user:token@github.com/user/repo → github.com/user/repo
34
+ */
35
+ export function normalizeRemoteUrl(url: string): string {
36
+ let normalized = url.trim();
37
+
38
+ // SSH shorthand: git@host:user/repo.git → host/user/repo
39
+ const sshMatch = normalized.match(/^[\w.-]+@([\w.-]+):(.+)$/);
40
+ if (sshMatch) {
41
+ normalized = `${sshMatch[1]}/${sshMatch[2]}`;
42
+ } else {
43
+ // Strip protocol (https://, http://, ssh://, git://)
44
+ normalized = normalized.replace(/^[\w+]+:\/\//, "");
45
+ // Strip auth (user@, user:pass@)
46
+ normalized = normalized.replace(/^[^@/]+@/, "");
47
+ }
48
+
49
+ // Strip .git suffix
50
+ normalized = normalized.replace(/\.git$/, "");
51
+ // Strip trailing slashes
52
+ normalized = normalized.replace(/\/+$/, "");
53
+ // Lowercase the host portion for case-insensitive comparison.
54
+ // Host is everything before the first `/`.
55
+ const slashIdx = normalized.indexOf("/");
56
+ if (slashIdx > 0) {
57
+ normalized =
58
+ normalized.slice(0, slashIdx).toLowerCase() + normalized.slice(slashIdx);
59
+ } else {
60
+ normalized = normalized.toLowerCase();
61
+ }
62
+
63
+ return normalized;
64
+ }
65
+
66
+ // ---------------------------------------------------------------------------
67
+ // Remote extraction
68
+ // ---------------------------------------------------------------------------
69
+
70
+ /**
71
+ * In-memory cache for git remote lookups. Keyed by absolute path, values are
72
+ * normalized remote URLs (or null for non-git directories). Prevents repeated
73
+ * subprocess spawns for the same path within a single process lifetime.
74
+ */
75
+ const gitRemoteCache = new Map<string, string | null>();
76
+
77
+ /**
78
+ * Clear the in-memory git remote cache.
79
+ *
80
+ * Intended for test harnesses that need deterministic behavior across
81
+ * test cases without leaking cached results.
82
+ */
83
+ export function clearGitRemoteCache(): void {
84
+ gitRemoteCache.clear();
85
+ }
86
+
87
+ /**
88
+ * Get the canonical git remote URL for a repository at the given path.
89
+ *
90
+ * Prefers `upstream` remote (for forks) over `origin`, then falls back
91
+ * to any other remote. Returns null if the path is not in a git repo
92
+ * or has no remotes configured.
93
+ *
94
+ * Results are cached in-memory for the process lifetime to avoid repeated
95
+ * subprocess calls — `git remote -v` only runs once per unique path.
96
+ */
97
+ export function getGitRemote(path: string): string | null {
98
+ const cached = gitRemoteCache.get(path);
99
+ if (cached !== undefined) return cached;
100
+
101
+ try {
102
+ // git remote -v outputs lines like:
103
+ // origin git@github.com:user/repo.git (fetch)
104
+ // upstream https://github.com/org/repo.git (fetch)
105
+ const output = execSync("git remote -v", {
106
+ cwd: path,
107
+ encoding: "utf-8",
108
+ timeout: 5000,
109
+ stdio: ["pipe", "pipe", "pipe"], // suppress stderr
110
+ });
111
+
112
+ const remotes = new Map<string, string>();
113
+ for (const line of output.split("\n")) {
114
+ // Only parse fetch URLs (avoid duplicates from push lines)
115
+ const match = line.match(/^(\S+)\s+(\S+)\s+\(fetch\)$/);
116
+ if (match) {
117
+ remotes.set(match[1], match[2]);
118
+ }
119
+ }
120
+
121
+ if (remotes.size === 0) {
122
+ gitRemoteCache.set(path, null);
123
+ return null;
124
+ }
125
+
126
+ // Prefer upstream (fork source) > origin > any other
127
+ const url =
128
+ remotes.get("upstream") ??
129
+ remotes.get("origin") ??
130
+ remotes.values().next().value;
131
+ if (!url) {
132
+ gitRemoteCache.set(path, null);
133
+ return null;
134
+ }
135
+
136
+ const result = normalizeRemoteUrl(url);
137
+ gitRemoteCache.set(path, result);
138
+ return result;
139
+ } catch {
140
+ // Not a git repo, git not installed, timeout, etc.
141
+ gitRemoteCache.set(path, null);
142
+ return null;
143
+ }
144
+ }
package/src/gradient.ts CHANGED
@@ -46,6 +46,127 @@ let maxLayer0Tokens = 0;
46
46
 
47
47
  const MIN_LAYER0_FLOOR = 40_000;
48
48
 
49
+ // ---------------------------------------------------------------------------
50
+ // Cost-aware context token cap (layer 1+)
51
+ //
52
+ // Limits total tokens (distilled + raw) to keep per-bust cache write cost
53
+ // bounded. For opus-4-6 at $6.25/M write, a $1.00 target yields a 160K cap.
54
+ // For sonnet-4 at $3.75/M write, the cap is 267K (effectively uncapped).
55
+ //
56
+ // The cap is further adjusted dynamically per session via bust rate EMA and
57
+ // inter-bust interval tracking: tighten when busts are frequent, relax when
58
+ // the cache is working well. Asymmetric rates: tighten fast, relax slowly.
59
+ // ---------------------------------------------------------------------------
60
+
61
+ /** Static ceiling for total context tokens, derived from model pricing.
62
+ * 0 = disabled (no cap). Set via setMaxContextTokens(). */
63
+ let maxContextTokensCeiling = 0;
64
+
65
+ const MIN_CONTEXT_FLOOR = 130_000;
66
+
67
+ /** Compute the context ceiling from a per-bust cost target and cache-write price per token. */
68
+ export function computeContextCap(
69
+ targetBustCost: number,
70
+ cacheWriteCostPerToken: number,
71
+ ): number {
72
+ if (targetBustCost <= 0 || cacheWriteCostPerToken <= 0) return 0;
73
+ return Math.max(MIN_CONTEXT_FLOOR, Math.floor(targetBustCost / cacheWriteCostPerToken));
74
+ }
75
+
76
+ /** Set the static context ceiling. Called by the host adapter after computing
77
+ * from model pricing. The effective per-session cap may be lower due to
78
+ * dynamic adaptation (bust rate EMA). */
79
+ export function setMaxContextTokens(tokens: number) {
80
+ maxContextTokensCeiling = Math.max(0, Math.floor(tokens));
81
+ }
82
+
83
+ /** Returns the current static ceiling (for external callers / tests). */
84
+ export function getMaxContextTokens(): number {
85
+ return maxContextTokensCeiling;
86
+ }
87
+
88
+ /**
89
+ * Feed cache usage data after each API response. Updates the per-session
90
+ * bust rate EMA and inter-bust interval, which adjust the effective context
91
+ * cap dynamically.
92
+ *
93
+ * @param cacheWrite - cache_creation_input_tokens from the API response
94
+ * @param cacheRead - cache_read_input_tokens from the API response
95
+ * @param sessionID - session that produced this response
96
+ */
97
+ export function updateBustRate(
98
+ cacheWrite: number,
99
+ cacheRead: number,
100
+ sessionID?: string,
101
+ ): void {
102
+ if (!sessionID) return;
103
+ const state = getSessionState(sessionID);
104
+ const total = cacheWrite + cacheRead;
105
+ if (total === 0) return;
106
+
107
+ // Bust ratio: fraction of total input that was cache-written (0 = all reads, 1 = all writes)
108
+ const bustRatio = cacheWrite / total;
109
+
110
+ // EMA update (α = 0.3 for smoothing — responsive but not twitchy)
111
+ state.bustRateEMA =
112
+ state.bustRateEMA < 0
113
+ ? bustRatio // first observation
114
+ : state.bustRateEMA * 0.7 + bustRatio * 0.3;
115
+
116
+ // Inter-bust interval tracking: a "bust" is when >50% of input is writes
117
+ const now = Date.now();
118
+ if (bustRatio > 0.5) {
119
+ if (state.lastBustAt > 0) {
120
+ const interval = now - state.lastBustAt;
121
+ state.interBustIntervalEMA =
122
+ state.interBustIntervalEMA < 0
123
+ ? interval
124
+ : state.interBustIntervalEMA * 0.7 + interval * 0.3;
125
+ }
126
+ state.lastBustAt = now;
127
+ }
128
+
129
+ // Adapt per-session cap based on bust rate and interval
130
+ adaptContextCap(state);
131
+ }
132
+
133
+ /** Adapt the per-session context cap based on bust rate and break frequency. */
134
+ function adaptContextCap(state: SessionState): void {
135
+ if (maxContextTokensCeiling <= 0) return; // disabled
136
+
137
+ const cap = state.dynamicContextCap > 0
138
+ ? state.dynamicContextCap
139
+ : maxContextTokensCeiling;
140
+
141
+ let newCap = cap;
142
+
143
+ // Primary signal: bust rate EMA
144
+ if (state.bustRateEMA > 0.8) {
145
+ // Mostly writes — tighten by 10%
146
+ newCap = Math.floor(cap * 0.90);
147
+ } else if (state.bustRateEMA < 0.3) {
148
+ // Mostly reads — relax by 5% (slower than tightening)
149
+ newCap = Math.floor(cap * 1.05);
150
+ }
151
+
152
+ // Secondary signal: inter-bust interval
153
+ if (state.interBustIntervalEMA > 0) {
154
+ if (state.interBustIntervalEMA < 2 * 60_000) {
155
+ // Busts less than 2 min apart — proactively tighten by extra 5%
156
+ newCap = Math.floor(newCap * 0.95);
157
+ } else if (state.interBustIntervalEMA > 10 * 60_000) {
158
+ // Busts more than 10 min apart — allow extra relaxation
159
+ newCap = Math.floor(newCap * 1.03);
160
+ }
161
+ }
162
+
163
+ // Clamp to [floor, ceiling]
164
+ state.dynamicContextCap = Math.max(
165
+ MIN_CONTEXT_FLOOR,
166
+ Math.min(maxContextTokensCeiling, newCap),
167
+ );
168
+ }
169
+
49
170
  // Conservative overhead reserve for first-turn (before calibration):
50
171
  // accounts for provider system prompt + AGENTS.md + tool definitions + env info
51
172
  const FIRST_TURN_OVERHEAD = 15_000;
@@ -133,6 +254,18 @@ type SessionState = {
133
254
  /** Consecutive turns at layer >= 2. When >= 3, log a compaction hint. */
134
255
  consecutiveHighLayer: number;
135
256
 
257
+ // --- Cost-aware context cap dynamic state ---
258
+
259
+ /** EMA of bust ratio (cacheWrite / total). -1 = uninitialized. */
260
+ bustRateEMA: number;
261
+ /** EMA of time between full busts (ms). -1 = uninitialized. */
262
+ interBustIntervalEMA: number;
263
+ /** Epoch ms of the last full bust (cacheWrite > 50% of total). 0 = never. */
264
+ lastBustAt: number;
265
+ /** Per-session dynamic context cap (tokens). Adjusted by adaptContextCap().
266
+ * 0 = use the static ceiling (maxContextTokensCeiling). */
267
+ dynamicContextCap: number;
268
+
136
269
  /**
137
270
  * Distillation row snapshot — cached to avoid hitting the DB on every
138
271
  * transform() call. Refreshed only at turn boundaries (when a new user
@@ -166,6 +299,11 @@ function makeSessionState(): SessionState {
166
299
  postIdleCompact: false,
167
300
  consecutiveHighLayer: 0,
168
301
 
302
+ bustRateEMA: -1,
303
+ interBustIntervalEMA: -1,
304
+ lastBustAt: 0,
305
+ dynamicContextCap: 0,
306
+
169
307
  distillationSnapshot: null,
170
308
  };
171
309
  }
@@ -978,7 +1116,7 @@ function buildPrefixMessages(formatted: string): MessageWithParts[] {
978
1116
  sessionID: "",
979
1117
  messageID: "lore-distilled-assistant",
980
1118
  type: "text" as const,
981
- text: formatted + "\n\nI'm ready to continue.",
1119
+ text: formatted,
982
1120
  time: { start: 0, end: 0 },
983
1121
  },
984
1122
  ],
@@ -1291,11 +1429,13 @@ export type TransformResult = {
1291
1429
  rawBudget: number;
1292
1430
  };
1293
1431
 
1294
- // Signal that we need urgent distillation
1295
- let urgentDistillation = false;
1296
- export function needsUrgentDistillation(): boolean {
1297
- const v = urgentDistillation;
1298
- urgentDistillation = false;
1432
+ // Per-session urgent distillation tracking.
1433
+ // Keyed by sessionID. Set by layer returns in transformInner(),
1434
+ // consumed (read + delete) by needsUrgentDistillation(sessionID).
1435
+ const urgentDistillationMap = new Map<string, boolean>();
1436
+ export function needsUrgentDistillation(sessionID: string): boolean {
1437
+ const v = urgentDistillationMap.get(sessionID) ?? false;
1438
+ urgentDistillationMap.delete(sessionID);
1299
1439
  return v;
1300
1440
  }
1301
1441
 
@@ -1315,10 +1455,23 @@ function transformInner(input: {
1315
1455
  // minus LTM tokens already injected into the system prompt this turn.
1316
1456
  // Read LTM tokens from per-session state to avoid cross-session contamination.
1317
1457
  const sessLtmTokens = sid ? sessState.ltmTokens : ltmTokensFallback;
1318
- const usable = Math.max(
1458
+ const usableRaw = Math.max(
1319
1459
  0,
1320
1460
  contextLimit - outputReserved - overhead - sessLtmTokens,
1321
1461
  );
1462
+
1463
+ // Cost-aware context cap: limit total distilled + raw tokens to keep
1464
+ // per-bust cache write cost bounded. On opus-4-6 at $6.25/M, a $1.00
1465
+ // target yields a 160K ceiling; on sonnet-4 at $3.75/M, 267K (effectively
1466
+ // uncapped at 200K context). Per-session dynamic adaptation may reduce
1467
+ // this further based on observed bust rate and break frequency.
1468
+ const effectiveCap = sid && sessState.dynamicContextCap > 0
1469
+ ? sessState.dynamicContextCap
1470
+ : maxContextTokensCeiling;
1471
+ const usable = effectiveCap > 0 && usableRaw > effectiveCap
1472
+ ? effectiveCap
1473
+ : usableRaw;
1474
+
1322
1475
  const distilledBudget = Math.floor(usable * cfg.budget.distilled);
1323
1476
  // Base raw budget. May be overridden below for post-idle compact mode.
1324
1477
  let rawBudget = Math.floor(usable * cfg.budget.raw);
@@ -1385,12 +1538,11 @@ function transformInner(input: {
1385
1538
  sessState.postIdleCompact = false;
1386
1539
  // Skip layer 0 — don't pass through all raw messages on a cold cache.
1387
1540
  effectiveMinLayer = Math.max(effectiveMinLayer, 1) as SafetyLayer;
1388
- // Use a tighter raw budget: 20% of usable instead of the normal 40%.
1389
- // The distilled prefix covers the older history; the raw window only
1390
- // needs the current turn + minimal recent context. This reduces the
1391
- // total cold-cache write cost by up to 20% of usable (~29K tokens on
1392
- // a 200K context model).
1393
- rawBudget = Math.floor(usable * 0.20);
1541
+ // Use a tighter raw budget. When the cost-aware context cap is active,
1542
+ // total write size is already bounded use a moderate 30%. Without
1543
+ // the cap, use a tighter 20% to limit cold-write cost directly.
1544
+ const postIdleRawFraction = effectiveCap > 0 ? 0.30 : 0.20;
1545
+ rawBudget = Math.floor(usable * postIdleRawFraction);
1394
1546
  log.info(
1395
1547
  `post-idle compact: session=${sid} rawBudget=${rawBudget}` +
1396
1548
  ` (${Math.floor(usable * cfg.budget.raw)}→${rawBudget})`,
@@ -1500,7 +1652,12 @@ function transformInner(input: {
1500
1652
  rawBudget,
1501
1653
  strip: "none",
1502
1654
  });
1503
- if (fitsWithSafetyMargin(layer1)) return { ...layer1!, layer: 1, usable, distilledBudget, rawBudget };
1655
+ if (fitsWithSafetyMargin(layer1)) {
1656
+ if (cached.tokens === 0 && sid) {
1657
+ urgentDistillationMap.set(sid, true);
1658
+ }
1659
+ return { ...layer1!, layer: 1, usable, distilledBudget, rawBudget };
1660
+ }
1504
1661
  }
1505
1662
 
1506
1663
  // Layer 1 didn't fit (or was force-skipped) — reset the raw window cache.
@@ -1520,7 +1677,7 @@ function transformInner(input: {
1520
1677
  protectedTurns: 2,
1521
1678
  });
1522
1679
  if (fitsWithSafetyMargin(layer2)) {
1523
- urgentDistillation = true;
1680
+ if (sid) urgentDistillationMap.set(sid, true);
1524
1681
  return { ...layer2!, layer: 2, usable, distilledBudget, rawBudget };
1525
1682
  }
1526
1683
  }
@@ -1541,7 +1698,7 @@ function transformInner(input: {
1541
1698
  strip: "all-tools",
1542
1699
  });
1543
1700
  if (fitsWithSafetyMargin(layer3)) {
1544
- urgentDistillation = true;
1701
+ if (sid) urgentDistillationMap.set(sid, true);
1545
1702
  return { ...layer3!, layer: 3, usable, distilledBudget, rawBudget };
1546
1703
  }
1547
1704
 
@@ -1558,7 +1715,7 @@ function transformInner(input: {
1558
1715
  // if it alone exceeds the tail budget — layer 4 is the terminal layer
1559
1716
  // and must always return. Remaining budget is filled backward with older
1560
1717
  // messages.
1561
- urgentDistillation = true;
1718
+ if (sid) urgentDistillationMap.set(sid, true);
1562
1719
  const nuclearDistillations = distillations.slice(-2);
1563
1720
  const nuclearPrefix = distilledPrefix(nuclearDistillations);
1564
1721
  const nuclearPrefixTokens = nuclearPrefix.reduce(
package/src/index.ts CHANGED
@@ -11,21 +11,27 @@
11
11
 
12
12
  export * as temporal from "./temporal";
13
13
  export * as ltm from "./ltm";
14
+ export * as data from "./data";
14
15
  export * as distillation from "./distillation";
15
16
  export * as curator from "./curator";
16
17
  export * as embedding from "./embedding";
18
+ export * as embeddingVendor from "./embedding-vendor";
17
19
  export * as latReader from "./lat-reader";
18
20
  export * as patternExtract from "./pattern-extract";
19
21
  export * as log from "./log";
20
22
 
21
23
  export {
22
24
  runRecall,
25
+ searchRecall,
26
+ recallById,
23
27
  RECALL_TOOL_DESCRIPTION,
24
28
  RECALL_PARAM_DESCRIPTIONS,
25
29
  type RecallInput,
26
30
  type RecallResult,
27
31
  type RecallScope,
28
32
  type ScoredDistillation,
33
+ type TaggedResult,
34
+ type ScoredTaggedResult,
29
35
  } from "./recall";
30
36
 
31
37
  export type {
@@ -50,22 +56,33 @@ export { isTextPart, isReasoningPart, isToolPart } from "./types";
50
56
  export { load, config, type LoreConfig } from "./config";
51
57
  export {
52
58
  db,
59
+ dbPath,
53
60
  ensureProject,
54
61
  isFirstRun,
55
62
  projectId,
56
63
  projectName,
64
+ mergeProjectInternal,
57
65
  loadForceMinLayer,
58
66
  saveForceMinLayer,
67
+ saveSessionCosts,
68
+ loadSessionCosts,
69
+ loadAllSessionCosts,
70
+ type SessionCostSnapshot,
59
71
  getMeta,
60
72
  setMeta,
61
73
  getInstanceId,
62
74
  close,
63
75
  } from "./db";
76
+ export { normalizeRemoteUrl, getGitRemote, clearGitRemoteCache } from "./git";
64
77
  export {
65
78
  transform,
66
79
  setModelLimits,
67
80
  setMaxLayer0Tokens,
68
81
  computeLayer0Cap,
82
+ setMaxContextTokens,
83
+ computeContextCap,
84
+ getMaxContextTokens,
85
+ updateBustRate,
69
86
  needsUrgentDistillation,
70
87
  calibrate,
71
88
  setLtmTokens,
@@ -106,6 +123,7 @@ export {
106
123
  importLoreFile,
107
124
  shouldImportLoreFile,
108
125
  loreFileExists,
126
+ clearLoreFileCache,
109
127
  LORE_FILE,
110
128
  } from "./agents-file";
111
129
  export { workerSessionIDs, isWorkerSession } from "./worker";
@@ -113,6 +131,7 @@ export * as workerModel from "./worker-model";
113
131
  export {
114
132
  ftsQuery,
115
133
  ftsQueryOr,
134
+ ftsQueryRelaxed,
116
135
  EMPTY_QUERY,
117
136
  reciprocalRankFusion,
118
137
  expandQuery,
@@ -133,4 +152,5 @@ export {
133
152
  normalize,
134
153
  sanitizeSurrogates,
135
154
  unescapeMarkdown,
155
+ renderMarkdown,
136
156
  } from "./markdown";
package/src/lat-reader.ts CHANGED
@@ -15,7 +15,7 @@ import { remark } from "remark";
15
15
  import type { Root, Heading, Paragraph, Text } from "mdast";
16
16
  import { db, ensureProject } from "./db";
17
17
  import { sha256 } from "#db/driver";
18
- import { ftsQuery, ftsQueryOr, extractTopTerms, EMPTY_QUERY } from "./search";
18
+ import { ftsQuery, extractTopTerms, EMPTY_QUERY, runRelaxedSearch } from "./search";
19
19
  import * as log from "./log";
20
20
 
21
21
  const processor = remark();
@@ -274,7 +274,7 @@ export function refresh(projectPath: string): number {
274
274
 
275
275
  /**
276
276
  * Search lat sections by FTS5 with BM25 scoring.
277
- * Uses AND-then-OR fallback (same pattern as knowledge search).
277
+ * Uses progressive AND relaxation before falling back to OR.
278
278
  */
279
279
  export function searchScored(input: {
280
280
  query: string;
@@ -282,8 +282,6 @@ export function searchScored(input: {
282
282
  limit?: number;
283
283
  }): ScoredLatSection[] {
284
284
  const limit = input.limit ?? 10;
285
- const q = ftsQuery(input.query);
286
- if (q === EMPTY_QUERY) return [];
287
285
 
288
286
  const pid = ensureProject(input.projectPath);
289
287
 
@@ -297,13 +295,9 @@ export function searchScored(input: {
297
295
  ORDER BY rank LIMIT ?`;
298
296
 
299
297
  try {
300
- const results = db().query(ftsSQL).all(q, pid, limit) as ScoredLatSection[];
301
- if (results.length) return results;
302
-
303
- // AND returned nothing — try OR fallback
304
- const qOr = ftsQueryOr(input.query);
305
- if (qOr === EMPTY_QUERY) return [];
306
- return db().query(ftsSQL).all(qOr, pid, limit) as ScoredLatSection[];
298
+ return runRelaxedSearch(input.query, (matchExpr) =>
299
+ db().query(ftsSQL).all(matchExpr, pid, limit) as ScoredLatSection[],
300
+ );
307
301
  } catch {
308
302
  return [];
309
303
  }
package/src/ltm.ts CHANGED
@@ -1,7 +1,7 @@
1
1
  import { uuidv7 } from "uuidv7";
2
2
  import { db, ensureProject } from "./db";
3
3
  import { config } from "./config";
4
- import { ftsQuery, ftsQueryOr, EMPTY_QUERY, extractTopTerms } from "./search";
4
+ import { ftsQuery, EMPTY_QUERY, extractTopTerms, runRelaxedSearch } from "./search";
5
5
  import * as embedding from "./embedding";
6
6
  import * as latReader from "./lat-reader";
7
7
  import * as log from "./log";
@@ -454,8 +454,6 @@ export function search(input: {
454
454
  limit?: number;
455
455
  }): KnowledgeEntry[] {
456
456
  const limit = input.limit ?? 20;
457
- const q = ftsQuery(input.query);
458
- if (q === EMPTY_QUERY) return [];
459
457
 
460
458
  const pid = input.projectPath ? ensureProject(input.projectPath) : null;
461
459
 
@@ -473,22 +471,14 @@ export function search(input: {
473
471
  ORDER BY bm25(knowledge_fts, ?, ?, ?) LIMIT ?`;
474
472
 
475
473
  const { title, content, category } = ftsWeights();
476
- const ftsParams = pid
477
- ? [q, pid, title, content, category, limit]
478
- : [q, title, content, category, limit];
479
474
 
480
475
  try {
481
- const results = db().query(ftsSQL).all(...ftsParams) as KnowledgeEntry[];
482
- if (results.length) return results;
483
-
484
- // AND returned nothing try OR fallback for broader recall
485
- const qOr = ftsQueryOr(input.query);
486
- if (qOr === EMPTY_QUERY) return [];
487
-
488
- const ftsParamsOr = pid
489
- ? [qOr, pid, title, content, category, limit]
490
- : [qOr, title, content, category, limit];
491
- return db().query(ftsSQL).all(...ftsParamsOr) as KnowledgeEntry[];
476
+ return runRelaxedSearch(input.query, (matchExpr) => {
477
+ const params = pid
478
+ ? [matchExpr, pid, title, content, category, limit]
479
+ : [matchExpr, title, content, category, limit];
480
+ return db().query(ftsSQL).all(...params) as KnowledgeEntry[];
481
+ });
492
482
  } catch {
493
483
  return searchLike({
494
484
  query: input.query,
@@ -510,8 +500,6 @@ export function searchScored(input: {
510
500
  limit?: number;
511
501
  }): ScoredKnowledgeEntry[] {
512
502
  const limit = input.limit ?? 20;
513
- const q = ftsQuery(input.query);
514
- if (q === EMPTY_QUERY) return [];
515
503
 
516
504
  const pid = input.projectPath ? ensureProject(input.projectPath) : null;
517
505
  const { title, content, category } = ftsWeights();
@@ -529,20 +517,13 @@ export function searchScored(input: {
529
517
  AND k.confidence > 0.2
530
518
  ORDER BY rank LIMIT ?`;
531
519
 
532
- const ftsParams = pid
533
- ? [title, content, category, q, pid, limit]
534
- : [title, content, category, q, limit];
535
-
536
520
  try {
537
- const results = db().query(ftsSQL).all(...ftsParams) as ScoredKnowledgeEntry[];
538
- if (results.length) return results;
539
-
540
- const qOr = ftsQueryOr(input.query);
541
- if (qOr === EMPTY_QUERY) return [];
542
- const ftsParamsOr = pid
543
- ? [title, content, category, qOr, pid, limit]
544
- : [title, content, category, qOr, limit];
545
- return db().query(ftsSQL).all(...ftsParamsOr) as ScoredKnowledgeEntry[];
521
+ return runRelaxedSearch(input.query, (matchExpr) => {
522
+ const params = pid
523
+ ? [title, content, category, matchExpr, pid, limit]
524
+ : [title, content, category, matchExpr, limit];
525
+ return db().query(ftsSQL).all(...params) as ScoredKnowledgeEntry[];
526
+ });
546
527
  } catch {
547
528
  return [];
548
529
  }
@@ -560,8 +541,6 @@ export function searchScoredOtherProjects(input: {
560
541
  limit?: number;
561
542
  }): ScoredKnowledgeEntry[] {
562
543
  const limit = input.limit ?? 10;
563
- const q = ftsQuery(input.query);
564
- if (q === EMPTY_QUERY) return [];
565
544
 
566
545
  const excludePid = ensureProject(input.excludeProjectPath);
567
546
  const { title, content, category } = ftsWeights();
@@ -578,17 +557,11 @@ export function searchScoredOtherProjects(input: {
578
557
  AND k.confidence > 0.2
579
558
  ORDER BY rank LIMIT ?`;
580
559
 
581
- const ftsParams = [title, content, category, q, excludePid, limit];
582
-
583
560
  try {
584
- const results = db().query(ftsSQL).all(...ftsParams) as ScoredKnowledgeEntry[];
585
- if (results.length) return results;
586
-
587
- // AND returned nothing — try OR fallback
588
- const qOr = ftsQueryOr(input.query);
589
- if (qOr === EMPTY_QUERY) return [];
590
- const ftsParamsOr = [title, content, category, qOr, excludePid, limit];
591
- return db().query(ftsSQL).all(...ftsParamsOr) as ScoredKnowledgeEntry[];
561
+ return runRelaxedSearch(input.query, (matchExpr) => {
562
+ const params = [title, content, category, matchExpr, excludePid, limit];
563
+ return db().query(ftsSQL).all(...params) as ScoredKnowledgeEntry[];
564
+ });
592
565
  } catch {
593
566
  return [];
594
567
  }
package/src/markdown.ts CHANGED
@@ -1,3 +1,4 @@
1
+ import { micromark } from "micromark";
1
2
  import { remark } from "remark";
2
3
  import type {
3
4
  Root,
@@ -127,3 +128,17 @@ export function strong(value: string): Strong {
127
128
  export function root(...children: Root["children"]): Root {
128
129
  return { type: "root", children };
129
130
  }
131
+
132
+ /**
133
+ * Render a markdown string to sanitized HTML.
134
+ *
135
+ * Uses micromark with default options:
136
+ * - Raw HTML in input is escaped (no allowDangerousHtml)
137
+ * - Only safe URL protocols are permitted (no allowDangerousProtocol)
138
+ *
139
+ * The output is safe to embed directly in an HTML page without
140
+ * additional escaping.
141
+ */
142
+ export function renderMarkdown(md: string): string {
143
+ return micromark(md);
144
+ }
package/src/prompt.ts CHANGED
@@ -446,8 +446,7 @@ Rules:
446
446
  - Keep every section, even when empty.
447
447
  - Use terse bullets, not prose paragraphs.
448
448
  - Preserve exact file paths, commands, error strings, and identifiers when known.
449
- - Do not mention the summary process or that context was compacted.
450
- - End with "I'm ready to continue." on its own line after the closing "---".`;
449
+ - Do not mention the summary process or that context was compacted.`;
451
450
 
452
451
  // Build the user-facing prompt passed to the compaction agent during /compact.
453
452
  // Lore injects pre-computed distillations as context separately; this prompt