@loreai/core 0.16.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -0
- package/dist/bun/agents-file.d.ts +13 -1
- package/dist/bun/agents-file.d.ts.map +1 -1
- package/dist/bun/config.d.ts +20 -1
- package/dist/bun/config.d.ts.map +1 -1
- package/dist/bun/data.d.ts +174 -0
- package/dist/bun/data.d.ts.map +1 -0
- package/dist/bun/db.d.ts +65 -0
- package/dist/bun/db.d.ts.map +1 -1
- package/dist/bun/distillation.d.ts +49 -6
- package/dist/bun/distillation.d.ts.map +1 -1
- package/dist/bun/embedding-vendor.d.ts +66 -0
- package/dist/bun/embedding-vendor.d.ts.map +1 -0
- package/dist/bun/embedding-worker-types.d.ts +66 -0
- package/dist/bun/embedding-worker-types.d.ts.map +1 -0
- package/dist/bun/embedding-worker.d.ts +16 -0
- package/dist/bun/embedding-worker.d.ts.map +1 -0
- package/dist/bun/embedding-worker.js +100 -0
- package/dist/bun/embedding-worker.js.map +7 -0
- package/dist/bun/embedding.d.ts +91 -8
- package/dist/bun/embedding.d.ts.map +1 -1
- package/dist/bun/git.d.ts +47 -0
- package/dist/bun/git.d.ts.map +1 -0
- package/dist/bun/gradient.d.ts +19 -1
- package/dist/bun/gradient.d.ts.map +1 -1
- package/dist/bun/index.d.ts +9 -6
- package/dist/bun/index.d.ts.map +1 -1
- package/dist/bun/index.js +13029 -10885
- package/dist/bun/index.js.map +4 -4
- package/dist/bun/lat-reader.d.ts +1 -1
- package/dist/bun/lat-reader.d.ts.map +1 -1
- package/dist/bun/ltm.d.ts.map +1 -1
- package/dist/bun/markdown.d.ts +11 -0
- package/dist/bun/markdown.d.ts.map +1 -1
- package/dist/bun/prompt.d.ts +1 -1
- package/dist/bun/prompt.d.ts.map +1 -1
- package/dist/bun/recall.d.ts +53 -0
- package/dist/bun/recall.d.ts.map +1 -1
- package/dist/bun/search.d.ts +29 -0
- package/dist/bun/search.d.ts.map +1 -1
- package/dist/bun/temporal.d.ts +2 -0
- package/dist/bun/temporal.d.ts.map +1 -1
- package/dist/bun/types.d.ts +15 -0
- package/dist/bun/types.d.ts.map +1 -1
- package/dist/bun/worker-model.d.ts +12 -9
- package/dist/bun/worker-model.d.ts.map +1 -1
- package/dist/node/agents-file.d.ts +13 -1
- package/dist/node/agents-file.d.ts.map +1 -1
- package/dist/node/config.d.ts +20 -1
- package/dist/node/config.d.ts.map +1 -1
- package/dist/node/data.d.ts +174 -0
- package/dist/node/data.d.ts.map +1 -0
- package/dist/node/db.d.ts +65 -0
- package/dist/node/db.d.ts.map +1 -1
- package/dist/node/distillation.d.ts +49 -6
- package/dist/node/distillation.d.ts.map +1 -1
- package/dist/node/embedding-vendor.d.ts +66 -0
- package/dist/node/embedding-vendor.d.ts.map +1 -0
- package/dist/node/embedding-worker-types.d.ts +66 -0
- package/dist/node/embedding-worker-types.d.ts.map +1 -0
- package/dist/node/embedding-worker.d.ts +16 -0
- package/dist/node/embedding-worker.d.ts.map +1 -0
- package/dist/node/embedding-worker.js +100 -0
- package/dist/node/embedding-worker.js.map +7 -0
- package/dist/node/embedding.d.ts +91 -8
- package/dist/node/embedding.d.ts.map +1 -1
- package/dist/node/git.d.ts +47 -0
- package/dist/node/git.d.ts.map +1 -0
- package/dist/node/gradient.d.ts +19 -1
- package/dist/node/gradient.d.ts.map +1 -1
- package/dist/node/index.d.ts +9 -6
- package/dist/node/index.d.ts.map +1 -1
- package/dist/node/index.js +13029 -10885
- package/dist/node/index.js.map +4 -4
- package/dist/node/lat-reader.d.ts +1 -1
- package/dist/node/lat-reader.d.ts.map +1 -1
- package/dist/node/ltm.d.ts.map +1 -1
- package/dist/node/markdown.d.ts +11 -0
- package/dist/node/markdown.d.ts.map +1 -1
- package/dist/node/prompt.d.ts +1 -1
- package/dist/node/prompt.d.ts.map +1 -1
- package/dist/node/recall.d.ts +53 -0
- package/dist/node/recall.d.ts.map +1 -1
- package/dist/node/search.d.ts +29 -0
- package/dist/node/search.d.ts.map +1 -1
- package/dist/node/temporal.d.ts +2 -0
- package/dist/node/temporal.d.ts.map +1 -1
- package/dist/node/types.d.ts +15 -0
- package/dist/node/types.d.ts.map +1 -1
- package/dist/node/worker-model.d.ts +12 -9
- package/dist/node/worker-model.d.ts.map +1 -1
- package/dist/types/agents-file.d.ts +13 -1
- package/dist/types/agents-file.d.ts.map +1 -1
- package/dist/types/config.d.ts +20 -1
- package/dist/types/config.d.ts.map +1 -1
- package/dist/types/data.d.ts +174 -0
- package/dist/types/data.d.ts.map +1 -0
- package/dist/types/db.d.ts +65 -0
- package/dist/types/db.d.ts.map +1 -1
- package/dist/types/distillation.d.ts +49 -6
- package/dist/types/distillation.d.ts.map +1 -1
- package/dist/types/embedding-vendor.d.ts +66 -0
- package/dist/types/embedding-vendor.d.ts.map +1 -0
- package/dist/types/embedding-worker-types.d.ts +66 -0
- package/dist/types/embedding-worker-types.d.ts.map +1 -0
- package/dist/types/embedding-worker.d.ts +16 -0
- package/dist/types/embedding-worker.d.ts.map +1 -0
- package/dist/types/embedding.d.ts +91 -8
- package/dist/types/embedding.d.ts.map +1 -1
- package/dist/types/git.d.ts +47 -0
- package/dist/types/git.d.ts.map +1 -0
- package/dist/types/gradient.d.ts +19 -1
- package/dist/types/gradient.d.ts.map +1 -1
- package/dist/types/index.d.ts +9 -6
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/lat-reader.d.ts +1 -1
- package/dist/types/lat-reader.d.ts.map +1 -1
- package/dist/types/ltm.d.ts.map +1 -1
- package/dist/types/markdown.d.ts +11 -0
- package/dist/types/markdown.d.ts.map +1 -1
- package/dist/types/prompt.d.ts +1 -1
- package/dist/types/prompt.d.ts.map +1 -1
- package/dist/types/recall.d.ts +53 -0
- package/dist/types/recall.d.ts.map +1 -1
- package/dist/types/search.d.ts +29 -0
- package/dist/types/search.d.ts.map +1 -1
- package/dist/types/temporal.d.ts +2 -0
- package/dist/types/temporal.d.ts.map +1 -1
- package/dist/types/types.d.ts +15 -0
- package/dist/types/types.d.ts.map +1 -1
- package/dist/types/worker-model.d.ts +12 -9
- package/dist/types/worker-model.d.ts.map +1 -1
- package/package.json +5 -2
- package/src/agents-file.ts +87 -4
- package/src/config.ts +68 -5
- package/src/curator.ts +2 -2
- package/src/data.ts +768 -0
- package/src/db.ts +386 -7
- package/src/distillation.ts +178 -35
- package/src/embedding-vendor.ts +102 -0
- package/src/embedding-worker-types.ts +82 -0
- package/src/embedding-worker.ts +185 -0
- package/src/embedding.ts +607 -61
- package/src/git.ts +144 -0
- package/src/gradient.ts +174 -17
- package/src/index.ts +20 -0
- package/src/lat-reader.ts +5 -11
- package/src/ltm.ts +17 -44
- package/src/markdown.ts +15 -0
- package/src/prompt.ts +1 -2
- package/src/recall.ts +401 -70
- package/src/search.ts +71 -1
- package/src/temporal.ts +42 -35
- package/src/types.ts +15 -0
- package/src/worker-model.ts +14 -9
package/src/git.ts
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* git.ts — Git repository identification utilities.
|
|
3
|
+
*
|
|
4
|
+
* Extracts and normalizes git remote URLs to identify projects by their
|
|
5
|
+
* repository identity rather than filesystem path. This enables:
|
|
6
|
+
* - Worktree awareness: main checkout and worktrees share one project
|
|
7
|
+
* - Clone deduplication: same repo cloned to different paths is one project
|
|
8
|
+
* - Fork awareness: prefers `upstream` remote to unify forks with their source
|
|
9
|
+
*
|
|
10
|
+
* Remote URL normalization strips protocol, auth, and `.git` suffix to produce
|
|
11
|
+
* a stable canonical identifier (e.g. "github.com/user/repo") regardless of
|
|
12
|
+
* how the remote was configured (SSH, HTTPS, git://).
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { execSync } from "child_process";
|
|
16
|
+
|
|
17
|
+
// ---------------------------------------------------------------------------
|
|
18
|
+
// URL normalization
|
|
19
|
+
// ---------------------------------------------------------------------------
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Normalize a git remote URL to a canonical form for comparison.
|
|
23
|
+
*
|
|
24
|
+
* Strips protocol, auth, `.git` suffix, and normalizes SSH ↔ HTTPS
|
|
25
|
+
* to produce a stable identifier regardless of how the remote was
|
|
26
|
+
* configured.
|
|
27
|
+
*
|
|
28
|
+
* Examples:
|
|
29
|
+
* git@github.com:user/repo.git → github.com/user/repo
|
|
30
|
+
* https://github.com/user/repo.git → github.com/user/repo
|
|
31
|
+
* ssh://git@github.com/user/repo → github.com/user/repo
|
|
32
|
+
* git://github.com/user/repo.git → github.com/user/repo
|
|
33
|
+
* https://user:token@github.com/user/repo → github.com/user/repo
|
|
34
|
+
*/
|
|
35
|
+
export function normalizeRemoteUrl(url: string): string {
|
|
36
|
+
let normalized = url.trim();
|
|
37
|
+
|
|
38
|
+
// SSH shorthand: git@host:user/repo.git → host/user/repo
|
|
39
|
+
const sshMatch = normalized.match(/^[\w.-]+@([\w.-]+):(.+)$/);
|
|
40
|
+
if (sshMatch) {
|
|
41
|
+
normalized = `${sshMatch[1]}/${sshMatch[2]}`;
|
|
42
|
+
} else {
|
|
43
|
+
// Strip protocol (https://, http://, ssh://, git://)
|
|
44
|
+
normalized = normalized.replace(/^[\w+]+:\/\//, "");
|
|
45
|
+
// Strip auth (user@, user:pass@)
|
|
46
|
+
normalized = normalized.replace(/^[^@/]+@/, "");
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// Strip .git suffix
|
|
50
|
+
normalized = normalized.replace(/\.git$/, "");
|
|
51
|
+
// Strip trailing slashes
|
|
52
|
+
normalized = normalized.replace(/\/+$/, "");
|
|
53
|
+
// Lowercase the host portion for case-insensitive comparison.
|
|
54
|
+
// Host is everything before the first `/`.
|
|
55
|
+
const slashIdx = normalized.indexOf("/");
|
|
56
|
+
if (slashIdx > 0) {
|
|
57
|
+
normalized =
|
|
58
|
+
normalized.slice(0, slashIdx).toLowerCase() + normalized.slice(slashIdx);
|
|
59
|
+
} else {
|
|
60
|
+
normalized = normalized.toLowerCase();
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
return normalized;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
// ---------------------------------------------------------------------------
|
|
67
|
+
// Remote extraction
|
|
68
|
+
// ---------------------------------------------------------------------------
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* In-memory cache for git remote lookups. Keyed by absolute path, values are
|
|
72
|
+
* normalized remote URLs (or null for non-git directories). Prevents repeated
|
|
73
|
+
* subprocess spawns for the same path within a single process lifetime.
|
|
74
|
+
*/
|
|
75
|
+
const gitRemoteCache = new Map<string, string | null>();
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Clear the in-memory git remote cache.
|
|
79
|
+
*
|
|
80
|
+
* Intended for test harnesses that need deterministic behavior across
|
|
81
|
+
* test cases without leaking cached results.
|
|
82
|
+
*/
|
|
83
|
+
export function clearGitRemoteCache(): void {
|
|
84
|
+
gitRemoteCache.clear();
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Get the canonical git remote URL for a repository at the given path.
|
|
89
|
+
*
|
|
90
|
+
* Prefers `upstream` remote (for forks) over `origin`, then falls back
|
|
91
|
+
* to any other remote. Returns null if the path is not in a git repo
|
|
92
|
+
* or has no remotes configured.
|
|
93
|
+
*
|
|
94
|
+
* Results are cached in-memory for the process lifetime to avoid repeated
|
|
95
|
+
* subprocess calls — `git remote -v` only runs once per unique path.
|
|
96
|
+
*/
|
|
97
|
+
export function getGitRemote(path: string): string | null {
|
|
98
|
+
const cached = gitRemoteCache.get(path);
|
|
99
|
+
if (cached !== undefined) return cached;
|
|
100
|
+
|
|
101
|
+
try {
|
|
102
|
+
// git remote -v outputs lines like:
|
|
103
|
+
// origin git@github.com:user/repo.git (fetch)
|
|
104
|
+
// upstream https://github.com/org/repo.git (fetch)
|
|
105
|
+
const output = execSync("git remote -v", {
|
|
106
|
+
cwd: path,
|
|
107
|
+
encoding: "utf-8",
|
|
108
|
+
timeout: 5000,
|
|
109
|
+
stdio: ["pipe", "pipe", "pipe"], // suppress stderr
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
const remotes = new Map<string, string>();
|
|
113
|
+
for (const line of output.split("\n")) {
|
|
114
|
+
// Only parse fetch URLs (avoid duplicates from push lines)
|
|
115
|
+
const match = line.match(/^(\S+)\s+(\S+)\s+\(fetch\)$/);
|
|
116
|
+
if (match) {
|
|
117
|
+
remotes.set(match[1], match[2]);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
if (remotes.size === 0) {
|
|
122
|
+
gitRemoteCache.set(path, null);
|
|
123
|
+
return null;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
// Prefer upstream (fork source) > origin > any other
|
|
127
|
+
const url =
|
|
128
|
+
remotes.get("upstream") ??
|
|
129
|
+
remotes.get("origin") ??
|
|
130
|
+
remotes.values().next().value;
|
|
131
|
+
if (!url) {
|
|
132
|
+
gitRemoteCache.set(path, null);
|
|
133
|
+
return null;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
const result = normalizeRemoteUrl(url);
|
|
137
|
+
gitRemoteCache.set(path, result);
|
|
138
|
+
return result;
|
|
139
|
+
} catch {
|
|
140
|
+
// Not a git repo, git not installed, timeout, etc.
|
|
141
|
+
gitRemoteCache.set(path, null);
|
|
142
|
+
return null;
|
|
143
|
+
}
|
|
144
|
+
}
|
package/src/gradient.ts
CHANGED
|
@@ -46,6 +46,127 @@ let maxLayer0Tokens = 0;
|
|
|
46
46
|
|
|
47
47
|
const MIN_LAYER0_FLOOR = 40_000;
|
|
48
48
|
|
|
49
|
+
// ---------------------------------------------------------------------------
|
|
50
|
+
// Cost-aware context token cap (layer 1+)
|
|
51
|
+
//
|
|
52
|
+
// Limits total tokens (distilled + raw) to keep per-bust cache write cost
|
|
53
|
+
// bounded. For opus-4-6 at $6.25/M write, a $1.00 target yields a 160K cap.
|
|
54
|
+
// For sonnet-4 at $3.75/M write, the cap is 267K (effectively uncapped).
|
|
55
|
+
//
|
|
56
|
+
// The cap is further adjusted dynamically per session via bust rate EMA and
|
|
57
|
+
// inter-bust interval tracking: tighten when busts are frequent, relax when
|
|
58
|
+
// the cache is working well. Asymmetric rates: tighten fast, relax slowly.
|
|
59
|
+
// ---------------------------------------------------------------------------
|
|
60
|
+
|
|
61
|
+
/** Static ceiling for total context tokens, derived from model pricing.
|
|
62
|
+
* 0 = disabled (no cap). Set via setMaxContextTokens(). */
|
|
63
|
+
let maxContextTokensCeiling = 0;
|
|
64
|
+
|
|
65
|
+
const MIN_CONTEXT_FLOOR = 130_000;
|
|
66
|
+
|
|
67
|
+
/** Compute the context ceiling from a per-bust cost target and cache-write price per token. */
|
|
68
|
+
export function computeContextCap(
|
|
69
|
+
targetBustCost: number,
|
|
70
|
+
cacheWriteCostPerToken: number,
|
|
71
|
+
): number {
|
|
72
|
+
if (targetBustCost <= 0 || cacheWriteCostPerToken <= 0) return 0;
|
|
73
|
+
return Math.max(MIN_CONTEXT_FLOOR, Math.floor(targetBustCost / cacheWriteCostPerToken));
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/** Set the static context ceiling. Called by the host adapter after computing
|
|
77
|
+
* from model pricing. The effective per-session cap may be lower due to
|
|
78
|
+
* dynamic adaptation (bust rate EMA). */
|
|
79
|
+
export function setMaxContextTokens(tokens: number) {
|
|
80
|
+
maxContextTokensCeiling = Math.max(0, Math.floor(tokens));
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** Returns the current static ceiling (for external callers / tests). */
|
|
84
|
+
export function getMaxContextTokens(): number {
|
|
85
|
+
return maxContextTokensCeiling;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Feed cache usage data after each API response. Updates the per-session
|
|
90
|
+
* bust rate EMA and inter-bust interval, which adjust the effective context
|
|
91
|
+
* cap dynamically.
|
|
92
|
+
*
|
|
93
|
+
* @param cacheWrite - cache_creation_input_tokens from the API response
|
|
94
|
+
* @param cacheRead - cache_read_input_tokens from the API response
|
|
95
|
+
* @param sessionID - session that produced this response
|
|
96
|
+
*/
|
|
97
|
+
export function updateBustRate(
|
|
98
|
+
cacheWrite: number,
|
|
99
|
+
cacheRead: number,
|
|
100
|
+
sessionID?: string,
|
|
101
|
+
): void {
|
|
102
|
+
if (!sessionID) return;
|
|
103
|
+
const state = getSessionState(sessionID);
|
|
104
|
+
const total = cacheWrite + cacheRead;
|
|
105
|
+
if (total === 0) return;
|
|
106
|
+
|
|
107
|
+
// Bust ratio: fraction of total input that was cache-written (0 = all reads, 1 = all writes)
|
|
108
|
+
const bustRatio = cacheWrite / total;
|
|
109
|
+
|
|
110
|
+
// EMA update (α = 0.3 for smoothing — responsive but not twitchy)
|
|
111
|
+
state.bustRateEMA =
|
|
112
|
+
state.bustRateEMA < 0
|
|
113
|
+
? bustRatio // first observation
|
|
114
|
+
: state.bustRateEMA * 0.7 + bustRatio * 0.3;
|
|
115
|
+
|
|
116
|
+
// Inter-bust interval tracking: a "bust" is when >50% of input is writes
|
|
117
|
+
const now = Date.now();
|
|
118
|
+
if (bustRatio > 0.5) {
|
|
119
|
+
if (state.lastBustAt > 0) {
|
|
120
|
+
const interval = now - state.lastBustAt;
|
|
121
|
+
state.interBustIntervalEMA =
|
|
122
|
+
state.interBustIntervalEMA < 0
|
|
123
|
+
? interval
|
|
124
|
+
: state.interBustIntervalEMA * 0.7 + interval * 0.3;
|
|
125
|
+
}
|
|
126
|
+
state.lastBustAt = now;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
// Adapt per-session cap based on bust rate and interval
|
|
130
|
+
adaptContextCap(state);
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/** Adapt the per-session context cap based on bust rate and break frequency. */
|
|
134
|
+
function adaptContextCap(state: SessionState): void {
|
|
135
|
+
if (maxContextTokensCeiling <= 0) return; // disabled
|
|
136
|
+
|
|
137
|
+
const cap = state.dynamicContextCap > 0
|
|
138
|
+
? state.dynamicContextCap
|
|
139
|
+
: maxContextTokensCeiling;
|
|
140
|
+
|
|
141
|
+
let newCap = cap;
|
|
142
|
+
|
|
143
|
+
// Primary signal: bust rate EMA
|
|
144
|
+
if (state.bustRateEMA > 0.8) {
|
|
145
|
+
// Mostly writes — tighten by 10%
|
|
146
|
+
newCap = Math.floor(cap * 0.90);
|
|
147
|
+
} else if (state.bustRateEMA < 0.3) {
|
|
148
|
+
// Mostly reads — relax by 5% (slower than tightening)
|
|
149
|
+
newCap = Math.floor(cap * 1.05);
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
// Secondary signal: inter-bust interval
|
|
153
|
+
if (state.interBustIntervalEMA > 0) {
|
|
154
|
+
if (state.interBustIntervalEMA < 2 * 60_000) {
|
|
155
|
+
// Busts less than 2 min apart — proactively tighten by extra 5%
|
|
156
|
+
newCap = Math.floor(newCap * 0.95);
|
|
157
|
+
} else if (state.interBustIntervalEMA > 10 * 60_000) {
|
|
158
|
+
// Busts more than 10 min apart — allow extra relaxation
|
|
159
|
+
newCap = Math.floor(newCap * 1.03);
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
// Clamp to [floor, ceiling]
|
|
164
|
+
state.dynamicContextCap = Math.max(
|
|
165
|
+
MIN_CONTEXT_FLOOR,
|
|
166
|
+
Math.min(maxContextTokensCeiling, newCap),
|
|
167
|
+
);
|
|
168
|
+
}
|
|
169
|
+
|
|
49
170
|
// Conservative overhead reserve for first-turn (before calibration):
|
|
50
171
|
// accounts for provider system prompt + AGENTS.md + tool definitions + env info
|
|
51
172
|
const FIRST_TURN_OVERHEAD = 15_000;
|
|
@@ -133,6 +254,18 @@ type SessionState = {
|
|
|
133
254
|
/** Consecutive turns at layer >= 2. When >= 3, log a compaction hint. */
|
|
134
255
|
consecutiveHighLayer: number;
|
|
135
256
|
|
|
257
|
+
// --- Cost-aware context cap dynamic state ---
|
|
258
|
+
|
|
259
|
+
/** EMA of bust ratio (cacheWrite / total). -1 = uninitialized. */
|
|
260
|
+
bustRateEMA: number;
|
|
261
|
+
/** EMA of time between full busts (ms). -1 = uninitialized. */
|
|
262
|
+
interBustIntervalEMA: number;
|
|
263
|
+
/** Epoch ms of the last full bust (cacheWrite > 50% of total). 0 = never. */
|
|
264
|
+
lastBustAt: number;
|
|
265
|
+
/** Per-session dynamic context cap (tokens). Adjusted by adaptContextCap().
|
|
266
|
+
* 0 = use the static ceiling (maxContextTokensCeiling). */
|
|
267
|
+
dynamicContextCap: number;
|
|
268
|
+
|
|
136
269
|
/**
|
|
137
270
|
* Distillation row snapshot — cached to avoid hitting the DB on every
|
|
138
271
|
* transform() call. Refreshed only at turn boundaries (when a new user
|
|
@@ -166,6 +299,11 @@ function makeSessionState(): SessionState {
|
|
|
166
299
|
postIdleCompact: false,
|
|
167
300
|
consecutiveHighLayer: 0,
|
|
168
301
|
|
|
302
|
+
bustRateEMA: -1,
|
|
303
|
+
interBustIntervalEMA: -1,
|
|
304
|
+
lastBustAt: 0,
|
|
305
|
+
dynamicContextCap: 0,
|
|
306
|
+
|
|
169
307
|
distillationSnapshot: null,
|
|
170
308
|
};
|
|
171
309
|
}
|
|
@@ -978,7 +1116,7 @@ function buildPrefixMessages(formatted: string): MessageWithParts[] {
|
|
|
978
1116
|
sessionID: "",
|
|
979
1117
|
messageID: "lore-distilled-assistant",
|
|
980
1118
|
type: "text" as const,
|
|
981
|
-
text: formatted
|
|
1119
|
+
text: formatted,
|
|
982
1120
|
time: { start: 0, end: 0 },
|
|
983
1121
|
},
|
|
984
1122
|
],
|
|
@@ -1291,11 +1429,13 @@ export type TransformResult = {
|
|
|
1291
1429
|
rawBudget: number;
|
|
1292
1430
|
};
|
|
1293
1431
|
|
|
1294
|
-
//
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1432
|
+
// Per-session urgent distillation tracking.
|
|
1433
|
+
// Keyed by sessionID. Set by layer returns in transformInner(),
|
|
1434
|
+
// consumed (read + delete) by needsUrgentDistillation(sessionID).
|
|
1435
|
+
const urgentDistillationMap = new Map<string, boolean>();
|
|
1436
|
+
export function needsUrgentDistillation(sessionID: string): boolean {
|
|
1437
|
+
const v = urgentDistillationMap.get(sessionID) ?? false;
|
|
1438
|
+
urgentDistillationMap.delete(sessionID);
|
|
1299
1439
|
return v;
|
|
1300
1440
|
}
|
|
1301
1441
|
|
|
@@ -1315,10 +1455,23 @@ function transformInner(input: {
|
|
|
1315
1455
|
// minus LTM tokens already injected into the system prompt this turn.
|
|
1316
1456
|
// Read LTM tokens from per-session state to avoid cross-session contamination.
|
|
1317
1457
|
const sessLtmTokens = sid ? sessState.ltmTokens : ltmTokensFallback;
|
|
1318
|
-
const
|
|
1458
|
+
const usableRaw = Math.max(
|
|
1319
1459
|
0,
|
|
1320
1460
|
contextLimit - outputReserved - overhead - sessLtmTokens,
|
|
1321
1461
|
);
|
|
1462
|
+
|
|
1463
|
+
// Cost-aware context cap: limit total distilled + raw tokens to keep
|
|
1464
|
+
// per-bust cache write cost bounded. On opus-4-6 at $6.25/M, a $1.00
|
|
1465
|
+
// target yields a 160K ceiling; on sonnet-4 at $3.75/M, 267K (effectively
|
|
1466
|
+
// uncapped at 200K context). Per-session dynamic adaptation may reduce
|
|
1467
|
+
// this further based on observed bust rate and break frequency.
|
|
1468
|
+
const effectiveCap = sid && sessState.dynamicContextCap > 0
|
|
1469
|
+
? sessState.dynamicContextCap
|
|
1470
|
+
: maxContextTokensCeiling;
|
|
1471
|
+
const usable = effectiveCap > 0 && usableRaw > effectiveCap
|
|
1472
|
+
? effectiveCap
|
|
1473
|
+
: usableRaw;
|
|
1474
|
+
|
|
1322
1475
|
const distilledBudget = Math.floor(usable * cfg.budget.distilled);
|
|
1323
1476
|
// Base raw budget. May be overridden below for post-idle compact mode.
|
|
1324
1477
|
let rawBudget = Math.floor(usable * cfg.budget.raw);
|
|
@@ -1385,12 +1538,11 @@ function transformInner(input: {
|
|
|
1385
1538
|
sessState.postIdleCompact = false;
|
|
1386
1539
|
// Skip layer 0 — don't pass through all raw messages on a cold cache.
|
|
1387
1540
|
effectiveMinLayer = Math.max(effectiveMinLayer, 1) as SafetyLayer;
|
|
1388
|
-
// Use a tighter raw budget
|
|
1389
|
-
//
|
|
1390
|
-
//
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
rawBudget = Math.floor(usable * 0.20);
|
|
1541
|
+
// Use a tighter raw budget. When the cost-aware context cap is active,
|
|
1542
|
+
// total write size is already bounded — use a moderate 30%. Without
|
|
1543
|
+
// the cap, use a tighter 20% to limit cold-write cost directly.
|
|
1544
|
+
const postIdleRawFraction = effectiveCap > 0 ? 0.30 : 0.20;
|
|
1545
|
+
rawBudget = Math.floor(usable * postIdleRawFraction);
|
|
1394
1546
|
log.info(
|
|
1395
1547
|
`post-idle compact: session=${sid} rawBudget=${rawBudget}` +
|
|
1396
1548
|
` (${Math.floor(usable * cfg.budget.raw)}→${rawBudget})`,
|
|
@@ -1500,7 +1652,12 @@ function transformInner(input: {
|
|
|
1500
1652
|
rawBudget,
|
|
1501
1653
|
strip: "none",
|
|
1502
1654
|
});
|
|
1503
|
-
if (fitsWithSafetyMargin(layer1))
|
|
1655
|
+
if (fitsWithSafetyMargin(layer1)) {
|
|
1656
|
+
if (cached.tokens === 0 && sid) {
|
|
1657
|
+
urgentDistillationMap.set(sid, true);
|
|
1658
|
+
}
|
|
1659
|
+
return { ...layer1!, layer: 1, usable, distilledBudget, rawBudget };
|
|
1660
|
+
}
|
|
1504
1661
|
}
|
|
1505
1662
|
|
|
1506
1663
|
// Layer 1 didn't fit (or was force-skipped) — reset the raw window cache.
|
|
@@ -1520,7 +1677,7 @@ function transformInner(input: {
|
|
|
1520
1677
|
protectedTurns: 2,
|
|
1521
1678
|
});
|
|
1522
1679
|
if (fitsWithSafetyMargin(layer2)) {
|
|
1523
|
-
|
|
1680
|
+
if (sid) urgentDistillationMap.set(sid, true);
|
|
1524
1681
|
return { ...layer2!, layer: 2, usable, distilledBudget, rawBudget };
|
|
1525
1682
|
}
|
|
1526
1683
|
}
|
|
@@ -1541,7 +1698,7 @@ function transformInner(input: {
|
|
|
1541
1698
|
strip: "all-tools",
|
|
1542
1699
|
});
|
|
1543
1700
|
if (fitsWithSafetyMargin(layer3)) {
|
|
1544
|
-
|
|
1701
|
+
if (sid) urgentDistillationMap.set(sid, true);
|
|
1545
1702
|
return { ...layer3!, layer: 3, usable, distilledBudget, rawBudget };
|
|
1546
1703
|
}
|
|
1547
1704
|
|
|
@@ -1558,7 +1715,7 @@ function transformInner(input: {
|
|
|
1558
1715
|
// if it alone exceeds the tail budget — layer 4 is the terminal layer
|
|
1559
1716
|
// and must always return. Remaining budget is filled backward with older
|
|
1560
1717
|
// messages.
|
|
1561
|
-
|
|
1718
|
+
if (sid) urgentDistillationMap.set(sid, true);
|
|
1562
1719
|
const nuclearDistillations = distillations.slice(-2);
|
|
1563
1720
|
const nuclearPrefix = distilledPrefix(nuclearDistillations);
|
|
1564
1721
|
const nuclearPrefixTokens = nuclearPrefix.reduce(
|
package/src/index.ts
CHANGED
|
@@ -11,21 +11,27 @@
|
|
|
11
11
|
|
|
12
12
|
export * as temporal from "./temporal";
|
|
13
13
|
export * as ltm from "./ltm";
|
|
14
|
+
export * as data from "./data";
|
|
14
15
|
export * as distillation from "./distillation";
|
|
15
16
|
export * as curator from "./curator";
|
|
16
17
|
export * as embedding from "./embedding";
|
|
18
|
+
export * as embeddingVendor from "./embedding-vendor";
|
|
17
19
|
export * as latReader from "./lat-reader";
|
|
18
20
|
export * as patternExtract from "./pattern-extract";
|
|
19
21
|
export * as log from "./log";
|
|
20
22
|
|
|
21
23
|
export {
|
|
22
24
|
runRecall,
|
|
25
|
+
searchRecall,
|
|
26
|
+
recallById,
|
|
23
27
|
RECALL_TOOL_DESCRIPTION,
|
|
24
28
|
RECALL_PARAM_DESCRIPTIONS,
|
|
25
29
|
type RecallInput,
|
|
26
30
|
type RecallResult,
|
|
27
31
|
type RecallScope,
|
|
28
32
|
type ScoredDistillation,
|
|
33
|
+
type TaggedResult,
|
|
34
|
+
type ScoredTaggedResult,
|
|
29
35
|
} from "./recall";
|
|
30
36
|
|
|
31
37
|
export type {
|
|
@@ -50,22 +56,33 @@ export { isTextPart, isReasoningPart, isToolPart } from "./types";
|
|
|
50
56
|
export { load, config, type LoreConfig } from "./config";
|
|
51
57
|
export {
|
|
52
58
|
db,
|
|
59
|
+
dbPath,
|
|
53
60
|
ensureProject,
|
|
54
61
|
isFirstRun,
|
|
55
62
|
projectId,
|
|
56
63
|
projectName,
|
|
64
|
+
mergeProjectInternal,
|
|
57
65
|
loadForceMinLayer,
|
|
58
66
|
saveForceMinLayer,
|
|
67
|
+
saveSessionCosts,
|
|
68
|
+
loadSessionCosts,
|
|
69
|
+
loadAllSessionCosts,
|
|
70
|
+
type SessionCostSnapshot,
|
|
59
71
|
getMeta,
|
|
60
72
|
setMeta,
|
|
61
73
|
getInstanceId,
|
|
62
74
|
close,
|
|
63
75
|
} from "./db";
|
|
76
|
+
export { normalizeRemoteUrl, getGitRemote, clearGitRemoteCache } from "./git";
|
|
64
77
|
export {
|
|
65
78
|
transform,
|
|
66
79
|
setModelLimits,
|
|
67
80
|
setMaxLayer0Tokens,
|
|
68
81
|
computeLayer0Cap,
|
|
82
|
+
setMaxContextTokens,
|
|
83
|
+
computeContextCap,
|
|
84
|
+
getMaxContextTokens,
|
|
85
|
+
updateBustRate,
|
|
69
86
|
needsUrgentDistillation,
|
|
70
87
|
calibrate,
|
|
71
88
|
setLtmTokens,
|
|
@@ -106,6 +123,7 @@ export {
|
|
|
106
123
|
importLoreFile,
|
|
107
124
|
shouldImportLoreFile,
|
|
108
125
|
loreFileExists,
|
|
126
|
+
clearLoreFileCache,
|
|
109
127
|
LORE_FILE,
|
|
110
128
|
} from "./agents-file";
|
|
111
129
|
export { workerSessionIDs, isWorkerSession } from "./worker";
|
|
@@ -113,6 +131,7 @@ export * as workerModel from "./worker-model";
|
|
|
113
131
|
export {
|
|
114
132
|
ftsQuery,
|
|
115
133
|
ftsQueryOr,
|
|
134
|
+
ftsQueryRelaxed,
|
|
116
135
|
EMPTY_QUERY,
|
|
117
136
|
reciprocalRankFusion,
|
|
118
137
|
expandQuery,
|
|
@@ -133,4 +152,5 @@ export {
|
|
|
133
152
|
normalize,
|
|
134
153
|
sanitizeSurrogates,
|
|
135
154
|
unescapeMarkdown,
|
|
155
|
+
renderMarkdown,
|
|
136
156
|
} from "./markdown";
|
package/src/lat-reader.ts
CHANGED
|
@@ -15,7 +15,7 @@ import { remark } from "remark";
|
|
|
15
15
|
import type { Root, Heading, Paragraph, Text } from "mdast";
|
|
16
16
|
import { db, ensureProject } from "./db";
|
|
17
17
|
import { sha256 } from "#db/driver";
|
|
18
|
-
import { ftsQuery,
|
|
18
|
+
import { ftsQuery, extractTopTerms, EMPTY_QUERY, runRelaxedSearch } from "./search";
|
|
19
19
|
import * as log from "./log";
|
|
20
20
|
|
|
21
21
|
const processor = remark();
|
|
@@ -274,7 +274,7 @@ export function refresh(projectPath: string): number {
|
|
|
274
274
|
|
|
275
275
|
/**
|
|
276
276
|
* Search lat sections by FTS5 with BM25 scoring.
|
|
277
|
-
* Uses AND
|
|
277
|
+
* Uses progressive AND relaxation before falling back to OR.
|
|
278
278
|
*/
|
|
279
279
|
export function searchScored(input: {
|
|
280
280
|
query: string;
|
|
@@ -282,8 +282,6 @@ export function searchScored(input: {
|
|
|
282
282
|
limit?: number;
|
|
283
283
|
}): ScoredLatSection[] {
|
|
284
284
|
const limit = input.limit ?? 10;
|
|
285
|
-
const q = ftsQuery(input.query);
|
|
286
|
-
if (q === EMPTY_QUERY) return [];
|
|
287
285
|
|
|
288
286
|
const pid = ensureProject(input.projectPath);
|
|
289
287
|
|
|
@@ -297,13 +295,9 @@ export function searchScored(input: {
|
|
|
297
295
|
ORDER BY rank LIMIT ?`;
|
|
298
296
|
|
|
299
297
|
try {
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
// AND returned nothing — try OR fallback
|
|
304
|
-
const qOr = ftsQueryOr(input.query);
|
|
305
|
-
if (qOr === EMPTY_QUERY) return [];
|
|
306
|
-
return db().query(ftsSQL).all(qOr, pid, limit) as ScoredLatSection[];
|
|
298
|
+
return runRelaxedSearch(input.query, (matchExpr) =>
|
|
299
|
+
db().query(ftsSQL).all(matchExpr, pid, limit) as ScoredLatSection[],
|
|
300
|
+
);
|
|
307
301
|
} catch {
|
|
308
302
|
return [];
|
|
309
303
|
}
|
package/src/ltm.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { uuidv7 } from "uuidv7";
|
|
2
2
|
import { db, ensureProject } from "./db";
|
|
3
3
|
import { config } from "./config";
|
|
4
|
-
import { ftsQuery,
|
|
4
|
+
import { ftsQuery, EMPTY_QUERY, extractTopTerms, runRelaxedSearch } from "./search";
|
|
5
5
|
import * as embedding from "./embedding";
|
|
6
6
|
import * as latReader from "./lat-reader";
|
|
7
7
|
import * as log from "./log";
|
|
@@ -454,8 +454,6 @@ export function search(input: {
|
|
|
454
454
|
limit?: number;
|
|
455
455
|
}): KnowledgeEntry[] {
|
|
456
456
|
const limit = input.limit ?? 20;
|
|
457
|
-
const q = ftsQuery(input.query);
|
|
458
|
-
if (q === EMPTY_QUERY) return [];
|
|
459
457
|
|
|
460
458
|
const pid = input.projectPath ? ensureProject(input.projectPath) : null;
|
|
461
459
|
|
|
@@ -473,22 +471,14 @@ export function search(input: {
|
|
|
473
471
|
ORDER BY bm25(knowledge_fts, ?, ?, ?) LIMIT ?`;
|
|
474
472
|
|
|
475
473
|
const { title, content, category } = ftsWeights();
|
|
476
|
-
const ftsParams = pid
|
|
477
|
-
? [q, pid, title, content, category, limit]
|
|
478
|
-
: [q, title, content, category, limit];
|
|
479
474
|
|
|
480
475
|
try {
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
const ftsParamsOr = pid
|
|
489
|
-
? [qOr, pid, title, content, category, limit]
|
|
490
|
-
: [qOr, title, content, category, limit];
|
|
491
|
-
return db().query(ftsSQL).all(...ftsParamsOr) as KnowledgeEntry[];
|
|
476
|
+
return runRelaxedSearch(input.query, (matchExpr) => {
|
|
477
|
+
const params = pid
|
|
478
|
+
? [matchExpr, pid, title, content, category, limit]
|
|
479
|
+
: [matchExpr, title, content, category, limit];
|
|
480
|
+
return db().query(ftsSQL).all(...params) as KnowledgeEntry[];
|
|
481
|
+
});
|
|
492
482
|
} catch {
|
|
493
483
|
return searchLike({
|
|
494
484
|
query: input.query,
|
|
@@ -510,8 +500,6 @@ export function searchScored(input: {
|
|
|
510
500
|
limit?: number;
|
|
511
501
|
}): ScoredKnowledgeEntry[] {
|
|
512
502
|
const limit = input.limit ?? 20;
|
|
513
|
-
const q = ftsQuery(input.query);
|
|
514
|
-
if (q === EMPTY_QUERY) return [];
|
|
515
503
|
|
|
516
504
|
const pid = input.projectPath ? ensureProject(input.projectPath) : null;
|
|
517
505
|
const { title, content, category } = ftsWeights();
|
|
@@ -529,20 +517,13 @@ export function searchScored(input: {
|
|
|
529
517
|
AND k.confidence > 0.2
|
|
530
518
|
ORDER BY rank LIMIT ?`;
|
|
531
519
|
|
|
532
|
-
const ftsParams = pid
|
|
533
|
-
? [title, content, category, q, pid, limit]
|
|
534
|
-
: [title, content, category, q, limit];
|
|
535
|
-
|
|
536
520
|
try {
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
? [title, content, category, qOr, pid, limit]
|
|
544
|
-
: [title, content, category, qOr, limit];
|
|
545
|
-
return db().query(ftsSQL).all(...ftsParamsOr) as ScoredKnowledgeEntry[];
|
|
521
|
+
return runRelaxedSearch(input.query, (matchExpr) => {
|
|
522
|
+
const params = pid
|
|
523
|
+
? [title, content, category, matchExpr, pid, limit]
|
|
524
|
+
: [title, content, category, matchExpr, limit];
|
|
525
|
+
return db().query(ftsSQL).all(...params) as ScoredKnowledgeEntry[];
|
|
526
|
+
});
|
|
546
527
|
} catch {
|
|
547
528
|
return [];
|
|
548
529
|
}
|
|
@@ -560,8 +541,6 @@ export function searchScoredOtherProjects(input: {
|
|
|
560
541
|
limit?: number;
|
|
561
542
|
}): ScoredKnowledgeEntry[] {
|
|
562
543
|
const limit = input.limit ?? 10;
|
|
563
|
-
const q = ftsQuery(input.query);
|
|
564
|
-
if (q === EMPTY_QUERY) return [];
|
|
565
544
|
|
|
566
545
|
const excludePid = ensureProject(input.excludeProjectPath);
|
|
567
546
|
const { title, content, category } = ftsWeights();
|
|
@@ -578,17 +557,11 @@ export function searchScoredOtherProjects(input: {
|
|
|
578
557
|
AND k.confidence > 0.2
|
|
579
558
|
ORDER BY rank LIMIT ?`;
|
|
580
559
|
|
|
581
|
-
const ftsParams = [title, content, category, q, excludePid, limit];
|
|
582
|
-
|
|
583
560
|
try {
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
const qOr = ftsQueryOr(input.query);
|
|
589
|
-
if (qOr === EMPTY_QUERY) return [];
|
|
590
|
-
const ftsParamsOr = [title, content, category, qOr, excludePid, limit];
|
|
591
|
-
return db().query(ftsSQL).all(...ftsParamsOr) as ScoredKnowledgeEntry[];
|
|
561
|
+
return runRelaxedSearch(input.query, (matchExpr) => {
|
|
562
|
+
const params = [title, content, category, matchExpr, excludePid, limit];
|
|
563
|
+
return db().query(ftsSQL).all(...params) as ScoredKnowledgeEntry[];
|
|
564
|
+
});
|
|
592
565
|
} catch {
|
|
593
566
|
return [];
|
|
594
567
|
}
|
package/src/markdown.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { micromark } from "micromark";
|
|
1
2
|
import { remark } from "remark";
|
|
2
3
|
import type {
|
|
3
4
|
Root,
|
|
@@ -127,3 +128,17 @@ export function strong(value: string): Strong {
|
|
|
127
128
|
export function root(...children: Root["children"]): Root {
|
|
128
129
|
return { type: "root", children };
|
|
129
130
|
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Render a markdown string to sanitized HTML.
|
|
134
|
+
*
|
|
135
|
+
* Uses micromark with default options:
|
|
136
|
+
* - Raw HTML in input is escaped (no allowDangerousHtml)
|
|
137
|
+
* - Only safe URL protocols are permitted (no allowDangerousProtocol)
|
|
138
|
+
*
|
|
139
|
+
* The output is safe to embed directly in an HTML page without
|
|
140
|
+
* additional escaping.
|
|
141
|
+
*/
|
|
142
|
+
export function renderMarkdown(md: string): string {
|
|
143
|
+
return micromark(md);
|
|
144
|
+
}
|
package/src/prompt.ts
CHANGED
|
@@ -446,8 +446,7 @@ Rules:
|
|
|
446
446
|
- Keep every section, even when empty.
|
|
447
447
|
- Use terse bullets, not prose paragraphs.
|
|
448
448
|
- Preserve exact file paths, commands, error strings, and identifiers when known.
|
|
449
|
-
- Do not mention the summary process or that context was compacted
|
|
450
|
-
- End with "I'm ready to continue." on its own line after the closing "---".`;
|
|
449
|
+
- Do not mention the summary process or that context was compacted.`;
|
|
451
450
|
|
|
452
451
|
// Build the user-facing prompt passed to the compaction agent during /compact.
|
|
453
452
|
// Lore injects pre-computed distillations as context separately; this prompt
|