@loreai/core 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bun/agents-file.d.ts +29 -8
- package/dist/bun/agents-file.d.ts.map +1 -1
- package/dist/bun/config.d.ts +1 -0
- package/dist/bun/config.d.ts.map +1 -1
- package/dist/bun/db.d.ts.map +1 -1
- package/dist/bun/distillation.d.ts +29 -0
- package/dist/bun/distillation.d.ts.map +1 -1
- package/dist/bun/embedding.d.ts +15 -1
- package/dist/bun/embedding.d.ts.map +1 -1
- package/dist/bun/gradient.d.ts +53 -5
- package/dist/bun/gradient.d.ts.map +1 -1
- package/dist/bun/index.d.ts +4 -4
- package/dist/bun/index.d.ts.map +1 -1
- package/dist/bun/index.js +696 -243
- package/dist/bun/index.js.map +4 -4
- package/dist/bun/pattern-extract.d.ts +36 -0
- package/dist/bun/pattern-extract.d.ts.map +1 -0
- package/dist/bun/recall.d.ts +1 -0
- package/dist/bun/recall.d.ts.map +1 -1
- package/dist/bun/search.d.ts +13 -1
- package/dist/bun/search.d.ts.map +1 -1
- package/dist/bun/types.d.ts +41 -1
- package/dist/bun/types.d.ts.map +1 -1
- package/dist/bun/worker-model.d.ts +22 -0
- package/dist/bun/worker-model.d.ts.map +1 -1
- package/dist/node/agents-file.d.ts +29 -8
- package/dist/node/agents-file.d.ts.map +1 -1
- package/dist/node/config.d.ts +1 -0
- package/dist/node/config.d.ts.map +1 -1
- package/dist/node/db.d.ts.map +1 -1
- package/dist/node/distillation.d.ts +29 -0
- package/dist/node/distillation.d.ts.map +1 -1
- package/dist/node/embedding.d.ts +15 -1
- package/dist/node/embedding.d.ts.map +1 -1
- package/dist/node/gradient.d.ts +53 -5
- package/dist/node/gradient.d.ts.map +1 -1
- package/dist/node/index.d.ts +4 -4
- package/dist/node/index.d.ts.map +1 -1
- package/dist/node/index.js +696 -243
- package/dist/node/index.js.map +4 -4
- package/dist/node/pattern-extract.d.ts +36 -0
- package/dist/node/pattern-extract.d.ts.map +1 -0
- package/dist/node/recall.d.ts +1 -0
- package/dist/node/recall.d.ts.map +1 -1
- package/dist/node/search.d.ts +13 -1
- package/dist/node/search.d.ts.map +1 -1
- package/dist/node/types.d.ts +41 -1
- package/dist/node/types.d.ts.map +1 -1
- package/dist/node/worker-model.d.ts +22 -0
- package/dist/node/worker-model.d.ts.map +1 -1
- package/dist/types/agents-file.d.ts +29 -8
- package/dist/types/agents-file.d.ts.map +1 -1
- package/dist/types/config.d.ts +1 -0
- package/dist/types/config.d.ts.map +1 -1
- package/dist/types/db.d.ts.map +1 -1
- package/dist/types/distillation.d.ts +29 -0
- package/dist/types/distillation.d.ts.map +1 -1
- package/dist/types/embedding.d.ts +15 -1
- package/dist/types/embedding.d.ts.map +1 -1
- package/dist/types/gradient.d.ts +53 -5
- package/dist/types/gradient.d.ts.map +1 -1
- package/dist/types/index.d.ts +4 -4
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/pattern-extract.d.ts +36 -0
- package/dist/types/pattern-extract.d.ts.map +1 -0
- package/dist/types/recall.d.ts +1 -0
- package/dist/types/recall.d.ts.map +1 -1
- package/dist/types/search.d.ts +13 -1
- package/dist/types/search.d.ts.map +1 -1
- package/dist/types/types.d.ts +41 -1
- package/dist/types/types.d.ts.map +1 -1
- package/dist/types/worker-model.d.ts +22 -0
- package/dist/types/worker-model.d.ts.map +1 -1
- package/package.json +3 -2
- package/src/agents-file.ts +111 -28
- package/src/config.ts +25 -18
- package/src/curator.ts +2 -2
- package/src/db.ts +19 -2
- package/src/distillation.ts +152 -15
- package/src/embedding.ts +158 -14
- package/src/gradient.ts +398 -227
- package/src/index.ts +13 -5
- package/src/pattern-extract.ts +108 -0
- package/src/recall.ts +124 -6
- package/src/search.ts +37 -1
- package/src/types.ts +41 -1
- package/src/worker-model.ts +142 -5
package/dist/node/index.js
CHANGED
|
@@ -163,6 +163,7 @@ function sha256(input) {
|
|
|
163
163
|
// src/db.ts
|
|
164
164
|
import { join, dirname } from "path";
|
|
165
165
|
import { mkdirSync } from "fs";
|
|
166
|
+
import { homedir } from "os";
|
|
166
167
|
var MIGRATIONS = [
|
|
167
168
|
`
|
|
168
169
|
-- Version 1: Initial schema
|
|
@@ -491,11 +492,27 @@ var MIGRATIONS = [
|
|
|
491
492
|
)
|
|
492
493
|
WHERE content LIKE '%' || char(10) || '[tool:%'
|
|
493
494
|
OR content LIKE '%' || char(10) || '[reasoning] %';
|
|
495
|
+
`,
|
|
496
|
+
`
|
|
497
|
+
-- Version 12: Context health diagnostic columns on distillations.
|
|
498
|
+
--
|
|
499
|
+
-- r_compression: k/\u221AN where k = distilled token count, N = source token
|
|
500
|
+
-- count. Values < 1.0 signal likely lossy compression. NULL for rows
|
|
501
|
+
-- created before this migration or for meta-distillations (gen > 0)
|
|
502
|
+
-- where the metric is not computed.
|
|
503
|
+
--
|
|
504
|
+
-- c_norm: normalized variance of relative-existence weights over source
|
|
505
|
+
-- message timestamps. Range [0, 1]; 0 = uniform distribution, 1 = attention
|
|
506
|
+
-- dominated by distant past. NULL for pre-migration rows or meta-distillations.
|
|
507
|
+
--
|
|
508
|
+
-- Both columns are nullable REALs \u2014 cheap to add, no backfill needed.
|
|
509
|
+
ALTER TABLE distillations ADD COLUMN r_compression REAL;
|
|
510
|
+
ALTER TABLE distillations ADD COLUMN c_norm REAL;
|
|
494
511
|
`
|
|
495
512
|
];
|
|
496
513
|
function dataDir() {
|
|
497
514
|
const xdg = process.env.XDG_DATA_HOME;
|
|
498
|
-
const base = xdg || join(
|
|
515
|
+
const base = xdg || join(homedir(), ".local", "share");
|
|
499
516
|
return join(base, "opencode-lore");
|
|
500
517
|
}
|
|
501
518
|
var instance;
|
|
@@ -11291,14 +11308,24 @@ function reciprocalRankFusion(lists, k = 60) {
|
|
|
11291
11308
|
}
|
|
11292
11309
|
return [...scores.values()].sort((a, b) => b.score - a.score);
|
|
11293
11310
|
}
|
|
11294
|
-
|
|
11311
|
+
function exactTermMatchRank(items, getText, query) {
|
|
11312
|
+
const terms = filterTerms(query).map((t2) => t2.toLowerCase());
|
|
11313
|
+
if (!terms.length) return [];
|
|
11314
|
+
const scored = items.map((item) => {
|
|
11315
|
+
const text4 = getText(item).toLowerCase();
|
|
11316
|
+
const matches = terms.filter((t2) => text4.includes(t2)).length;
|
|
11317
|
+
return { item, matches };
|
|
11318
|
+
}).filter((s) => s.matches > 0).sort((a, b) => b.matches - a.matches);
|
|
11319
|
+
return scored.map((s) => s.item);
|
|
11320
|
+
}
|
|
11321
|
+
async function expandQuery(llm, query, model, sessionID) {
|
|
11295
11322
|
const TIMEOUT_MS = 3e3;
|
|
11296
11323
|
try {
|
|
11297
11324
|
const responseText = await Promise.race([
|
|
11298
11325
|
llm.prompt(
|
|
11299
11326
|
QUERY_EXPANSION_SYSTEM,
|
|
11300
11327
|
`Input: "${query}"`,
|
|
11301
|
-
{ model, workerID: "lore-query-expand" }
|
|
11328
|
+
{ model, workerID: "lore-query-expand", thinking: false, urgent: true, sessionID }
|
|
11302
11329
|
),
|
|
11303
11330
|
new Promise((resolve) => setTimeout(() => resolve(null), TIMEOUT_MS))
|
|
11304
11331
|
]);
|
|
@@ -25708,11 +25735,15 @@ var LoreConfig = external_exports.object({
|
|
|
25708
25735
|
* Anthropic's April 23 postmortem identified dropping reasoning blocks as
|
|
25709
25736
|
* the root cause of forgetfulness/repetition.
|
|
25710
25737
|
*
|
|
25711
|
-
* `idleResumeMinutes` is the threshold in minutes. Default
|
|
25712
|
-
* Anthropic's
|
|
25738
|
+
* `idleResumeMinutes` is the threshold in minutes. Default 5 — matches
|
|
25739
|
+
* Anthropic's default-tier prompt cache TTL. After 5 min of inactivity the
|
|
25740
|
+
* upstream cache is cold, so preserving byte-identity wastes cache-write cost
|
|
25741
|
+
* for no benefit. Refreshing the caches on resume produces a better-fitting
|
|
25742
|
+
* window at the same cold-write price. Users on Anthropic's extended-cache
|
|
25743
|
+
* tier (1 h TTL) should set this to 60 in `.lore.json`.
|
|
25713
25744
|
* Set to 0 to disable the feature.
|
|
25714
25745
|
*/
|
|
25715
|
-
idleResumeMinutes: external_exports.number().min(0).max(24 * 60).default(
|
|
25746
|
+
idleResumeMinutes: external_exports.number().min(0).max(24 * 60).default(5),
|
|
25716
25747
|
distillation: external_exports.object({
|
|
25717
25748
|
minMessages: external_exports.number().min(3).default(5),
|
|
25718
25749
|
maxSegment: external_exports.number().min(5).default(30),
|
|
@@ -25763,34 +25794,37 @@ var LoreConfig = external_exports.object({
|
|
|
25763
25794
|
* before search, improving recall for ambiguous queries. */
|
|
25764
25795
|
queryExpansion: external_exports.boolean().default(false),
|
|
25765
25796
|
/** Vector embedding search.
|
|
25766
|
-
* Supports multiple providers:
|
|
25767
|
-
* "
|
|
25768
|
-
*
|
|
25769
|
-
*
|
|
25797
|
+
* Supports multiple providers:
|
|
25798
|
+
* - "local" (default): fastembed + ONNX Runtime, no API key needed.
|
|
25799
|
+
* Uses bge-small-en-v1.5 (384 dims). Model downloaded on first use (~33MB),
|
|
25800
|
+
* cached in ~/.cache/fastembed. ~150ms per query embed.
|
|
25801
|
+
* - "voyage": Voyage AI (VOYAGE_API_KEY, voyage-code-3, 1024 dims)
|
|
25802
|
+
* - "openai": OpenAI (OPENAI_API_KEY, text-embedding-3-small, 1536 dims)
|
|
25803
|
+
* Set enabled: false to explicitly disable even with a provider available. */
|
|
25770
25804
|
embeddings: external_exports.object({
|
|
25771
25805
|
/** Enable/disable vector embedding search. Default: true.
|
|
25772
|
-
* Set to false to explicitly disable
|
|
25806
|
+
* Set to false to explicitly disable. */
|
|
25773
25807
|
enabled: external_exports.boolean().default(true),
|
|
25774
|
-
/** Embedding provider. Default: "
|
|
25775
|
-
*
|
|
25808
|
+
/** Embedding provider. Default: "local".
|
|
25809
|
+
* - "local": fastembed + ONNX Runtime, no API key (default model: bge-small-en-v1.5, 384 dims)
|
|
25776
25810
|
* - "voyage": VOYAGE_API_KEY (default model: voyage-code-3, 1024 dims)
|
|
25777
25811
|
* - "openai": OPENAI_API_KEY (default model: text-embedding-3-small, 1536 dims) */
|
|
25778
|
-
provider: external_exports.enum(["voyage", "openai"]).default("
|
|
25812
|
+
provider: external_exports.enum(["local", "voyage", "openai"]).default("local"),
|
|
25779
25813
|
/** Model ID for the embedding provider. Default depends on provider. */
|
|
25780
|
-
model: external_exports.string().default("
|
|
25781
|
-
/** Embedding dimensions. Default: 1024. */
|
|
25782
|
-
dimensions: external_exports.number().min(
|
|
25814
|
+
model: external_exports.string().default("BGESmallENV15"),
|
|
25815
|
+
/** Embedding dimensions. Default: 384 (local) / 1024 (voyage) / 1536 (openai). */
|
|
25816
|
+
dimensions: external_exports.number().min(64).max(2048).default(384)
|
|
25783
25817
|
}).default({
|
|
25784
25818
|
enabled: true,
|
|
25785
|
-
provider: "
|
|
25786
|
-
model: "
|
|
25787
|
-
dimensions:
|
|
25819
|
+
provider: "local",
|
|
25820
|
+
model: "BGESmallENV15",
|
|
25821
|
+
dimensions: 384
|
|
25788
25822
|
})
|
|
25789
25823
|
}).default({
|
|
25790
25824
|
ftsWeights: { title: 6, content: 2, category: 3 },
|
|
25791
25825
|
recallLimit: 10,
|
|
25792
25826
|
queryExpansion: false,
|
|
25793
|
-
embeddings: { enabled: true, provider: "
|
|
25827
|
+
embeddings: { enabled: true, provider: "local", model: "BGESmallENV15", dimensions: 384 }
|
|
25794
25828
|
}),
|
|
25795
25829
|
crossProject: external_exports.boolean().default(false),
|
|
25796
25830
|
agentsFile: external_exports.object({
|
|
@@ -25828,6 +25862,7 @@ __export(embedding_exports, {
|
|
|
25828
25862
|
fromBlob: () => fromBlob,
|
|
25829
25863
|
isAvailable: () => isAvailable,
|
|
25830
25864
|
resetProvider: () => resetProvider,
|
|
25865
|
+
runStartupBackfill: () => runStartupBackfill,
|
|
25831
25866
|
toBlob: () => toBlob,
|
|
25832
25867
|
vectorSearch: () => vectorSearch,
|
|
25833
25868
|
vectorSearchDistillations: () => vectorSearchDistillations
|
|
@@ -25905,9 +25940,43 @@ var OpenAIProvider = class {
|
|
|
25905
25940
|
return sorted.map((d) => new Float32Array(d.embedding));
|
|
25906
25941
|
}
|
|
25907
25942
|
};
|
|
25908
|
-
var
|
|
25909
|
-
|
|
25910
|
-
|
|
25943
|
+
var LocalProvider = class {
|
|
25944
|
+
maxBatchSize = 256;
|
|
25945
|
+
model = null;
|
|
25946
|
+
initPromise = null;
|
|
25947
|
+
modelName;
|
|
25948
|
+
constructor(modelName) {
|
|
25949
|
+
this.modelName = modelName;
|
|
25950
|
+
}
|
|
25951
|
+
async getModel() {
|
|
25952
|
+
if (this.model) return this.model;
|
|
25953
|
+
if (!this.initPromise) {
|
|
25954
|
+
this.initPromise = (async () => {
|
|
25955
|
+
const { EmbeddingModel, FlagEmbedding } = await import("fastembed");
|
|
25956
|
+
const enumValue = EmbeddingModel[this.modelName];
|
|
25957
|
+
const m = await FlagEmbedding.init({
|
|
25958
|
+
model: enumValue ?? this.modelName
|
|
25959
|
+
});
|
|
25960
|
+
this.model = m;
|
|
25961
|
+
return m;
|
|
25962
|
+
})();
|
|
25963
|
+
}
|
|
25964
|
+
return this.initPromise;
|
|
25965
|
+
}
|
|
25966
|
+
async embed(texts, inputType) {
|
|
25967
|
+
const model = await this.getModel();
|
|
25968
|
+
if (inputType === "query" && texts.length === 1) {
|
|
25969
|
+
const vec = await model.queryEmbed(texts[0]);
|
|
25970
|
+
return [new Float32Array(vec)];
|
|
25971
|
+
}
|
|
25972
|
+
const results = [];
|
|
25973
|
+
for await (const batch of model.passageEmbed(texts)) {
|
|
25974
|
+
for (const vec of batch) {
|
|
25975
|
+
results.push(new Float32Array(vec));
|
|
25976
|
+
}
|
|
25977
|
+
}
|
|
25978
|
+
return results;
|
|
25979
|
+
}
|
|
25911
25980
|
};
|
|
25912
25981
|
var PROVIDER_ENV_KEYS = {
|
|
25913
25982
|
voyage: "VOYAGE_API_KEY",
|
|
@@ -25926,21 +25995,35 @@ function getProvider() {
|
|
|
25926
25995
|
return null;
|
|
25927
25996
|
}
|
|
25928
25997
|
const providerName = cfg.provider;
|
|
25929
|
-
const
|
|
25930
|
-
if (!apiKey) {
|
|
25931
|
-
cachedProvider = null;
|
|
25932
|
-
return null;
|
|
25933
|
-
}
|
|
25934
|
-
const defaults = PROVIDER_DEFAULTS[providerName];
|
|
25935
|
-
const model = cfg.model === defaults?.model ? cfg.model : cfg.model;
|
|
25936
|
-
const dimensions = cfg.dimensions;
|
|
25998
|
+
const model = cfg.model;
|
|
25937
25999
|
switch (providerName) {
|
|
25938
|
-
case "
|
|
25939
|
-
|
|
26000
|
+
case "local": {
|
|
26001
|
+
try {
|
|
26002
|
+
cachedProvider = new LocalProvider(model);
|
|
26003
|
+
} catch {
|
|
26004
|
+
info("local embedding provider unavailable (fastembed not installed)");
|
|
26005
|
+
cachedProvider = null;
|
|
26006
|
+
}
|
|
25940
26007
|
break;
|
|
25941
|
-
|
|
25942
|
-
|
|
26008
|
+
}
|
|
26009
|
+
case "voyage": {
|
|
26010
|
+
const apiKey = getProviderApiKey(providerName);
|
|
26011
|
+
if (!apiKey) {
|
|
26012
|
+
cachedProvider = null;
|
|
26013
|
+
return null;
|
|
26014
|
+
}
|
|
26015
|
+
cachedProvider = new VoyageProvider(apiKey, model, cfg.dimensions);
|
|
26016
|
+
break;
|
|
26017
|
+
}
|
|
26018
|
+
case "openai": {
|
|
26019
|
+
const apiKey = getProviderApiKey(providerName);
|
|
26020
|
+
if (!apiKey) {
|
|
26021
|
+
cachedProvider = null;
|
|
26022
|
+
return null;
|
|
26023
|
+
}
|
|
26024
|
+
cachedProvider = new OpenAIProvider(apiKey, model, cfg.dimensions);
|
|
25943
26025
|
break;
|
|
26026
|
+
}
|
|
25944
26027
|
default:
|
|
25945
26028
|
info(`unknown embedding provider: ${providerName}`);
|
|
25946
26029
|
cachedProvider = null;
|
|
@@ -26045,6 +26128,29 @@ function checkConfigChange() {
|
|
|
26045
26128
|
).run(EMBEDDING_CONFIG_KEY, current2, current2);
|
|
26046
26129
|
return true;
|
|
26047
26130
|
}
|
|
26131
|
+
async function runStartupBackfill() {
|
|
26132
|
+
if (!isAvailable()) return;
|
|
26133
|
+
const knowledgeEmbedded = await backfillEmbeddings();
|
|
26134
|
+
const distillationEmbedded = await backfillDistillationEmbeddings();
|
|
26135
|
+
const kTotal = db().query("SELECT COUNT(*) as n FROM knowledge WHERE confidence > 0.2").get().n;
|
|
26136
|
+
const kWithEmb = db().query(
|
|
26137
|
+
"SELECT COUNT(*) as n FROM knowledge WHERE embedding IS NOT NULL AND confidence > 0.2"
|
|
26138
|
+
).get().n;
|
|
26139
|
+
const dTotal = db().query(
|
|
26140
|
+
"SELECT COUNT(*) as n FROM distillations WHERE archived = 0 AND observations != ''"
|
|
26141
|
+
).get().n;
|
|
26142
|
+
const dWithEmb = db().query(
|
|
26143
|
+
"SELECT COUNT(*) as n FROM distillations WHERE embedding IS NOT NULL AND archived = 0"
|
|
26144
|
+
).get().n;
|
|
26145
|
+
const parts = [];
|
|
26146
|
+
if (knowledgeEmbedded > 0 || distillationEmbedded > 0) {
|
|
26147
|
+
parts.push(`backfilled ${knowledgeEmbedded} knowledge + ${distillationEmbedded} distillations`);
|
|
26148
|
+
}
|
|
26149
|
+
parts.push(
|
|
26150
|
+
`coverage: knowledge ${kWithEmb}/${kTotal}, distillations ${dWithEmb}/${dTotal}`
|
|
26151
|
+
);
|
|
26152
|
+
info(`embedding startup: ${parts.join("; ")}`);
|
|
26153
|
+
}
|
|
26048
26154
|
async function backfillEmbeddings() {
|
|
26049
26155
|
checkConfigChange();
|
|
26050
26156
|
const provider = getProvider();
|
|
@@ -26801,6 +26907,7 @@ function check2(projectPath) {
|
|
|
26801
26907
|
// src/distillation.ts
|
|
26802
26908
|
var distillation_exports = {};
|
|
26803
26909
|
__export(distillation_exports, {
|
|
26910
|
+
backfillMetrics: () => backfillMetrics,
|
|
26804
26911
|
compressionRatio: () => compressionRatio,
|
|
26805
26912
|
detectSegments: () => detectSegments,
|
|
26806
26913
|
latestMetaObservations: () => latestMetaObservations,
|
|
@@ -26813,6 +26920,72 @@ __export(distillation_exports, {
|
|
|
26813
26920
|
workerSessionIDs: () => workerSessionIDs
|
|
26814
26921
|
});
|
|
26815
26922
|
|
|
26923
|
+
// src/pattern-extract.ts
|
|
26924
|
+
var pattern_extract_exports = {};
|
|
26925
|
+
__export(pattern_extract_exports, {
|
|
26926
|
+
extractPatterns: () => extractPatterns
|
|
26927
|
+
});
|
|
26928
|
+
var PATTERNS = [
|
|
26929
|
+
// Decision patterns
|
|
26930
|
+
{
|
|
26931
|
+
regex: /decided to (?:use |switch to |go with |adopt )(.+?)(?:\.|,|$)/gi,
|
|
26932
|
+
category: "decision",
|
|
26933
|
+
titleFn: (m) => `Decided to use ${m[1].trim()}`
|
|
26934
|
+
},
|
|
26935
|
+
{
|
|
26936
|
+
regex: /chose (.+?) over (.+?)(?:\.|,|$)/gi,
|
|
26937
|
+
category: "decision",
|
|
26938
|
+
titleFn: (m) => `Chose ${m[1].trim()} over ${m[2].trim()}`
|
|
26939
|
+
},
|
|
26940
|
+
{
|
|
26941
|
+
regex: /switched from (.+?) to (.+?)(?:\.|,|$)/gi,
|
|
26942
|
+
category: "decision",
|
|
26943
|
+
titleFn: (m) => `Switched from ${m[1].trim()} to ${m[2].trim()}`
|
|
26944
|
+
},
|
|
26945
|
+
{
|
|
26946
|
+
regex: /going with (.+?) (?:because|for|due to)(.+?)(?:\.|,|$)/gi,
|
|
26947
|
+
category: "decision",
|
|
26948
|
+
titleFn: (m) => `Going with ${m[1].trim()}`
|
|
26949
|
+
},
|
|
26950
|
+
{
|
|
26951
|
+
regex: /migrat(?:ed|ing) (?:from .+? )?to (.+?)(?:\.|,|$)/gi,
|
|
26952
|
+
category: "decision",
|
|
26953
|
+
titleFn: (m) => `Migrated to ${m[1].trim()}`
|
|
26954
|
+
},
|
|
26955
|
+
{
|
|
26956
|
+
regex: /adopted (.+?) (?:for|as|instead)(.+?)(?:\.|,|$)/gi,
|
|
26957
|
+
category: "decision",
|
|
26958
|
+
titleFn: (m) => `Adopted ${m[1].trim()}`
|
|
26959
|
+
},
|
|
26960
|
+
// Preference patterns
|
|
26961
|
+
{
|
|
26962
|
+
regex: /prefers? (.+?) (?:over|to|instead of|rather than) (.+?)(?:\.|,|$)/gi,
|
|
26963
|
+
category: "preference",
|
|
26964
|
+
titleFn: (m) => `Prefers ${m[1].trim()} over ${m[2].trim()}`
|
|
26965
|
+
},
|
|
26966
|
+
{
|
|
26967
|
+
regex: /(?:user |team |we )(?:always |usually |typically )(?:use|prefer|go with) (.+?)(?:\.|,|$)/gi,
|
|
26968
|
+
category: "preference",
|
|
26969
|
+
titleFn: (m) => `Typically uses ${m[1].trim()}`
|
|
26970
|
+
}
|
|
26971
|
+
];
|
|
26972
|
+
function extractPatterns(observations) {
|
|
26973
|
+
const results = [];
|
|
26974
|
+
const seen = /* @__PURE__ */ new Set();
|
|
26975
|
+
for (const { regex, category, titleFn } of PATTERNS) {
|
|
26976
|
+
regex.lastIndex = 0;
|
|
26977
|
+
let match;
|
|
26978
|
+
while ((match = regex.exec(observations)) !== null) {
|
|
26979
|
+
const title = titleFn(match);
|
|
26980
|
+
const key = title.toLowerCase();
|
|
26981
|
+
if (seen.has(key)) continue;
|
|
26982
|
+
seen.add(key);
|
|
26983
|
+
results.push({ category, title, content: match[0].trim() });
|
|
26984
|
+
}
|
|
26985
|
+
}
|
|
26986
|
+
return results;
|
|
26987
|
+
}
|
|
26988
|
+
|
|
26816
26989
|
// src/gradient.ts
|
|
26817
26990
|
function estimate2(text4) {
|
|
26818
26991
|
return Math.ceil(text4.length / 3);
|
|
@@ -26848,12 +27021,17 @@ function makeSessionState() {
|
|
|
26848
27021
|
lastWindowMessageIDs: /* @__PURE__ */ new Set(),
|
|
26849
27022
|
forceMinLayer: 0,
|
|
26850
27023
|
lastTransformEstimate: 0,
|
|
27024
|
+
ltmTokens: 0,
|
|
26851
27025
|
prefixCache: null,
|
|
26852
27026
|
rawWindowCache: null,
|
|
26853
27027
|
lastTurnAt: 0,
|
|
26854
27028
|
cameOutOfIdle: false,
|
|
27029
|
+
postIdleCompact: false,
|
|
26855
27030
|
consecutiveHighLayer: 0,
|
|
26856
|
-
lastPrefixHash: ""
|
|
27031
|
+
lastPrefixHash: "",
|
|
27032
|
+
bustCount: 0,
|
|
27033
|
+
transformCount: 0,
|
|
27034
|
+
distillationSnapshot: null
|
|
26857
27035
|
};
|
|
26858
27036
|
}
|
|
26859
27037
|
var sessionStates = /* @__PURE__ */ new Map();
|
|
@@ -26874,16 +27052,21 @@ function onIdleResume(sessionID, thresholdMs, now = Date.now()) {
|
|
|
26874
27052
|
if (idleMs < thresholdMs) return { triggered: false };
|
|
26875
27053
|
state.prefixCache = null;
|
|
26876
27054
|
state.rawWindowCache = null;
|
|
27055
|
+
state.distillationSnapshot = null;
|
|
26877
27056
|
state.cameOutOfIdle = true;
|
|
27057
|
+
state.postIdleCompact = true;
|
|
26878
27058
|
return { triggered: true, idleMs };
|
|
26879
27059
|
}
|
|
27060
|
+
function getLastTurnAt(sessionID) {
|
|
27061
|
+
return sessionStates.get(sessionID)?.lastTurnAt ?? 0;
|
|
27062
|
+
}
|
|
26880
27063
|
function consumeCameOutOfIdle(sessionID) {
|
|
26881
27064
|
const state = sessionStates.get(sessionID);
|
|
26882
27065
|
if (!state || !state.cameOutOfIdle) return false;
|
|
26883
27066
|
state.cameOutOfIdle = false;
|
|
26884
27067
|
return true;
|
|
26885
27068
|
}
|
|
26886
|
-
var
|
|
27069
|
+
var ltmTokensFallback = 0;
|
|
26887
27070
|
function setModelLimits(limits) {
|
|
26888
27071
|
contextLimit = limits.context || 2e5;
|
|
26889
27072
|
outputReserved = Math.min(limits.output || 32e3, 32e3);
|
|
@@ -26896,11 +27079,18 @@ function computeLayer0Cap(targetCostPerTurn, cacheReadCostPerToken) {
|
|
|
26896
27079
|
const rawCap = Math.floor(targetCostPerTurn / cacheReadCostPerToken);
|
|
26897
27080
|
return Math.max(rawCap, MIN_LAYER0_FLOOR);
|
|
26898
27081
|
}
|
|
26899
|
-
function setLtmTokens(tokens) {
|
|
26900
|
-
|
|
27082
|
+
function setLtmTokens(tokens, sessionID) {
|
|
27083
|
+
if (sessionID) {
|
|
27084
|
+
getSessionState(sessionID).ltmTokens = tokens;
|
|
27085
|
+
}
|
|
27086
|
+
ltmTokensFallback = tokens;
|
|
26901
27087
|
}
|
|
26902
|
-
function getLtmTokens() {
|
|
26903
|
-
|
|
27088
|
+
function getLtmTokens(sessionID) {
|
|
27089
|
+
if (sessionID) {
|
|
27090
|
+
const state = sessionStates.get(sessionID);
|
|
27091
|
+
if (state) return state.ltmTokens;
|
|
27092
|
+
}
|
|
27093
|
+
return ltmTokensFallback;
|
|
26904
27094
|
}
|
|
26905
27095
|
function getLtmBudget(ltmFraction) {
|
|
26906
27096
|
const overhead = calibratedOverhead ?? FIRST_TURN_OVERHEAD;
|
|
@@ -26916,7 +27106,7 @@ function calibrate(actualInput, sessionID, messageCount) {
|
|
|
26916
27106
|
if (sessionID !== void 0) {
|
|
26917
27107
|
const state = getSessionState(sessionID);
|
|
26918
27108
|
state.lastKnownInput = actualInput;
|
|
26919
|
-
state.lastKnownLtm = ltmTokens;
|
|
27109
|
+
state.lastKnownLtm = state.ltmTokens;
|
|
26920
27110
|
if (messageCount !== void 0) state.lastKnownMessageCount = messageCount;
|
|
26921
27111
|
}
|
|
26922
27112
|
}
|
|
@@ -26947,7 +27137,9 @@ function inspectSessionState(sessionID) {
|
|
|
26947
27137
|
hasPrefixCache: state.prefixCache !== null,
|
|
26948
27138
|
hasRawWindowCache: state.rawWindowCache !== null,
|
|
26949
27139
|
cameOutOfIdle: state.cameOutOfIdle,
|
|
26950
|
-
|
|
27140
|
+
postIdleCompact: state.postIdleCompact,
|
|
27141
|
+
lastTurnAt: state.lastTurnAt,
|
|
27142
|
+
distillationSnapshot: state.distillationSnapshot
|
|
26951
27143
|
};
|
|
26952
27144
|
}
|
|
26953
27145
|
function setLastTurnAtForTest(sessionID, ms) {
|
|
@@ -26959,6 +27151,25 @@ function loadDistillations(projectPath, sessionID) {
|
|
|
26959
27151
|
const params = sessionID ? [pid, sessionID] : [pid];
|
|
26960
27152
|
return db().query(query).all(...params);
|
|
26961
27153
|
}
|
|
27154
|
+
function loadDistillationsCached(projectPath, sessionID, messages, sessState) {
|
|
27155
|
+
let lastUserMsgId = null;
|
|
27156
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
27157
|
+
if (messages[i].info.role === "user") {
|
|
27158
|
+
lastUserMsgId = messages[i].info.id;
|
|
27159
|
+
break;
|
|
27160
|
+
}
|
|
27161
|
+
}
|
|
27162
|
+
const snapshot = sessState.distillationSnapshot;
|
|
27163
|
+
if (snapshot && snapshot.lastUserMsgId === lastUserMsgId) {
|
|
27164
|
+
return snapshot.rows;
|
|
27165
|
+
}
|
|
27166
|
+
const rows = loadDistillations(projectPath, sessionID);
|
|
27167
|
+
sessState.distillationSnapshot = { rows, lastUserMsgId };
|
|
27168
|
+
info(
|
|
27169
|
+
`distillation refresh: ${rows.length} rows (user msg ${lastUserMsgId?.substring(0, 16) ?? "none"})`
|
|
27170
|
+
);
|
|
27171
|
+
return rows;
|
|
27172
|
+
}
|
|
26962
27173
|
function stripSystemReminders(text4) {
|
|
26963
27174
|
return text4.replace(/<system-reminder>[\s\S]*?<\/system-reminder>\n?/g, (match) => {
|
|
26964
27175
|
const inner = match.match(
|
|
@@ -27011,24 +27222,51 @@ function simpleHash(str) {
|
|
|
27011
27222
|
}
|
|
27012
27223
|
return hash2;
|
|
27013
27224
|
}
|
|
27014
|
-
function
|
|
27225
|
+
function extractReadRange(input) {
|
|
27015
27226
|
try {
|
|
27016
27227
|
const parsed = JSON.parse(input);
|
|
27017
|
-
|
|
27228
|
+
const path = parsed.path || parsed.filePath || parsed.file;
|
|
27229
|
+
if (!path) return void 0;
|
|
27230
|
+
const offset = typeof parsed.offset === "number" ? parsed.offset : void 0;
|
|
27231
|
+
const limit = typeof parsed.limit === "number" ? parsed.limit : void 0;
|
|
27232
|
+
return { path, offset, limit };
|
|
27018
27233
|
} catch {
|
|
27019
27234
|
const match = input.match(/(?:[\w.-]+\/)+[\w.-]+\.\w{1,5}/);
|
|
27020
|
-
return
|
|
27235
|
+
if (!match) return void 0;
|
|
27236
|
+
return { path: match[0], offset: void 0, limit: void 0 };
|
|
27021
27237
|
}
|
|
27022
27238
|
}
|
|
27023
|
-
function
|
|
27239
|
+
function laterReadCovers(later, earlier) {
|
|
27240
|
+
if (later.path !== earlier.path) return false;
|
|
27241
|
+
if (later.offset === void 0 && later.limit === void 0) return true;
|
|
27242
|
+
if (earlier.offset === void 0 && earlier.limit === void 0) return false;
|
|
27243
|
+
const laterStart = later.offset ?? 1;
|
|
27244
|
+
const earlierStart = earlier.offset ?? 1;
|
|
27245
|
+
if (later.limit === void 0) return laterStart <= earlierStart;
|
|
27246
|
+
if (earlier.limit === void 0) return false;
|
|
27247
|
+
const laterEnd = laterStart + later.limit;
|
|
27248
|
+
const earlierEnd = earlierStart + earlier.limit;
|
|
27249
|
+
return laterStart <= earlierStart && laterEnd >= earlierEnd;
|
|
27250
|
+
}
|
|
27251
|
+
function rangeLabel(range) {
|
|
27252
|
+
if (range.offset !== void 0 && range.limit !== void 0) {
|
|
27253
|
+
return ` lines ${range.offset}-${range.offset + range.limit - 1}`;
|
|
27254
|
+
}
|
|
27255
|
+
if (range.offset !== void 0) {
|
|
27256
|
+
return ` from line ${range.offset}`;
|
|
27257
|
+
}
|
|
27258
|
+
return "";
|
|
27259
|
+
}
|
|
27260
|
+
function dedupAnnotation(toolName, filePath, range) {
|
|
27024
27261
|
if (filePath) {
|
|
27025
|
-
|
|
27262
|
+
const rl = range ? rangeLabel(range) : "";
|
|
27263
|
+
return `[earlier read of ${filePath}${rl} \u2014 see latest read below for current content]`;
|
|
27026
27264
|
}
|
|
27027
27265
|
return `[duplicate output \u2014 same content as later ${toolName} in this session \u2014 use recall for details]`;
|
|
27028
27266
|
}
|
|
27029
27267
|
function deduplicateToolOutputs(messages, currentTurnIdx) {
|
|
27030
27268
|
const contentLatest = /* @__PURE__ */ new Map();
|
|
27031
|
-
const
|
|
27269
|
+
const fileReads = /* @__PURE__ */ new Map();
|
|
27032
27270
|
for (let i = 0; i < messages.length; i++) {
|
|
27033
27271
|
for (const part of messages[i].parts) {
|
|
27034
27272
|
if (!isToolPart(part) || part.state.status !== "completed") continue;
|
|
@@ -27038,8 +27276,15 @@ function deduplicateToolOutputs(messages, currentTurnIdx) {
|
|
|
27038
27276
|
contentLatest.set(key, i);
|
|
27039
27277
|
if (part.tool === "read_file" || part.tool === "read") {
|
|
27040
27278
|
const inputStr = typeof part.state.input === "string" ? part.state.input : JSON.stringify(part.state.input);
|
|
27041
|
-
const
|
|
27042
|
-
if (
|
|
27279
|
+
const range = extractReadRange(inputStr);
|
|
27280
|
+
if (range) {
|
|
27281
|
+
let entries = fileReads.get(range.path);
|
|
27282
|
+
if (!entries) {
|
|
27283
|
+
entries = [];
|
|
27284
|
+
fileReads.set(range.path, entries);
|
|
27285
|
+
}
|
|
27286
|
+
entries.push({ range, msgIdx: i });
|
|
27287
|
+
}
|
|
27043
27288
|
}
|
|
27044
27289
|
}
|
|
27045
27290
|
}
|
|
@@ -27053,20 +27298,30 @@ function deduplicateToolOutputs(messages, currentTurnIdx) {
|
|
|
27053
27298
|
if (!output || output.length < DEDUP_MIN_CHARS) return part;
|
|
27054
27299
|
const contentKey = `${part.tool}:${simpleHash(output)}`;
|
|
27055
27300
|
const isLatestContent = contentLatest.get(contentKey) === msgIdx;
|
|
27056
|
-
let
|
|
27057
|
-
let
|
|
27301
|
+
let readRange;
|
|
27302
|
+
let coveredByLater = false;
|
|
27058
27303
|
if (part.tool === "read_file" || part.tool === "read") {
|
|
27059
27304
|
const inputStr = typeof part.state.input === "string" ? part.state.input : JSON.stringify(part.state.input);
|
|
27060
|
-
|
|
27061
|
-
if (
|
|
27305
|
+
readRange = extractReadRange(inputStr);
|
|
27306
|
+
if (readRange) {
|
|
27307
|
+
const entries = fileReads.get(readRange.path);
|
|
27308
|
+
if (entries) {
|
|
27309
|
+
for (const entry of entries) {
|
|
27310
|
+
if (entry.msgIdx > msgIdx && laterReadCovers(entry.range, readRange)) {
|
|
27311
|
+
coveredByLater = true;
|
|
27312
|
+
break;
|
|
27313
|
+
}
|
|
27314
|
+
}
|
|
27315
|
+
}
|
|
27316
|
+
}
|
|
27062
27317
|
}
|
|
27063
|
-
if (isLatestContent &&
|
|
27318
|
+
if (isLatestContent && !coveredByLater) return part;
|
|
27064
27319
|
partsChanged = true;
|
|
27065
27320
|
return {
|
|
27066
27321
|
...part,
|
|
27067
27322
|
state: {
|
|
27068
27323
|
...part.state,
|
|
27069
|
-
output: dedupAnnotation(part.tool,
|
|
27324
|
+
output: dedupAnnotation(part.tool, readRange?.path, readRange)
|
|
27070
27325
|
}
|
|
27071
27326
|
};
|
|
27072
27327
|
});
|
|
@@ -27086,7 +27341,7 @@ function sanitizeToolParts(messages) {
|
|
|
27086
27341
|
const { status } = part.state;
|
|
27087
27342
|
if (status === "completed" || status === "error") return part;
|
|
27088
27343
|
partsChanged = true;
|
|
27089
|
-
const
|
|
27344
|
+
const existingStart = "time" in part.state ? part.state.time.start : 0;
|
|
27090
27345
|
return {
|
|
27091
27346
|
...part,
|
|
27092
27347
|
state: {
|
|
@@ -27095,8 +27350,8 @@ function sanitizeToolParts(messages) {
|
|
|
27095
27350
|
error: "[tool execution interrupted \u2014 session recovered]",
|
|
27096
27351
|
metadata: "metadata" in part.state ? part.state.metadata : void 0,
|
|
27097
27352
|
time: {
|
|
27098
|
-
start:
|
|
27099
|
-
end:
|
|
27353
|
+
start: existingStart,
|
|
27354
|
+
end: existingStart
|
|
27100
27355
|
}
|
|
27101
27356
|
}
|
|
27102
27357
|
};
|
|
@@ -27120,97 +27375,6 @@ function stripToolOutputs(parts) {
|
|
|
27120
27375
|
};
|
|
27121
27376
|
});
|
|
27122
27377
|
}
|
|
27123
|
-
function formatRelativeTime(date5, now) {
|
|
27124
|
-
const diffMs = now.getTime() - date5.getTime();
|
|
27125
|
-
const diffDays = Math.floor(diffMs / (1e3 * 60 * 60 * 24));
|
|
27126
|
-
if (diffDays === 0) return "today";
|
|
27127
|
-
if (diffDays === 1) return "yesterday";
|
|
27128
|
-
if (diffDays < 7) return `${diffDays} days ago`;
|
|
27129
|
-
if (diffDays < 14) return "1 week ago";
|
|
27130
|
-
if (diffDays < 30) return `${Math.floor(diffDays / 7)} weeks ago`;
|
|
27131
|
-
if (diffDays < 60) return "1 month ago";
|
|
27132
|
-
if (diffDays < 365) return `${Math.floor(diffDays / 30)} months ago`;
|
|
27133
|
-
return `${Math.floor(diffDays / 365)} year${Math.floor(diffDays / 365) > 1 ? "s" : ""} ago`;
|
|
27134
|
-
}
|
|
27135
|
-
function parseDateFromContent(s) {
|
|
27136
|
-
const simple = s.match(/([A-Z][a-z]+)\s+(\d{1,2}),?\s+(\d{4})/);
|
|
27137
|
-
if (simple) {
|
|
27138
|
-
const d = /* @__PURE__ */ new Date(`${simple[1]} ${simple[2]}, ${simple[3]}`);
|
|
27139
|
-
if (!isNaN(d.getTime())) return d;
|
|
27140
|
-
}
|
|
27141
|
-
const range = s.match(/([A-Z][a-z]+)\s+(\d{1,2})-\d{1,2},?\s+(\d{4})/);
|
|
27142
|
-
if (range) {
|
|
27143
|
-
const d = /* @__PURE__ */ new Date(`${range[1]} ${range[2]}, ${range[3]}`);
|
|
27144
|
-
if (!isNaN(d.getTime())) return d;
|
|
27145
|
-
}
|
|
27146
|
-
const vague = s.match(/(late|early|mid)[- ]?([A-Z][a-z]+)\s+(\d{4})/i);
|
|
27147
|
-
if (vague) {
|
|
27148
|
-
const day = vague[1].toLowerCase() === "early" ? 7 : vague[1].toLowerCase() === "late" ? 23 : 15;
|
|
27149
|
-
const d = /* @__PURE__ */ new Date(`${vague[2]} ${day}, ${vague[3]}`);
|
|
27150
|
-
if (!isNaN(d.getTime())) return d;
|
|
27151
|
-
}
|
|
27152
|
-
return null;
|
|
27153
|
-
}
|
|
27154
|
-
function expandInlineEstimatedDates(text4, now) {
|
|
27155
|
-
return text4.replace(
|
|
27156
|
-
/\(((?:meaning|estimated)\s+)([^)]+\d{4})\)/gi,
|
|
27157
|
-
(match, prefix, dateContent) => {
|
|
27158
|
-
const d = parseDateFromContent(dateContent);
|
|
27159
|
-
if (!d) return match;
|
|
27160
|
-
const rel = formatRelativeTime(d, now);
|
|
27161
|
-
const matchIdx = text4.indexOf(match);
|
|
27162
|
-
const lineStart = text4.lastIndexOf("\n", matchIdx) + 1;
|
|
27163
|
-
const linePrefix = text4.slice(lineStart, matchIdx);
|
|
27164
|
-
const isFutureIntent = /\b(?:will|plans?\s+to|planning\s+to|going\s+to|intends?\s+to)\b/i.test(
|
|
27165
|
-
linePrefix
|
|
27166
|
-
);
|
|
27167
|
-
if (d < now && isFutureIntent)
|
|
27168
|
-
return `(${prefix}${dateContent} \u2014 ${rel}, likely already happened)`;
|
|
27169
|
-
return `(${prefix}${dateContent} \u2014 ${rel})`;
|
|
27170
|
-
}
|
|
27171
|
-
);
|
|
27172
|
-
}
|
|
27173
|
-
function addRelativeTimeToObservations(text4, now) {
|
|
27174
|
-
const withInline = expandInlineEstimatedDates(text4, now);
|
|
27175
|
-
const dateHeaderRe = /^(Date:\s*)([A-Z][a-z]+ \d{1,2}, \d{4})$/gm;
|
|
27176
|
-
const found = [];
|
|
27177
|
-
let m;
|
|
27178
|
-
while ((m = dateHeaderRe.exec(withInline)) !== null) {
|
|
27179
|
-
const d = new Date(m[2]);
|
|
27180
|
-
if (!isNaN(d.getTime()))
|
|
27181
|
-
found.push({
|
|
27182
|
-
index: m.index,
|
|
27183
|
-
date: d,
|
|
27184
|
-
full: m[0],
|
|
27185
|
-
prefix: m[1],
|
|
27186
|
-
ds: m[2]
|
|
27187
|
-
});
|
|
27188
|
-
}
|
|
27189
|
-
if (!found.length) return withInline;
|
|
27190
|
-
let result = "";
|
|
27191
|
-
let last = 0;
|
|
27192
|
-
for (let i = 0; i < found.length; i++) {
|
|
27193
|
-
const curr = found[i];
|
|
27194
|
-
const prev = found[i - 1];
|
|
27195
|
-
result += withInline.slice(last, curr.index);
|
|
27196
|
-
if (prev) {
|
|
27197
|
-
const gapDays = Math.floor(
|
|
27198
|
-
(curr.date.getTime() - prev.date.getTime()) / 864e5
|
|
27199
|
-
);
|
|
27200
|
-
if (gapDays > 1) {
|
|
27201
|
-
const gap = gapDays < 7 ? `[${gapDays} days later]` : gapDays < 14 ? "[1 week later]" : gapDays < 30 ? `[${Math.floor(gapDays / 7)} weeks later]` : gapDays < 60 ? "[1 month later]" : `[${Math.floor(gapDays / 30)} months later]`;
|
|
27202
|
-
result += `
|
|
27203
|
-
${gap}
|
|
27204
|
-
|
|
27205
|
-
`;
|
|
27206
|
-
}
|
|
27207
|
-
}
|
|
27208
|
-
result += `${curr.prefix}${curr.ds} (${formatRelativeTime(curr.date, now)})`;
|
|
27209
|
-
last = curr.index + curr.full.length;
|
|
27210
|
-
}
|
|
27211
|
-
result += withInline.slice(last);
|
|
27212
|
-
return result;
|
|
27213
|
-
}
|
|
27214
27378
|
function buildPrefixMessages(formatted) {
|
|
27215
27379
|
return [
|
|
27216
27380
|
{
|
|
@@ -27267,12 +27431,7 @@ function buildPrefixMessages(formatted) {
|
|
|
27267
27431
|
}
|
|
27268
27432
|
function distilledPrefix(distillations) {
|
|
27269
27433
|
if (!distillations.length) return [];
|
|
27270
|
-
const
|
|
27271
|
-
const annotated = distillations.map((d) => ({
|
|
27272
|
-
...d,
|
|
27273
|
-
observations: addRelativeTimeToObservations(d.observations, now)
|
|
27274
|
-
}));
|
|
27275
|
-
const formatted = formatDistillations(annotated);
|
|
27434
|
+
const formatted = formatDistillations(distillations);
|
|
27276
27435
|
if (!formatted) return [];
|
|
27277
27436
|
return buildPrefixMessages(formatted);
|
|
27278
27437
|
}
|
|
@@ -27292,12 +27451,7 @@ function distilledPrefixCached(distillations, sessionID, sessState) {
|
|
|
27292
27451
|
};
|
|
27293
27452
|
}
|
|
27294
27453
|
const newRows = distillations.slice(prefixCache.rowCount);
|
|
27295
|
-
const
|
|
27296
|
-
const annotated2 = newRows.map((d) => ({
|
|
27297
|
-
...d,
|
|
27298
|
-
observations: addRelativeTimeToObservations(d.observations, now2)
|
|
27299
|
-
}));
|
|
27300
|
-
const deltaText = formatDistillations(annotated2);
|
|
27454
|
+
const deltaText = formatDistillations(newRows);
|
|
27301
27455
|
if (deltaText) {
|
|
27302
27456
|
const fullText2 = prefixCache.cachedText + "\n\n" + deltaText;
|
|
27303
27457
|
const messages2 = buildPrefixMessages(fullText2);
|
|
@@ -27313,12 +27467,7 @@ function distilledPrefixCached(distillations, sessionID, sessState) {
|
|
|
27313
27467
|
return { messages: messages2, tokens: tokens2 };
|
|
27314
27468
|
}
|
|
27315
27469
|
}
|
|
27316
|
-
const
|
|
27317
|
-
const annotated = distillations.map((d) => ({
|
|
27318
|
-
...d,
|
|
27319
|
-
observations: addRelativeTimeToObservations(d.observations, now)
|
|
27320
|
-
}));
|
|
27321
|
-
const fullText = formatDistillations(annotated);
|
|
27470
|
+
const fullText = formatDistillations(distillations);
|
|
27322
27471
|
if (!fullText) {
|
|
27323
27472
|
sessState.prefixCache = null;
|
|
27324
27473
|
return { messages: [], tokens: 0 };
|
|
@@ -27341,29 +27490,40 @@ function tryFitStable(input) {
|
|
|
27341
27490
|
const rawWindowCache = input.sessState.rawWindowCache;
|
|
27342
27491
|
const cacheValid = rawWindowCache !== null && rawWindowCache.sessionID === input.sessionID;
|
|
27343
27492
|
if (cacheValid) {
|
|
27344
|
-
const
|
|
27345
|
-
|
|
27493
|
+
const newMessages = Math.max(0, input.messages.length - rawWindowCache.pinnedTotalCount);
|
|
27494
|
+
const windowSize = rawWindowCache.pinnedRawCount + newMessages;
|
|
27495
|
+
const pinnedIdx = Math.max(0, input.messages.length - windowSize);
|
|
27496
|
+
const pinnedWindow = input.messages.slice(pinnedIdx);
|
|
27497
|
+
const pinnedTokens = pinnedWindow.reduce(
|
|
27498
|
+
(sum, m) => sum + estimateMessage(m),
|
|
27499
|
+
0
|
|
27346
27500
|
);
|
|
27347
|
-
|
|
27348
|
-
|
|
27349
|
-
|
|
27350
|
-
|
|
27351
|
-
|
|
27352
|
-
|
|
27353
|
-
|
|
27354
|
-
|
|
27355
|
-
|
|
27356
|
-
return parts !== msg.parts ? { info: msg.info, parts } : msg;
|
|
27357
|
-
});
|
|
27358
|
-
const total = input.prefixTokens + pinnedTokens;
|
|
27359
|
-
return {
|
|
27360
|
-
messages: [...input.prefix, ...processed],
|
|
27361
|
-
distilledTokens: input.prefixTokens,
|
|
27362
|
-
rawTokens: pinnedTokens,
|
|
27363
|
-
totalTokens: total
|
|
27501
|
+
const highWaterBudget = Math.max(rawWindowCache.pinnedBudget, input.rawBudget);
|
|
27502
|
+
const effectiveBudget = highWaterBudget * 1.15;
|
|
27503
|
+
if (pinnedTokens <= effectiveBudget) {
|
|
27504
|
+
if (pinnedTokens > rawWindowCache.pinnedBudget * 1.15) {
|
|
27505
|
+
input.sessState.rawWindowCache = {
|
|
27506
|
+
...rawWindowCache,
|
|
27507
|
+
pinnedRawCount: pinnedWindow.length,
|
|
27508
|
+
pinnedTotalCount: input.messages.length,
|
|
27509
|
+
pinnedBudget: input.rawBudget
|
|
27364
27510
|
};
|
|
27365
27511
|
}
|
|
27512
|
+
const processed = pinnedWindow.map((msg) => {
|
|
27513
|
+
const parts = cleanParts(msg.parts);
|
|
27514
|
+
return parts !== msg.parts ? { info: msg.info, parts } : msg;
|
|
27515
|
+
});
|
|
27516
|
+
const total = input.prefixTokens + pinnedTokens;
|
|
27517
|
+
return {
|
|
27518
|
+
messages: [...input.prefix, ...processed],
|
|
27519
|
+
distilledTokens: input.prefixTokens,
|
|
27520
|
+
rawTokens: pinnedTokens,
|
|
27521
|
+
totalTokens: total
|
|
27522
|
+
};
|
|
27366
27523
|
}
|
|
27524
|
+
info(
|
|
27525
|
+
`pin-overflow: session=${input.sessionID} pinnedTokens=${pinnedTokens} pinnedBudget=${rawWindowCache.pinnedBudget} effectiveBudget=${Math.round(effectiveBudget)} currentRawBudget=${input.rawBudget} windowSize=${pinnedWindow.length}`
|
|
27526
|
+
);
|
|
27367
27527
|
}
|
|
27368
27528
|
const result = tryFit({
|
|
27369
27529
|
messages: input.messages,
|
|
@@ -27374,11 +27534,13 @@ function tryFitStable(input) {
|
|
|
27374
27534
|
strip: "none"
|
|
27375
27535
|
});
|
|
27376
27536
|
if (result) {
|
|
27377
|
-
const
|
|
27378
|
-
if (
|
|
27537
|
+
const rawMessageCount = result.messages.length - input.prefix.length;
|
|
27538
|
+
if (rawMessageCount > 0) {
|
|
27379
27539
|
input.sessState.rawWindowCache = {
|
|
27380
27540
|
sessionID: input.sessionID,
|
|
27381
|
-
|
|
27541
|
+
pinnedRawCount: rawMessageCount,
|
|
27542
|
+
pinnedTotalCount: input.messages.length,
|
|
27543
|
+
pinnedBudget: input.rawBudget
|
|
27382
27544
|
};
|
|
27383
27545
|
}
|
|
27384
27546
|
}
|
|
@@ -27393,14 +27555,15 @@ function needsUrgentDistillation() {
|
|
|
27393
27555
|
function transformInner(input) {
|
|
27394
27556
|
const cfg = config2();
|
|
27395
27557
|
const overhead = getOverhead();
|
|
27558
|
+
const sid = input.sessionID ?? input.messages[0]?.info.sessionID;
|
|
27559
|
+
const sessState = sid ? getSessionState(sid) : makeSessionState();
|
|
27560
|
+
const sessLtmTokens = sid ? sessState.ltmTokens : ltmTokensFallback;
|
|
27396
27561
|
const usable = Math.max(
|
|
27397
27562
|
0,
|
|
27398
|
-
contextLimit - outputReserved - overhead -
|
|
27563
|
+
contextLimit - outputReserved - overhead - sessLtmTokens
|
|
27399
27564
|
);
|
|
27400
27565
|
const distilledBudget = Math.floor(usable * cfg.budget.distilled);
|
|
27401
|
-
|
|
27402
|
-
const sid = input.sessionID ?? input.messages[0]?.info.sessionID;
|
|
27403
|
-
const sessState = sid ? getSessionState(sid) : makeSessionState();
|
|
27566
|
+
let rawBudget = Math.floor(usable * cfg.budget.raw);
|
|
27404
27567
|
let effectiveMinLayer = sessState.forceMinLayer;
|
|
27405
27568
|
sessState.forceMinLayer = 0;
|
|
27406
27569
|
if (sid && effectiveMinLayer > 0) saveForceMinLayer(sid, 0);
|
|
@@ -27413,17 +27576,26 @@ function transformInner(input) {
|
|
|
27413
27576
|
return result.totalTokens * UNCALIBRATED_SAFETY <= maxInput;
|
|
27414
27577
|
}
|
|
27415
27578
|
if (calibrated && sessState.lastLayer >= 1 && input.messages.length >= sessState.lastKnownMessageCount) {
|
|
27579
|
+
effectiveMinLayer = Math.max(effectiveMinLayer, sessState.lastLayer);
|
|
27580
|
+
}
|
|
27581
|
+
const postIdleCompact = sessState.postIdleCompact;
|
|
27582
|
+
if (postIdleCompact) {
|
|
27583
|
+
sessState.postIdleCompact = false;
|
|
27416
27584
|
effectiveMinLayer = Math.max(effectiveMinLayer, 1);
|
|
27585
|
+
rawBudget = Math.floor(usable * 0.2);
|
|
27586
|
+
info(
|
|
27587
|
+
`post-idle compact: session=${sid} rawBudget=${rawBudget} (${Math.floor(usable * cfg.budget.raw)}\u2192${rawBudget})`
|
|
27588
|
+
);
|
|
27417
27589
|
}
|
|
27418
27590
|
let expectedInput;
|
|
27419
27591
|
if (calibrated) {
|
|
27420
27592
|
const newMessages = sessState.lastWindowMessageIDs.size > 0 ? input.messages.filter((m) => !sessState.lastWindowMessageIDs.has(m.info.id)) : input.messages.slice(-Math.max(0, input.messages.length - sessState.lastKnownMessageCount));
|
|
27421
27593
|
const newMsgTokens = newMessages.reduce((s, m) => s + estimateMessage(m), 0);
|
|
27422
|
-
const ltmDelta =
|
|
27594
|
+
const ltmDelta = sessLtmTokens - sessState.lastKnownLtm;
|
|
27423
27595
|
expectedInput = sessState.lastKnownInput + newMsgTokens + ltmDelta;
|
|
27424
27596
|
} else {
|
|
27425
27597
|
const messageTokens = input.messages.reduce((s, m) => s + estimateMessage(m), 0);
|
|
27426
|
-
expectedInput = messageTokens + overhead +
|
|
27598
|
+
expectedInput = messageTokens + overhead + sessLtmTokens;
|
|
27427
27599
|
}
|
|
27428
27600
|
const layer0Input = calibrated ? expectedInput : expectedInput * UNCALIBRATED_SAFETY;
|
|
27429
27601
|
let layer0Ceiling = maxLayer0Tokens > 0 ? Math.min(maxInput, maxLayer0Tokens) : maxInput;
|
|
@@ -27431,7 +27603,7 @@ function transformInner(input) {
|
|
|
27431
27603
|
layer0Ceiling = Math.floor(layer0Ceiling * 0.7);
|
|
27432
27604
|
}
|
|
27433
27605
|
if (effectiveMinLayer === 0 && layer0Input <= layer0Ceiling) {
|
|
27434
|
-
const messageTokens = calibrated ? expectedInput - (
|
|
27606
|
+
const messageTokens = calibrated ? expectedInput - (sessLtmTokens - sessState.lastKnownLtm) : expectedInput - overhead - sessLtmTokens;
|
|
27435
27607
|
return {
|
|
27436
27608
|
messages: input.messages,
|
|
27437
27609
|
layer: 0,
|
|
@@ -27445,7 +27617,7 @@ function transformInner(input) {
|
|
|
27445
27617
|
}
|
|
27446
27618
|
const turnStart = currentTurnStart(input.messages);
|
|
27447
27619
|
const dedupMessages = deduplicateToolOutputs(input.messages, turnStart);
|
|
27448
|
-
const distillations = sid ?
|
|
27620
|
+
const distillations = sid ? loadDistillationsCached(input.projectPath, sid, input.messages, sessState) : [];
|
|
27449
27621
|
const cached2 = sid ? distilledPrefixCached(distillations, sid, sessState) : (() => {
|
|
27450
27622
|
const msgs = distilledPrefix(distillations);
|
|
27451
27623
|
return { messages: msgs, tokens: msgs.reduce((sum, m) => sum + estimateMessage(m), 0) };
|
|
@@ -27558,12 +27730,27 @@ function transform2(input) {
|
|
|
27558
27730
|
state.lastLayer = result.layer;
|
|
27559
27731
|
state.lastWindowMessageIDs = new Set(result.messages.map((m) => m.info.id));
|
|
27560
27732
|
state.lastTurnAt = Date.now();
|
|
27561
|
-
const
|
|
27562
|
-
|
|
27733
|
+
const prefixFingerprint = result.messages.slice(0, 5).map((m) => {
|
|
27734
|
+
const text4 = m.parts.map((p3) => {
|
|
27735
|
+
if (isTextPart(p3)) return p3.text?.slice(0, 40) ?? "";
|
|
27736
|
+
if (isReasoningPart(p3)) return p3.text?.slice(0, 40) ?? "";
|
|
27737
|
+
return p3.type;
|
|
27738
|
+
}).join("|");
|
|
27739
|
+
return `${m.info.role}:${text4.slice(0, 60)}`;
|
|
27740
|
+
}).join(",");
|
|
27741
|
+
const prefixHash = `${result.layer}:${prefixFingerprint}`;
|
|
27742
|
+
state.transformCount++;
|
|
27563
27743
|
if (state.lastPrefixHash && state.lastPrefixHash !== prefixHash) {
|
|
27744
|
+
state.bustCount++;
|
|
27745
|
+
const rate = state.bustCount / state.transformCount;
|
|
27564
27746
|
info(
|
|
27565
|
-
`cache-bust
|
|
27747
|
+
`cache-bust #${state.bustCount} (${(rate * 100).toFixed(0)}%): session=${sid} layer=${state.lastLayer}\u2192${result.layer} msgs=${state.lastTransformedCount}\u2192${result.messages.length} prefix=${state.lastPrefixHash.slice(0, 30)}\u2192${prefixHash.slice(0, 30)}`
|
|
27566
27748
|
);
|
|
27749
|
+
if (state.transformCount >= 20 && rate > 0.5) {
|
|
27750
|
+
warn(
|
|
27751
|
+
`HIGH BUST RATE: session ${sid} has ${(rate * 100).toFixed(0)}% bust rate (${state.bustCount}/${state.transformCount} transforms)`
|
|
27752
|
+
);
|
|
27753
|
+
}
|
|
27567
27754
|
}
|
|
27568
27755
|
state.lastPrefixHash = prefixHash;
|
|
27569
27756
|
if (result.layer >= 2) {
|
|
@@ -27782,7 +27969,7 @@ function parseSourceIds(raw) {
|
|
|
27782
27969
|
}
|
|
27783
27970
|
function loadForSession(projectPath, sessionID, includeArchived = false) {
|
|
27784
27971
|
const pid = ensureProject(projectPath);
|
|
27785
|
-
const sql = includeArchived ? "SELECT id, project_id, session_id, observations, source_ids, generation, token_count, created_at FROM distillations WHERE project_id = ? AND session_id = ? ORDER BY created_at ASC" : "SELECT id, project_id, session_id, observations, source_ids, generation, token_count, created_at FROM distillations WHERE project_id = ? AND session_id = ? AND archived = 0 ORDER BY created_at ASC";
|
|
27972
|
+
const sql = includeArchived ? "SELECT id, project_id, session_id, observations, source_ids, generation, token_count, created_at, r_compression, c_norm FROM distillations WHERE project_id = ? AND session_id = ? ORDER BY created_at ASC" : "SELECT id, project_id, session_id, observations, source_ids, generation, token_count, created_at, r_compression, c_norm FROM distillations WHERE project_id = ? AND session_id = ? AND archived = 0 ORDER BY created_at ASC";
|
|
27786
27973
|
const rows = db().query(sql).all(pid, sessionID);
|
|
27787
27974
|
return rows.map((r) => ({
|
|
27788
27975
|
...r,
|
|
@@ -27795,8 +27982,8 @@ function storeDistillation(input) {
|
|
|
27795
27982
|
const sourceJson = JSON.stringify(input.sourceIDs);
|
|
27796
27983
|
const tokens = Math.ceil(input.observations.length / 3);
|
|
27797
27984
|
db().query(
|
|
27798
|
-
`INSERT INTO distillations (id, project_id, session_id, narrative, facts, observations, source_ids, generation, token_count, created_at)
|
|
27799
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
27985
|
+
`INSERT INTO distillations (id, project_id, session_id, narrative, facts, observations, source_ids, generation, token_count, created_at, r_compression, c_norm)
|
|
27986
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
27800
27987
|
).run(
|
|
27801
27988
|
id,
|
|
27802
27989
|
pid,
|
|
@@ -27809,7 +27996,9 @@ function storeDistillation(input) {
|
|
|
27809
27996
|
sourceJson,
|
|
27810
27997
|
input.generation,
|
|
27811
27998
|
tokens,
|
|
27812
|
-
Date.now()
|
|
27999
|
+
Date.now(),
|
|
28000
|
+
input.rCompression ?? null,
|
|
28001
|
+
input.cNorm ?? null
|
|
27813
28002
|
);
|
|
27814
28003
|
return id;
|
|
27815
28004
|
}
|
|
@@ -27822,7 +28011,7 @@ function gen0Count(projectPath, sessionID) {
|
|
|
27822
28011
|
function loadGen0(projectPath, sessionID) {
|
|
27823
28012
|
const pid = ensureProject(projectPath);
|
|
27824
28013
|
const rows = db().query(
|
|
27825
|
-
"SELECT id, project_id, session_id, observations, source_ids, generation, token_count, created_at FROM distillations WHERE project_id = ? AND session_id = ? AND generation = 0 AND archived = 0 ORDER BY created_at ASC"
|
|
28014
|
+
"SELECT id, project_id, session_id, observations, source_ids, generation, token_count, created_at, r_compression, c_norm FROM distillations WHERE project_id = ? AND session_id = ? AND generation = 0 AND archived = 0 ORDER BY created_at ASC"
|
|
27826
28015
|
).all(pid, sessionID);
|
|
27827
28016
|
return rows.map((r) => ({
|
|
27828
28017
|
...r,
|
|
@@ -27889,7 +28078,8 @@ async function run(input) {
|
|
|
27889
28078
|
projectPath: input.projectPath,
|
|
27890
28079
|
sessionID: input.sessionID,
|
|
27891
28080
|
messages: segment,
|
|
27892
|
-
model: input.model
|
|
28081
|
+
model: input.model,
|
|
28082
|
+
urgent: input.urgent
|
|
27893
28083
|
});
|
|
27894
28084
|
if (result) {
|
|
27895
28085
|
distilled += segment.length;
|
|
@@ -27897,12 +28087,13 @@ async function run(input) {
|
|
|
27897
28087
|
}
|
|
27898
28088
|
}
|
|
27899
28089
|
}
|
|
27900
|
-
if (gen0Count(input.projectPath, input.sessionID) >= cfg.distillation.metaThreshold) {
|
|
28090
|
+
if (!input.skipMeta && gen0Count(input.projectPath, input.sessionID) >= cfg.distillation.metaThreshold) {
|
|
27901
28091
|
await metaDistill({
|
|
27902
28092
|
llm: input.llm,
|
|
27903
28093
|
projectPath: input.projectPath,
|
|
27904
28094
|
sessionID: input.sessionID,
|
|
27905
|
-
model: input.model
|
|
28095
|
+
model: input.model,
|
|
28096
|
+
urgent: input.urgent
|
|
27906
28097
|
});
|
|
27907
28098
|
rounds++;
|
|
27908
28099
|
}
|
|
@@ -27928,29 +28119,46 @@ async function distillSegment(input) {
|
|
|
27928
28119
|
const responseText = await input.llm.prompt(
|
|
27929
28120
|
DISTILLATION_SYSTEM,
|
|
27930
28121
|
userContent,
|
|
27931
|
-
{ model, workerID: "lore-distill" }
|
|
28122
|
+
{ model, workerID: "lore-distill", thinking: false, urgent: input.urgent, sessionID: input.sessionID }
|
|
27932
28123
|
);
|
|
27933
28124
|
if (!responseText) return null;
|
|
27934
28125
|
const result = parseDistillationResult(responseText);
|
|
27935
28126
|
if (!result) return null;
|
|
28127
|
+
const distilledTokens = Math.ceil(result.observations.length / 3);
|
|
28128
|
+
const sourceTokens = input.messages.reduce((sum, m) => sum + m.tokens, 0);
|
|
28129
|
+
const rComp = compressionRatio(distilledTokens, sourceTokens);
|
|
28130
|
+
const cNorm = temporalCnorm(input.messages.map((m) => m.created_at));
|
|
27936
28131
|
const distillId = storeDistillation({
|
|
27937
28132
|
projectPath: input.projectPath,
|
|
27938
28133
|
sessionID: input.sessionID,
|
|
27939
28134
|
observations: result.observations,
|
|
27940
28135
|
sourceIDs: input.messages.map((m) => m.id),
|
|
27941
|
-
generation: 0
|
|
28136
|
+
generation: 0,
|
|
28137
|
+
rCompression: rComp,
|
|
28138
|
+
cNorm
|
|
27942
28139
|
});
|
|
27943
28140
|
markDistilled(input.messages.map((m) => m.id));
|
|
27944
|
-
const distilledTokens = Math.ceil(result.observations.length / 3);
|
|
27945
|
-
const sourceTokens = input.messages.reduce((sum, m) => sum + m.tokens, 0);
|
|
27946
|
-
const rComp = compressionRatio(distilledTokens, sourceTokens);
|
|
27947
|
-
const cNorm = temporalCnorm(input.messages.map((m) => m.created_at));
|
|
27948
28141
|
info(
|
|
27949
28142
|
`distill segment: ${input.messages.length} msgs, ${sourceTokens}\u2192${distilledTokens} tokens, R=${rComp.toFixed(2)}, C_norm=${cNorm.toFixed(3)}`
|
|
27950
28143
|
);
|
|
27951
28144
|
if (isAvailable()) {
|
|
27952
28145
|
embedDistillation(distillId, result.observations);
|
|
27953
28146
|
}
|
|
28147
|
+
if (config2().knowledge.enabled) {
|
|
28148
|
+
for (const pat of extractPatterns(result.observations)) {
|
|
28149
|
+
try {
|
|
28150
|
+
create({
|
|
28151
|
+
projectPath: input.projectPath,
|
|
28152
|
+
category: pat.category,
|
|
28153
|
+
title: pat.title,
|
|
28154
|
+
content: pat.content,
|
|
28155
|
+
session: input.sessionID,
|
|
28156
|
+
scope: "project"
|
|
28157
|
+
});
|
|
28158
|
+
} catch {
|
|
28159
|
+
}
|
|
28160
|
+
}
|
|
28161
|
+
}
|
|
27954
28162
|
return result;
|
|
27955
28163
|
}
|
|
27956
28164
|
async function metaDistill(input) {
|
|
@@ -27966,7 +28174,7 @@ async function metaDistill(input) {
|
|
|
27966
28174
|
const responseText = await input.llm.prompt(
|
|
27967
28175
|
RECURSIVE_SYSTEM,
|
|
27968
28176
|
userContent,
|
|
27969
|
-
{ model, workerID: "lore-distill" }
|
|
28177
|
+
{ model, workerID: "lore-distill", thinking: false, urgent: input.urgent, sessionID: input.sessionID }
|
|
27970
28178
|
);
|
|
27971
28179
|
if (!responseText) return null;
|
|
27972
28180
|
const result = parseDistillationResult(responseText);
|
|
@@ -27995,8 +28203,54 @@ async function metaDistill(input) {
|
|
|
27995
28203
|
if (isAvailable()) {
|
|
27996
28204
|
embedDistillation(metaId, result.observations);
|
|
27997
28205
|
}
|
|
28206
|
+
if (config2().knowledge.enabled) {
|
|
28207
|
+
for (const pat of extractPatterns(result.observations)) {
|
|
28208
|
+
try {
|
|
28209
|
+
create({
|
|
28210
|
+
projectPath: input.projectPath,
|
|
28211
|
+
category: pat.category,
|
|
28212
|
+
title: pat.title,
|
|
28213
|
+
content: pat.content,
|
|
28214
|
+
session: input.sessionID,
|
|
28215
|
+
scope: "project"
|
|
28216
|
+
});
|
|
28217
|
+
} catch {
|
|
28218
|
+
}
|
|
28219
|
+
}
|
|
28220
|
+
}
|
|
27998
28221
|
return result;
|
|
27999
28222
|
}
|
|
28223
|
+
function backfillMetrics() {
|
|
28224
|
+
const rows = db().query(
|
|
28225
|
+
"SELECT id, source_ids, token_count FROM distillations WHERE r_compression IS NULL"
|
|
28226
|
+
).all();
|
|
28227
|
+
if (!rows.length) return 0;
|
|
28228
|
+
const update2 = db().prepare(
|
|
28229
|
+
"UPDATE distillations SET r_compression = ?, c_norm = ? WHERE id = ?"
|
|
28230
|
+
);
|
|
28231
|
+
let updated = 0;
|
|
28232
|
+
for (const row of rows) {
|
|
28233
|
+
const sourceIds = parseSourceIds(row.source_ids);
|
|
28234
|
+
if (!sourceIds.length) continue;
|
|
28235
|
+
const placeholders = sourceIds.map(() => "?").join(",");
|
|
28236
|
+
const sources = db().query(
|
|
28237
|
+
`SELECT tokens, created_at FROM temporal_messages WHERE id IN (${placeholders})`
|
|
28238
|
+
).all(...sourceIds);
|
|
28239
|
+
if (!sources.length) continue;
|
|
28240
|
+
const sourceTokens = sources.reduce((sum, s) => sum + s.tokens, 0);
|
|
28241
|
+
const timestamps = sources.map((s) => s.created_at);
|
|
28242
|
+
const rComp = compressionRatio(row.token_count, sourceTokens);
|
|
28243
|
+
const cNorm = temporalCnorm(timestamps);
|
|
28244
|
+
update2.run(rComp, cNorm, row.id);
|
|
28245
|
+
updated++;
|
|
28246
|
+
}
|
|
28247
|
+
if (updated > 0) {
|
|
28248
|
+
info(
|
|
28249
|
+
`backfilled metrics for ${updated} distillations (${rows.length - updated} skipped \u2014 missing sources)`
|
|
28250
|
+
);
|
|
28251
|
+
}
|
|
28252
|
+
return updated;
|
|
28253
|
+
}
|
|
28000
28254
|
|
|
28001
28255
|
// src/curator.ts
|
|
28002
28256
|
var curator_exports = {};
|
|
@@ -28042,7 +28296,7 @@ async function run2(input) {
|
|
|
28042
28296
|
const responseText = await input.llm.prompt(
|
|
28043
28297
|
CURATOR_SYSTEM,
|
|
28044
28298
|
userContent,
|
|
28045
|
-
{ model, workerID: "lore-curator" }
|
|
28299
|
+
{ model, workerID: "lore-curator", thinking: false, sessionID: input.sessionID }
|
|
28046
28300
|
);
|
|
28047
28301
|
if (!responseText) return { created: 0, updated: 0, deleted: 0 };
|
|
28048
28302
|
const ops = parseOps(responseText);
|
|
@@ -28112,7 +28366,7 @@ async function consolidate(input) {
|
|
|
28112
28366
|
const responseText = await input.llm.prompt(
|
|
28113
28367
|
CONSOLIDATION_SYSTEM,
|
|
28114
28368
|
userContent,
|
|
28115
|
-
{ model, workerID: "lore-curator" }
|
|
28369
|
+
{ model, workerID: "lore-curator", thinking: false, sessionID: input.sessionID }
|
|
28116
28370
|
);
|
|
28117
28371
|
if (!responseText) return { updated: 0, deleted: 0 };
|
|
28118
28372
|
const ops = parseOps(responseText);
|
|
@@ -28138,12 +28392,39 @@ async function consolidate(input) {
|
|
|
28138
28392
|
}
|
|
28139
28393
|
|
|
28140
28394
|
// src/recall.ts
|
|
28395
|
+
function getTaggedText(tagged) {
|
|
28396
|
+
switch (tagged.source) {
|
|
28397
|
+
case "knowledge":
|
|
28398
|
+
case "cross-knowledge":
|
|
28399
|
+
return `${tagged.item.title} ${tagged.item.content}`;
|
|
28400
|
+
case "distillation":
|
|
28401
|
+
return tagged.item.observations;
|
|
28402
|
+
case "temporal":
|
|
28403
|
+
return tagged.item.content;
|
|
28404
|
+
case "lat-section":
|
|
28405
|
+
return `${tagged.item.heading} ${tagged.item.content}`;
|
|
28406
|
+
}
|
|
28407
|
+
}
|
|
28408
|
+
function taggedResultKey(r) {
|
|
28409
|
+
switch (r.source) {
|
|
28410
|
+
case "knowledge":
|
|
28411
|
+
return `k:${r.item.id}`;
|
|
28412
|
+
case "cross-knowledge":
|
|
28413
|
+
return `xk:${r.item.id}`;
|
|
28414
|
+
case "distillation":
|
|
28415
|
+
return `d:${r.item.id}`;
|
|
28416
|
+
case "temporal":
|
|
28417
|
+
return `t:${r.item.id}`;
|
|
28418
|
+
case "lat-section":
|
|
28419
|
+
return `lat:${r.item.id}`;
|
|
28420
|
+
}
|
|
28421
|
+
}
|
|
28141
28422
|
function searchDistillationsLike(input) {
|
|
28142
28423
|
const terms = input.query.toLowerCase().split(/\s+/).filter((term) => term.length > 1);
|
|
28143
28424
|
if (!terms.length) return [];
|
|
28144
28425
|
const conditions = terms.map(() => "LOWER(observations) LIKE ?").join(" AND ");
|
|
28145
28426
|
const likeParams = terms.map((term) => `%${term}%`);
|
|
28146
|
-
const sql = input.sessionID ? `SELECT id, observations, generation, created_at, session_id FROM distillations WHERE project_id = ? AND session_id = ? AND ${conditions} ORDER BY created_at DESC LIMIT ?` : `SELECT id, observations, generation, created_at, session_id FROM distillations WHERE project_id = ? AND ${conditions} ORDER BY created_at DESC LIMIT ?`;
|
|
28427
|
+
const sql = input.sessionID ? `SELECT id, observations, generation, created_at, session_id, c_norm FROM distillations WHERE project_id = ? AND session_id = ? AND ${conditions} ORDER BY created_at DESC LIMIT ?` : `SELECT id, observations, generation, created_at, session_id, c_norm FROM distillations WHERE project_id = ? AND ${conditions} ORDER BY created_at DESC LIMIT ?`;
|
|
28147
28428
|
const allParams = input.sessionID ? [input.pid, input.sessionID, ...likeParams, input.limit] : [input.pid, ...likeParams, input.limit];
|
|
28148
28429
|
return db().query(sql).all(...allParams);
|
|
28149
28430
|
}
|
|
@@ -28152,12 +28433,12 @@ function searchDistillationsScored(input) {
|
|
|
28152
28433
|
const limit = input.limit ?? 10;
|
|
28153
28434
|
const q = ftsQuery(input.query);
|
|
28154
28435
|
if (q === EMPTY_QUERY) return [];
|
|
28155
|
-
const ftsSQL = input.sessionID ? `SELECT d.id, d.observations, d.generation, d.created_at, d.session_id, rank
|
|
28436
|
+
const ftsSQL = input.sessionID ? `SELECT d.id, d.observations, d.generation, d.created_at, d.session_id, d.c_norm, rank
|
|
28156
28437
|
FROM distillation_fts f
|
|
28157
28438
|
CROSS JOIN distillations d ON d.rowid = f.rowid
|
|
28158
28439
|
WHERE distillation_fts MATCH ?
|
|
28159
28440
|
AND d.project_id = ? AND d.session_id = ?
|
|
28160
|
-
ORDER BY rank LIMIT ?` : `SELECT d.id, d.observations, d.generation, d.created_at, d.session_id, rank
|
|
28441
|
+
ORDER BY rank LIMIT ?` : `SELECT d.id, d.observations, d.generation, d.created_at, d.session_id, d.c_norm, rank
|
|
28161
28442
|
FROM distillation_fts f
|
|
28162
28443
|
CROSS JOIN distillations d ON d.rowid = f.rowid
|
|
28163
28444
|
WHERE distillation_fts MATCH ?
|
|
@@ -28242,7 +28523,7 @@ async function runRecall(input) {
|
|
|
28242
28523
|
let queries = [query];
|
|
28243
28524
|
if (searchConfig?.queryExpansion && llm) {
|
|
28244
28525
|
try {
|
|
28245
|
-
queries = await expandQuery(llm, query);
|
|
28526
|
+
queries = await expandQuery(llm, query, void 0, sessionID);
|
|
28246
28527
|
} catch (err) {
|
|
28247
28528
|
info("recall: query expansion failed, using original:", err);
|
|
28248
28529
|
}
|
|
@@ -28351,7 +28632,7 @@ async function runRecall(input) {
|
|
|
28351
28632
|
const distVectorHits = vectorSearchDistillations(queryVec, limit);
|
|
28352
28633
|
const distVectorTagged = distVectorHits.map((hit) => {
|
|
28353
28634
|
const row = db().query(
|
|
28354
|
-
"SELECT id, observations, generation, created_at, session_id FROM distillations WHERE id = ?"
|
|
28635
|
+
"SELECT id, observations, generation, created_at, session_id, c_norm FROM distillations WHERE id = ?"
|
|
28355
28636
|
).get(hit.id);
|
|
28356
28637
|
if (!row) return null;
|
|
28357
28638
|
return {
|
|
@@ -28414,6 +28695,57 @@ async function runRecall(input) {
|
|
|
28414
28695
|
info("recall: cross-project knowledge search failed:", err);
|
|
28415
28696
|
}
|
|
28416
28697
|
}
|
|
28698
|
+
{
|
|
28699
|
+
const distillationCandidates = [];
|
|
28700
|
+
for (const list4 of allRrfLists) {
|
|
28701
|
+
for (const item of list4.items) {
|
|
28702
|
+
if (item.source !== "distillation") continue;
|
|
28703
|
+
const key = `d:${item.item.id}`;
|
|
28704
|
+
const d = item.item;
|
|
28705
|
+
const cNorm = d.c_norm ?? 0;
|
|
28706
|
+
const ageDays = Math.min(
|
|
28707
|
+
(Date.now() - d.created_at) / 864e5,
|
|
28708
|
+
90
|
|
28709
|
+
);
|
|
28710
|
+
const score = cNorm + ageDays / 90 * 0.1;
|
|
28711
|
+
distillationCandidates.push({ tagged: item, key, qualityScore: score });
|
|
28712
|
+
}
|
|
28713
|
+
}
|
|
28714
|
+
if (distillationCandidates.length > 1) {
|
|
28715
|
+
const seen = /* @__PURE__ */ new Set();
|
|
28716
|
+
const unique = distillationCandidates.filter((c) => {
|
|
28717
|
+
if (seen.has(c.key)) return false;
|
|
28718
|
+
seen.add(c.key);
|
|
28719
|
+
return true;
|
|
28720
|
+
});
|
|
28721
|
+
unique.sort((a, b) => a.qualityScore - b.qualityScore);
|
|
28722
|
+
allRrfLists.push({
|
|
28723
|
+
items: unique.map((c) => c.tagged),
|
|
28724
|
+
key: (r) => `d:${r.item.id}`
|
|
28725
|
+
});
|
|
28726
|
+
}
|
|
28727
|
+
}
|
|
28728
|
+
if (filterTerms(query).length > 0 && allRrfLists.length > 0) {
|
|
28729
|
+
const allCandidates = /* @__PURE__ */ new Map();
|
|
28730
|
+
for (const list4 of allRrfLists) {
|
|
28731
|
+
for (const item of list4.items) {
|
|
28732
|
+
const key = list4.key(item);
|
|
28733
|
+
if (!allCandidates.has(key)) allCandidates.set(key, item);
|
|
28734
|
+
}
|
|
28735
|
+
}
|
|
28736
|
+
const candidateEntries = [...allCandidates.entries()];
|
|
28737
|
+
const exactRanked = exactTermMatchRank(
|
|
28738
|
+
candidateEntries,
|
|
28739
|
+
([, tagged]) => getTaggedText(tagged),
|
|
28740
|
+
query
|
|
28741
|
+
);
|
|
28742
|
+
if (exactRanked.length) {
|
|
28743
|
+
allRrfLists.push({
|
|
28744
|
+
items: exactRanked.map(([, item]) => item),
|
|
28745
|
+
key: taggedResultKey
|
|
28746
|
+
});
|
|
28747
|
+
}
|
|
28748
|
+
}
|
|
28417
28749
|
const fused = reciprocalRankFusion(allRrfLists);
|
|
28418
28750
|
return formatFusedResults(fused, 20);
|
|
28419
28751
|
}
|
|
@@ -28425,7 +28757,7 @@ var RECALL_PARAM_DESCRIPTIONS = {
|
|
|
28425
28757
|
|
|
28426
28758
|
// src/agents-file.ts
|
|
28427
28759
|
import { existsSync as existsSync3, readFileSync as readFileSync3, writeFileSync, mkdirSync as mkdirSync2 } from "fs";
|
|
28428
|
-
import { dirname as dirname2 } from "path";
|
|
28760
|
+
import { dirname as dirname2, join as join5 } from "path";
|
|
28429
28761
|
var LORE_SECTION_START = "<!-- This section is maintained by the coding agent via lore (https://github.com/BYK/loreai) -->";
|
|
28430
28762
|
var LORE_SECTION_END = "<!-- End lore-managed section -->";
|
|
28431
28763
|
var ALL_START_MARKERS = [
|
|
@@ -28434,6 +28766,8 @@ var ALL_START_MARKERS = [
|
|
|
28434
28766
|
"<!-- This section is maintained by the coding agent via lore (https://github.com/BYK/opencode-lore) -->",
|
|
28435
28767
|
"<!-- This section is auto-maintained by lore (https://github.com/BYK/opencode-lore) -->"
|
|
28436
28768
|
];
|
|
28769
|
+
var LORE_FILE = ".lore.md";
|
|
28770
|
+
var LORE_FILE_HEADER = "<!-- Managed by lore (https://github.com/BYK/loreai) \u2014 manual edits are imported on next session. -->";
|
|
28437
28771
|
var UUID_RE2 = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
|
28438
28772
|
var MARKER_RE = /^<!--\s*lore:([0-9a-f-]+)\s*-->$/;
|
|
28439
28773
|
function splitFile(fileContent) {
|
|
@@ -28543,8 +28877,9 @@ function buildSection(projectPath) {
|
|
|
28543
28877
|
return out.join("\n");
|
|
28544
28878
|
}
|
|
28545
28879
|
function exportToFile(input) {
|
|
28546
|
-
|
|
28547
|
-
const
|
|
28880
|
+
exportLoreFile(input.projectPath);
|
|
28881
|
+
const pointerBody = "\n## Long-term Knowledge\n\nFor long-term knowledge entries managed by [lore](https://github.com/BYK/loreai) (gotchas, patterns, decisions, architecture), see [`.lore.md`](.lore.md) in the project root.\n";
|
|
28882
|
+
const newSection = LORE_SECTION_START + pointerBody + LORE_SECTION_END + "\n";
|
|
28548
28883
|
let fileContent = "";
|
|
28549
28884
|
if (existsSync3(input.filePath)) {
|
|
28550
28885
|
fileContent = readFileSync3(input.filePath, "utf8");
|
|
@@ -28568,15 +28903,9 @@ function shouldImport(input) {
|
|
|
28568
28903
|
const expected = buildSection(input.projectPath);
|
|
28569
28904
|
return hashSection(section) !== hashSection(expected);
|
|
28570
28905
|
}
|
|
28571
|
-
function
|
|
28572
|
-
if (!existsSync3(input.filePath)) return;
|
|
28573
|
-
const fileContent = readFileSync3(input.filePath, "utf8");
|
|
28574
|
-
const { section, before } = splitFile(fileContent);
|
|
28575
|
-
const textToParse = section ?? fileContent;
|
|
28576
|
-
const fileEntries = parseEntriesFromSection(textToParse);
|
|
28577
|
-
if (!fileEntries.length) return;
|
|
28906
|
+
function _importEntries(entries, projectPath) {
|
|
28578
28907
|
const seenIds = /* @__PURE__ */ new Set();
|
|
28579
|
-
for (const entry of
|
|
28908
|
+
for (const entry of entries) {
|
|
28580
28909
|
if (entry.id !== null) {
|
|
28581
28910
|
if (seenIds.has(entry.id)) continue;
|
|
28582
28911
|
seenIds.add(entry.id);
|
|
@@ -28587,7 +28916,7 @@ function importFromFile(input) {
|
|
|
28587
28916
|
}
|
|
28588
28917
|
} else {
|
|
28589
28918
|
create({
|
|
28590
|
-
projectPath
|
|
28919
|
+
projectPath,
|
|
28591
28920
|
category: entry.category,
|
|
28592
28921
|
title: entry.title,
|
|
28593
28922
|
content: entry.content,
|
|
@@ -28597,13 +28926,13 @@ function importFromFile(input) {
|
|
|
28597
28926
|
});
|
|
28598
28927
|
}
|
|
28599
28928
|
} else {
|
|
28600
|
-
const existing = forProject(
|
|
28929
|
+
const existing = forProject(projectPath, true);
|
|
28601
28930
|
const titleMatch = existing.find(
|
|
28602
28931
|
(e) => e.title.toLowerCase() === entry.title.toLowerCase()
|
|
28603
28932
|
);
|
|
28604
28933
|
if (!titleMatch) {
|
|
28605
28934
|
create({
|
|
28606
|
-
projectPath
|
|
28935
|
+
projectPath,
|
|
28607
28936
|
category: entry.category,
|
|
28608
28937
|
title: entry.title,
|
|
28609
28938
|
content: entry.content,
|
|
@@ -28614,16 +28943,50 @@ function importFromFile(input) {
|
|
|
28614
28943
|
}
|
|
28615
28944
|
}
|
|
28616
28945
|
}
|
|
28946
|
+
function importFromFile(input) {
|
|
28947
|
+
if (!existsSync3(input.filePath)) return;
|
|
28948
|
+
const fileContent = readFileSync3(input.filePath, "utf8");
|
|
28949
|
+
const { section } = splitFile(fileContent);
|
|
28950
|
+
const textToParse = section ?? fileContent;
|
|
28951
|
+
const fileEntries = parseEntriesFromSection(textToParse);
|
|
28952
|
+
if (!fileEntries.length) return;
|
|
28953
|
+
_importEntries(fileEntries, input.projectPath);
|
|
28954
|
+
}
|
|
28955
|
+
function loreFileExists(projectPath) {
|
|
28956
|
+
return existsSync3(join5(projectPath, LORE_FILE));
|
|
28957
|
+
}
|
|
28958
|
+
function exportLoreFile(projectPath) {
|
|
28959
|
+
const sectionBody = buildSection(projectPath);
|
|
28960
|
+
const content3 = LORE_FILE_HEADER + "\n" + sectionBody;
|
|
28961
|
+
writeFileSync(join5(projectPath, LORE_FILE), content3, "utf8");
|
|
28962
|
+
}
|
|
28963
|
+
function shouldImportLoreFile(projectPath) {
|
|
28964
|
+
const fp = join5(projectPath, LORE_FILE);
|
|
28965
|
+
if (!existsSync3(fp)) return false;
|
|
28966
|
+
const fileContent = readFileSync3(fp, "utf8");
|
|
28967
|
+
const expected = LORE_FILE_HEADER + "\n" + buildSection(projectPath);
|
|
28968
|
+
return hashSection(fileContent) !== hashSection(expected);
|
|
28969
|
+
}
|
|
28970
|
+
function importLoreFile(projectPath) {
|
|
28971
|
+
const fp = join5(projectPath, LORE_FILE);
|
|
28972
|
+
if (!existsSync3(fp)) return;
|
|
28973
|
+
const fileContent = readFileSync3(fp, "utf8");
|
|
28974
|
+
const fileEntries = parseEntriesFromSection(fileContent);
|
|
28975
|
+
if (!fileEntries.length) return;
|
|
28976
|
+
_importEntries(fileEntries, projectPath);
|
|
28977
|
+
}
|
|
28617
28978
|
|
|
28618
28979
|
// src/worker-model.ts
|
|
28619
28980
|
var worker_model_exports = {};
|
|
28620
28981
|
__export(worker_model_exports, {
|
|
28621
28982
|
WORKER_JUDGE_SYSTEM: () => WORKER_JUDGE_SYSTEM,
|
|
28983
|
+
clearValidatedWorkerModel: () => clearValidatedWorkerModel,
|
|
28622
28984
|
computeModelFingerprint: () => computeModelFingerprint,
|
|
28623
28985
|
getValidatedWorkerModel: () => getValidatedWorkerModel,
|
|
28624
28986
|
isValidationStale: () => isValidationStale,
|
|
28625
28987
|
parseJudgeScore: () => parseJudgeScore,
|
|
28626
28988
|
resolveWorkerModel: () => resolveWorkerModel,
|
|
28989
|
+
runValidation: () => runValidation,
|
|
28627
28990
|
selectWorkerCandidates: () => selectWorkerCandidates,
|
|
28628
28991
|
storeValidatedWorkerModel: () => storeValidatedWorkerModel,
|
|
28629
28992
|
structuralCheck: () => structuralCheck,
|
|
@@ -28635,7 +28998,13 @@ function selectWorkerCandidates(sessionModel, providerModels) {
|
|
|
28635
28998
|
(m) => m.providerID === sessionModel.providerID && m.status === "active" && m.capabilities.input.text
|
|
28636
28999
|
);
|
|
28637
29000
|
if (eligible.length === 0) return [];
|
|
28638
|
-
const sorted = [...eligible].sort((a, b) =>
|
|
29001
|
+
const sorted = [...eligible].sort((a, b) => {
|
|
29002
|
+
const costDiff = a.cost.input - b.cost.input;
|
|
29003
|
+
if (costDiff !== 0) return costDiff;
|
|
29004
|
+
const aReasoning = a.capabilities.reasoning ? 1 : 0;
|
|
29005
|
+
const bReasoning = b.capabilities.reasoning ? 1 : 0;
|
|
29006
|
+
return aReasoning - bReasoning;
|
|
29007
|
+
});
|
|
28639
29008
|
const cheapest = sorted[0];
|
|
28640
29009
|
const belowSession = sorted.filter((m) => m.cost.input < sessionModel.cost.input).pop();
|
|
28641
29010
|
const candidates = /* @__PURE__ */ new Map();
|
|
@@ -28670,6 +29039,9 @@ function storeValidatedWorkerModel(result) {
|
|
|
28670
29039
|
"INSERT INTO kv_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = ?"
|
|
28671
29040
|
).run(key, value, value);
|
|
28672
29041
|
}
|
|
29042
|
+
function clearValidatedWorkerModel(providerID) {
|
|
29043
|
+
db().query("DELETE FROM kv_meta WHERE key = ?").run(`${KV_PREFIX}${providerID}`);
|
|
29044
|
+
}
|
|
28673
29045
|
function isValidationStale(stored, currentFingerprint) {
|
|
28674
29046
|
if (!stored) return true;
|
|
28675
29047
|
return stored.fingerprint !== currentFingerprint;
|
|
@@ -28728,10 +29100,85 @@ function parseJudgeScore(response) {
|
|
|
28728
29100
|
if (!match) return null;
|
|
28729
29101
|
return parseInt(match[1], 10);
|
|
28730
29102
|
}
|
|
29103
|
+
async function runValidation(input) {
|
|
29104
|
+
const { llm, candidates, referenceObservations, sourceMessagesText, date: date5 } = input;
|
|
29105
|
+
const userPrompt = distillationUser({
|
|
29106
|
+
messages: sourceMessagesText,
|
|
29107
|
+
date: date5
|
|
29108
|
+
});
|
|
29109
|
+
for (const candidate of candidates) {
|
|
29110
|
+
if (candidate.id === input.sessionModelID) continue;
|
|
29111
|
+
let candidateObservations = null;
|
|
29112
|
+
try {
|
|
29113
|
+
const raw = await llm.prompt(DISTILLATION_SYSTEM, userPrompt, {
|
|
29114
|
+
model: { providerID: candidate.providerID, modelID: candidate.id },
|
|
29115
|
+
workerID: "lore-distill",
|
|
29116
|
+
thinking: false
|
|
29117
|
+
});
|
|
29118
|
+
if (raw) {
|
|
29119
|
+
const match = raw.match(/<observations>([\s\S]*?)<\/observations>/);
|
|
29120
|
+
candidateObservations = match ? match[1].trim() : raw.trim();
|
|
29121
|
+
}
|
|
29122
|
+
} catch (e) {
|
|
29123
|
+
warn(`worker model validation: candidate ${candidate.id} failed:`, e);
|
|
29124
|
+
continue;
|
|
29125
|
+
}
|
|
29126
|
+
const structural = structuralCheck(candidateObservations, referenceObservations);
|
|
29127
|
+
if (!structural.passed) {
|
|
29128
|
+
info(
|
|
29129
|
+
`worker model validation: ${candidate.id} failed structural check: ${structural.reason}`
|
|
29130
|
+
);
|
|
29131
|
+
continue;
|
|
29132
|
+
}
|
|
29133
|
+
let judgeScore = null;
|
|
29134
|
+
try {
|
|
29135
|
+
const judgeResponse = await llm.prompt(
|
|
29136
|
+
WORKER_JUDGE_SYSTEM,
|
|
29137
|
+
workerJudgeUser(referenceObservations, candidateObservations),
|
|
29138
|
+
{ workerID: "lore-distill", thinking: false }
|
|
29139
|
+
// use session model (no model override)
|
|
29140
|
+
);
|
|
29141
|
+
if (judgeResponse) {
|
|
29142
|
+
judgeScore = parseJudgeScore(judgeResponse);
|
|
29143
|
+
}
|
|
29144
|
+
} catch (e) {
|
|
29145
|
+
warn(`worker model validation: judge call failed for ${candidate.id}:`, e);
|
|
29146
|
+
}
|
|
29147
|
+
if (judgeScore !== null && judgeScore < 3) {
|
|
29148
|
+
info(
|
|
29149
|
+
`worker model validation: ${candidate.id} failed judge (score=${judgeScore})`
|
|
29150
|
+
);
|
|
29151
|
+
continue;
|
|
29152
|
+
}
|
|
29153
|
+
const fingerprint = computeModelFingerprint(
|
|
29154
|
+
input.providerID,
|
|
29155
|
+
input.sessionModelID,
|
|
29156
|
+
candidates.map((c) => c.id)
|
|
29157
|
+
);
|
|
29158
|
+
const result = {
|
|
29159
|
+
modelID: candidate.id,
|
|
29160
|
+
providerID: candidate.providerID,
|
|
29161
|
+
fingerprint,
|
|
29162
|
+
validatedAt: Date.now(),
|
|
29163
|
+
judgeScore
|
|
29164
|
+
};
|
|
29165
|
+
storeValidatedWorkerModel(result);
|
|
29166
|
+
info(
|
|
29167
|
+
`worker model validated: ${candidate.id} (judge=${judgeScore}) for provider ${input.providerID}`
|
|
29168
|
+
);
|
|
29169
|
+
return result;
|
|
29170
|
+
}
|
|
29171
|
+
clearValidatedWorkerModel(input.providerID);
|
|
29172
|
+
info(
|
|
29173
|
+
`worker model validation: no candidate passed for ${input.providerID} \u2014 cleared stale entry`
|
|
29174
|
+
);
|
|
29175
|
+
return null;
|
|
29176
|
+
}
|
|
28731
29177
|
function resolveWorkerModel(providerID, configWorkerModel, configModel) {
|
|
28732
29178
|
if (configWorkerModel) return configWorkerModel;
|
|
28733
29179
|
const validated = getValidatedWorkerModel(providerID);
|
|
28734
|
-
|
|
29180
|
+
const MAX_AGE_MS = 24 * 60 * 60 * 1e3;
|
|
29181
|
+
if (validated && Date.now() - validated.validatedAt <= MAX_AGE_MS) {
|
|
28735
29182
|
return { providerID: validated.providerID, modelID: validated.modelID };
|
|
28736
29183
|
}
|
|
28737
29184
|
return configModel;
|
|
@@ -28742,11 +29189,11 @@ export {
|
|
|
28742
29189
|
CURATOR_SYSTEM,
|
|
28743
29190
|
DISTILLATION_SYSTEM,
|
|
28744
29191
|
EMPTY_QUERY,
|
|
29192
|
+
LORE_FILE,
|
|
28745
29193
|
QUERY_EXPANSION_SYSTEM,
|
|
28746
29194
|
RECALL_PARAM_DESCRIPTIONS,
|
|
28747
29195
|
RECALL_TOOL_DESCRIPTION,
|
|
28748
29196
|
RECURSIVE_SYSTEM,
|
|
28749
|
-
WORKER_JUDGE_SYSTEM,
|
|
28750
29197
|
buildCompactPrompt,
|
|
28751
29198
|
calibrate,
|
|
28752
29199
|
close,
|
|
@@ -28761,7 +29208,9 @@ export {
|
|
|
28761
29208
|
distillationUser,
|
|
28762
29209
|
embedding_exports as embedding,
|
|
28763
29210
|
ensureProject,
|
|
29211
|
+
exactTermMatchRank,
|
|
28764
29212
|
expandQuery,
|
|
29213
|
+
exportLoreFile,
|
|
28765
29214
|
exportToFile,
|
|
28766
29215
|
extractTopTerms,
|
|
28767
29216
|
formatDistillations,
|
|
@@ -28770,10 +29219,12 @@ export {
|
|
|
28770
29219
|
ftsQueryOr,
|
|
28771
29220
|
getLastTransformEstimate,
|
|
28772
29221
|
getLastTransformedCount,
|
|
29222
|
+
getLastTurnAt,
|
|
28773
29223
|
getLtmBudget,
|
|
28774
29224
|
getLtmTokens,
|
|
28775
29225
|
h,
|
|
28776
29226
|
importFromFile,
|
|
29227
|
+
importLoreFile,
|
|
28777
29228
|
inline,
|
|
28778
29229
|
inspectSessionState,
|
|
28779
29230
|
isFirstRun,
|
|
@@ -28787,11 +29238,13 @@ export {
|
|
|
28787
29238
|
load,
|
|
28788
29239
|
loadForceMinLayer,
|
|
28789
29240
|
log_exports as log,
|
|
29241
|
+
loreFileExists,
|
|
28790
29242
|
ltm_exports as ltm,
|
|
28791
29243
|
needsUrgentDistillation,
|
|
28792
29244
|
normalize,
|
|
28793
29245
|
onIdleResume,
|
|
28794
29246
|
p,
|
|
29247
|
+
pattern_extract_exports as patternExtract,
|
|
28795
29248
|
projectId,
|
|
28796
29249
|
projectName,
|
|
28797
29250
|
reciprocalRankFusion,
|
|
@@ -28807,6 +29260,7 @@ export {
|
|
|
28807
29260
|
setMaxLayer0Tokens,
|
|
28808
29261
|
setModelLimits,
|
|
28809
29262
|
shouldImport,
|
|
29263
|
+
shouldImportLoreFile,
|
|
28810
29264
|
strong2 as strong,
|
|
28811
29265
|
t,
|
|
28812
29266
|
temporal_exports as temporal,
|
|
@@ -28814,7 +29268,6 @@ export {
|
|
|
28814
29268
|
transform2 as transform,
|
|
28815
29269
|
ul,
|
|
28816
29270
|
unescapeMarkdown,
|
|
28817
|
-
workerJudgeUser,
|
|
28818
29271
|
worker_model_exports as workerModel,
|
|
28819
29272
|
workerSessionIDs
|
|
28820
29273
|
};
|