@loreai/core 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bun/agents-file.d.ts +29 -8
- package/dist/bun/agents-file.d.ts.map +1 -1
- package/dist/bun/config.d.ts +1 -0
- package/dist/bun/config.d.ts.map +1 -1
- package/dist/bun/db.d.ts.map +1 -1
- package/dist/bun/distillation.d.ts +29 -0
- package/dist/bun/distillation.d.ts.map +1 -1
- package/dist/bun/embedding.d.ts +15 -1
- package/dist/bun/embedding.d.ts.map +1 -1
- package/dist/bun/gradient.d.ts +53 -5
- package/dist/bun/gradient.d.ts.map +1 -1
- package/dist/bun/index.d.ts +4 -4
- package/dist/bun/index.d.ts.map +1 -1
- package/dist/bun/index.js +696 -243
- package/dist/bun/index.js.map +4 -4
- package/dist/bun/pattern-extract.d.ts +36 -0
- package/dist/bun/pattern-extract.d.ts.map +1 -0
- package/dist/bun/recall.d.ts +1 -0
- package/dist/bun/recall.d.ts.map +1 -1
- package/dist/bun/search.d.ts +13 -1
- package/dist/bun/search.d.ts.map +1 -1
- package/dist/bun/types.d.ts +41 -1
- package/dist/bun/types.d.ts.map +1 -1
- package/dist/bun/worker-model.d.ts +22 -0
- package/dist/bun/worker-model.d.ts.map +1 -1
- package/dist/node/agents-file.d.ts +29 -8
- package/dist/node/agents-file.d.ts.map +1 -1
- package/dist/node/config.d.ts +1 -0
- package/dist/node/config.d.ts.map +1 -1
- package/dist/node/db.d.ts.map +1 -1
- package/dist/node/distillation.d.ts +29 -0
- package/dist/node/distillation.d.ts.map +1 -1
- package/dist/node/embedding.d.ts +15 -1
- package/dist/node/embedding.d.ts.map +1 -1
- package/dist/node/gradient.d.ts +53 -5
- package/dist/node/gradient.d.ts.map +1 -1
- package/dist/node/index.d.ts +4 -4
- package/dist/node/index.d.ts.map +1 -1
- package/dist/node/index.js +696 -243
- package/dist/node/index.js.map +4 -4
- package/dist/node/pattern-extract.d.ts +36 -0
- package/dist/node/pattern-extract.d.ts.map +1 -0
- package/dist/node/recall.d.ts +1 -0
- package/dist/node/recall.d.ts.map +1 -1
- package/dist/node/search.d.ts +13 -1
- package/dist/node/search.d.ts.map +1 -1
- package/dist/node/types.d.ts +41 -1
- package/dist/node/types.d.ts.map +1 -1
- package/dist/node/worker-model.d.ts +22 -0
- package/dist/node/worker-model.d.ts.map +1 -1
- package/dist/types/agents-file.d.ts +29 -8
- package/dist/types/agents-file.d.ts.map +1 -1
- package/dist/types/config.d.ts +1 -0
- package/dist/types/config.d.ts.map +1 -1
- package/dist/types/db.d.ts.map +1 -1
- package/dist/types/distillation.d.ts +29 -0
- package/dist/types/distillation.d.ts.map +1 -1
- package/dist/types/embedding.d.ts +15 -1
- package/dist/types/embedding.d.ts.map +1 -1
- package/dist/types/gradient.d.ts +53 -5
- package/dist/types/gradient.d.ts.map +1 -1
- package/dist/types/index.d.ts +4 -4
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/pattern-extract.d.ts +36 -0
- package/dist/types/pattern-extract.d.ts.map +1 -0
- package/dist/types/recall.d.ts +1 -0
- package/dist/types/recall.d.ts.map +1 -1
- package/dist/types/search.d.ts +13 -1
- package/dist/types/search.d.ts.map +1 -1
- package/dist/types/types.d.ts +41 -1
- package/dist/types/types.d.ts.map +1 -1
- package/dist/types/worker-model.d.ts +22 -0
- package/dist/types/worker-model.d.ts.map +1 -1
- package/package.json +3 -2
- package/src/agents-file.ts +111 -28
- package/src/config.ts +25 -18
- package/src/curator.ts +2 -2
- package/src/db.ts +19 -2
- package/src/distillation.ts +152 -15
- package/src/embedding.ts +158 -14
- package/src/gradient.ts +398 -227
- package/src/index.ts +13 -5
- package/src/pattern-extract.ts +108 -0
- package/src/recall.ts +124 -6
- package/src/search.ts +37 -1
- package/src/types.ts +41 -1
- package/src/worker-model.ts +142 -5
package/dist/bun/index.js
CHANGED
|
@@ -146,6 +146,7 @@ function sha256(input) {
|
|
|
146
146
|
// src/db.ts
|
|
147
147
|
import { join, dirname } from "path";
|
|
148
148
|
import { mkdirSync } from "fs";
|
|
149
|
+
import { homedir } from "os";
|
|
149
150
|
var MIGRATIONS = [
|
|
150
151
|
`
|
|
151
152
|
-- Version 1: Initial schema
|
|
@@ -474,11 +475,27 @@ var MIGRATIONS = [
|
|
|
474
475
|
)
|
|
475
476
|
WHERE content LIKE '%' || char(10) || '[tool:%'
|
|
476
477
|
OR content LIKE '%' || char(10) || '[reasoning] %';
|
|
478
|
+
`,
|
|
479
|
+
`
|
|
480
|
+
-- Version 12: Context health diagnostic columns on distillations.
|
|
481
|
+
--
|
|
482
|
+
-- r_compression: k/\u221AN where k = distilled token count, N = source token
|
|
483
|
+
-- count. Values < 1.0 signal likely lossy compression. NULL for rows
|
|
484
|
+
-- created before this migration or for meta-distillations (gen > 0)
|
|
485
|
+
-- where the metric is not computed.
|
|
486
|
+
--
|
|
487
|
+
-- c_norm: normalized variance of relative-existence weights over source
|
|
488
|
+
-- message timestamps. Range [0, 1]; 0 = uniform distribution, 1 = attention
|
|
489
|
+
-- dominated by distant past. NULL for pre-migration rows or meta-distillations.
|
|
490
|
+
--
|
|
491
|
+
-- Both columns are nullable REALs \u2014 cheap to add, no backfill needed.
|
|
492
|
+
ALTER TABLE distillations ADD COLUMN r_compression REAL;
|
|
493
|
+
ALTER TABLE distillations ADD COLUMN c_norm REAL;
|
|
477
494
|
`
|
|
478
495
|
];
|
|
479
496
|
function dataDir() {
|
|
480
497
|
const xdg = process.env.XDG_DATA_HOME;
|
|
481
|
-
const base = xdg || join(
|
|
498
|
+
const base = xdg || join(homedir(), ".local", "share");
|
|
482
499
|
return join(base, "opencode-lore");
|
|
483
500
|
}
|
|
484
501
|
var instance;
|
|
@@ -11274,14 +11291,24 @@ function reciprocalRankFusion(lists, k = 60) {
|
|
|
11274
11291
|
}
|
|
11275
11292
|
return [...scores.values()].sort((a, b) => b.score - a.score);
|
|
11276
11293
|
}
|
|
11277
|
-
|
|
11294
|
+
function exactTermMatchRank(items, getText, query) {
|
|
11295
|
+
const terms = filterTerms(query).map((t2) => t2.toLowerCase());
|
|
11296
|
+
if (!terms.length) return [];
|
|
11297
|
+
const scored = items.map((item) => {
|
|
11298
|
+
const text4 = getText(item).toLowerCase();
|
|
11299
|
+
const matches = terms.filter((t2) => text4.includes(t2)).length;
|
|
11300
|
+
return { item, matches };
|
|
11301
|
+
}).filter((s) => s.matches > 0).sort((a, b) => b.matches - a.matches);
|
|
11302
|
+
return scored.map((s) => s.item);
|
|
11303
|
+
}
|
|
11304
|
+
async function expandQuery(llm, query, model, sessionID) {
|
|
11278
11305
|
const TIMEOUT_MS = 3e3;
|
|
11279
11306
|
try {
|
|
11280
11307
|
const responseText = await Promise.race([
|
|
11281
11308
|
llm.prompt(
|
|
11282
11309
|
QUERY_EXPANSION_SYSTEM,
|
|
11283
11310
|
`Input: "${query}"`,
|
|
11284
|
-
{ model, workerID: "lore-query-expand" }
|
|
11311
|
+
{ model, workerID: "lore-query-expand", thinking: false, urgent: true, sessionID }
|
|
11285
11312
|
),
|
|
11286
11313
|
new Promise((resolve) => setTimeout(() => resolve(null), TIMEOUT_MS))
|
|
11287
11314
|
]);
|
|
@@ -25691,11 +25718,15 @@ var LoreConfig = external_exports.object({
|
|
|
25691
25718
|
* Anthropic's April 23 postmortem identified dropping reasoning blocks as
|
|
25692
25719
|
* the root cause of forgetfulness/repetition.
|
|
25693
25720
|
*
|
|
25694
|
-
* `idleResumeMinutes` is the threshold in minutes. Default
|
|
25695
|
-
* Anthropic's
|
|
25721
|
+
* `idleResumeMinutes` is the threshold in minutes. Default 5 — matches
|
|
25722
|
+
* Anthropic's default-tier prompt cache TTL. After 5 min of inactivity the
|
|
25723
|
+
* upstream cache is cold, so preserving byte-identity wastes cache-write cost
|
|
25724
|
+
* for no benefit. Refreshing the caches on resume produces a better-fitting
|
|
25725
|
+
* window at the same cold-write price. Users on Anthropic's extended-cache
|
|
25726
|
+
* tier (1 h TTL) should set this to 60 in `.lore.json`.
|
|
25696
25727
|
* Set to 0 to disable the feature.
|
|
25697
25728
|
*/
|
|
25698
|
-
idleResumeMinutes: external_exports.number().min(0).max(24 * 60).default(
|
|
25729
|
+
idleResumeMinutes: external_exports.number().min(0).max(24 * 60).default(5),
|
|
25699
25730
|
distillation: external_exports.object({
|
|
25700
25731
|
minMessages: external_exports.number().min(3).default(5),
|
|
25701
25732
|
maxSegment: external_exports.number().min(5).default(30),
|
|
@@ -25746,34 +25777,37 @@ var LoreConfig = external_exports.object({
|
|
|
25746
25777
|
* before search, improving recall for ambiguous queries. */
|
|
25747
25778
|
queryExpansion: external_exports.boolean().default(false),
|
|
25748
25779
|
/** Vector embedding search.
|
|
25749
|
-
* Supports multiple providers:
|
|
25750
|
-
* "
|
|
25751
|
-
*
|
|
25752
|
-
*
|
|
25780
|
+
* Supports multiple providers:
|
|
25781
|
+
* - "local" (default): fastembed + ONNX Runtime, no API key needed.
|
|
25782
|
+
* Uses bge-small-en-v1.5 (384 dims). Model downloaded on first use (~33MB),
|
|
25783
|
+
* cached in ~/.cache/fastembed. ~150ms per query embed.
|
|
25784
|
+
* - "voyage": Voyage AI (VOYAGE_API_KEY, voyage-code-3, 1024 dims)
|
|
25785
|
+
* - "openai": OpenAI (OPENAI_API_KEY, text-embedding-3-small, 1536 dims)
|
|
25786
|
+
* Set enabled: false to explicitly disable even with a provider available. */
|
|
25753
25787
|
embeddings: external_exports.object({
|
|
25754
25788
|
/** Enable/disable vector embedding search. Default: true.
|
|
25755
|
-
* Set to false to explicitly disable
|
|
25789
|
+
* Set to false to explicitly disable. */
|
|
25756
25790
|
enabled: external_exports.boolean().default(true),
|
|
25757
|
-
/** Embedding provider. Default: "
|
|
25758
|
-
*
|
|
25791
|
+
/** Embedding provider. Default: "local".
|
|
25792
|
+
* - "local": fastembed + ONNX Runtime, no API key (default model: bge-small-en-v1.5, 384 dims)
|
|
25759
25793
|
* - "voyage": VOYAGE_API_KEY (default model: voyage-code-3, 1024 dims)
|
|
25760
25794
|
* - "openai": OPENAI_API_KEY (default model: text-embedding-3-small, 1536 dims) */
|
|
25761
|
-
provider: external_exports.enum(["voyage", "openai"]).default("
|
|
25795
|
+
provider: external_exports.enum(["local", "voyage", "openai"]).default("local"),
|
|
25762
25796
|
/** Model ID for the embedding provider. Default depends on provider. */
|
|
25763
|
-
model: external_exports.string().default("
|
|
25764
|
-
/** Embedding dimensions. Default: 1024. */
|
|
25765
|
-
dimensions: external_exports.number().min(
|
|
25797
|
+
model: external_exports.string().default("BGESmallENV15"),
|
|
25798
|
+
/** Embedding dimensions. Default: 384 (local) / 1024 (voyage) / 1536 (openai). */
|
|
25799
|
+
dimensions: external_exports.number().min(64).max(2048).default(384)
|
|
25766
25800
|
}).default({
|
|
25767
25801
|
enabled: true,
|
|
25768
|
-
provider: "
|
|
25769
|
-
model: "
|
|
25770
|
-
dimensions:
|
|
25802
|
+
provider: "local",
|
|
25803
|
+
model: "BGESmallENV15",
|
|
25804
|
+
dimensions: 384
|
|
25771
25805
|
})
|
|
25772
25806
|
}).default({
|
|
25773
25807
|
ftsWeights: { title: 6, content: 2, category: 3 },
|
|
25774
25808
|
recallLimit: 10,
|
|
25775
25809
|
queryExpansion: false,
|
|
25776
|
-
embeddings: { enabled: true, provider: "
|
|
25810
|
+
embeddings: { enabled: true, provider: "local", model: "BGESmallENV15", dimensions: 384 }
|
|
25777
25811
|
}),
|
|
25778
25812
|
crossProject: external_exports.boolean().default(false),
|
|
25779
25813
|
agentsFile: external_exports.object({
|
|
@@ -25811,6 +25845,7 @@ __export(embedding_exports, {
|
|
|
25811
25845
|
fromBlob: () => fromBlob,
|
|
25812
25846
|
isAvailable: () => isAvailable,
|
|
25813
25847
|
resetProvider: () => resetProvider,
|
|
25848
|
+
runStartupBackfill: () => runStartupBackfill,
|
|
25814
25849
|
toBlob: () => toBlob,
|
|
25815
25850
|
vectorSearch: () => vectorSearch,
|
|
25816
25851
|
vectorSearchDistillations: () => vectorSearchDistillations
|
|
@@ -25888,9 +25923,43 @@ var OpenAIProvider = class {
|
|
|
25888
25923
|
return sorted.map((d) => new Float32Array(d.embedding));
|
|
25889
25924
|
}
|
|
25890
25925
|
};
|
|
25891
|
-
var
|
|
25892
|
-
|
|
25893
|
-
|
|
25926
|
+
var LocalProvider = class {
|
|
25927
|
+
maxBatchSize = 256;
|
|
25928
|
+
model = null;
|
|
25929
|
+
initPromise = null;
|
|
25930
|
+
modelName;
|
|
25931
|
+
constructor(modelName) {
|
|
25932
|
+
this.modelName = modelName;
|
|
25933
|
+
}
|
|
25934
|
+
async getModel() {
|
|
25935
|
+
if (this.model) return this.model;
|
|
25936
|
+
if (!this.initPromise) {
|
|
25937
|
+
this.initPromise = (async () => {
|
|
25938
|
+
const { EmbeddingModel, FlagEmbedding } = await import("fastembed");
|
|
25939
|
+
const enumValue = EmbeddingModel[this.modelName];
|
|
25940
|
+
const m = await FlagEmbedding.init({
|
|
25941
|
+
model: enumValue ?? this.modelName
|
|
25942
|
+
});
|
|
25943
|
+
this.model = m;
|
|
25944
|
+
return m;
|
|
25945
|
+
})();
|
|
25946
|
+
}
|
|
25947
|
+
return this.initPromise;
|
|
25948
|
+
}
|
|
25949
|
+
async embed(texts, inputType) {
|
|
25950
|
+
const model = await this.getModel();
|
|
25951
|
+
if (inputType === "query" && texts.length === 1) {
|
|
25952
|
+
const vec = await model.queryEmbed(texts[0]);
|
|
25953
|
+
return [new Float32Array(vec)];
|
|
25954
|
+
}
|
|
25955
|
+
const results = [];
|
|
25956
|
+
for await (const batch of model.passageEmbed(texts)) {
|
|
25957
|
+
for (const vec of batch) {
|
|
25958
|
+
results.push(new Float32Array(vec));
|
|
25959
|
+
}
|
|
25960
|
+
}
|
|
25961
|
+
return results;
|
|
25962
|
+
}
|
|
25894
25963
|
};
|
|
25895
25964
|
var PROVIDER_ENV_KEYS = {
|
|
25896
25965
|
voyage: "VOYAGE_API_KEY",
|
|
@@ -25909,21 +25978,35 @@ function getProvider() {
|
|
|
25909
25978
|
return null;
|
|
25910
25979
|
}
|
|
25911
25980
|
const providerName = cfg.provider;
|
|
25912
|
-
const
|
|
25913
|
-
if (!apiKey) {
|
|
25914
|
-
cachedProvider = null;
|
|
25915
|
-
return null;
|
|
25916
|
-
}
|
|
25917
|
-
const defaults = PROVIDER_DEFAULTS[providerName];
|
|
25918
|
-
const model = cfg.model === defaults?.model ? cfg.model : cfg.model;
|
|
25919
|
-
const dimensions = cfg.dimensions;
|
|
25981
|
+
const model = cfg.model;
|
|
25920
25982
|
switch (providerName) {
|
|
25921
|
-
case "
|
|
25922
|
-
|
|
25983
|
+
case "local": {
|
|
25984
|
+
try {
|
|
25985
|
+
cachedProvider = new LocalProvider(model);
|
|
25986
|
+
} catch {
|
|
25987
|
+
info("local embedding provider unavailable (fastembed not installed)");
|
|
25988
|
+
cachedProvider = null;
|
|
25989
|
+
}
|
|
25923
25990
|
break;
|
|
25924
|
-
|
|
25925
|
-
|
|
25991
|
+
}
|
|
25992
|
+
case "voyage": {
|
|
25993
|
+
const apiKey = getProviderApiKey(providerName);
|
|
25994
|
+
if (!apiKey) {
|
|
25995
|
+
cachedProvider = null;
|
|
25996
|
+
return null;
|
|
25997
|
+
}
|
|
25998
|
+
cachedProvider = new VoyageProvider(apiKey, model, cfg.dimensions);
|
|
25999
|
+
break;
|
|
26000
|
+
}
|
|
26001
|
+
case "openai": {
|
|
26002
|
+
const apiKey = getProviderApiKey(providerName);
|
|
26003
|
+
if (!apiKey) {
|
|
26004
|
+
cachedProvider = null;
|
|
26005
|
+
return null;
|
|
26006
|
+
}
|
|
26007
|
+
cachedProvider = new OpenAIProvider(apiKey, model, cfg.dimensions);
|
|
25926
26008
|
break;
|
|
26009
|
+
}
|
|
25927
26010
|
default:
|
|
25928
26011
|
info(`unknown embedding provider: ${providerName}`);
|
|
25929
26012
|
cachedProvider = null;
|
|
@@ -26028,6 +26111,29 @@ function checkConfigChange() {
|
|
|
26028
26111
|
).run(EMBEDDING_CONFIG_KEY, current2, current2);
|
|
26029
26112
|
return true;
|
|
26030
26113
|
}
|
|
26114
|
+
async function runStartupBackfill() {
|
|
26115
|
+
if (!isAvailable()) return;
|
|
26116
|
+
const knowledgeEmbedded = await backfillEmbeddings();
|
|
26117
|
+
const distillationEmbedded = await backfillDistillationEmbeddings();
|
|
26118
|
+
const kTotal = db().query("SELECT COUNT(*) as n FROM knowledge WHERE confidence > 0.2").get().n;
|
|
26119
|
+
const kWithEmb = db().query(
|
|
26120
|
+
"SELECT COUNT(*) as n FROM knowledge WHERE embedding IS NOT NULL AND confidence > 0.2"
|
|
26121
|
+
).get().n;
|
|
26122
|
+
const dTotal = db().query(
|
|
26123
|
+
"SELECT COUNT(*) as n FROM distillations WHERE archived = 0 AND observations != ''"
|
|
26124
|
+
).get().n;
|
|
26125
|
+
const dWithEmb = db().query(
|
|
26126
|
+
"SELECT COUNT(*) as n FROM distillations WHERE embedding IS NOT NULL AND archived = 0"
|
|
26127
|
+
).get().n;
|
|
26128
|
+
const parts = [];
|
|
26129
|
+
if (knowledgeEmbedded > 0 || distillationEmbedded > 0) {
|
|
26130
|
+
parts.push(`backfilled ${knowledgeEmbedded} knowledge + ${distillationEmbedded} distillations`);
|
|
26131
|
+
}
|
|
26132
|
+
parts.push(
|
|
26133
|
+
`coverage: knowledge ${kWithEmb}/${kTotal}, distillations ${dWithEmb}/${dTotal}`
|
|
26134
|
+
);
|
|
26135
|
+
info(`embedding startup: ${parts.join("; ")}`);
|
|
26136
|
+
}
|
|
26031
26137
|
async function backfillEmbeddings() {
|
|
26032
26138
|
checkConfigChange();
|
|
26033
26139
|
const provider = getProvider();
|
|
@@ -26784,6 +26890,7 @@ function check2(projectPath) {
|
|
|
26784
26890
|
// src/distillation.ts
|
|
26785
26891
|
var distillation_exports = {};
|
|
26786
26892
|
__export(distillation_exports, {
|
|
26893
|
+
backfillMetrics: () => backfillMetrics,
|
|
26787
26894
|
compressionRatio: () => compressionRatio,
|
|
26788
26895
|
detectSegments: () => detectSegments,
|
|
26789
26896
|
latestMetaObservations: () => latestMetaObservations,
|
|
@@ -26796,6 +26903,72 @@ __export(distillation_exports, {
|
|
|
26796
26903
|
workerSessionIDs: () => workerSessionIDs
|
|
26797
26904
|
});
|
|
26798
26905
|
|
|
26906
|
+
// src/pattern-extract.ts
|
|
26907
|
+
var pattern_extract_exports = {};
|
|
26908
|
+
__export(pattern_extract_exports, {
|
|
26909
|
+
extractPatterns: () => extractPatterns
|
|
26910
|
+
});
|
|
26911
|
+
var PATTERNS = [
|
|
26912
|
+
// Decision patterns
|
|
26913
|
+
{
|
|
26914
|
+
regex: /decided to (?:use |switch to |go with |adopt )(.+?)(?:\.|,|$)/gi,
|
|
26915
|
+
category: "decision",
|
|
26916
|
+
titleFn: (m) => `Decided to use ${m[1].trim()}`
|
|
26917
|
+
},
|
|
26918
|
+
{
|
|
26919
|
+
regex: /chose (.+?) over (.+?)(?:\.|,|$)/gi,
|
|
26920
|
+
category: "decision",
|
|
26921
|
+
titleFn: (m) => `Chose ${m[1].trim()} over ${m[2].trim()}`
|
|
26922
|
+
},
|
|
26923
|
+
{
|
|
26924
|
+
regex: /switched from (.+?) to (.+?)(?:\.|,|$)/gi,
|
|
26925
|
+
category: "decision",
|
|
26926
|
+
titleFn: (m) => `Switched from ${m[1].trim()} to ${m[2].trim()}`
|
|
26927
|
+
},
|
|
26928
|
+
{
|
|
26929
|
+
regex: /going with (.+?) (?:because|for|due to)(.+?)(?:\.|,|$)/gi,
|
|
26930
|
+
category: "decision",
|
|
26931
|
+
titleFn: (m) => `Going with ${m[1].trim()}`
|
|
26932
|
+
},
|
|
26933
|
+
{
|
|
26934
|
+
regex: /migrat(?:ed|ing) (?:from .+? )?to (.+?)(?:\.|,|$)/gi,
|
|
26935
|
+
category: "decision",
|
|
26936
|
+
titleFn: (m) => `Migrated to ${m[1].trim()}`
|
|
26937
|
+
},
|
|
26938
|
+
{
|
|
26939
|
+
regex: /adopted (.+?) (?:for|as|instead)(.+?)(?:\.|,|$)/gi,
|
|
26940
|
+
category: "decision",
|
|
26941
|
+
titleFn: (m) => `Adopted ${m[1].trim()}`
|
|
26942
|
+
},
|
|
26943
|
+
// Preference patterns
|
|
26944
|
+
{
|
|
26945
|
+
regex: /prefers? (.+?) (?:over|to|instead of|rather than) (.+?)(?:\.|,|$)/gi,
|
|
26946
|
+
category: "preference",
|
|
26947
|
+
titleFn: (m) => `Prefers ${m[1].trim()} over ${m[2].trim()}`
|
|
26948
|
+
},
|
|
26949
|
+
{
|
|
26950
|
+
regex: /(?:user |team |we )(?:always |usually |typically )(?:use|prefer|go with) (.+?)(?:\.|,|$)/gi,
|
|
26951
|
+
category: "preference",
|
|
26952
|
+
titleFn: (m) => `Typically uses ${m[1].trim()}`
|
|
26953
|
+
}
|
|
26954
|
+
];
|
|
26955
|
+
function extractPatterns(observations) {
|
|
26956
|
+
const results = [];
|
|
26957
|
+
const seen = /* @__PURE__ */ new Set();
|
|
26958
|
+
for (const { regex, category, titleFn } of PATTERNS) {
|
|
26959
|
+
regex.lastIndex = 0;
|
|
26960
|
+
let match;
|
|
26961
|
+
while ((match = regex.exec(observations)) !== null) {
|
|
26962
|
+
const title = titleFn(match);
|
|
26963
|
+
const key = title.toLowerCase();
|
|
26964
|
+
if (seen.has(key)) continue;
|
|
26965
|
+
seen.add(key);
|
|
26966
|
+
results.push({ category, title, content: match[0].trim() });
|
|
26967
|
+
}
|
|
26968
|
+
}
|
|
26969
|
+
return results;
|
|
26970
|
+
}
|
|
26971
|
+
|
|
26799
26972
|
// src/gradient.ts
|
|
26800
26973
|
function estimate2(text4) {
|
|
26801
26974
|
return Math.ceil(text4.length / 3);
|
|
@@ -26831,12 +27004,17 @@ function makeSessionState() {
|
|
|
26831
27004
|
lastWindowMessageIDs: /* @__PURE__ */ new Set(),
|
|
26832
27005
|
forceMinLayer: 0,
|
|
26833
27006
|
lastTransformEstimate: 0,
|
|
27007
|
+
ltmTokens: 0,
|
|
26834
27008
|
prefixCache: null,
|
|
26835
27009
|
rawWindowCache: null,
|
|
26836
27010
|
lastTurnAt: 0,
|
|
26837
27011
|
cameOutOfIdle: false,
|
|
27012
|
+
postIdleCompact: false,
|
|
26838
27013
|
consecutiveHighLayer: 0,
|
|
26839
|
-
lastPrefixHash: ""
|
|
27014
|
+
lastPrefixHash: "",
|
|
27015
|
+
bustCount: 0,
|
|
27016
|
+
transformCount: 0,
|
|
27017
|
+
distillationSnapshot: null
|
|
26840
27018
|
};
|
|
26841
27019
|
}
|
|
26842
27020
|
var sessionStates = /* @__PURE__ */ new Map();
|
|
@@ -26857,16 +27035,21 @@ function onIdleResume(sessionID, thresholdMs, now = Date.now()) {
|
|
|
26857
27035
|
if (idleMs < thresholdMs) return { triggered: false };
|
|
26858
27036
|
state.prefixCache = null;
|
|
26859
27037
|
state.rawWindowCache = null;
|
|
27038
|
+
state.distillationSnapshot = null;
|
|
26860
27039
|
state.cameOutOfIdle = true;
|
|
27040
|
+
state.postIdleCompact = true;
|
|
26861
27041
|
return { triggered: true, idleMs };
|
|
26862
27042
|
}
|
|
27043
|
+
function getLastTurnAt(sessionID) {
|
|
27044
|
+
return sessionStates.get(sessionID)?.lastTurnAt ?? 0;
|
|
27045
|
+
}
|
|
26863
27046
|
function consumeCameOutOfIdle(sessionID) {
|
|
26864
27047
|
const state = sessionStates.get(sessionID);
|
|
26865
27048
|
if (!state || !state.cameOutOfIdle) return false;
|
|
26866
27049
|
state.cameOutOfIdle = false;
|
|
26867
27050
|
return true;
|
|
26868
27051
|
}
|
|
26869
|
-
var
|
|
27052
|
+
var ltmTokensFallback = 0;
|
|
26870
27053
|
function setModelLimits(limits) {
|
|
26871
27054
|
contextLimit = limits.context || 2e5;
|
|
26872
27055
|
outputReserved = Math.min(limits.output || 32e3, 32e3);
|
|
@@ -26879,11 +27062,18 @@ function computeLayer0Cap(targetCostPerTurn, cacheReadCostPerToken) {
|
|
|
26879
27062
|
const rawCap = Math.floor(targetCostPerTurn / cacheReadCostPerToken);
|
|
26880
27063
|
return Math.max(rawCap, MIN_LAYER0_FLOOR);
|
|
26881
27064
|
}
|
|
26882
|
-
function setLtmTokens(tokens) {
|
|
26883
|
-
|
|
27065
|
+
function setLtmTokens(tokens, sessionID) {
|
|
27066
|
+
if (sessionID) {
|
|
27067
|
+
getSessionState(sessionID).ltmTokens = tokens;
|
|
27068
|
+
}
|
|
27069
|
+
ltmTokensFallback = tokens;
|
|
26884
27070
|
}
|
|
26885
|
-
function getLtmTokens() {
|
|
26886
|
-
|
|
27071
|
+
function getLtmTokens(sessionID) {
|
|
27072
|
+
if (sessionID) {
|
|
27073
|
+
const state = sessionStates.get(sessionID);
|
|
27074
|
+
if (state) return state.ltmTokens;
|
|
27075
|
+
}
|
|
27076
|
+
return ltmTokensFallback;
|
|
26887
27077
|
}
|
|
26888
27078
|
function getLtmBudget(ltmFraction) {
|
|
26889
27079
|
const overhead = calibratedOverhead ?? FIRST_TURN_OVERHEAD;
|
|
@@ -26899,7 +27089,7 @@ function calibrate(actualInput, sessionID, messageCount) {
|
|
|
26899
27089
|
if (sessionID !== void 0) {
|
|
26900
27090
|
const state = getSessionState(sessionID);
|
|
26901
27091
|
state.lastKnownInput = actualInput;
|
|
26902
|
-
state.lastKnownLtm = ltmTokens;
|
|
27092
|
+
state.lastKnownLtm = state.ltmTokens;
|
|
26903
27093
|
if (messageCount !== void 0) state.lastKnownMessageCount = messageCount;
|
|
26904
27094
|
}
|
|
26905
27095
|
}
|
|
@@ -26930,7 +27120,9 @@ function inspectSessionState(sessionID) {
|
|
|
26930
27120
|
hasPrefixCache: state.prefixCache !== null,
|
|
26931
27121
|
hasRawWindowCache: state.rawWindowCache !== null,
|
|
26932
27122
|
cameOutOfIdle: state.cameOutOfIdle,
|
|
26933
|
-
|
|
27123
|
+
postIdleCompact: state.postIdleCompact,
|
|
27124
|
+
lastTurnAt: state.lastTurnAt,
|
|
27125
|
+
distillationSnapshot: state.distillationSnapshot
|
|
26934
27126
|
};
|
|
26935
27127
|
}
|
|
26936
27128
|
function setLastTurnAtForTest(sessionID, ms) {
|
|
@@ -26942,6 +27134,25 @@ function loadDistillations(projectPath, sessionID) {
|
|
|
26942
27134
|
const params = sessionID ? [pid, sessionID] : [pid];
|
|
26943
27135
|
return db().query(query).all(...params);
|
|
26944
27136
|
}
|
|
27137
|
+
function loadDistillationsCached(projectPath, sessionID, messages, sessState) {
|
|
27138
|
+
let lastUserMsgId = null;
|
|
27139
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
27140
|
+
if (messages[i].info.role === "user") {
|
|
27141
|
+
lastUserMsgId = messages[i].info.id;
|
|
27142
|
+
break;
|
|
27143
|
+
}
|
|
27144
|
+
}
|
|
27145
|
+
const snapshot = sessState.distillationSnapshot;
|
|
27146
|
+
if (snapshot && snapshot.lastUserMsgId === lastUserMsgId) {
|
|
27147
|
+
return snapshot.rows;
|
|
27148
|
+
}
|
|
27149
|
+
const rows = loadDistillations(projectPath, sessionID);
|
|
27150
|
+
sessState.distillationSnapshot = { rows, lastUserMsgId };
|
|
27151
|
+
info(
|
|
27152
|
+
`distillation refresh: ${rows.length} rows (user msg ${lastUserMsgId?.substring(0, 16) ?? "none"})`
|
|
27153
|
+
);
|
|
27154
|
+
return rows;
|
|
27155
|
+
}
|
|
26945
27156
|
function stripSystemReminders(text4) {
|
|
26946
27157
|
return text4.replace(/<system-reminder>[\s\S]*?<\/system-reminder>\n?/g, (match) => {
|
|
26947
27158
|
const inner = match.match(
|
|
@@ -26994,24 +27205,51 @@ function simpleHash(str) {
|
|
|
26994
27205
|
}
|
|
26995
27206
|
return hash2;
|
|
26996
27207
|
}
|
|
26997
|
-
function
|
|
27208
|
+
function extractReadRange(input) {
|
|
26998
27209
|
try {
|
|
26999
27210
|
const parsed = JSON.parse(input);
|
|
27000
|
-
|
|
27211
|
+
const path = parsed.path || parsed.filePath || parsed.file;
|
|
27212
|
+
if (!path) return void 0;
|
|
27213
|
+
const offset = typeof parsed.offset === "number" ? parsed.offset : void 0;
|
|
27214
|
+
const limit = typeof parsed.limit === "number" ? parsed.limit : void 0;
|
|
27215
|
+
return { path, offset, limit };
|
|
27001
27216
|
} catch {
|
|
27002
27217
|
const match = input.match(/(?:[\w.-]+\/)+[\w.-]+\.\w{1,5}/);
|
|
27003
|
-
return
|
|
27218
|
+
if (!match) return void 0;
|
|
27219
|
+
return { path: match[0], offset: void 0, limit: void 0 };
|
|
27004
27220
|
}
|
|
27005
27221
|
}
|
|
27006
|
-
function
|
|
27222
|
+
function laterReadCovers(later, earlier) {
|
|
27223
|
+
if (later.path !== earlier.path) return false;
|
|
27224
|
+
if (later.offset === void 0 && later.limit === void 0) return true;
|
|
27225
|
+
if (earlier.offset === void 0 && earlier.limit === void 0) return false;
|
|
27226
|
+
const laterStart = later.offset ?? 1;
|
|
27227
|
+
const earlierStart = earlier.offset ?? 1;
|
|
27228
|
+
if (later.limit === void 0) return laterStart <= earlierStart;
|
|
27229
|
+
if (earlier.limit === void 0) return false;
|
|
27230
|
+
const laterEnd = laterStart + later.limit;
|
|
27231
|
+
const earlierEnd = earlierStart + earlier.limit;
|
|
27232
|
+
return laterStart <= earlierStart && laterEnd >= earlierEnd;
|
|
27233
|
+
}
|
|
27234
|
+
function rangeLabel(range) {
|
|
27235
|
+
if (range.offset !== void 0 && range.limit !== void 0) {
|
|
27236
|
+
return ` lines ${range.offset}-${range.offset + range.limit - 1}`;
|
|
27237
|
+
}
|
|
27238
|
+
if (range.offset !== void 0) {
|
|
27239
|
+
return ` from line ${range.offset}`;
|
|
27240
|
+
}
|
|
27241
|
+
return "";
|
|
27242
|
+
}
|
|
27243
|
+
function dedupAnnotation(toolName, filePath, range) {
|
|
27007
27244
|
if (filePath) {
|
|
27008
|
-
|
|
27245
|
+
const rl = range ? rangeLabel(range) : "";
|
|
27246
|
+
return `[earlier read of ${filePath}${rl} \u2014 see latest read below for current content]`;
|
|
27009
27247
|
}
|
|
27010
27248
|
return `[duplicate output \u2014 same content as later ${toolName} in this session \u2014 use recall for details]`;
|
|
27011
27249
|
}
|
|
27012
27250
|
function deduplicateToolOutputs(messages, currentTurnIdx) {
|
|
27013
27251
|
const contentLatest = /* @__PURE__ */ new Map();
|
|
27014
|
-
const
|
|
27252
|
+
const fileReads = /* @__PURE__ */ new Map();
|
|
27015
27253
|
for (let i = 0; i < messages.length; i++) {
|
|
27016
27254
|
for (const part of messages[i].parts) {
|
|
27017
27255
|
if (!isToolPart(part) || part.state.status !== "completed") continue;
|
|
@@ -27021,8 +27259,15 @@ function deduplicateToolOutputs(messages, currentTurnIdx) {
|
|
|
27021
27259
|
contentLatest.set(key, i);
|
|
27022
27260
|
if (part.tool === "read_file" || part.tool === "read") {
|
|
27023
27261
|
const inputStr = typeof part.state.input === "string" ? part.state.input : JSON.stringify(part.state.input);
|
|
27024
|
-
const
|
|
27025
|
-
if (
|
|
27262
|
+
const range = extractReadRange(inputStr);
|
|
27263
|
+
if (range) {
|
|
27264
|
+
let entries = fileReads.get(range.path);
|
|
27265
|
+
if (!entries) {
|
|
27266
|
+
entries = [];
|
|
27267
|
+
fileReads.set(range.path, entries);
|
|
27268
|
+
}
|
|
27269
|
+
entries.push({ range, msgIdx: i });
|
|
27270
|
+
}
|
|
27026
27271
|
}
|
|
27027
27272
|
}
|
|
27028
27273
|
}
|
|
@@ -27036,20 +27281,30 @@ function deduplicateToolOutputs(messages, currentTurnIdx) {
|
|
|
27036
27281
|
if (!output || output.length < DEDUP_MIN_CHARS) return part;
|
|
27037
27282
|
const contentKey = `${part.tool}:${simpleHash(output)}`;
|
|
27038
27283
|
const isLatestContent = contentLatest.get(contentKey) === msgIdx;
|
|
27039
|
-
let
|
|
27040
|
-
let
|
|
27284
|
+
let readRange;
|
|
27285
|
+
let coveredByLater = false;
|
|
27041
27286
|
if (part.tool === "read_file" || part.tool === "read") {
|
|
27042
27287
|
const inputStr = typeof part.state.input === "string" ? part.state.input : JSON.stringify(part.state.input);
|
|
27043
|
-
|
|
27044
|
-
if (
|
|
27288
|
+
readRange = extractReadRange(inputStr);
|
|
27289
|
+
if (readRange) {
|
|
27290
|
+
const entries = fileReads.get(readRange.path);
|
|
27291
|
+
if (entries) {
|
|
27292
|
+
for (const entry of entries) {
|
|
27293
|
+
if (entry.msgIdx > msgIdx && laterReadCovers(entry.range, readRange)) {
|
|
27294
|
+
coveredByLater = true;
|
|
27295
|
+
break;
|
|
27296
|
+
}
|
|
27297
|
+
}
|
|
27298
|
+
}
|
|
27299
|
+
}
|
|
27045
27300
|
}
|
|
27046
|
-
if (isLatestContent &&
|
|
27301
|
+
if (isLatestContent && !coveredByLater) return part;
|
|
27047
27302
|
partsChanged = true;
|
|
27048
27303
|
return {
|
|
27049
27304
|
...part,
|
|
27050
27305
|
state: {
|
|
27051
27306
|
...part.state,
|
|
27052
|
-
output: dedupAnnotation(part.tool,
|
|
27307
|
+
output: dedupAnnotation(part.tool, readRange?.path, readRange)
|
|
27053
27308
|
}
|
|
27054
27309
|
};
|
|
27055
27310
|
});
|
|
@@ -27069,7 +27324,7 @@ function sanitizeToolParts(messages) {
|
|
|
27069
27324
|
const { status } = part.state;
|
|
27070
27325
|
if (status === "completed" || status === "error") return part;
|
|
27071
27326
|
partsChanged = true;
|
|
27072
|
-
const
|
|
27327
|
+
const existingStart = "time" in part.state ? part.state.time.start : 0;
|
|
27073
27328
|
return {
|
|
27074
27329
|
...part,
|
|
27075
27330
|
state: {
|
|
@@ -27078,8 +27333,8 @@ function sanitizeToolParts(messages) {
|
|
|
27078
27333
|
error: "[tool execution interrupted \u2014 session recovered]",
|
|
27079
27334
|
metadata: "metadata" in part.state ? part.state.metadata : void 0,
|
|
27080
27335
|
time: {
|
|
27081
|
-
start:
|
|
27082
|
-
end:
|
|
27336
|
+
start: existingStart,
|
|
27337
|
+
end: existingStart
|
|
27083
27338
|
}
|
|
27084
27339
|
}
|
|
27085
27340
|
};
|
|
@@ -27103,97 +27358,6 @@ function stripToolOutputs(parts) {
|
|
|
27103
27358
|
};
|
|
27104
27359
|
});
|
|
27105
27360
|
}
|
|
27106
|
-
function formatRelativeTime(date5, now) {
|
|
27107
|
-
const diffMs = now.getTime() - date5.getTime();
|
|
27108
|
-
const diffDays = Math.floor(diffMs / (1e3 * 60 * 60 * 24));
|
|
27109
|
-
if (diffDays === 0) return "today";
|
|
27110
|
-
if (diffDays === 1) return "yesterday";
|
|
27111
|
-
if (diffDays < 7) return `${diffDays} days ago`;
|
|
27112
|
-
if (diffDays < 14) return "1 week ago";
|
|
27113
|
-
if (diffDays < 30) return `${Math.floor(diffDays / 7)} weeks ago`;
|
|
27114
|
-
if (diffDays < 60) return "1 month ago";
|
|
27115
|
-
if (diffDays < 365) return `${Math.floor(diffDays / 30)} months ago`;
|
|
27116
|
-
return `${Math.floor(diffDays / 365)} year${Math.floor(diffDays / 365) > 1 ? "s" : ""} ago`;
|
|
27117
|
-
}
|
|
27118
|
-
function parseDateFromContent(s) {
|
|
27119
|
-
const simple = s.match(/([A-Z][a-z]+)\s+(\d{1,2}),?\s+(\d{4})/);
|
|
27120
|
-
if (simple) {
|
|
27121
|
-
const d = /* @__PURE__ */ new Date(`${simple[1]} ${simple[2]}, ${simple[3]}`);
|
|
27122
|
-
if (!isNaN(d.getTime())) return d;
|
|
27123
|
-
}
|
|
27124
|
-
const range = s.match(/([A-Z][a-z]+)\s+(\d{1,2})-\d{1,2},?\s+(\d{4})/);
|
|
27125
|
-
if (range) {
|
|
27126
|
-
const d = /* @__PURE__ */ new Date(`${range[1]} ${range[2]}, ${range[3]}`);
|
|
27127
|
-
if (!isNaN(d.getTime())) return d;
|
|
27128
|
-
}
|
|
27129
|
-
const vague = s.match(/(late|early|mid)[- ]?([A-Z][a-z]+)\s+(\d{4})/i);
|
|
27130
|
-
if (vague) {
|
|
27131
|
-
const day = vague[1].toLowerCase() === "early" ? 7 : vague[1].toLowerCase() === "late" ? 23 : 15;
|
|
27132
|
-
const d = /* @__PURE__ */ new Date(`${vague[2]} ${day}, ${vague[3]}`);
|
|
27133
|
-
if (!isNaN(d.getTime())) return d;
|
|
27134
|
-
}
|
|
27135
|
-
return null;
|
|
27136
|
-
}
|
|
27137
|
-
function expandInlineEstimatedDates(text4, now) {
|
|
27138
|
-
return text4.replace(
|
|
27139
|
-
/\(((?:meaning|estimated)\s+)([^)]+\d{4})\)/gi,
|
|
27140
|
-
(match, prefix, dateContent) => {
|
|
27141
|
-
const d = parseDateFromContent(dateContent);
|
|
27142
|
-
if (!d) return match;
|
|
27143
|
-
const rel = formatRelativeTime(d, now);
|
|
27144
|
-
const matchIdx = text4.indexOf(match);
|
|
27145
|
-
const lineStart = text4.lastIndexOf("\n", matchIdx) + 1;
|
|
27146
|
-
const linePrefix = text4.slice(lineStart, matchIdx);
|
|
27147
|
-
const isFutureIntent = /\b(?:will|plans?\s+to|planning\s+to|going\s+to|intends?\s+to)\b/i.test(
|
|
27148
|
-
linePrefix
|
|
27149
|
-
);
|
|
27150
|
-
if (d < now && isFutureIntent)
|
|
27151
|
-
return `(${prefix}${dateContent} \u2014 ${rel}, likely already happened)`;
|
|
27152
|
-
return `(${prefix}${dateContent} \u2014 ${rel})`;
|
|
27153
|
-
}
|
|
27154
|
-
);
|
|
27155
|
-
}
|
|
27156
|
-
function addRelativeTimeToObservations(text4, now) {
|
|
27157
|
-
const withInline = expandInlineEstimatedDates(text4, now);
|
|
27158
|
-
const dateHeaderRe = /^(Date:\s*)([A-Z][a-z]+ \d{1,2}, \d{4})$/gm;
|
|
27159
|
-
const found = [];
|
|
27160
|
-
let m;
|
|
27161
|
-
while ((m = dateHeaderRe.exec(withInline)) !== null) {
|
|
27162
|
-
const d = new Date(m[2]);
|
|
27163
|
-
if (!isNaN(d.getTime()))
|
|
27164
|
-
found.push({
|
|
27165
|
-
index: m.index,
|
|
27166
|
-
date: d,
|
|
27167
|
-
full: m[0],
|
|
27168
|
-
prefix: m[1],
|
|
27169
|
-
ds: m[2]
|
|
27170
|
-
});
|
|
27171
|
-
}
|
|
27172
|
-
if (!found.length) return withInline;
|
|
27173
|
-
let result = "";
|
|
27174
|
-
let last = 0;
|
|
27175
|
-
for (let i = 0; i < found.length; i++) {
|
|
27176
|
-
const curr = found[i];
|
|
27177
|
-
const prev = found[i - 1];
|
|
27178
|
-
result += withInline.slice(last, curr.index);
|
|
27179
|
-
if (prev) {
|
|
27180
|
-
const gapDays = Math.floor(
|
|
27181
|
-
(curr.date.getTime() - prev.date.getTime()) / 864e5
|
|
27182
|
-
);
|
|
27183
|
-
if (gapDays > 1) {
|
|
27184
|
-
const gap = gapDays < 7 ? `[${gapDays} days later]` : gapDays < 14 ? "[1 week later]" : gapDays < 30 ? `[${Math.floor(gapDays / 7)} weeks later]` : gapDays < 60 ? "[1 month later]" : `[${Math.floor(gapDays / 30)} months later]`;
|
|
27185
|
-
result += `
|
|
27186
|
-
${gap}
|
|
27187
|
-
|
|
27188
|
-
`;
|
|
27189
|
-
}
|
|
27190
|
-
}
|
|
27191
|
-
result += `${curr.prefix}${curr.ds} (${formatRelativeTime(curr.date, now)})`;
|
|
27192
|
-
last = curr.index + curr.full.length;
|
|
27193
|
-
}
|
|
27194
|
-
result += withInline.slice(last);
|
|
27195
|
-
return result;
|
|
27196
|
-
}
|
|
27197
27361
|
function buildPrefixMessages(formatted) {
|
|
27198
27362
|
return [
|
|
27199
27363
|
{
|
|
@@ -27250,12 +27414,7 @@ function buildPrefixMessages(formatted) {
|
|
|
27250
27414
|
}
|
|
27251
27415
|
function distilledPrefix(distillations) {
|
|
27252
27416
|
if (!distillations.length) return [];
|
|
27253
|
-
const
|
|
27254
|
-
const annotated = distillations.map((d) => ({
|
|
27255
|
-
...d,
|
|
27256
|
-
observations: addRelativeTimeToObservations(d.observations, now)
|
|
27257
|
-
}));
|
|
27258
|
-
const formatted = formatDistillations(annotated);
|
|
27417
|
+
const formatted = formatDistillations(distillations);
|
|
27259
27418
|
if (!formatted) return [];
|
|
27260
27419
|
return buildPrefixMessages(formatted);
|
|
27261
27420
|
}
|
|
@@ -27275,12 +27434,7 @@ function distilledPrefixCached(distillations, sessionID, sessState) {
|
|
|
27275
27434
|
};
|
|
27276
27435
|
}
|
|
27277
27436
|
const newRows = distillations.slice(prefixCache.rowCount);
|
|
27278
|
-
const
|
|
27279
|
-
const annotated2 = newRows.map((d) => ({
|
|
27280
|
-
...d,
|
|
27281
|
-
observations: addRelativeTimeToObservations(d.observations, now2)
|
|
27282
|
-
}));
|
|
27283
|
-
const deltaText = formatDistillations(annotated2);
|
|
27437
|
+
const deltaText = formatDistillations(newRows);
|
|
27284
27438
|
if (deltaText) {
|
|
27285
27439
|
const fullText2 = prefixCache.cachedText + "\n\n" + deltaText;
|
|
27286
27440
|
const messages2 = buildPrefixMessages(fullText2);
|
|
@@ -27296,12 +27450,7 @@ function distilledPrefixCached(distillations, sessionID, sessState) {
|
|
|
27296
27450
|
return { messages: messages2, tokens: tokens2 };
|
|
27297
27451
|
}
|
|
27298
27452
|
}
|
|
27299
|
-
const
|
|
27300
|
-
const annotated = distillations.map((d) => ({
|
|
27301
|
-
...d,
|
|
27302
|
-
observations: addRelativeTimeToObservations(d.observations, now)
|
|
27303
|
-
}));
|
|
27304
|
-
const fullText = formatDistillations(annotated);
|
|
27453
|
+
const fullText = formatDistillations(distillations);
|
|
27305
27454
|
if (!fullText) {
|
|
27306
27455
|
sessState.prefixCache = null;
|
|
27307
27456
|
return { messages: [], tokens: 0 };
|
|
@@ -27324,29 +27473,40 @@ function tryFitStable(input) {
|
|
|
27324
27473
|
const rawWindowCache = input.sessState.rawWindowCache;
|
|
27325
27474
|
const cacheValid = rawWindowCache !== null && rawWindowCache.sessionID === input.sessionID;
|
|
27326
27475
|
if (cacheValid) {
|
|
27327
|
-
const
|
|
27328
|
-
|
|
27476
|
+
const newMessages = Math.max(0, input.messages.length - rawWindowCache.pinnedTotalCount);
|
|
27477
|
+
const windowSize = rawWindowCache.pinnedRawCount + newMessages;
|
|
27478
|
+
const pinnedIdx = Math.max(0, input.messages.length - windowSize);
|
|
27479
|
+
const pinnedWindow = input.messages.slice(pinnedIdx);
|
|
27480
|
+
const pinnedTokens = pinnedWindow.reduce(
|
|
27481
|
+
(sum, m) => sum + estimateMessage(m),
|
|
27482
|
+
0
|
|
27329
27483
|
);
|
|
27330
|
-
|
|
27331
|
-
|
|
27332
|
-
|
|
27333
|
-
|
|
27334
|
-
|
|
27335
|
-
|
|
27336
|
-
|
|
27337
|
-
|
|
27338
|
-
|
|
27339
|
-
return parts !== msg.parts ? { info: msg.info, parts } : msg;
|
|
27340
|
-
});
|
|
27341
|
-
const total = input.prefixTokens + pinnedTokens;
|
|
27342
|
-
return {
|
|
27343
|
-
messages: [...input.prefix, ...processed],
|
|
27344
|
-
distilledTokens: input.prefixTokens,
|
|
27345
|
-
rawTokens: pinnedTokens,
|
|
27346
|
-
totalTokens: total
|
|
27484
|
+
const highWaterBudget = Math.max(rawWindowCache.pinnedBudget, input.rawBudget);
|
|
27485
|
+
const effectiveBudget = highWaterBudget * 1.15;
|
|
27486
|
+
if (pinnedTokens <= effectiveBudget) {
|
|
27487
|
+
if (pinnedTokens > rawWindowCache.pinnedBudget * 1.15) {
|
|
27488
|
+
input.sessState.rawWindowCache = {
|
|
27489
|
+
...rawWindowCache,
|
|
27490
|
+
pinnedRawCount: pinnedWindow.length,
|
|
27491
|
+
pinnedTotalCount: input.messages.length,
|
|
27492
|
+
pinnedBudget: input.rawBudget
|
|
27347
27493
|
};
|
|
27348
27494
|
}
|
|
27495
|
+
const processed = pinnedWindow.map((msg) => {
|
|
27496
|
+
const parts = cleanParts(msg.parts);
|
|
27497
|
+
return parts !== msg.parts ? { info: msg.info, parts } : msg;
|
|
27498
|
+
});
|
|
27499
|
+
const total = input.prefixTokens + pinnedTokens;
|
|
27500
|
+
return {
|
|
27501
|
+
messages: [...input.prefix, ...processed],
|
|
27502
|
+
distilledTokens: input.prefixTokens,
|
|
27503
|
+
rawTokens: pinnedTokens,
|
|
27504
|
+
totalTokens: total
|
|
27505
|
+
};
|
|
27349
27506
|
}
|
|
27507
|
+
info(
|
|
27508
|
+
`pin-overflow: session=${input.sessionID} pinnedTokens=${pinnedTokens} pinnedBudget=${rawWindowCache.pinnedBudget} effectiveBudget=${Math.round(effectiveBudget)} currentRawBudget=${input.rawBudget} windowSize=${pinnedWindow.length}`
|
|
27509
|
+
);
|
|
27350
27510
|
}
|
|
27351
27511
|
const result = tryFit({
|
|
27352
27512
|
messages: input.messages,
|
|
@@ -27357,11 +27517,13 @@ function tryFitStable(input) {
|
|
|
27357
27517
|
strip: "none"
|
|
27358
27518
|
});
|
|
27359
27519
|
if (result) {
|
|
27360
|
-
const
|
|
27361
|
-
if (
|
|
27520
|
+
const rawMessageCount = result.messages.length - input.prefix.length;
|
|
27521
|
+
if (rawMessageCount > 0) {
|
|
27362
27522
|
input.sessState.rawWindowCache = {
|
|
27363
27523
|
sessionID: input.sessionID,
|
|
27364
|
-
|
|
27524
|
+
pinnedRawCount: rawMessageCount,
|
|
27525
|
+
pinnedTotalCount: input.messages.length,
|
|
27526
|
+
pinnedBudget: input.rawBudget
|
|
27365
27527
|
};
|
|
27366
27528
|
}
|
|
27367
27529
|
}
|
|
@@ -27376,14 +27538,15 @@ function needsUrgentDistillation() {
|
|
|
27376
27538
|
function transformInner(input) {
|
|
27377
27539
|
const cfg = config2();
|
|
27378
27540
|
const overhead = getOverhead();
|
|
27541
|
+
const sid = input.sessionID ?? input.messages[0]?.info.sessionID;
|
|
27542
|
+
const sessState = sid ? getSessionState(sid) : makeSessionState();
|
|
27543
|
+
const sessLtmTokens = sid ? sessState.ltmTokens : ltmTokensFallback;
|
|
27379
27544
|
const usable = Math.max(
|
|
27380
27545
|
0,
|
|
27381
|
-
contextLimit - outputReserved - overhead -
|
|
27546
|
+
contextLimit - outputReserved - overhead - sessLtmTokens
|
|
27382
27547
|
);
|
|
27383
27548
|
const distilledBudget = Math.floor(usable * cfg.budget.distilled);
|
|
27384
|
-
|
|
27385
|
-
const sid = input.sessionID ?? input.messages[0]?.info.sessionID;
|
|
27386
|
-
const sessState = sid ? getSessionState(sid) : makeSessionState();
|
|
27549
|
+
let rawBudget = Math.floor(usable * cfg.budget.raw);
|
|
27387
27550
|
let effectiveMinLayer = sessState.forceMinLayer;
|
|
27388
27551
|
sessState.forceMinLayer = 0;
|
|
27389
27552
|
if (sid && effectiveMinLayer > 0) saveForceMinLayer(sid, 0);
|
|
@@ -27396,17 +27559,26 @@ function transformInner(input) {
|
|
|
27396
27559
|
return result.totalTokens * UNCALIBRATED_SAFETY <= maxInput;
|
|
27397
27560
|
}
|
|
27398
27561
|
if (calibrated && sessState.lastLayer >= 1 && input.messages.length >= sessState.lastKnownMessageCount) {
|
|
27562
|
+
effectiveMinLayer = Math.max(effectiveMinLayer, sessState.lastLayer);
|
|
27563
|
+
}
|
|
27564
|
+
const postIdleCompact = sessState.postIdleCompact;
|
|
27565
|
+
if (postIdleCompact) {
|
|
27566
|
+
sessState.postIdleCompact = false;
|
|
27399
27567
|
effectiveMinLayer = Math.max(effectiveMinLayer, 1);
|
|
27568
|
+
rawBudget = Math.floor(usable * 0.2);
|
|
27569
|
+
info(
|
|
27570
|
+
`post-idle compact: session=${sid} rawBudget=${rawBudget} (${Math.floor(usable * cfg.budget.raw)}\u2192${rawBudget})`
|
|
27571
|
+
);
|
|
27400
27572
|
}
|
|
27401
27573
|
let expectedInput;
|
|
27402
27574
|
if (calibrated) {
|
|
27403
27575
|
const newMessages = sessState.lastWindowMessageIDs.size > 0 ? input.messages.filter((m) => !sessState.lastWindowMessageIDs.has(m.info.id)) : input.messages.slice(-Math.max(0, input.messages.length - sessState.lastKnownMessageCount));
|
|
27404
27576
|
const newMsgTokens = newMessages.reduce((s, m) => s + estimateMessage(m), 0);
|
|
27405
|
-
const ltmDelta =
|
|
27577
|
+
const ltmDelta = sessLtmTokens - sessState.lastKnownLtm;
|
|
27406
27578
|
expectedInput = sessState.lastKnownInput + newMsgTokens + ltmDelta;
|
|
27407
27579
|
} else {
|
|
27408
27580
|
const messageTokens = input.messages.reduce((s, m) => s + estimateMessage(m), 0);
|
|
27409
|
-
expectedInput = messageTokens + overhead +
|
|
27581
|
+
expectedInput = messageTokens + overhead + sessLtmTokens;
|
|
27410
27582
|
}
|
|
27411
27583
|
const layer0Input = calibrated ? expectedInput : expectedInput * UNCALIBRATED_SAFETY;
|
|
27412
27584
|
let layer0Ceiling = maxLayer0Tokens > 0 ? Math.min(maxInput, maxLayer0Tokens) : maxInput;
|
|
@@ -27414,7 +27586,7 @@ function transformInner(input) {
|
|
|
27414
27586
|
layer0Ceiling = Math.floor(layer0Ceiling * 0.7);
|
|
27415
27587
|
}
|
|
27416
27588
|
if (effectiveMinLayer === 0 && layer0Input <= layer0Ceiling) {
|
|
27417
|
-
const messageTokens = calibrated ? expectedInput - (
|
|
27589
|
+
const messageTokens = calibrated ? expectedInput - (sessLtmTokens - sessState.lastKnownLtm) : expectedInput - overhead - sessLtmTokens;
|
|
27418
27590
|
return {
|
|
27419
27591
|
messages: input.messages,
|
|
27420
27592
|
layer: 0,
|
|
@@ -27428,7 +27600,7 @@ function transformInner(input) {
|
|
|
27428
27600
|
}
|
|
27429
27601
|
const turnStart = currentTurnStart(input.messages);
|
|
27430
27602
|
const dedupMessages = deduplicateToolOutputs(input.messages, turnStart);
|
|
27431
|
-
const distillations = sid ?
|
|
27603
|
+
const distillations = sid ? loadDistillationsCached(input.projectPath, sid, input.messages, sessState) : [];
|
|
27432
27604
|
const cached2 = sid ? distilledPrefixCached(distillations, sid, sessState) : (() => {
|
|
27433
27605
|
const msgs = distilledPrefix(distillations);
|
|
27434
27606
|
return { messages: msgs, tokens: msgs.reduce((sum, m) => sum + estimateMessage(m), 0) };
|
|
@@ -27541,12 +27713,27 @@ function transform2(input) {
|
|
|
27541
27713
|
state.lastLayer = result.layer;
|
|
27542
27714
|
state.lastWindowMessageIDs = new Set(result.messages.map((m) => m.info.id));
|
|
27543
27715
|
state.lastTurnAt = Date.now();
|
|
27544
|
-
const
|
|
27545
|
-
|
|
27716
|
+
const prefixFingerprint = result.messages.slice(0, 5).map((m) => {
|
|
27717
|
+
const text4 = m.parts.map((p3) => {
|
|
27718
|
+
if (isTextPart(p3)) return p3.text?.slice(0, 40) ?? "";
|
|
27719
|
+
if (isReasoningPart(p3)) return p3.text?.slice(0, 40) ?? "";
|
|
27720
|
+
return p3.type;
|
|
27721
|
+
}).join("|");
|
|
27722
|
+
return `${m.info.role}:${text4.slice(0, 60)}`;
|
|
27723
|
+
}).join(",");
|
|
27724
|
+
const prefixHash = `${result.layer}:${prefixFingerprint}`;
|
|
27725
|
+
state.transformCount++;
|
|
27546
27726
|
if (state.lastPrefixHash && state.lastPrefixHash !== prefixHash) {
|
|
27727
|
+
state.bustCount++;
|
|
27728
|
+
const rate = state.bustCount / state.transformCount;
|
|
27547
27729
|
info(
|
|
27548
|
-
`cache-bust
|
|
27730
|
+
`cache-bust #${state.bustCount} (${(rate * 100).toFixed(0)}%): session=${sid} layer=${state.lastLayer}\u2192${result.layer} msgs=${state.lastTransformedCount}\u2192${result.messages.length} prefix=${state.lastPrefixHash.slice(0, 30)}\u2192${prefixHash.slice(0, 30)}`
|
|
27549
27731
|
);
|
|
27732
|
+
if (state.transformCount >= 20 && rate > 0.5) {
|
|
27733
|
+
warn(
|
|
27734
|
+
`HIGH BUST RATE: session ${sid} has ${(rate * 100).toFixed(0)}% bust rate (${state.bustCount}/${state.transformCount} transforms)`
|
|
27735
|
+
);
|
|
27736
|
+
}
|
|
27550
27737
|
}
|
|
27551
27738
|
state.lastPrefixHash = prefixHash;
|
|
27552
27739
|
if (result.layer >= 2) {
|
|
@@ -27765,7 +27952,7 @@ function parseSourceIds(raw) {
|
|
|
27765
27952
|
}
|
|
27766
27953
|
function loadForSession(projectPath, sessionID, includeArchived = false) {
|
|
27767
27954
|
const pid = ensureProject(projectPath);
|
|
27768
|
-
const sql = includeArchived ? "SELECT id, project_id, session_id, observations, source_ids, generation, token_count, created_at FROM distillations WHERE project_id = ? AND session_id = ? ORDER BY created_at ASC" : "SELECT id, project_id, session_id, observations, source_ids, generation, token_count, created_at FROM distillations WHERE project_id = ? AND session_id = ? AND archived = 0 ORDER BY created_at ASC";
|
|
27955
|
+
const sql = includeArchived ? "SELECT id, project_id, session_id, observations, source_ids, generation, token_count, created_at, r_compression, c_norm FROM distillations WHERE project_id = ? AND session_id = ? ORDER BY created_at ASC" : "SELECT id, project_id, session_id, observations, source_ids, generation, token_count, created_at, r_compression, c_norm FROM distillations WHERE project_id = ? AND session_id = ? AND archived = 0 ORDER BY created_at ASC";
|
|
27769
27956
|
const rows = db().query(sql).all(pid, sessionID);
|
|
27770
27957
|
return rows.map((r) => ({
|
|
27771
27958
|
...r,
|
|
@@ -27778,8 +27965,8 @@ function storeDistillation(input) {
|
|
|
27778
27965
|
const sourceJson = JSON.stringify(input.sourceIDs);
|
|
27779
27966
|
const tokens = Math.ceil(input.observations.length / 3);
|
|
27780
27967
|
db().query(
|
|
27781
|
-
`INSERT INTO distillations (id, project_id, session_id, narrative, facts, observations, source_ids, generation, token_count, created_at)
|
|
27782
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
27968
|
+
`INSERT INTO distillations (id, project_id, session_id, narrative, facts, observations, source_ids, generation, token_count, created_at, r_compression, c_norm)
|
|
27969
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
27783
27970
|
).run(
|
|
27784
27971
|
id,
|
|
27785
27972
|
pid,
|
|
@@ -27792,7 +27979,9 @@ function storeDistillation(input) {
|
|
|
27792
27979
|
sourceJson,
|
|
27793
27980
|
input.generation,
|
|
27794
27981
|
tokens,
|
|
27795
|
-
Date.now()
|
|
27982
|
+
Date.now(),
|
|
27983
|
+
input.rCompression ?? null,
|
|
27984
|
+
input.cNorm ?? null
|
|
27796
27985
|
);
|
|
27797
27986
|
return id;
|
|
27798
27987
|
}
|
|
@@ -27805,7 +27994,7 @@ function gen0Count(projectPath, sessionID) {
|
|
|
27805
27994
|
function loadGen0(projectPath, sessionID) {
|
|
27806
27995
|
const pid = ensureProject(projectPath);
|
|
27807
27996
|
const rows = db().query(
|
|
27808
|
-
"SELECT id, project_id, session_id, observations, source_ids, generation, token_count, created_at FROM distillations WHERE project_id = ? AND session_id = ? AND generation = 0 AND archived = 0 ORDER BY created_at ASC"
|
|
27997
|
+
"SELECT id, project_id, session_id, observations, source_ids, generation, token_count, created_at, r_compression, c_norm FROM distillations WHERE project_id = ? AND session_id = ? AND generation = 0 AND archived = 0 ORDER BY created_at ASC"
|
|
27809
27998
|
).all(pid, sessionID);
|
|
27810
27999
|
return rows.map((r) => ({
|
|
27811
28000
|
...r,
|
|
@@ -27872,7 +28061,8 @@ async function run(input) {
|
|
|
27872
28061
|
projectPath: input.projectPath,
|
|
27873
28062
|
sessionID: input.sessionID,
|
|
27874
28063
|
messages: segment,
|
|
27875
|
-
model: input.model
|
|
28064
|
+
model: input.model,
|
|
28065
|
+
urgent: input.urgent
|
|
27876
28066
|
});
|
|
27877
28067
|
if (result) {
|
|
27878
28068
|
distilled += segment.length;
|
|
@@ -27880,12 +28070,13 @@ async function run(input) {
|
|
|
27880
28070
|
}
|
|
27881
28071
|
}
|
|
27882
28072
|
}
|
|
27883
|
-
if (gen0Count(input.projectPath, input.sessionID) >= cfg.distillation.metaThreshold) {
|
|
28073
|
+
if (!input.skipMeta && gen0Count(input.projectPath, input.sessionID) >= cfg.distillation.metaThreshold) {
|
|
27884
28074
|
await metaDistill({
|
|
27885
28075
|
llm: input.llm,
|
|
27886
28076
|
projectPath: input.projectPath,
|
|
27887
28077
|
sessionID: input.sessionID,
|
|
27888
|
-
model: input.model
|
|
28078
|
+
model: input.model,
|
|
28079
|
+
urgent: input.urgent
|
|
27889
28080
|
});
|
|
27890
28081
|
rounds++;
|
|
27891
28082
|
}
|
|
@@ -27911,29 +28102,46 @@ async function distillSegment(input) {
|
|
|
27911
28102
|
const responseText = await input.llm.prompt(
|
|
27912
28103
|
DISTILLATION_SYSTEM,
|
|
27913
28104
|
userContent,
|
|
27914
|
-
{ model, workerID: "lore-distill" }
|
|
28105
|
+
{ model, workerID: "lore-distill", thinking: false, urgent: input.urgent, sessionID: input.sessionID }
|
|
27915
28106
|
);
|
|
27916
28107
|
if (!responseText) return null;
|
|
27917
28108
|
const result = parseDistillationResult(responseText);
|
|
27918
28109
|
if (!result) return null;
|
|
28110
|
+
const distilledTokens = Math.ceil(result.observations.length / 3);
|
|
28111
|
+
const sourceTokens = input.messages.reduce((sum, m) => sum + m.tokens, 0);
|
|
28112
|
+
const rComp = compressionRatio(distilledTokens, sourceTokens);
|
|
28113
|
+
const cNorm = temporalCnorm(input.messages.map((m) => m.created_at));
|
|
27919
28114
|
const distillId = storeDistillation({
|
|
27920
28115
|
projectPath: input.projectPath,
|
|
27921
28116
|
sessionID: input.sessionID,
|
|
27922
28117
|
observations: result.observations,
|
|
27923
28118
|
sourceIDs: input.messages.map((m) => m.id),
|
|
27924
|
-
generation: 0
|
|
28119
|
+
generation: 0,
|
|
28120
|
+
rCompression: rComp,
|
|
28121
|
+
cNorm
|
|
27925
28122
|
});
|
|
27926
28123
|
markDistilled(input.messages.map((m) => m.id));
|
|
27927
|
-
const distilledTokens = Math.ceil(result.observations.length / 3);
|
|
27928
|
-
const sourceTokens = input.messages.reduce((sum, m) => sum + m.tokens, 0);
|
|
27929
|
-
const rComp = compressionRatio(distilledTokens, sourceTokens);
|
|
27930
|
-
const cNorm = temporalCnorm(input.messages.map((m) => m.created_at));
|
|
27931
28124
|
info(
|
|
27932
28125
|
`distill segment: ${input.messages.length} msgs, ${sourceTokens}\u2192${distilledTokens} tokens, R=${rComp.toFixed(2)}, C_norm=${cNorm.toFixed(3)}`
|
|
27933
28126
|
);
|
|
27934
28127
|
if (isAvailable()) {
|
|
27935
28128
|
embedDistillation(distillId, result.observations);
|
|
27936
28129
|
}
|
|
28130
|
+
if (config2().knowledge.enabled) {
|
|
28131
|
+
for (const pat of extractPatterns(result.observations)) {
|
|
28132
|
+
try {
|
|
28133
|
+
create({
|
|
28134
|
+
projectPath: input.projectPath,
|
|
28135
|
+
category: pat.category,
|
|
28136
|
+
title: pat.title,
|
|
28137
|
+
content: pat.content,
|
|
28138
|
+
session: input.sessionID,
|
|
28139
|
+
scope: "project"
|
|
28140
|
+
});
|
|
28141
|
+
} catch {
|
|
28142
|
+
}
|
|
28143
|
+
}
|
|
28144
|
+
}
|
|
27937
28145
|
return result;
|
|
27938
28146
|
}
|
|
27939
28147
|
async function metaDistill(input) {
|
|
@@ -27949,7 +28157,7 @@ async function metaDistill(input) {
|
|
|
27949
28157
|
const responseText = await input.llm.prompt(
|
|
27950
28158
|
RECURSIVE_SYSTEM,
|
|
27951
28159
|
userContent,
|
|
27952
|
-
{ model, workerID: "lore-distill" }
|
|
28160
|
+
{ model, workerID: "lore-distill", thinking: false, urgent: input.urgent, sessionID: input.sessionID }
|
|
27953
28161
|
);
|
|
27954
28162
|
if (!responseText) return null;
|
|
27955
28163
|
const result = parseDistillationResult(responseText);
|
|
@@ -27978,8 +28186,54 @@ async function metaDistill(input) {
|
|
|
27978
28186
|
if (isAvailable()) {
|
|
27979
28187
|
embedDistillation(metaId, result.observations);
|
|
27980
28188
|
}
|
|
28189
|
+
if (config2().knowledge.enabled) {
|
|
28190
|
+
for (const pat of extractPatterns(result.observations)) {
|
|
28191
|
+
try {
|
|
28192
|
+
create({
|
|
28193
|
+
projectPath: input.projectPath,
|
|
28194
|
+
category: pat.category,
|
|
28195
|
+
title: pat.title,
|
|
28196
|
+
content: pat.content,
|
|
28197
|
+
session: input.sessionID,
|
|
28198
|
+
scope: "project"
|
|
28199
|
+
});
|
|
28200
|
+
} catch {
|
|
28201
|
+
}
|
|
28202
|
+
}
|
|
28203
|
+
}
|
|
27981
28204
|
return result;
|
|
27982
28205
|
}
|
|
28206
|
+
function backfillMetrics() {
|
|
28207
|
+
const rows = db().query(
|
|
28208
|
+
"SELECT id, source_ids, token_count FROM distillations WHERE r_compression IS NULL"
|
|
28209
|
+
).all();
|
|
28210
|
+
if (!rows.length) return 0;
|
|
28211
|
+
const update2 = db().prepare(
|
|
28212
|
+
"UPDATE distillations SET r_compression = ?, c_norm = ? WHERE id = ?"
|
|
28213
|
+
);
|
|
28214
|
+
let updated = 0;
|
|
28215
|
+
for (const row of rows) {
|
|
28216
|
+
const sourceIds = parseSourceIds(row.source_ids);
|
|
28217
|
+
if (!sourceIds.length) continue;
|
|
28218
|
+
const placeholders = sourceIds.map(() => "?").join(",");
|
|
28219
|
+
const sources = db().query(
|
|
28220
|
+
`SELECT tokens, created_at FROM temporal_messages WHERE id IN (${placeholders})`
|
|
28221
|
+
).all(...sourceIds);
|
|
28222
|
+
if (!sources.length) continue;
|
|
28223
|
+
const sourceTokens = sources.reduce((sum, s) => sum + s.tokens, 0);
|
|
28224
|
+
const timestamps = sources.map((s) => s.created_at);
|
|
28225
|
+
const rComp = compressionRatio(row.token_count, sourceTokens);
|
|
28226
|
+
const cNorm = temporalCnorm(timestamps);
|
|
28227
|
+
update2.run(rComp, cNorm, row.id);
|
|
28228
|
+
updated++;
|
|
28229
|
+
}
|
|
28230
|
+
if (updated > 0) {
|
|
28231
|
+
info(
|
|
28232
|
+
`backfilled metrics for ${updated} distillations (${rows.length - updated} skipped \u2014 missing sources)`
|
|
28233
|
+
);
|
|
28234
|
+
}
|
|
28235
|
+
return updated;
|
|
28236
|
+
}
|
|
27983
28237
|
|
|
27984
28238
|
// src/curator.ts
|
|
27985
28239
|
var curator_exports = {};
|
|
@@ -28025,7 +28279,7 @@ async function run2(input) {
|
|
|
28025
28279
|
const responseText = await input.llm.prompt(
|
|
28026
28280
|
CURATOR_SYSTEM,
|
|
28027
28281
|
userContent,
|
|
28028
|
-
{ model, workerID: "lore-curator" }
|
|
28282
|
+
{ model, workerID: "lore-curator", thinking: false, sessionID: input.sessionID }
|
|
28029
28283
|
);
|
|
28030
28284
|
if (!responseText) return { created: 0, updated: 0, deleted: 0 };
|
|
28031
28285
|
const ops = parseOps(responseText);
|
|
@@ -28095,7 +28349,7 @@ async function consolidate(input) {
|
|
|
28095
28349
|
const responseText = await input.llm.prompt(
|
|
28096
28350
|
CONSOLIDATION_SYSTEM,
|
|
28097
28351
|
userContent,
|
|
28098
|
-
{ model, workerID: "lore-curator" }
|
|
28352
|
+
{ model, workerID: "lore-curator", thinking: false, sessionID: input.sessionID }
|
|
28099
28353
|
);
|
|
28100
28354
|
if (!responseText) return { updated: 0, deleted: 0 };
|
|
28101
28355
|
const ops = parseOps(responseText);
|
|
@@ -28121,12 +28375,39 @@ async function consolidate(input) {
|
|
|
28121
28375
|
}
|
|
28122
28376
|
|
|
28123
28377
|
// src/recall.ts
|
|
28378
|
+
function getTaggedText(tagged) {
|
|
28379
|
+
switch (tagged.source) {
|
|
28380
|
+
case "knowledge":
|
|
28381
|
+
case "cross-knowledge":
|
|
28382
|
+
return `${tagged.item.title} ${tagged.item.content}`;
|
|
28383
|
+
case "distillation":
|
|
28384
|
+
return tagged.item.observations;
|
|
28385
|
+
case "temporal":
|
|
28386
|
+
return tagged.item.content;
|
|
28387
|
+
case "lat-section":
|
|
28388
|
+
return `${tagged.item.heading} ${tagged.item.content}`;
|
|
28389
|
+
}
|
|
28390
|
+
}
|
|
28391
|
+
function taggedResultKey(r) {
|
|
28392
|
+
switch (r.source) {
|
|
28393
|
+
case "knowledge":
|
|
28394
|
+
return `k:${r.item.id}`;
|
|
28395
|
+
case "cross-knowledge":
|
|
28396
|
+
return `xk:${r.item.id}`;
|
|
28397
|
+
case "distillation":
|
|
28398
|
+
return `d:${r.item.id}`;
|
|
28399
|
+
case "temporal":
|
|
28400
|
+
return `t:${r.item.id}`;
|
|
28401
|
+
case "lat-section":
|
|
28402
|
+
return `lat:${r.item.id}`;
|
|
28403
|
+
}
|
|
28404
|
+
}
|
|
28124
28405
|
function searchDistillationsLike(input) {
|
|
28125
28406
|
const terms = input.query.toLowerCase().split(/\s+/).filter((term) => term.length > 1);
|
|
28126
28407
|
if (!terms.length) return [];
|
|
28127
28408
|
const conditions = terms.map(() => "LOWER(observations) LIKE ?").join(" AND ");
|
|
28128
28409
|
const likeParams = terms.map((term) => `%${term}%`);
|
|
28129
|
-
const sql = input.sessionID ? `SELECT id, observations, generation, created_at, session_id FROM distillations WHERE project_id = ? AND session_id = ? AND ${conditions} ORDER BY created_at DESC LIMIT ?` : `SELECT id, observations, generation, created_at, session_id FROM distillations WHERE project_id = ? AND ${conditions} ORDER BY created_at DESC LIMIT ?`;
|
|
28410
|
+
const sql = input.sessionID ? `SELECT id, observations, generation, created_at, session_id, c_norm FROM distillations WHERE project_id = ? AND session_id = ? AND ${conditions} ORDER BY created_at DESC LIMIT ?` : `SELECT id, observations, generation, created_at, session_id, c_norm FROM distillations WHERE project_id = ? AND ${conditions} ORDER BY created_at DESC LIMIT ?`;
|
|
28130
28411
|
const allParams = input.sessionID ? [input.pid, input.sessionID, ...likeParams, input.limit] : [input.pid, ...likeParams, input.limit];
|
|
28131
28412
|
return db().query(sql).all(...allParams);
|
|
28132
28413
|
}
|
|
@@ -28135,12 +28416,12 @@ function searchDistillationsScored(input) {
|
|
|
28135
28416
|
const limit = input.limit ?? 10;
|
|
28136
28417
|
const q = ftsQuery(input.query);
|
|
28137
28418
|
if (q === EMPTY_QUERY) return [];
|
|
28138
|
-
const ftsSQL = input.sessionID ? `SELECT d.id, d.observations, d.generation, d.created_at, d.session_id, rank
|
|
28419
|
+
const ftsSQL = input.sessionID ? `SELECT d.id, d.observations, d.generation, d.created_at, d.session_id, d.c_norm, rank
|
|
28139
28420
|
FROM distillation_fts f
|
|
28140
28421
|
CROSS JOIN distillations d ON d.rowid = f.rowid
|
|
28141
28422
|
WHERE distillation_fts MATCH ?
|
|
28142
28423
|
AND d.project_id = ? AND d.session_id = ?
|
|
28143
|
-
ORDER BY rank LIMIT ?` : `SELECT d.id, d.observations, d.generation, d.created_at, d.session_id, rank
|
|
28424
|
+
ORDER BY rank LIMIT ?` : `SELECT d.id, d.observations, d.generation, d.created_at, d.session_id, d.c_norm, rank
|
|
28144
28425
|
FROM distillation_fts f
|
|
28145
28426
|
CROSS JOIN distillations d ON d.rowid = f.rowid
|
|
28146
28427
|
WHERE distillation_fts MATCH ?
|
|
@@ -28225,7 +28506,7 @@ async function runRecall(input) {
|
|
|
28225
28506
|
let queries = [query];
|
|
28226
28507
|
if (searchConfig?.queryExpansion && llm) {
|
|
28227
28508
|
try {
|
|
28228
|
-
queries = await expandQuery(llm, query);
|
|
28509
|
+
queries = await expandQuery(llm, query, void 0, sessionID);
|
|
28229
28510
|
} catch (err) {
|
|
28230
28511
|
info("recall: query expansion failed, using original:", err);
|
|
28231
28512
|
}
|
|
@@ -28334,7 +28615,7 @@ async function runRecall(input) {
|
|
|
28334
28615
|
const distVectorHits = vectorSearchDistillations(queryVec, limit);
|
|
28335
28616
|
const distVectorTagged = distVectorHits.map((hit) => {
|
|
28336
28617
|
const row = db().query(
|
|
28337
|
-
"SELECT id, observations, generation, created_at, session_id FROM distillations WHERE id = ?"
|
|
28618
|
+
"SELECT id, observations, generation, created_at, session_id, c_norm FROM distillations WHERE id = ?"
|
|
28338
28619
|
).get(hit.id);
|
|
28339
28620
|
if (!row) return null;
|
|
28340
28621
|
return {
|
|
@@ -28397,6 +28678,57 @@ async function runRecall(input) {
|
|
|
28397
28678
|
info("recall: cross-project knowledge search failed:", err);
|
|
28398
28679
|
}
|
|
28399
28680
|
}
|
|
28681
|
+
{
|
|
28682
|
+
const distillationCandidates = [];
|
|
28683
|
+
for (const list4 of allRrfLists) {
|
|
28684
|
+
for (const item of list4.items) {
|
|
28685
|
+
if (item.source !== "distillation") continue;
|
|
28686
|
+
const key = `d:${item.item.id}`;
|
|
28687
|
+
const d = item.item;
|
|
28688
|
+
const cNorm = d.c_norm ?? 0;
|
|
28689
|
+
const ageDays = Math.min(
|
|
28690
|
+
(Date.now() - d.created_at) / 864e5,
|
|
28691
|
+
90
|
|
28692
|
+
);
|
|
28693
|
+
const score = cNorm + ageDays / 90 * 0.1;
|
|
28694
|
+
distillationCandidates.push({ tagged: item, key, qualityScore: score });
|
|
28695
|
+
}
|
|
28696
|
+
}
|
|
28697
|
+
if (distillationCandidates.length > 1) {
|
|
28698
|
+
const seen = /* @__PURE__ */ new Set();
|
|
28699
|
+
const unique = distillationCandidates.filter((c) => {
|
|
28700
|
+
if (seen.has(c.key)) return false;
|
|
28701
|
+
seen.add(c.key);
|
|
28702
|
+
return true;
|
|
28703
|
+
});
|
|
28704
|
+
unique.sort((a, b) => a.qualityScore - b.qualityScore);
|
|
28705
|
+
allRrfLists.push({
|
|
28706
|
+
items: unique.map((c) => c.tagged),
|
|
28707
|
+
key: (r) => `d:${r.item.id}`
|
|
28708
|
+
});
|
|
28709
|
+
}
|
|
28710
|
+
}
|
|
28711
|
+
if (filterTerms(query).length > 0 && allRrfLists.length > 0) {
|
|
28712
|
+
const allCandidates = /* @__PURE__ */ new Map();
|
|
28713
|
+
for (const list4 of allRrfLists) {
|
|
28714
|
+
for (const item of list4.items) {
|
|
28715
|
+
const key = list4.key(item);
|
|
28716
|
+
if (!allCandidates.has(key)) allCandidates.set(key, item);
|
|
28717
|
+
}
|
|
28718
|
+
}
|
|
28719
|
+
const candidateEntries = [...allCandidates.entries()];
|
|
28720
|
+
const exactRanked = exactTermMatchRank(
|
|
28721
|
+
candidateEntries,
|
|
28722
|
+
([, tagged]) => getTaggedText(tagged),
|
|
28723
|
+
query
|
|
28724
|
+
);
|
|
28725
|
+
if (exactRanked.length) {
|
|
28726
|
+
allRrfLists.push({
|
|
28727
|
+
items: exactRanked.map(([, item]) => item),
|
|
28728
|
+
key: taggedResultKey
|
|
28729
|
+
});
|
|
28730
|
+
}
|
|
28731
|
+
}
|
|
28400
28732
|
const fused = reciprocalRankFusion(allRrfLists);
|
|
28401
28733
|
return formatFusedResults(fused, 20);
|
|
28402
28734
|
}
|
|
@@ -28408,7 +28740,7 @@ var RECALL_PARAM_DESCRIPTIONS = {
|
|
|
28408
28740
|
|
|
28409
28741
|
// src/agents-file.ts
|
|
28410
28742
|
import { existsSync as existsSync3, readFileSync as readFileSync3, writeFileSync, mkdirSync as mkdirSync2 } from "fs";
|
|
28411
|
-
import { dirname as dirname2 } from "path";
|
|
28743
|
+
import { dirname as dirname2, join as join5 } from "path";
|
|
28412
28744
|
var LORE_SECTION_START = "<!-- This section is maintained by the coding agent via lore (https://github.com/BYK/loreai) -->";
|
|
28413
28745
|
var LORE_SECTION_END = "<!-- End lore-managed section -->";
|
|
28414
28746
|
var ALL_START_MARKERS = [
|
|
@@ -28417,6 +28749,8 @@ var ALL_START_MARKERS = [
|
|
|
28417
28749
|
"<!-- This section is maintained by the coding agent via lore (https://github.com/BYK/opencode-lore) -->",
|
|
28418
28750
|
"<!-- This section is auto-maintained by lore (https://github.com/BYK/opencode-lore) -->"
|
|
28419
28751
|
];
|
|
28752
|
+
var LORE_FILE = ".lore.md";
|
|
28753
|
+
var LORE_FILE_HEADER = "<!-- Managed by lore (https://github.com/BYK/loreai) \u2014 manual edits are imported on next session. -->";
|
|
28420
28754
|
var UUID_RE2 = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
|
28421
28755
|
var MARKER_RE = /^<!--\s*lore:([0-9a-f-]+)\s*-->$/;
|
|
28422
28756
|
function splitFile(fileContent) {
|
|
@@ -28526,8 +28860,9 @@ function buildSection(projectPath) {
|
|
|
28526
28860
|
return out.join("\n");
|
|
28527
28861
|
}
|
|
28528
28862
|
function exportToFile(input) {
|
|
28529
|
-
|
|
28530
|
-
const
|
|
28863
|
+
exportLoreFile(input.projectPath);
|
|
28864
|
+
const pointerBody = "\n## Long-term Knowledge\n\nFor long-term knowledge entries managed by [lore](https://github.com/BYK/loreai) (gotchas, patterns, decisions, architecture), see [`.lore.md`](.lore.md) in the project root.\n";
|
|
28865
|
+
const newSection = LORE_SECTION_START + pointerBody + LORE_SECTION_END + "\n";
|
|
28531
28866
|
let fileContent = "";
|
|
28532
28867
|
if (existsSync3(input.filePath)) {
|
|
28533
28868
|
fileContent = readFileSync3(input.filePath, "utf8");
|
|
@@ -28551,15 +28886,9 @@ function shouldImport(input) {
|
|
|
28551
28886
|
const expected = buildSection(input.projectPath);
|
|
28552
28887
|
return hashSection(section) !== hashSection(expected);
|
|
28553
28888
|
}
|
|
28554
|
-
function
|
|
28555
|
-
if (!existsSync3(input.filePath)) return;
|
|
28556
|
-
const fileContent = readFileSync3(input.filePath, "utf8");
|
|
28557
|
-
const { section, before } = splitFile(fileContent);
|
|
28558
|
-
const textToParse = section ?? fileContent;
|
|
28559
|
-
const fileEntries = parseEntriesFromSection(textToParse);
|
|
28560
|
-
if (!fileEntries.length) return;
|
|
28889
|
+
function _importEntries(entries, projectPath) {
|
|
28561
28890
|
const seenIds = /* @__PURE__ */ new Set();
|
|
28562
|
-
for (const entry of
|
|
28891
|
+
for (const entry of entries) {
|
|
28563
28892
|
if (entry.id !== null) {
|
|
28564
28893
|
if (seenIds.has(entry.id)) continue;
|
|
28565
28894
|
seenIds.add(entry.id);
|
|
@@ -28570,7 +28899,7 @@ function importFromFile(input) {
|
|
|
28570
28899
|
}
|
|
28571
28900
|
} else {
|
|
28572
28901
|
create({
|
|
28573
|
-
projectPath
|
|
28902
|
+
projectPath,
|
|
28574
28903
|
category: entry.category,
|
|
28575
28904
|
title: entry.title,
|
|
28576
28905
|
content: entry.content,
|
|
@@ -28580,13 +28909,13 @@ function importFromFile(input) {
|
|
|
28580
28909
|
});
|
|
28581
28910
|
}
|
|
28582
28911
|
} else {
|
|
28583
|
-
const existing = forProject(
|
|
28912
|
+
const existing = forProject(projectPath, true);
|
|
28584
28913
|
const titleMatch = existing.find(
|
|
28585
28914
|
(e) => e.title.toLowerCase() === entry.title.toLowerCase()
|
|
28586
28915
|
);
|
|
28587
28916
|
if (!titleMatch) {
|
|
28588
28917
|
create({
|
|
28589
|
-
projectPath
|
|
28918
|
+
projectPath,
|
|
28590
28919
|
category: entry.category,
|
|
28591
28920
|
title: entry.title,
|
|
28592
28921
|
content: entry.content,
|
|
@@ -28597,16 +28926,50 @@ function importFromFile(input) {
|
|
|
28597
28926
|
}
|
|
28598
28927
|
}
|
|
28599
28928
|
}
|
|
28929
|
+
function importFromFile(input) {
|
|
28930
|
+
if (!existsSync3(input.filePath)) return;
|
|
28931
|
+
const fileContent = readFileSync3(input.filePath, "utf8");
|
|
28932
|
+
const { section } = splitFile(fileContent);
|
|
28933
|
+
const textToParse = section ?? fileContent;
|
|
28934
|
+
const fileEntries = parseEntriesFromSection(textToParse);
|
|
28935
|
+
if (!fileEntries.length) return;
|
|
28936
|
+
_importEntries(fileEntries, input.projectPath);
|
|
28937
|
+
}
|
|
28938
|
+
function loreFileExists(projectPath) {
|
|
28939
|
+
return existsSync3(join5(projectPath, LORE_FILE));
|
|
28940
|
+
}
|
|
28941
|
+
function exportLoreFile(projectPath) {
|
|
28942
|
+
const sectionBody = buildSection(projectPath);
|
|
28943
|
+
const content3 = LORE_FILE_HEADER + "\n" + sectionBody;
|
|
28944
|
+
writeFileSync(join5(projectPath, LORE_FILE), content3, "utf8");
|
|
28945
|
+
}
|
|
28946
|
+
function shouldImportLoreFile(projectPath) {
|
|
28947
|
+
const fp = join5(projectPath, LORE_FILE);
|
|
28948
|
+
if (!existsSync3(fp)) return false;
|
|
28949
|
+
const fileContent = readFileSync3(fp, "utf8");
|
|
28950
|
+
const expected = LORE_FILE_HEADER + "\n" + buildSection(projectPath);
|
|
28951
|
+
return hashSection(fileContent) !== hashSection(expected);
|
|
28952
|
+
}
|
|
28953
|
+
function importLoreFile(projectPath) {
|
|
28954
|
+
const fp = join5(projectPath, LORE_FILE);
|
|
28955
|
+
if (!existsSync3(fp)) return;
|
|
28956
|
+
const fileContent = readFileSync3(fp, "utf8");
|
|
28957
|
+
const fileEntries = parseEntriesFromSection(fileContent);
|
|
28958
|
+
if (!fileEntries.length) return;
|
|
28959
|
+
_importEntries(fileEntries, projectPath);
|
|
28960
|
+
}
|
|
28600
28961
|
|
|
28601
28962
|
// src/worker-model.ts
|
|
28602
28963
|
var worker_model_exports = {};
|
|
28603
28964
|
__export(worker_model_exports, {
|
|
28604
28965
|
WORKER_JUDGE_SYSTEM: () => WORKER_JUDGE_SYSTEM,
|
|
28966
|
+
clearValidatedWorkerModel: () => clearValidatedWorkerModel,
|
|
28605
28967
|
computeModelFingerprint: () => computeModelFingerprint,
|
|
28606
28968
|
getValidatedWorkerModel: () => getValidatedWorkerModel,
|
|
28607
28969
|
isValidationStale: () => isValidationStale,
|
|
28608
28970
|
parseJudgeScore: () => parseJudgeScore,
|
|
28609
28971
|
resolveWorkerModel: () => resolveWorkerModel,
|
|
28972
|
+
runValidation: () => runValidation,
|
|
28610
28973
|
selectWorkerCandidates: () => selectWorkerCandidates,
|
|
28611
28974
|
storeValidatedWorkerModel: () => storeValidatedWorkerModel,
|
|
28612
28975
|
structuralCheck: () => structuralCheck,
|
|
@@ -28618,7 +28981,13 @@ function selectWorkerCandidates(sessionModel, providerModels) {
|
|
|
28618
28981
|
(m) => m.providerID === sessionModel.providerID && m.status === "active" && m.capabilities.input.text
|
|
28619
28982
|
);
|
|
28620
28983
|
if (eligible.length === 0) return [];
|
|
28621
|
-
const sorted = [...eligible].sort((a, b) =>
|
|
28984
|
+
const sorted = [...eligible].sort((a, b) => {
|
|
28985
|
+
const costDiff = a.cost.input - b.cost.input;
|
|
28986
|
+
if (costDiff !== 0) return costDiff;
|
|
28987
|
+
const aReasoning = a.capabilities.reasoning ? 1 : 0;
|
|
28988
|
+
const bReasoning = b.capabilities.reasoning ? 1 : 0;
|
|
28989
|
+
return aReasoning - bReasoning;
|
|
28990
|
+
});
|
|
28622
28991
|
const cheapest = sorted[0];
|
|
28623
28992
|
const belowSession = sorted.filter((m) => m.cost.input < sessionModel.cost.input).pop();
|
|
28624
28993
|
const candidates = /* @__PURE__ */ new Map();
|
|
@@ -28653,6 +29022,9 @@ function storeValidatedWorkerModel(result) {
|
|
|
28653
29022
|
"INSERT INTO kv_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = ?"
|
|
28654
29023
|
).run(key, value, value);
|
|
28655
29024
|
}
|
|
29025
|
+
function clearValidatedWorkerModel(providerID) {
|
|
29026
|
+
db().query("DELETE FROM kv_meta WHERE key = ?").run(`${KV_PREFIX}${providerID}`);
|
|
29027
|
+
}
|
|
28656
29028
|
function isValidationStale(stored, currentFingerprint) {
|
|
28657
29029
|
if (!stored) return true;
|
|
28658
29030
|
return stored.fingerprint !== currentFingerprint;
|
|
@@ -28711,10 +29083,85 @@ function parseJudgeScore(response) {
|
|
|
28711
29083
|
if (!match) return null;
|
|
28712
29084
|
return parseInt(match[1], 10);
|
|
28713
29085
|
}
|
|
29086
|
+
async function runValidation(input) {
|
|
29087
|
+
const { llm, candidates, referenceObservations, sourceMessagesText, date: date5 } = input;
|
|
29088
|
+
const userPrompt = distillationUser({
|
|
29089
|
+
messages: sourceMessagesText,
|
|
29090
|
+
date: date5
|
|
29091
|
+
});
|
|
29092
|
+
for (const candidate of candidates) {
|
|
29093
|
+
if (candidate.id === input.sessionModelID) continue;
|
|
29094
|
+
let candidateObservations = null;
|
|
29095
|
+
try {
|
|
29096
|
+
const raw = await llm.prompt(DISTILLATION_SYSTEM, userPrompt, {
|
|
29097
|
+
model: { providerID: candidate.providerID, modelID: candidate.id },
|
|
29098
|
+
workerID: "lore-distill",
|
|
29099
|
+
thinking: false
|
|
29100
|
+
});
|
|
29101
|
+
if (raw) {
|
|
29102
|
+
const match = raw.match(/<observations>([\s\S]*?)<\/observations>/);
|
|
29103
|
+
candidateObservations = match ? match[1].trim() : raw.trim();
|
|
29104
|
+
}
|
|
29105
|
+
} catch (e) {
|
|
29106
|
+
warn(`worker model validation: candidate ${candidate.id} failed:`, e);
|
|
29107
|
+
continue;
|
|
29108
|
+
}
|
|
29109
|
+
const structural = structuralCheck(candidateObservations, referenceObservations);
|
|
29110
|
+
if (!structural.passed) {
|
|
29111
|
+
info(
|
|
29112
|
+
`worker model validation: ${candidate.id} failed structural check: ${structural.reason}`
|
|
29113
|
+
);
|
|
29114
|
+
continue;
|
|
29115
|
+
}
|
|
29116
|
+
let judgeScore = null;
|
|
29117
|
+
try {
|
|
29118
|
+
const judgeResponse = await llm.prompt(
|
|
29119
|
+
WORKER_JUDGE_SYSTEM,
|
|
29120
|
+
workerJudgeUser(referenceObservations, candidateObservations),
|
|
29121
|
+
{ workerID: "lore-distill", thinking: false }
|
|
29122
|
+
// use session model (no model override)
|
|
29123
|
+
);
|
|
29124
|
+
if (judgeResponse) {
|
|
29125
|
+
judgeScore = parseJudgeScore(judgeResponse);
|
|
29126
|
+
}
|
|
29127
|
+
} catch (e) {
|
|
29128
|
+
warn(`worker model validation: judge call failed for ${candidate.id}:`, e);
|
|
29129
|
+
}
|
|
29130
|
+
if (judgeScore !== null && judgeScore < 3) {
|
|
29131
|
+
info(
|
|
29132
|
+
`worker model validation: ${candidate.id} failed judge (score=${judgeScore})`
|
|
29133
|
+
);
|
|
29134
|
+
continue;
|
|
29135
|
+
}
|
|
29136
|
+
const fingerprint = computeModelFingerprint(
|
|
29137
|
+
input.providerID,
|
|
29138
|
+
input.sessionModelID,
|
|
29139
|
+
candidates.map((c) => c.id)
|
|
29140
|
+
);
|
|
29141
|
+
const result = {
|
|
29142
|
+
modelID: candidate.id,
|
|
29143
|
+
providerID: candidate.providerID,
|
|
29144
|
+
fingerprint,
|
|
29145
|
+
validatedAt: Date.now(),
|
|
29146
|
+
judgeScore
|
|
29147
|
+
};
|
|
29148
|
+
storeValidatedWorkerModel(result);
|
|
29149
|
+
info(
|
|
29150
|
+
`worker model validated: ${candidate.id} (judge=${judgeScore}) for provider ${input.providerID}`
|
|
29151
|
+
);
|
|
29152
|
+
return result;
|
|
29153
|
+
}
|
|
29154
|
+
clearValidatedWorkerModel(input.providerID);
|
|
29155
|
+
info(
|
|
29156
|
+
`worker model validation: no candidate passed for ${input.providerID} \u2014 cleared stale entry`
|
|
29157
|
+
);
|
|
29158
|
+
return null;
|
|
29159
|
+
}
|
|
28714
29160
|
function resolveWorkerModel(providerID, configWorkerModel, configModel) {
|
|
28715
29161
|
if (configWorkerModel) return configWorkerModel;
|
|
28716
29162
|
const validated = getValidatedWorkerModel(providerID);
|
|
28717
|
-
|
|
29163
|
+
const MAX_AGE_MS = 24 * 60 * 60 * 1e3;
|
|
29164
|
+
if (validated && Date.now() - validated.validatedAt <= MAX_AGE_MS) {
|
|
28718
29165
|
return { providerID: validated.providerID, modelID: validated.modelID };
|
|
28719
29166
|
}
|
|
28720
29167
|
return configModel;
|
|
@@ -28725,11 +29172,11 @@ export {
|
|
|
28725
29172
|
CURATOR_SYSTEM,
|
|
28726
29173
|
DISTILLATION_SYSTEM,
|
|
28727
29174
|
EMPTY_QUERY,
|
|
29175
|
+
LORE_FILE,
|
|
28728
29176
|
QUERY_EXPANSION_SYSTEM,
|
|
28729
29177
|
RECALL_PARAM_DESCRIPTIONS,
|
|
28730
29178
|
RECALL_TOOL_DESCRIPTION,
|
|
28731
29179
|
RECURSIVE_SYSTEM,
|
|
28732
|
-
WORKER_JUDGE_SYSTEM,
|
|
28733
29180
|
buildCompactPrompt,
|
|
28734
29181
|
calibrate,
|
|
28735
29182
|
close,
|
|
@@ -28744,7 +29191,9 @@ export {
|
|
|
28744
29191
|
distillationUser,
|
|
28745
29192
|
embedding_exports as embedding,
|
|
28746
29193
|
ensureProject,
|
|
29194
|
+
exactTermMatchRank,
|
|
28747
29195
|
expandQuery,
|
|
29196
|
+
exportLoreFile,
|
|
28748
29197
|
exportToFile,
|
|
28749
29198
|
extractTopTerms,
|
|
28750
29199
|
formatDistillations,
|
|
@@ -28753,10 +29202,12 @@ export {
|
|
|
28753
29202
|
ftsQueryOr,
|
|
28754
29203
|
getLastTransformEstimate,
|
|
28755
29204
|
getLastTransformedCount,
|
|
29205
|
+
getLastTurnAt,
|
|
28756
29206
|
getLtmBudget,
|
|
28757
29207
|
getLtmTokens,
|
|
28758
29208
|
h,
|
|
28759
29209
|
importFromFile,
|
|
29210
|
+
importLoreFile,
|
|
28760
29211
|
inline,
|
|
28761
29212
|
inspectSessionState,
|
|
28762
29213
|
isFirstRun,
|
|
@@ -28770,11 +29221,13 @@ export {
|
|
|
28770
29221
|
load,
|
|
28771
29222
|
loadForceMinLayer,
|
|
28772
29223
|
log_exports as log,
|
|
29224
|
+
loreFileExists,
|
|
28773
29225
|
ltm_exports as ltm,
|
|
28774
29226
|
needsUrgentDistillation,
|
|
28775
29227
|
normalize,
|
|
28776
29228
|
onIdleResume,
|
|
28777
29229
|
p,
|
|
29230
|
+
pattern_extract_exports as patternExtract,
|
|
28778
29231
|
projectId,
|
|
28779
29232
|
projectName,
|
|
28780
29233
|
reciprocalRankFusion,
|
|
@@ -28790,6 +29243,7 @@ export {
|
|
|
28790
29243
|
setMaxLayer0Tokens,
|
|
28791
29244
|
setModelLimits,
|
|
28792
29245
|
shouldImport,
|
|
29246
|
+
shouldImportLoreFile,
|
|
28793
29247
|
strong2 as strong,
|
|
28794
29248
|
t,
|
|
28795
29249
|
temporal_exports as temporal,
|
|
@@ -28797,7 +29251,6 @@ export {
|
|
|
28797
29251
|
transform2 as transform,
|
|
28798
29252
|
ul,
|
|
28799
29253
|
unescapeMarkdown,
|
|
28800
|
-
workerJudgeUser,
|
|
28801
29254
|
worker_model_exports as workerModel,
|
|
28802
29255
|
workerSessionIDs
|
|
28803
29256
|
};
|