@loreai/core 0.15.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bun/index.js +2 -200
- package/dist/bun/index.js.map +3 -3
- package/dist/bun/worker-model.d.ts +12 -80
- package/dist/bun/worker-model.d.ts.map +1 -1
- package/dist/node/index.js +2 -200
- package/dist/node/index.js.map +3 -3
- package/dist/node/worker-model.d.ts +12 -80
- package/dist/node/worker-model.d.ts.map +1 -1
- package/dist/types/worker-model.d.ts +12 -80
- package/dist/types/worker-model.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/worker-model.ts +13 -364
|
@@ -1,16 +1,17 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* Worker model resolution.
|
|
3
3
|
*
|
|
4
|
-
* Background workers (distillation, curation, query expansion)
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
* Phase 1: structural checks (parsability, observation count, token bounds)
|
|
8
|
-
* Phase 2: LLM judge (session model rates candidate output vs reference)
|
|
4
|
+
* Background workers (distillation, curation, query expansion) use the session
|
|
5
|
+
* model by default. An explicit `workerModel` config override is supported for
|
|
6
|
+
* cases where the user wants to pin background work to a specific model.
|
|
9
7
|
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
8
|
+
* Previously this module contained dynamic worker model selection with
|
|
9
|
+
* candidate discovery, two-phase validation (structural check + LLM judge),
|
|
10
|
+
* and fingerprint-based staleness detection. That complexity was removed in
|
|
11
|
+
* favor of always using the session model — A/B testing showed the quality
|
|
12
|
+
* gap on complex conversations wasn't worth the infrastructure cost.
|
|
12
13
|
*/
|
|
13
|
-
/** Minimal model info
|
|
14
|
+
/** Minimal model info — kept for downstream consumers. */
|
|
14
15
|
export type ModelInfo = {
|
|
15
16
|
id: string;
|
|
16
17
|
providerID: string;
|
|
@@ -26,80 +27,11 @@ export type ModelInfo = {
|
|
|
26
27
|
reasoning?: boolean;
|
|
27
28
|
};
|
|
28
29
|
};
|
|
29
|
-
/** Result of a worker model validation stored in kv_meta. */
|
|
30
|
-
export type WorkerModelResult = {
|
|
31
|
-
modelID: string;
|
|
32
|
-
providerID: string;
|
|
33
|
-
fingerprint: string;
|
|
34
|
-
validatedAt: number;
|
|
35
|
-
judgeScore: number | null;
|
|
36
|
-
};
|
|
37
|
-
/**
|
|
38
|
-
* Select worker model candidates from the available models.
|
|
39
|
-
*
|
|
40
|
-
* Returns up to 2 candidates: cheapest overall + one tier below the session
|
|
41
|
-
* model. The session model itself is included (if it's the cheapest, the list
|
|
42
|
-
* has 1 entry and no comparison is needed).
|
|
43
|
-
*/
|
|
44
|
-
export declare function selectWorkerCandidates(sessionModel: {
|
|
45
|
-
id: string;
|
|
46
|
-
providerID: string;
|
|
47
|
-
cost: {
|
|
48
|
-
input: number;
|
|
49
|
-
};
|
|
50
|
-
}, providerModels: ModelInfo[]): ModelInfo[];
|
|
51
|
-
/**
|
|
52
|
-
* Compute a fingerprint from the model landscape. Changes when:
|
|
53
|
-
* - Models are added or removed from the provider
|
|
54
|
-
* - The session model changes
|
|
55
|
-
*/
|
|
56
|
-
export declare function computeModelFingerprint(providerID: string, sessionModelID: string, activeModelIDs: string[]): string;
|
|
57
|
-
export declare function getValidatedWorkerModel(providerID: string): WorkerModelResult | null;
|
|
58
|
-
export declare function storeValidatedWorkerModel(result: WorkerModelResult): void;
|
|
59
|
-
/** Clear a stored worker model validation (e.g. when the model is deprecated). */
|
|
60
|
-
export declare function clearValidatedWorkerModel(providerID: string): void;
|
|
61
|
-
/**
|
|
62
|
-
* Check whether the stored validation is stale (fingerprint mismatch).
|
|
63
|
-
*/
|
|
64
|
-
export declare function isValidationStale(stored: WorkerModelResult | null, currentFingerprint: string): boolean;
|
|
65
|
-
export type StructuralCheckResult = {
|
|
66
|
-
passed: boolean;
|
|
67
|
-
observationCount: number;
|
|
68
|
-
tokenCount: number;
|
|
69
|
-
reason?: string;
|
|
70
|
-
};
|
|
71
|
-
/**
|
|
72
|
-
* Structural quality check: does the candidate distillation output meet
|
|
73
|
-
* minimum quality thresholds relative to the reference?
|
|
74
|
-
*/
|
|
75
|
-
export declare function structuralCheck(candidateObservations: string | null, referenceObservations: string): StructuralCheckResult;
|
|
76
|
-
export declare const WORKER_JUDGE_SYSTEM = "You are evaluating distillation quality. You will be given a REFERENCE distillation (produced by a capable model) and a CANDIDATE distillation (produced by a cheaper model) of the same conversation segment.\n\nRate the candidate on a scale of 1-5:\n5 = Captures all key facts and decisions, equivalent to reference\n4 = Captures most facts, minor omissions\n3 = Captures the essential facts, some detail loss acceptable\n2 = Missing important facts or technical details\n1 = Significantly incomplete or inaccurate\n\nRespond with ONLY a single digit (1-5).";
|
|
77
|
-
export declare function workerJudgeUser(reference: string, candidate: string): string;
|
|
78
|
-
/** Parse the judge's score from a response. Returns null on parse failure. */
|
|
79
|
-
export declare function parseJudgeScore(response: string): number | null;
|
|
80
|
-
import type { LLMClient } from "./types";
|
|
81
|
-
export type ValidationInput = {
|
|
82
|
-
llm: LLMClient;
|
|
83
|
-
providerID: string;
|
|
84
|
-
sessionModelID: string;
|
|
85
|
-
candidates: ModelInfo[];
|
|
86
|
-
/** Recent gen-0 distillation to use as reference (observations text). */
|
|
87
|
-
referenceObservations: string;
|
|
88
|
-
/** Source messages text for re-running distillation with candidates. */
|
|
89
|
-
sourceMessagesText: string;
|
|
90
|
-
/** Date string for the distillation prompt. */
|
|
91
|
-
date: string;
|
|
92
|
-
};
|
|
93
|
-
/**
|
|
94
|
-
* Run the two-phase quality validation for worker model candidates.
|
|
95
|
-
* Returns the cheapest passing candidate, or null if none pass.
|
|
96
|
-
*/
|
|
97
|
-
export declare function runValidation(input: ValidationInput): Promise<WorkerModelResult | null>;
|
|
98
30
|
/**
|
|
99
31
|
* Resolve the effective worker model for a given provider.
|
|
100
|
-
* Priority: explicit config
|
|
32
|
+
* Priority: explicit config override > session model (fallback).
|
|
101
33
|
*/
|
|
102
|
-
export declare function resolveWorkerModel(
|
|
34
|
+
export declare function resolveWorkerModel(_providerID: string, configWorkerModel?: {
|
|
103
35
|
providerID: string;
|
|
104
36
|
modelID: string;
|
|
105
37
|
}, configModel?: {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worker-model.d.ts","sourceRoot":"","sources":["../../src/worker-model.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"worker-model.d.ts","sourceRoot":"","sources":["../../src/worker-model.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAMH,0DAA0D;AAC1D,MAAM,MAAM,SAAS,GAAG;IACtB,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE;QAAE,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE;QACZ,KAAK,EAAE;YAAE,IAAI,EAAE,OAAO,CAAA;SAAE,CAAC;QACzB,+DAA+D;QAC/D,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB,CAAC;CACH,CAAC;AAMF;;;GAGG;AACH,wBAAgB,kBAAkB,CAChC,WAAW,EAAE,MAAM,EACnB,iBAAiB,CAAC,EAAE;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,EAC3D,WAAW,CAAC,EAAE;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GACpD;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,CAMrD"}
|
|
@@ -1,16 +1,17 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* Worker model resolution.
|
|
3
3
|
*
|
|
4
|
-
* Background workers (distillation, curation, query expansion)
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
* Phase 1: structural checks (parsability, observation count, token bounds)
|
|
8
|
-
* Phase 2: LLM judge (session model rates candidate output vs reference)
|
|
4
|
+
* Background workers (distillation, curation, query expansion) use the session
|
|
5
|
+
* model by default. An explicit `workerModel` config override is supported for
|
|
6
|
+
* cases where the user wants to pin background work to a specific model.
|
|
9
7
|
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
8
|
+
* Previously this module contained dynamic worker model selection with
|
|
9
|
+
* candidate discovery, two-phase validation (structural check + LLM judge),
|
|
10
|
+
* and fingerprint-based staleness detection. That complexity was removed in
|
|
11
|
+
* favor of always using the session model — A/B testing showed the quality
|
|
12
|
+
* gap on complex conversations wasn't worth the infrastructure cost.
|
|
12
13
|
*/
|
|
13
|
-
/** Minimal model info
|
|
14
|
+
/** Minimal model info — kept for downstream consumers. */
|
|
14
15
|
export type ModelInfo = {
|
|
15
16
|
id: string;
|
|
16
17
|
providerID: string;
|
|
@@ -26,80 +27,11 @@ export type ModelInfo = {
|
|
|
26
27
|
reasoning?: boolean;
|
|
27
28
|
};
|
|
28
29
|
};
|
|
29
|
-
/** Result of a worker model validation stored in kv_meta. */
|
|
30
|
-
export type WorkerModelResult = {
|
|
31
|
-
modelID: string;
|
|
32
|
-
providerID: string;
|
|
33
|
-
fingerprint: string;
|
|
34
|
-
validatedAt: number;
|
|
35
|
-
judgeScore: number | null;
|
|
36
|
-
};
|
|
37
|
-
/**
|
|
38
|
-
* Select worker model candidates from the available models.
|
|
39
|
-
*
|
|
40
|
-
* Returns up to 2 candidates: cheapest overall + one tier below the session
|
|
41
|
-
* model. The session model itself is included (if it's the cheapest, the list
|
|
42
|
-
* has 1 entry and no comparison is needed).
|
|
43
|
-
*/
|
|
44
|
-
export declare function selectWorkerCandidates(sessionModel: {
|
|
45
|
-
id: string;
|
|
46
|
-
providerID: string;
|
|
47
|
-
cost: {
|
|
48
|
-
input: number;
|
|
49
|
-
};
|
|
50
|
-
}, providerModels: ModelInfo[]): ModelInfo[];
|
|
51
|
-
/**
|
|
52
|
-
* Compute a fingerprint from the model landscape. Changes when:
|
|
53
|
-
* - Models are added or removed from the provider
|
|
54
|
-
* - The session model changes
|
|
55
|
-
*/
|
|
56
|
-
export declare function computeModelFingerprint(providerID: string, sessionModelID: string, activeModelIDs: string[]): string;
|
|
57
|
-
export declare function getValidatedWorkerModel(providerID: string): WorkerModelResult | null;
|
|
58
|
-
export declare function storeValidatedWorkerModel(result: WorkerModelResult): void;
|
|
59
|
-
/** Clear a stored worker model validation (e.g. when the model is deprecated). */
|
|
60
|
-
export declare function clearValidatedWorkerModel(providerID: string): void;
|
|
61
|
-
/**
|
|
62
|
-
* Check whether the stored validation is stale (fingerprint mismatch).
|
|
63
|
-
*/
|
|
64
|
-
export declare function isValidationStale(stored: WorkerModelResult | null, currentFingerprint: string): boolean;
|
|
65
|
-
export type StructuralCheckResult = {
|
|
66
|
-
passed: boolean;
|
|
67
|
-
observationCount: number;
|
|
68
|
-
tokenCount: number;
|
|
69
|
-
reason?: string;
|
|
70
|
-
};
|
|
71
|
-
/**
|
|
72
|
-
* Structural quality check: does the candidate distillation output meet
|
|
73
|
-
* minimum quality thresholds relative to the reference?
|
|
74
|
-
*/
|
|
75
|
-
export declare function structuralCheck(candidateObservations: string | null, referenceObservations: string): StructuralCheckResult;
|
|
76
|
-
export declare const WORKER_JUDGE_SYSTEM = "You are evaluating distillation quality. You will be given a REFERENCE distillation (produced by a capable model) and a CANDIDATE distillation (produced by a cheaper model) of the same conversation segment.\n\nRate the candidate on a scale of 1-5:\n5 = Captures all key facts and decisions, equivalent to reference\n4 = Captures most facts, minor omissions\n3 = Captures the essential facts, some detail loss acceptable\n2 = Missing important facts or technical details\n1 = Significantly incomplete or inaccurate\n\nRespond with ONLY a single digit (1-5).";
|
|
77
|
-
export declare function workerJudgeUser(reference: string, candidate: string): string;
|
|
78
|
-
/** Parse the judge's score from a response. Returns null on parse failure. */
|
|
79
|
-
export declare function parseJudgeScore(response: string): number | null;
|
|
80
|
-
import type { LLMClient } from "./types";
|
|
81
|
-
export type ValidationInput = {
|
|
82
|
-
llm: LLMClient;
|
|
83
|
-
providerID: string;
|
|
84
|
-
sessionModelID: string;
|
|
85
|
-
candidates: ModelInfo[];
|
|
86
|
-
/** Recent gen-0 distillation to use as reference (observations text). */
|
|
87
|
-
referenceObservations: string;
|
|
88
|
-
/** Source messages text for re-running distillation with candidates. */
|
|
89
|
-
sourceMessagesText: string;
|
|
90
|
-
/** Date string for the distillation prompt. */
|
|
91
|
-
date: string;
|
|
92
|
-
};
|
|
93
|
-
/**
|
|
94
|
-
* Run the two-phase quality validation for worker model candidates.
|
|
95
|
-
* Returns the cheapest passing candidate, or null if none pass.
|
|
96
|
-
*/
|
|
97
|
-
export declare function runValidation(input: ValidationInput): Promise<WorkerModelResult | null>;
|
|
98
30
|
/**
|
|
99
31
|
* Resolve the effective worker model for a given provider.
|
|
100
|
-
* Priority: explicit config
|
|
32
|
+
* Priority: explicit config override > session model (fallback).
|
|
101
33
|
*/
|
|
102
|
-
export declare function resolveWorkerModel(
|
|
34
|
+
export declare function resolveWorkerModel(_providerID: string, configWorkerModel?: {
|
|
103
35
|
providerID: string;
|
|
104
36
|
modelID: string;
|
|
105
37
|
}, configModel?: {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"worker-model.d.ts","sourceRoot":"","sources":["../../src/worker-model.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"worker-model.d.ts","sourceRoot":"","sources":["../../src/worker-model.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAMH,0DAA0D;AAC1D,MAAM,MAAM,SAAS,GAAG;IACtB,EAAE,EAAE,MAAM,CAAC;IACX,UAAU,EAAE,MAAM,CAAC;IACnB,IAAI,EAAE;QAAE,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE;QACZ,KAAK,EAAE;YAAE,IAAI,EAAE,OAAO,CAAA;SAAE,CAAC;QACzB,+DAA+D;QAC/D,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB,CAAC;CACH,CAAC;AAMF;;;GAGG;AACH,wBAAgB,kBAAkB,CAChC,WAAW,EAAE,MAAM,EACnB,iBAAiB,CAAC,EAAE;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,EAC3D,WAAW,CAAC,EAAE;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GACpD;IAAE,UAAU,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,CAMrD"}
|
package/package.json
CHANGED
package/src/worker-model.ts
CHANGED
|
@@ -1,25 +1,22 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* Worker model resolution.
|
|
3
3
|
*
|
|
4
|
-
* Background workers (distillation, curation, query expansion)
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
* Phase 1: structural checks (parsability, observation count, token bounds)
|
|
8
|
-
* Phase 2: LLM judge (session model rates candidate output vs reference)
|
|
4
|
+
* Background workers (distillation, curation, query expansion) use the session
|
|
5
|
+
* model by default. An explicit `workerModel` config override is supported for
|
|
6
|
+
* cases where the user wants to pin background work to a specific model.
|
|
9
7
|
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
8
|
+
* Previously this module contained dynamic worker model selection with
|
|
9
|
+
* candidate discovery, two-phase validation (structural check + LLM judge),
|
|
10
|
+
* and fingerprint-based staleness detection. That complexity was removed in
|
|
11
|
+
* favor of always using the session model — A/B testing showed the quality
|
|
12
|
+
* gap on complex conversations wasn't worth the infrastructure cost.
|
|
12
13
|
*/
|
|
13
14
|
|
|
14
|
-
import { db } from "./db";
|
|
15
|
-
import { sha256 } from "#db/driver";
|
|
16
|
-
import * as log from "./log";
|
|
17
|
-
|
|
18
15
|
// ---------------------------------------------------------------------------
|
|
19
|
-
// Types
|
|
16
|
+
// Types (kept for config compatibility)
|
|
20
17
|
// ---------------------------------------------------------------------------
|
|
21
18
|
|
|
22
|
-
/** Minimal model info
|
|
19
|
+
/** Minimal model info — kept for downstream consumers. */
|
|
23
20
|
export type ModelInfo = {
|
|
24
21
|
id: string;
|
|
25
22
|
providerID: string;
|
|
@@ -32,370 +29,22 @@ export type ModelInfo = {
|
|
|
32
29
|
};
|
|
33
30
|
};
|
|
34
31
|
|
|
35
|
-
/** Result of a worker model validation stored in kv_meta. */
|
|
36
|
-
export type WorkerModelResult = {
|
|
37
|
-
modelID: string;
|
|
38
|
-
providerID: string;
|
|
39
|
-
fingerprint: string;
|
|
40
|
-
validatedAt: number;
|
|
41
|
-
judgeScore: number | null; // null = structural-only (no judge run yet)
|
|
42
|
-
};
|
|
43
|
-
|
|
44
|
-
const KV_PREFIX = "lore:worker_model:";
|
|
45
|
-
|
|
46
|
-
// ---------------------------------------------------------------------------
|
|
47
|
-
// Candidate selection
|
|
48
|
-
// ---------------------------------------------------------------------------
|
|
49
|
-
|
|
50
|
-
/**
|
|
51
|
-
* Select worker model candidates from the available models.
|
|
52
|
-
*
|
|
53
|
-
* Returns up to 2 candidates: cheapest overall + one tier below the session
|
|
54
|
-
* model. The session model itself is included (if it's the cheapest, the list
|
|
55
|
-
* has 1 entry and no comparison is needed).
|
|
56
|
-
*/
|
|
57
|
-
export function selectWorkerCandidates(
|
|
58
|
-
sessionModel: { id: string; providerID: string; cost: { input: number } },
|
|
59
|
-
providerModels: ModelInfo[],
|
|
60
|
-
): ModelInfo[] {
|
|
61
|
-
// Filter: same provider, active, text-capable
|
|
62
|
-
const eligible = providerModels.filter(
|
|
63
|
-
(m) =>
|
|
64
|
-
m.providerID === sessionModel.providerID &&
|
|
65
|
-
m.status === "active" &&
|
|
66
|
-
m.capabilities.input.text,
|
|
67
|
-
);
|
|
68
|
-
|
|
69
|
-
if (eligible.length === 0) return [];
|
|
70
|
-
|
|
71
|
-
// Sort by cost ascending, then prefer non-reasoning models at equal cost.
|
|
72
|
-
// Non-reasoning models don't produce thinking tokens, avoiding wasted spend
|
|
73
|
-
// on tokens that background workers discard.
|
|
74
|
-
const sorted = [...eligible].sort((a, b) => {
|
|
75
|
-
const costDiff = a.cost.input - b.cost.input;
|
|
76
|
-
if (costDiff !== 0) return costDiff;
|
|
77
|
-
// At equal cost, non-reasoning (0) sorts before reasoning (1)
|
|
78
|
-
const aReasoning = a.capabilities.reasoning ? 1 : 0;
|
|
79
|
-
const bReasoning = b.capabilities.reasoning ? 1 : 0;
|
|
80
|
-
return aReasoning - bReasoning;
|
|
81
|
-
});
|
|
82
|
-
|
|
83
|
-
// Cheapest overall
|
|
84
|
-
const cheapest = sorted[0];
|
|
85
|
-
|
|
86
|
-
// One tier below session model: the most expensive model that's still
|
|
87
|
-
// cheaper than the session model. If session IS cheapest, this is undefined.
|
|
88
|
-
const belowSession = sorted
|
|
89
|
-
.filter((m) => m.cost.input < sessionModel.cost.input)
|
|
90
|
-
.pop(); // last = most expensive among cheaper ones
|
|
91
|
-
|
|
92
|
-
// Deduplicate
|
|
93
|
-
const candidates = new Map<string, ModelInfo>();
|
|
94
|
-
candidates.set(cheapest.id, cheapest);
|
|
95
|
-
if (belowSession && belowSession.id !== cheapest.id) {
|
|
96
|
-
candidates.set(belowSession.id, belowSession);
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
// If session model is the cheapest, return just it
|
|
100
|
-
if (cheapest.id === sessionModel.id || cheapest.cost.input >= sessionModel.cost.input) {
|
|
101
|
-
return [cheapest];
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
return [...candidates.values()];
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
// ---------------------------------------------------------------------------
|
|
108
|
-
// Fingerprinting
|
|
109
|
-
// ---------------------------------------------------------------------------
|
|
110
|
-
|
|
111
|
-
/**
|
|
112
|
-
* Compute a fingerprint from the model landscape. Changes when:
|
|
113
|
-
* - Models are added or removed from the provider
|
|
114
|
-
* - The session model changes
|
|
115
|
-
*/
|
|
116
|
-
export function computeModelFingerprint(
|
|
117
|
-
providerID: string,
|
|
118
|
-
sessionModelID: string,
|
|
119
|
-
activeModelIDs: string[],
|
|
120
|
-
): string {
|
|
121
|
-
const sorted = [...activeModelIDs].sort();
|
|
122
|
-
return sha256(
|
|
123
|
-
JSON.stringify({ providerID, sessionModelID, modelIDs: sorted }),
|
|
124
|
-
);
|
|
125
|
-
}
|
|
126
|
-
|
|
127
|
-
// ---------------------------------------------------------------------------
|
|
128
|
-
// Persistence
|
|
129
|
-
// ---------------------------------------------------------------------------
|
|
130
|
-
|
|
131
|
-
export function getValidatedWorkerModel(
|
|
132
|
-
providerID: string,
|
|
133
|
-
): WorkerModelResult | null {
|
|
134
|
-
const row = db()
|
|
135
|
-
.query("SELECT value FROM kv_meta WHERE key = ?")
|
|
136
|
-
.get(`${KV_PREFIX}${providerID}`) as { value: string } | null;
|
|
137
|
-
if (!row) return null;
|
|
138
|
-
try {
|
|
139
|
-
return JSON.parse(row.value) as WorkerModelResult;
|
|
140
|
-
} catch {
|
|
141
|
-
return null;
|
|
142
|
-
}
|
|
143
|
-
}
|
|
144
|
-
|
|
145
|
-
export function storeValidatedWorkerModel(result: WorkerModelResult): void {
|
|
146
|
-
const key = `${KV_PREFIX}${result.providerID}`;
|
|
147
|
-
const value = JSON.stringify(result);
|
|
148
|
-
db()
|
|
149
|
-
.query(
|
|
150
|
-
"INSERT INTO kv_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = ?",
|
|
151
|
-
)
|
|
152
|
-
.run(key, value, value);
|
|
153
|
-
}
|
|
154
|
-
|
|
155
|
-
/** Clear a stored worker model validation (e.g. when the model is deprecated). */
|
|
156
|
-
export function clearValidatedWorkerModel(providerID: string): void {
|
|
157
|
-
db().query("DELETE FROM kv_meta WHERE key = ?").run(`${KV_PREFIX}${providerID}`);
|
|
158
|
-
}
|
|
159
|
-
|
|
160
|
-
/**
|
|
161
|
-
* Check whether the stored validation is stale (fingerprint mismatch).
|
|
162
|
-
*/
|
|
163
|
-
export function isValidationStale(
|
|
164
|
-
stored: WorkerModelResult | null,
|
|
165
|
-
currentFingerprint: string,
|
|
166
|
-
): boolean {
|
|
167
|
-
if (!stored) return true;
|
|
168
|
-
return stored.fingerprint !== currentFingerprint;
|
|
169
|
-
}
|
|
170
|
-
|
|
171
|
-
// ---------------------------------------------------------------------------
|
|
172
|
-
// Structural validation
|
|
173
|
-
// ---------------------------------------------------------------------------
|
|
174
|
-
|
|
175
|
-
export type StructuralCheckResult = {
|
|
176
|
-
passed: boolean;
|
|
177
|
-
observationCount: number;
|
|
178
|
-
tokenCount: number;
|
|
179
|
-
reason?: string;
|
|
180
|
-
};
|
|
181
|
-
|
|
182
|
-
/**
|
|
183
|
-
* Structural quality check: does the candidate distillation output meet
|
|
184
|
-
* minimum quality thresholds relative to the reference?
|
|
185
|
-
*/
|
|
186
|
-
export function structuralCheck(
|
|
187
|
-
candidateObservations: string | null,
|
|
188
|
-
referenceObservations: string,
|
|
189
|
-
): StructuralCheckResult {
|
|
190
|
-
if (candidateObservations == null || candidateObservations.length === 0) {
|
|
191
|
-
return { passed: false, observationCount: 0, tokenCount: 0, reason: candidateObservations === null ? "parse_failed" : "empty" };
|
|
192
|
-
}
|
|
193
|
-
|
|
194
|
-
// Count observation lines (non-empty lines starting with common markers)
|
|
195
|
-
const countObs = (text: string) =>
|
|
196
|
-
text.split("\n").filter((l) => l.trim().length > 0).length;
|
|
197
|
-
|
|
198
|
-
const refCount = countObs(referenceObservations);
|
|
199
|
-
const candCount = countObs(candidateObservations);
|
|
200
|
-
const candTokens = Math.ceil(candidateObservations.length / 3);
|
|
201
|
-
|
|
202
|
-
// Observation count within ±50% of reference
|
|
203
|
-
if (refCount > 0 && (candCount < refCount * 0.5 || candCount > refCount * 1.5)) {
|
|
204
|
-
return {
|
|
205
|
-
passed: false,
|
|
206
|
-
observationCount: candCount,
|
|
207
|
-
tokenCount: candTokens,
|
|
208
|
-
reason: `observation_count_${candCount}_vs_ref_${refCount}`,
|
|
209
|
-
};
|
|
210
|
-
}
|
|
211
|
-
|
|
212
|
-
// Not degenerate: not empty, not >3x reference size
|
|
213
|
-
const refTokens = Math.ceil(referenceObservations.length / 3);
|
|
214
|
-
if (candTokens === 0) {
|
|
215
|
-
return { passed: false, observationCount: candCount, tokenCount: candTokens, reason: "empty" };
|
|
216
|
-
}
|
|
217
|
-
if (refTokens > 0 && candTokens > refTokens * 3) {
|
|
218
|
-
return {
|
|
219
|
-
passed: false,
|
|
220
|
-
observationCount: candCount,
|
|
221
|
-
tokenCount: candTokens,
|
|
222
|
-
reason: `token_count_${candTokens}_vs_ref_${refTokens}_3x`,
|
|
223
|
-
};
|
|
224
|
-
}
|
|
225
|
-
|
|
226
|
-
return { passed: true, observationCount: candCount, tokenCount: candTokens };
|
|
227
|
-
}
|
|
228
|
-
|
|
229
|
-
// ---------------------------------------------------------------------------
|
|
230
|
-
// Judge prompt
|
|
231
|
-
// ---------------------------------------------------------------------------
|
|
232
|
-
|
|
233
|
-
export const WORKER_JUDGE_SYSTEM = `You are evaluating distillation quality. You will be given a REFERENCE distillation (produced by a capable model) and a CANDIDATE distillation (produced by a cheaper model) of the same conversation segment.
|
|
234
|
-
|
|
235
|
-
Rate the candidate on a scale of 1-5:
|
|
236
|
-
5 = Captures all key facts and decisions, equivalent to reference
|
|
237
|
-
4 = Captures most facts, minor omissions
|
|
238
|
-
3 = Captures the essential facts, some detail loss acceptable
|
|
239
|
-
2 = Missing important facts or technical details
|
|
240
|
-
1 = Significantly incomplete or inaccurate
|
|
241
|
-
|
|
242
|
-
Respond with ONLY a single digit (1-5).`;
|
|
243
|
-
|
|
244
|
-
export function workerJudgeUser(
|
|
245
|
-
reference: string,
|
|
246
|
-
candidate: string,
|
|
247
|
-
): string {
|
|
248
|
-
return `<reference>\n${reference}\n</reference>\n\n<candidate>\n${candidate}\n</candidate>`;
|
|
249
|
-
}
|
|
250
|
-
|
|
251
|
-
/** Parse the judge's score from a response. Returns null on parse failure. */
|
|
252
|
-
export function parseJudgeScore(response: string): number | null {
|
|
253
|
-
const match = response.trim().match(/^([1-5])/);
|
|
254
|
-
if (!match) return null;
|
|
255
|
-
return parseInt(match[1], 10);
|
|
256
|
-
}
|
|
257
|
-
|
|
258
|
-
// ---------------------------------------------------------------------------
|
|
259
|
-
// Validation orchestration
|
|
260
|
-
// ---------------------------------------------------------------------------
|
|
261
|
-
|
|
262
|
-
import { DISTILLATION_SYSTEM, distillationUser } from "./prompt";
|
|
263
|
-
import type { LLMClient } from "./types";
|
|
264
|
-
|
|
265
|
-
export type ValidationInput = {
|
|
266
|
-
llm: LLMClient;
|
|
267
|
-
providerID: string;
|
|
268
|
-
sessionModelID: string;
|
|
269
|
-
candidates: ModelInfo[];
|
|
270
|
-
/** Recent gen-0 distillation to use as reference (observations text). */
|
|
271
|
-
referenceObservations: string;
|
|
272
|
-
/** Source messages text for re-running distillation with candidates. */
|
|
273
|
-
sourceMessagesText: string;
|
|
274
|
-
/** Date string for the distillation prompt. */
|
|
275
|
-
date: string;
|
|
276
|
-
};
|
|
277
|
-
|
|
278
|
-
/**
|
|
279
|
-
* Run the two-phase quality validation for worker model candidates.
|
|
280
|
-
* Returns the cheapest passing candidate, or null if none pass.
|
|
281
|
-
*/
|
|
282
|
-
export async function runValidation(
|
|
283
|
-
input: ValidationInput,
|
|
284
|
-
): Promise<WorkerModelResult | null> {
|
|
285
|
-
const { llm, candidates, referenceObservations, sourceMessagesText, date } = input;
|
|
286
|
-
|
|
287
|
-
const userPrompt = distillationUser({
|
|
288
|
-
messages: sourceMessagesText,
|
|
289
|
-
date,
|
|
290
|
-
});
|
|
291
|
-
|
|
292
|
-
for (const candidate of candidates) {
|
|
293
|
-
// Skip the session model — it produced the reference, no need to test
|
|
294
|
-
if (candidate.id === input.sessionModelID) continue;
|
|
295
|
-
|
|
296
|
-
// Phase 1: run distillation with candidate model
|
|
297
|
-
let candidateObservations: string | null = null;
|
|
298
|
-
try {
|
|
299
|
-
const raw = await llm.prompt(DISTILLATION_SYSTEM, userPrompt, {
|
|
300
|
-
model: { providerID: candidate.providerID, modelID: candidate.id },
|
|
301
|
-
workerID: "lore-distill",
|
|
302
|
-
thinking: false,
|
|
303
|
-
});
|
|
304
|
-
if (raw) {
|
|
305
|
-
// Parse <observations>...</observations> block
|
|
306
|
-
const match = raw.match(/<observations>([\s\S]*?)<\/observations>/);
|
|
307
|
-
candidateObservations = match ? match[1].trim() : raw.trim();
|
|
308
|
-
}
|
|
309
|
-
} catch (e) {
|
|
310
|
-
log.warn(`worker model validation: candidate ${candidate.id} failed:`, e);
|
|
311
|
-
continue;
|
|
312
|
-
}
|
|
313
|
-
|
|
314
|
-
const structural = structuralCheck(candidateObservations, referenceObservations);
|
|
315
|
-
if (!structural.passed) {
|
|
316
|
-
log.info(
|
|
317
|
-
`worker model validation: ${candidate.id} failed structural check: ${structural.reason}`,
|
|
318
|
-
);
|
|
319
|
-
continue;
|
|
320
|
-
}
|
|
321
|
-
|
|
322
|
-
// Phase 2: LLM judge (using session model)
|
|
323
|
-
let judgeScore: number | null = null;
|
|
324
|
-
try {
|
|
325
|
-
const judgeResponse = await llm.prompt(
|
|
326
|
-
WORKER_JUDGE_SYSTEM,
|
|
327
|
-
workerJudgeUser(referenceObservations, candidateObservations!),
|
|
328
|
-
{ workerID: "lore-distill", thinking: false }, // use session model (no model override)
|
|
329
|
-
);
|
|
330
|
-
if (judgeResponse) {
|
|
331
|
-
judgeScore = parseJudgeScore(judgeResponse);
|
|
332
|
-
}
|
|
333
|
-
} catch (e) {
|
|
334
|
-
log.warn(`worker model validation: judge call failed for ${candidate.id}:`, e);
|
|
335
|
-
}
|
|
336
|
-
|
|
337
|
-
if (judgeScore !== null && judgeScore < 3) {
|
|
338
|
-
log.info(
|
|
339
|
-
`worker model validation: ${candidate.id} failed judge (score=${judgeScore})`,
|
|
340
|
-
);
|
|
341
|
-
continue;
|
|
342
|
-
}
|
|
343
|
-
|
|
344
|
-
// Candidate passed both phases
|
|
345
|
-
const fingerprint = computeModelFingerprint(
|
|
346
|
-
input.providerID,
|
|
347
|
-
input.sessionModelID,
|
|
348
|
-
candidates.map((c) => c.id),
|
|
349
|
-
);
|
|
350
|
-
|
|
351
|
-
const result: WorkerModelResult = {
|
|
352
|
-
modelID: candidate.id,
|
|
353
|
-
providerID: candidate.providerID,
|
|
354
|
-
fingerprint,
|
|
355
|
-
validatedAt: Date.now(),
|
|
356
|
-
judgeScore,
|
|
357
|
-
};
|
|
358
|
-
storeValidatedWorkerModel(result);
|
|
359
|
-
log.info(
|
|
360
|
-
`worker model validated: ${candidate.id} (judge=${judgeScore}) for provider ${input.providerID}`,
|
|
361
|
-
);
|
|
362
|
-
return result;
|
|
363
|
-
}
|
|
364
|
-
|
|
365
|
-
// No candidate passed — clear any stale stored result so we don't keep
|
|
366
|
-
// routing worker calls to a potentially-deprecated model.
|
|
367
|
-
clearValidatedWorkerModel(input.providerID);
|
|
368
|
-
log.info(
|
|
369
|
-
`worker model validation: no candidate passed for ${input.providerID} — cleared stale entry`,
|
|
370
|
-
);
|
|
371
|
-
return null;
|
|
372
|
-
}
|
|
373
|
-
|
|
374
32
|
// ---------------------------------------------------------------------------
|
|
375
33
|
// Effective worker model resolution
|
|
376
34
|
// ---------------------------------------------------------------------------
|
|
377
35
|
|
|
378
36
|
/**
|
|
379
37
|
* Resolve the effective worker model for a given provider.
|
|
380
|
-
* Priority: explicit config
|
|
38
|
+
* Priority: explicit config override > session model (fallback).
|
|
381
39
|
*/
|
|
382
40
|
export function resolveWorkerModel(
|
|
383
|
-
|
|
41
|
+
_providerID: string,
|
|
384
42
|
configWorkerModel?: { providerID: string; modelID: string },
|
|
385
43
|
configModel?: { providerID: string; modelID: string },
|
|
386
44
|
): { providerID: string; modelID: string } | undefined {
|
|
387
45
|
// Explicit override wins
|
|
388
46
|
if (configWorkerModel) return configWorkerModel;
|
|
389
47
|
|
|
390
|
-
// Check for validated auto-selection.
|
|
391
|
-
// Don't trust entries older than 24h — model may have been deprecated.
|
|
392
|
-
// Validation will re-run on next idle cycle and either re-confirm or clear.
|
|
393
|
-
const validated = getValidatedWorkerModel(providerID);
|
|
394
|
-
const MAX_AGE_MS = 24 * 60 * 60 * 1000;
|
|
395
|
-
if (validated && Date.now() - validated.validatedAt <= MAX_AGE_MS) {
|
|
396
|
-
return { providerID: validated.providerID, modelID: validated.modelID };
|
|
397
|
-
}
|
|
398
|
-
|
|
399
48
|
// Fall back to the session model config (or undefined = host default)
|
|
400
49
|
return configModel;
|
|
401
50
|
}
|