@iworkforces/tracelattice 2.1.2 → 2.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -10
- package/dist/ServerConfig.d.ts +9 -5
- package/dist/ServerConfig.d.ts.map +1 -1
- package/dist/ServerConfig.js.map +1 -1
- package/dist/config/ConfigLoader.d.ts +20 -7
- package/dist/config/ConfigLoader.d.ts.map +1 -1
- package/dist/config/ConfigLoader.js.map +1 -1
- package/dist/contracts/PersistenceBackend.d.ts +10 -1
- package/dist/contracts/PersistenceBackend.d.ts.map +1 -1
- package/dist/contracts/calibrator.d.ts +9 -6
- package/dist/contracts/calibrator.d.ts.map +1 -1
- package/dist/contracts/interfaces.d.ts +6 -0
- package/dist/contracts/interfaces.d.ts.map +1 -1
- package/dist/contracts/persistence-work.d.ts +13 -1
- package/dist/contracts/persistence-work.d.ts.map +1 -1
- package/dist/contracts/strategy.d.ts +14 -9
- package/dist/contracts/strategy.d.ts.map +1 -1
- package/dist/core/DurableThoughtIdentityIndex.d.ts +35 -0
- package/dist/core/DurableThoughtIdentityIndex.d.ts.map +1 -0
- package/dist/core/DurableThoughtIdentityIndex.js +94 -0
- package/dist/core/DurableThoughtIdentityIndex.js.map +1 -0
- package/dist/core/HistoryManager.d.ts +5 -0
- package/dist/core/HistoryManager.d.ts.map +1 -1
- package/dist/core/HistoryManager.js +37 -4
- package/dist/core/HistoryManager.js.map +1 -1
- package/dist/core/IHistoryManager.d.ts +2 -0
- package/dist/core/IHistoryManager.d.ts.map +1 -1
- package/dist/core/PersistenceBuffer.d.ts +7 -1
- package/dist/core/PersistenceBuffer.d.ts.map +1 -1
- package/dist/core/PersistenceBuffer.js +13 -1
- package/dist/core/PersistenceBuffer.js.map +1 -1
- package/dist/core/PersistenceRestore.d.ts.map +1 -1
- package/dist/core/PersistenceRestore.js +11 -3
- package/dist/core/PersistenceRestore.js.map +1 -1
- package/dist/core/PersistenceWorkQueue.d.ts +13 -1
- package/dist/core/PersistenceWorkQueue.d.ts.map +1 -1
- package/dist/core/PersistenceWorkQueue.js +86 -11
- package/dist/core/PersistenceWorkQueue.js.map +1 -1
- package/dist/core/PersistenceWriter.d.ts.map +1 -1
- package/dist/core/PersistenceWriter.js +3 -0
- package/dist/core/PersistenceWriter.js.map +1 -1
- package/dist/core/ThoughtEvaluator.d.ts +7 -4
- package/dist/core/ThoughtEvaluator.d.ts.map +1 -1
- package/dist/core/ThoughtEvaluator.js +15 -7
- package/dist/core/ThoughtEvaluator.js.map +1 -1
- package/dist/core/ThoughtProcessor.d.ts.map +1 -1
- package/dist/core/ThoughtProcessor.js +19 -7
- package/dist/core/ThoughtProcessor.js.map +1 -1
- package/dist/core/ThoughtReferenceIndex.d.ts +4 -0
- package/dist/core/ThoughtReferenceIndex.d.ts.map +1 -1
- package/dist/core/ThoughtReferenceIndex.js +28 -1
- package/dist/core/ThoughtReferenceIndex.js.map +1 -1
- package/dist/core/VerificationOutcomeAdmission.d.ts +8 -1
- package/dist/core/VerificationOutcomeAdmission.d.ts.map +1 -1
- package/dist/core/VerificationOutcomeAdmission.js.map +1 -1
- package/dist/core/VerificationTargetRestore.d.ts +5 -0
- package/dist/core/VerificationTargetRestore.d.ts.map +1 -0
- package/dist/core/VerificationTargetRestore.js +61 -0
- package/dist/core/VerificationTargetRestore.js.map +1 -0
- package/dist/core/evaluator/Aggregator.d.ts +2 -1
- package/dist/core/evaluator/Aggregator.d.ts.map +1 -1
- package/dist/core/evaluator/Aggregator.js +4 -12
- package/dist/core/evaluator/Aggregator.js.map +1 -1
- package/dist/core/evaluator/Calibrator.d.ts +13 -9
- package/dist/core/evaluator/Calibrator.d.ts.map +1 -1
- package/dist/core/evaluator/Calibrator.js +25 -11
- package/dist/core/evaluator/Calibrator.js.map +1 -1
- package/dist/core/evaluator/PatternDetector.d.ts +2 -1
- package/dist/core/evaluator/PatternDetector.d.ts.map +1 -1
- package/dist/core/evaluator/PatternDetector.js +13 -12
- package/dist/core/evaluator/PatternDetector.js.map +1 -1
- package/dist/core/evaluator/SignalComputer.d.ts +3 -2
- package/dist/core/evaluator/SignalComputer.d.ts.map +1 -1
- package/dist/core/evaluator/SignalComputer.js +4 -9
- package/dist/core/evaluator/SignalComputer.js.map +1 -1
- package/dist/core/evaluator/VerificationLinks.d.ts +19 -0
- package/dist/core/evaluator/VerificationLinks.d.ts.map +1 -0
- package/dist/core/evaluator/VerificationLinks.js +113 -0
- package/dist/core/evaluator/VerificationLinks.js.map +1 -0
- package/dist/core/evaluator/calibration-math.d.ts +12 -2
- package/dist/core/evaluator/calibration-math.d.ts.map +1 -1
- package/dist/core/evaluator/calibration-math.js +11 -2
- package/dist/core/evaluator/calibration-math.js.map +1 -1
- package/dist/core/graph/EdgeEmitter.d.ts.map +1 -1
- package/dist/core/graph/EdgeEmitter.js +2 -1
- package/dist/core/graph/EdgeEmitter.js.map +1 -1
- package/dist/core/graph/GraphView.d.ts +5 -5
- package/dist/core/graph/GraphView.d.ts.map +1 -1
- package/dist/core/graph/GraphView.js +11 -11
- package/dist/core/graph/GraphView.js.map +1 -1
- package/dist/core/reasoning/ActiveEvidenceProjection.d.ts +20 -0
- package/dist/core/reasoning/ActiveEvidenceProjection.d.ts.map +1 -0
- package/dist/core/reasoning/ActiveEvidenceProjection.js +100 -0
- package/dist/core/reasoning/ActiveEvidenceProjection.js.map +1 -0
- package/dist/core/reasoning/strategies/TreeOfThoughtStrategy.d.ts +5 -4
- package/dist/core/reasoning/strategies/TreeOfThoughtStrategy.d.ts.map +1 -1
- package/dist/core/reasoning/strategies/TreeOfThoughtStrategy.js +24 -14
- package/dist/core/reasoning/strategies/TreeOfThoughtStrategy.js.map +1 -1
- package/dist/lib.d.ts.map +1 -1
- package/dist/lib.js +3 -1
- package/dist/lib.js.map +1 -1
- package/dist/metrics/metrics.impl.d.ts.map +1 -1
- package/dist/metrics/metrics.impl.js +19 -7
- package/dist/metrics/metrics.impl.js.map +1 -1
- package/dist/persistence/BacktrackPersistence.d.ts +13 -0
- package/dist/persistence/BacktrackPersistence.d.ts.map +1 -0
- package/dist/persistence/BacktrackPersistence.js +90 -0
- package/dist/persistence/BacktrackPersistence.js.map +1 -0
- package/dist/persistence/FilePersistence.d.ts +2 -1
- package/dist/persistence/FilePersistence.d.ts.map +1 -1
- package/dist/persistence/FilePersistence.js +28 -0
- package/dist/persistence/FilePersistence.js.map +1 -1
- package/dist/persistence/FileSnapshotV2.d.ts +2 -0
- package/dist/persistence/FileSnapshotV2.d.ts.map +1 -1
- package/dist/persistence/FileSnapshotV2.js +6 -2
- package/dist/persistence/FileSnapshotV2.js.map +1 -1
- package/dist/persistence/MemoryPersistence.d.ts +2 -1
- package/dist/persistence/MemoryPersistence.d.ts.map +1 -1
- package/dist/persistence/MemoryPersistence.js +21 -1
- package/dist/persistence/MemoryPersistence.js.map +1 -1
- package/dist/persistence/PersistenceCodec.d.ts.map +1 -1
- package/dist/persistence/PersistenceCodec.js +5 -0
- package/dist/persistence/PersistenceCodec.js.map +1 -1
- package/dist/persistence/PersistenceErrors.d.ts +1 -1
- package/dist/persistence/PersistenceErrors.d.ts.map +1 -1
- package/dist/persistence/PersistenceErrors.js.map +1 -1
- package/dist/persistence/PersistenceScope.d.ts +1 -1
- package/dist/persistence/PersistenceScope.d.ts.map +1 -1
- package/dist/persistence/PersistenceScope.js.map +1 -1
- package/dist/persistence/SqlitePersistence.d.ts +2 -1
- package/dist/persistence/SqlitePersistence.d.ts.map +1 -1
- package/dist/persistence/SqlitePersistence.js +18 -0
- package/dist/persistence/SqlitePersistence.js.map +1 -1
- package/package.json +12 -11
|
@@ -2,21 +2,25 @@
|
|
|
2
2
|
* Confidence calibration for sequential thinking reasoning.
|
|
3
3
|
*
|
|
4
4
|
* Provides the {@link Calibrator} class — maps raw model confidence values
|
|
5
|
-
* to calibrated probabilities using per-type
|
|
5
|
+
* to calibrated probabilities using per-type empirical evidence blending and
|
|
6
6
|
* grid-searched temperature scaling. Reports calibration quality through
|
|
7
7
|
* Brier score and 10-bin Expected Calibration Error (ECE).
|
|
8
8
|
*
|
|
9
9
|
* Math summary:
|
|
10
|
-
* - **Per-type
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
10
|
+
* - **Per-type evidence blend**: Raw confidence is the prior signal, weighted by
|
|
11
|
+
* `priorWeight = 10 / (10 + n)` where `n` is the type's outcome count. The
|
|
12
|
+
* remaining weight is applied to the observed per-type mean. An empty type
|
|
13
|
+
* defaults its mean to 0.5 but has zero empirical weight, so cold start returns raw.
|
|
14
14
|
* - **Temperature scaling**: Grid search T ∈ {0.5, 0.75, 1.0, 1.25, 1.5, 2.0}
|
|
15
|
-
* minimizing negative log-likelihood (NLL) over
|
|
16
|
-
* Requires
|
|
17
|
-
*
|
|
15
|
+
* minimizing negative log-likelihood (NLL) over per-type leave-one-out evidence blends.
|
|
16
|
+
* Exact ties retain T=1.0, then follow grid order. Requires at least 10 outcomes; below that
|
|
17
|
+
* threshold the evidence blend is returned unchanged.
|
|
18
|
+
* - **Brier score**: `mean((predicted - actual)^2)` over stored raw outcomes.
|
|
18
19
|
* - **ECE (10-bin)**: bucket predictions in 0.1 increments, compute weighted
|
|
19
|
-
* absolute deviation between bin mean confidence and bin accuracy.
|
|
20
|
+
* absolute deviation between raw bin mean confidence and bin accuracy.
|
|
21
|
+
*
|
|
22
|
+
* Stored outcomes and metrics remain raw. This transform does not guarantee factuality, better
|
|
23
|
+
* metrics for every dataset, or monotonic per-example changes.
|
|
20
24
|
*
|
|
21
25
|
* @module core/evaluator/Calibrator
|
|
22
26
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Calibrator.d.ts","sourceRoot":"","sources":["../../../src/core/evaluator/Calibrator.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"Calibrator.d.ts","sourceRoot":"","sources":["../../../src/core/evaluator/Calibrator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,KAAK,EACX,kBAAkB,EAClB,iBAAiB,EACjB,WAAW,EACX,MAAM,+BAA+B,CAAC;AACvC,OAAO,KAAK,EAAE,gBAAgB,EAAuB,MAAM,+BAA+B,CAAC;AAC3F,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oCAAoC,CAAC;AACtE,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AAoIxD;;;;;;;;;;;;;GAaG;AACH,qBAAa,UAAW,YAAW,WAAW;IAC7C,SAAgB,OAAO,EAAE,OAAO,CAAC;IACjC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;IAC7C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAgC;gBAElD,eAAe,EAAE,gBAAgB,EAAE,OAAO,EAAE,OAAO;IAKxD,SAAS,CACf,aAAa,EAAE,MAAM,EACrB,IAAI,EAAE,WAAW,EACjB,SAAS,EAAE,SAAS,GAClB,iBAAiB;IAmBb,OAAO,CAAC,SAAS,CAAC,EAAE,SAAS,GAAG,kBAAkB;IAclD,KAAK,CAAC,SAAS,EAAE,SAAS,GAAG,IAAI;IAMjC,YAAY,CAAC,SAAS,EAAE,SAAS,GAAG,IAAI;IAKxC,QAAQ,IAAI,IAAI;CAIvB"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { EPSILON, MIN_OUTCOMES_FOR_TEMPERATURE, applyTemperature, fitTemperature } from "./calibration-math.js";
|
|
1
|
+
import { EPSILON, MIN_OUTCOMES_FOR_TEMPERATURE, applyTemperature, blendWithRawPrior, fitTemperature } from "./calibration-math.js";
|
|
2
2
|
import { ALL_THOUGHT_TYPES } from "./internals.js";
|
|
3
3
|
const ECE_BINS = 10;
|
|
4
4
|
function aggregatePerType(outcomes) {
|
|
@@ -12,12 +12,26 @@ function aggregatePerType(outcomes) {
|
|
|
12
12
|
prev.count += 1;
|
|
13
13
|
sums.set(o.type, prev);
|
|
14
14
|
}
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
15
|
+
return sums;
|
|
16
|
+
}
|
|
17
|
+
function leaveOneOutBlendedOutcomes(outcomes) {
|
|
18
|
+
const perType = aggregatePerType(outcomes);
|
|
19
|
+
return outcomes.map((outcome)=>{
|
|
20
|
+
const typeStats = perType.get(outcome.type) ?? {
|
|
21
|
+
sum: outcome.actual,
|
|
22
|
+
count: 1
|
|
23
|
+
};
|
|
24
|
+
const sampleCount = typeStats.count - 1;
|
|
25
|
+
if (0 === sampleCount) return {
|
|
26
|
+
...outcome
|
|
27
|
+
};
|
|
28
|
+
const empiricalMean = (typeStats.sum - outcome.actual) / sampleCount;
|
|
29
|
+
const { blended } = blendWithRawPrior(outcome.predicted, empiricalMean, sampleCount);
|
|
30
|
+
return {
|
|
31
|
+
...outcome,
|
|
32
|
+
predicted: blended
|
|
33
|
+
};
|
|
19
34
|
});
|
|
20
|
-
return result;
|
|
21
35
|
}
|
|
22
36
|
function brierScore(outcomes) {
|
|
23
37
|
if (0 === outcomes.length) return null;
|
|
@@ -92,11 +106,10 @@ class Calibrator {
|
|
|
92
106
|
const perType = aggregatePerType(outcomes);
|
|
93
107
|
const typeStats = perType.get(type);
|
|
94
108
|
const n = typeStats?.count ?? 0;
|
|
95
|
-
const observedMean = typeStats
|
|
96
|
-
const priorWeight =
|
|
97
|
-
const shrunk = priorWeight * observedMean + (1 - priorWeight) * raw;
|
|
109
|
+
const observedMean = void 0 === typeStats ? 0.5 : typeStats.sum / typeStats.count;
|
|
110
|
+
const { blended, priorWeight } = blendWithRawPrior(raw, observedMean, n);
|
|
98
111
|
const temperature = this._temperatures.get(sessionId) ?? 1.0;
|
|
99
|
-
const calibrated = outcomes.length >= MIN_OUTCOMES_FOR_TEMPERATURE ? applyTemperature(
|
|
112
|
+
const calibrated = outcomes.length >= MIN_OUTCOMES_FOR_TEMPERATURE ? applyTemperature(blended, temperature) : blended;
|
|
100
113
|
return {
|
|
101
114
|
raw,
|
|
102
115
|
calibrated,
|
|
@@ -116,7 +129,8 @@ class Calibrator {
|
|
|
116
129
|
}
|
|
117
130
|
refit(sessionId) {
|
|
118
131
|
if (!this.enabled) return;
|
|
119
|
-
|
|
132
|
+
const outcomes = this._recorder.getOutcomes(sessionId);
|
|
133
|
+
this._temperatures.set(sessionId, fitTemperature(leaveOneOutBlendedOutcomes(outcomes)));
|
|
120
134
|
}
|
|
121
135
|
clearSession(sessionId) {
|
|
122
136
|
if (!this.enabled) return;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"core/evaluator/Calibrator.js","sources":["../../../src/core/evaluator/Calibrator.ts"],"sourcesContent":["/**\n * Confidence calibration for sequential thinking reasoning.\n *\n * Provides the {@link Calibrator} class — maps raw model confidence values\n * to calibrated probabilities using per-type Beta(2, 2) priors and\n * grid-searched temperature scaling. Reports calibration quality through\n * Brier score and 10-bin Expected Calibration Error (ECE).\n *\n * Math summary:\n * - **Per-type prior**: Beta(α=2, β=2), prior mean = 0.5. As outcomes accumulate\n * for a type, the calibrated value shrinks toward the observed mean of that\n * type with weight `priorWeight = 1 / (1 + n / 10)` where `n` is the type's\n * sample count.\n * - **Temperature scaling**: Grid search T ∈ {0.5, 0.75, 1.0, 1.25, 1.5, 2.0}\n * minimizing negative log-likelihood (NLL) over recorded outcomes. Default T=1.0.\n * Requires ≥10 outcomes; below that threshold only prior shrinkage is applied.\n * - **Brier score**: `mean((predicted - actual)^2)` over recorded outcomes.\n * - **ECE (10-bin)**: bucket predictions in 0.1 increments, compute weighted\n * absolute deviation between bin mean confidence and bin accuracy.\n *\n * @module core/evaluator/Calibrator\n */\n\nimport type {\n\tCalibrationMetrics,\n\tCalibrationResult,\n\tICalibrator,\n} from '../../contracts/calibrator.js';\nimport type { IOutcomeRecorder, VerificationOutcome } from '../../contracts/interfaces.js';\nimport type { ThoughtType } from '../../contracts/reasoning-types.js';\nimport type { SessionId } from '../../contracts/ids.js';\nimport {\n\tEPSILON,\n\tMIN_OUTCOMES_FOR_TEMPERATURE,\n\tapplyTemperature,\n\tfitTemperature,\n} from './calibration-math.js';\nimport { ALL_THOUGHT_TYPES } from './internals.js';\n\n/** Number of bins used by ECE (10-bin → 0.1 increments). */\nconst ECE_BINS = 10;\n\n/**\n * Build per-type empirical means + counts from a list of outcomes.\n *\n * @param outcomes - Outcomes to aggregate.\n * @returns Map of thought type to `{ mean, count }`. Types with no outcomes\n * are absent from the map.\n */\nfunction aggregatePerType(\n\toutcomes: readonly VerificationOutcome[]\n): Map<string, { mean: number; count: number }> {\n\tconst sums = new Map<string, { sum: number; count: number }>();\n\tfor (const o of outcomes) {\n\t\tconst prev = sums.get(o.type) ?? { sum: 0, count: 0 };\n\t\tprev.sum += o.actual;\n\t\tprev.count += 1;\n\t\tsums.set(o.type, prev);\n\t}\n\tconst result = new Map<string, { mean: number; count: number }>();\n\tfor (const [type, { sum, count }] of sums) {\n\t\tresult.set(type, { mean: sum / count, count });\n\t}\n\treturn result;\n}\n\n/**\n * Compute the Brier score over a list of outcomes.\n *\n * @param outcomes - Outcomes to score.\n * @returns Mean squared error between predicted probability and actual label,\n * or `null` if `outcomes` is empty.\n */\nfunction brierScore(outcomes: readonly VerificationOutcome[]): number | null {\n\tif (outcomes.length === 0) return null;\n\tlet sum = 0;\n\tfor (const o of outcomes) {\n\t\tconst diff = o.predicted - o.actual;\n\t\tsum += diff * diff;\n\t}\n\treturn sum / outcomes.length;\n}\n\n/**\n * Compute 10-bin Expected Calibration Error.\n *\n * Buckets predictions in 0.1-wide bins, then sums weighted absolute\n * differences between bin mean confidence and bin accuracy.\n *\n * @param outcomes - Outcomes to evaluate.\n * @returns ECE in `[0, 1]`, or `null` if `outcomes` is empty.\n */\nfunction expectedCalibrationError(outcomes: readonly VerificationOutcome[]): number | null {\n\tif (outcomes.length === 0) return null;\n\tconst binConfSum = new Array<number>(ECE_BINS).fill(0);\n\tconst binAccSum = new Array<number>(ECE_BINS).fill(0);\n\tconst binCount = new Array<number>(ECE_BINS).fill(0);\n\tfor (const o of outcomes) {\n\t\tconst p = Math.min(1 - EPSILON, Math.max(0, o.predicted));\n\t\tconst idx = Math.min(ECE_BINS - 1, Math.floor(p * ECE_BINS));\n\t\tbinConfSum[idx] = (binConfSum[idx] ?? 0) + p;\n\t\tbinAccSum[idx] = (binAccSum[idx] ?? 0) + o.actual;\n\t\tbinCount[idx] = (binCount[idx] ?? 0) + 1;\n\t}\n\tconst total = outcomes.length;\n\tlet ece = 0;\n\tfor (let i = 0; i < ECE_BINS; i++) {\n\t\tconst n = binCount[i] ?? 0;\n\t\tif (n === 0) continue;\n\t\tconst meanConf = (binConfSum[i] ?? 0) / n;\n\t\tconst meanAcc = (binAccSum[i] ?? 0) / n;\n\t\tece += (n / total) * Math.abs(meanConf - meanAcc);\n\t}\n\treturn ece;\n}\n\n/**\n * Build the per-type Brier score breakdown for {@link CalibrationMetrics}.\n *\n * @param outcomes - Outcomes to bucket by type.\n * @returns Record keyed by every {@link ThoughtType}; `null` for types with\n * no recorded outcomes.\n */\nfunction perTypeBrier(\n\toutcomes: readonly VerificationOutcome[]\n): Record<ThoughtType, number | null> {\n\tconst buckets = new Map<string, VerificationOutcome[]>();\n\tfor (const o of outcomes) {\n\t\tconst list = buckets.get(o.type) ?? [];\n\t\tlist.push(o);\n\t\tbuckets.set(o.type, list);\n\t}\n\tconst result = {} as Record<ThoughtType, number | null>;\n\tfor (const t of ALL_THOUGHT_TYPES) {\n\t\tresult[t] = brierScore(buckets.get(t) ?? []);\n\t}\n\treturn result;\n}\n\n/** Empty CalibrationMetrics returned when calibration is disabled. */\nfunction emptyMetrics(): CalibrationMetrics {\n\tconst perType = {} as Record<ThoughtType, number | null>;\n\tfor (const t of ALL_THOUGHT_TYPES) perType[t] = null;\n\treturn {\n\t\tbrierScore: null,\n\t\tece: null,\n\t\tsampleCount: 0,\n\t\tperTypeBrier: perType,\n\t};\n}\n\n/**\n * Confidence calibration service.\n *\n * Stateless w.r.t. outcome storage (delegated to {@link IOutcomeRecorder}),\n * but maintains a small per-session temperature cache that is recomputed\n * via {@link Calibrator.refit}.\n *\n * @example\n * ```typescript\n * const calibrator = new Calibrator(outcomeRecorder, true);\n * const result = calibrator.calibrate(0.9, 'hypothesis', 'session-1');\n * console.log(result.calibrated, result.temperature, result.priorWeight);\n * ```\n */\nexport class Calibrator implements ICalibrator {\n\tpublic readonly enabled: boolean;\n\tprivate readonly _recorder: IOutcomeRecorder;\n\tprivate readonly _temperatures = new Map<SessionId, number>();\n\n\tconstructor(outcomeRecorder: IOutcomeRecorder, enabled: boolean) {\n\t\tthis._recorder = outcomeRecorder;\n\t\tthis.enabled = enabled;\n\t}\n\n\tpublic calibrate(\n\t\trawConfidence: number,\n\t\ttype: ThoughtType,\n\t\tsessionId: SessionId\n\t): CalibrationResult {\n\t\tconst raw = Math.min(1, Math.max(0, rawConfidence));\n\t\tif (!this.enabled) {\n\t\t\treturn { raw, calibrated: raw, temperature: 1.0, priorWeight: 0 };\n\t\t}\n\t\tconst outcomes = this._recorder.getOutcomes(sessionId);\n\t\tconst perType = aggregatePerType(outcomes);\n\t\tconst typeStats = perType.get(type);\n\t\tconst n = typeStats?.count ?? 0;\n\t\tconst observedMean = typeStats?.mean ?? 0.5;\n\t\tconst priorWeight = 1 / (1 + n / 10);\n\t\tconst shrunk = priorWeight * observedMean + (1 - priorWeight) * raw;\n\t\tconst temperature = this._temperatures.get(sessionId) ?? 1.0;\n\t\tconst calibrated =\n\t\t\toutcomes.length >= MIN_OUTCOMES_FOR_TEMPERATURE\n\t\t\t\t? applyTemperature(shrunk, temperature)\n\t\t\t\t: shrunk;\n\t\treturn { raw, calibrated, temperature, priorWeight };\n\t}\n\n\tpublic metrics(sessionId?: SessionId): CalibrationMetrics {\n\t\tif (!this.enabled) return emptyMetrics();\n\t\tconst outcomes =\n\t\t\tsessionId === undefined\n\t\t\t\t? this._recorder.getAllOutcomes()\n\t\t\t\t: this._recorder.getOutcomes(sessionId);\n\t\treturn {\n\t\t\tbrierScore: brierScore(outcomes),\n\t\t\tece: expectedCalibrationError(outcomes),\n\t\t\tsampleCount: outcomes.length,\n\t\t\tperTypeBrier: perTypeBrier(outcomes),\n\t\t};\n\t}\n\n\tpublic refit(sessionId: SessionId): void {\n\t\tif (!this.enabled) return;\n\t\tthis._temperatures.set(sessionId, fitTemperature(this._recorder.getOutcomes(sessionId)));\n\t}\n\n\tpublic clearSession(sessionId: SessionId): void {\n\t\tif (!this.enabled) return;\n\t\tthis._temperatures.delete(sessionId);\n\t}\n\n\tpublic clearAll(): void {\n\t\tif (!this.enabled) return;\n\t\tthis._temperatures.clear();\n\t}\n}\n"],"names":["ECE_BINS","aggregatePerType","outcomes","sums","Map","o","prev","result","type","sum","count","brierScore","diff","expectedCalibrationError","binConfSum","Array","binAccSum","binCount","p","Math","EPSILON","idx","total","ece","i","n","meanConf","meanAcc","perTypeBrier","buckets","list","t","ALL_THOUGHT_TYPES","emptyMetrics","perType","Calibrator","outcomeRecorder","enabled","rawConfidence","sessionId","raw","typeStats","observedMean","priorWeight","shrunk","temperature","calibrated","MIN_OUTCOMES_FOR_TEMPERATURE","applyTemperature","undefined","fitTemperature"],"mappings":";;AAwCA,MAAMA,WAAW;AASjB,SAASC,iBACRC,QAAwC;IAExC,MAAMC,OAAO,IAAIC;IACjB,KAAK,MAAMC,KAAKH,SAAU;QACzB,MAAMI,OAAOH,KAAK,GAAG,CAACE,EAAE,IAAI,KAAK;YAAE,KAAK;YAAG,OAAO;QAAE;QACpDC,KAAK,GAAG,IAAID,EAAE,MAAM;QACpBC,KAAK,KAAK,IAAI;QACdH,KAAK,GAAG,CAACE,EAAE,IAAI,EAAEC;IAClB;IACA,MAAMC,SAAS,IAAIH;IACnB,KAAK,MAAM,CAACI,MAAM,EAAEC,GAAG,EAAEC,KAAK,EAAE,CAAC,IAAIP,KACpCI,OAAO,GAAG,CAACC,MAAM;QAAE,MAAMC,MAAMC;QAAOA;IAAM;IAE7C,OAAOH;AACR;AASA,SAASI,WAAWT,QAAwC;IAC3D,IAAIA,AAAoB,MAApBA,SAAS,MAAM,EAAQ,OAAO;IAClC,IAAIO,MAAM;IACV,KAAK,MAAMJ,KAAKH,SAAU;QACzB,MAAMU,OAAOP,EAAE,SAAS,GAAGA,EAAE,MAAM;QACnCI,OAAOG,OAAOA;IACf;IACA,OAAOH,MAAMP,SAAS,MAAM;AAC7B;AAWA,SAASW,yBAAyBX,QAAwC;IACzE,IAAIA,AAAoB,MAApBA,SAAS,MAAM,EAAQ,OAAO;IAClC,MAAMY,aAAa,IAAIC,MAAcf,UAAU,IAAI,CAAC;IACpD,MAAMgB,YAAY,IAAID,MAAcf,UAAU,IAAI,CAAC;IACnD,MAAMiB,WAAW,IAAIF,MAAcf,UAAU,IAAI,CAAC;IAClD,KAAK,MAAMK,KAAKH,SAAU;QACzB,MAAMgB,IAAIC,KAAK,GAAG,CAAC,IAAIC,SAASD,KAAK,GAAG,CAAC,GAAGd,EAAE,SAAS;QACvD,MAAMgB,MAAMF,KAAK,GAAG,CAACnB,WAAW,GAAGmB,KAAK,KAAK,CAACD,IAAIlB;QAClDc,UAAU,CAACO,IAAI,GAAIP,AAAAA,CAAAA,UAAU,CAACO,IAAI,IAAI,KAAKH;QAC3CF,SAAS,CAACK,IAAI,GAAIL,AAAAA,CAAAA,SAAS,CAACK,IAAI,IAAI,KAAKhB,EAAE,MAAM;QACjDY,QAAQ,CAACI,IAAI,GAAIJ,AAAAA,CAAAA,QAAQ,CAACI,IAAI,IAAI,KAAK;IACxC;IACA,MAAMC,QAAQpB,SAAS,MAAM;IAC7B,IAAIqB,MAAM;IACV,IAAK,IAAIC,IAAI,GAAGA,IAAIxB,UAAUwB,IAAK;QAClC,MAAMC,IAAIR,QAAQ,CAACO,EAAE,IAAI;QACzB,IAAIC,AAAM,MAANA,GAAS;QACb,MAAMC,WAAYZ,AAAAA,CAAAA,UAAU,CAACU,EAAE,IAAI,KAAKC;QACxC,MAAME,UAAWX,AAAAA,CAAAA,SAAS,CAACQ,EAAE,IAAI,KAAKC;QACtCF,OAAQE,IAAIH,QAASH,KAAK,GAAG,CAACO,WAAWC;IAC1C;IACA,OAAOJ;AACR;AASA,SAASK,aACR1B,QAAwC;IAExC,MAAM2B,UAAU,IAAIzB;IACpB,KAAK,MAAMC,KAAKH,SAAU;QACzB,MAAM4B,OAAOD,QAAQ,GAAG,CAACxB,EAAE,IAAI,KAAK,EAAE;QACtCyB,KAAK,IAAI,CAACzB;QACVwB,QAAQ,GAAG,CAACxB,EAAE,IAAI,EAAEyB;IACrB;IACA,MAAMvB,SAAS,CAAC;IAChB,KAAK,MAAMwB,KAAKC,kBACfzB,MAAM,CAACwB,EAAE,GAAGpB,WAAWkB,QAAQ,GAAG,CAACE,MAAM,EAAE;IAE5C,OAAOxB;AACR;AAGA,SAAS0B;IACR,MAAMC,UAAU,CAAC;IACjB,KAAK,MAAMH,KAAKC,kBAAmBE,OAAO,CAACH,EAAE,GAAG;IAChD,OAAO;QACN,YAAY;QACZ,KAAK;QACL,aAAa;QACb,cAAcG;IACf;AACD;AAgBO,MAAMC;IACI,QAAiB;IAChB,UAA4B;IAC5B,gBAAgB,IAAI/B,MAAyB;IAE9D,YAAYgC,eAAiC,EAAEC,OAAgB,CAAE;QAChE,IAAI,CAAC,SAAS,GAAGD;QACjB,IAAI,CAAC,OAAO,GAAGC;IAChB;IAEO,UACNC,aAAqB,EACrB9B,IAAiB,EACjB+B,SAAoB,EACA;QACpB,MAAMC,MAAMrB,KAAK,GAAG,CAAC,GAAGA,KAAK,GAAG,CAAC,GAAGmB;QACpC,IAAI,CAAC,IAAI,CAAC,OAAO,EAChB,OAAO;YAAEE;YAAK,YAAYA;YAAK,aAAa;YAAK,aAAa;QAAE;QAEjE,MAAMtC,WAAW,IAAI,CAAC,SAAS,CAAC,WAAW,CAACqC;QAC5C,MAAML,UAAUjC,iBAAiBC;QACjC,MAAMuC,YAAYP,QAAQ,GAAG,CAAC1B;QAC9B,MAAMiB,IAAIgB,WAAW,SAAS;QAC9B,MAAMC,eAAeD,WAAW,QAAQ;QACxC,MAAME,cAAc,IAAK,KAAIlB,IAAI,EAAC;QAClC,MAAMmB,SAASD,cAAcD,eAAgB,KAAIC,WAAU,IAAKH;QAChE,MAAMK,cAAc,IAAI,CAAC,aAAa,CAAC,GAAG,CAACN,cAAc;QACzD,MAAMO,aACL5C,SAAS,MAAM,IAAI6C,+BAChBC,iBAAiBJ,QAAQC,eACzBD;QACJ,OAAO;YAAEJ;YAAKM;YAAYD;YAAaF;QAAY;IACpD;IAEO,QAAQJ,SAAqB,EAAsB;QACzD,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,OAAON;QAC1B,MAAM/B,WACLqC,AAAcU,WAAdV,YACG,IAAI,CAAC,SAAS,CAAC,cAAc,KAC7B,IAAI,CAAC,SAAS,CAAC,WAAW,CAACA;QAC/B,OAAO;YACN,YAAY5B,WAAWT;YACvB,KAAKW,yBAAyBX;YAC9B,aAAaA,SAAS,MAAM;YAC5B,cAAc0B,aAAa1B;QAC5B;IACD;IAEO,MAAMqC,SAAoB,EAAQ;QACxC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;QACnB,IAAI,CAAC,aAAa,CAAC,GAAG,CAACA,WAAWW,eAAe,IAAI,CAAC,SAAS,CAAC,WAAW,CAACX;IAC7E;IAEO,aAAaA,SAAoB,EAAQ;QAC/C,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;QACnB,IAAI,CAAC,aAAa,CAAC,MAAM,CAACA;IAC3B;IAEO,WAAiB;QACvB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;QACnB,IAAI,CAAC,aAAa,CAAC,KAAK;IACzB;AACD"}
|
|
1
|
+
{"version":3,"file":"core/evaluator/Calibrator.js","sources":["../../../src/core/evaluator/Calibrator.ts"],"sourcesContent":["/**\n * Confidence calibration for sequential thinking reasoning.\n *\n * Provides the {@link Calibrator} class — maps raw model confidence values\n * to calibrated probabilities using per-type empirical evidence blending and\n * grid-searched temperature scaling. Reports calibration quality through\n * Brier score and 10-bin Expected Calibration Error (ECE).\n *\n * Math summary:\n * - **Per-type evidence blend**: Raw confidence is the prior signal, weighted by\n * `priorWeight = 10 / (10 + n)` where `n` is the type's outcome count. The\n * remaining weight is applied to the observed per-type mean. An empty type\n * defaults its mean to 0.5 but has zero empirical weight, so cold start returns raw.\n * - **Temperature scaling**: Grid search T ∈ {0.5, 0.75, 1.0, 1.25, 1.5, 2.0}\n * minimizing negative log-likelihood (NLL) over per-type leave-one-out evidence blends.\n * Exact ties retain T=1.0, then follow grid order. Requires at least 10 outcomes; below that\n * threshold the evidence blend is returned unchanged.\n * - **Brier score**: `mean((predicted - actual)^2)` over stored raw outcomes.\n * - **ECE (10-bin)**: bucket predictions in 0.1 increments, compute weighted\n * absolute deviation between raw bin mean confidence and bin accuracy.\n *\n * Stored outcomes and metrics remain raw. This transform does not guarantee factuality, better\n * metrics for every dataset, or monotonic per-example changes.\n *\n * @module core/evaluator/Calibrator\n */\n\nimport type {\n\tCalibrationMetrics,\n\tCalibrationResult,\n\tICalibrator,\n} from '../../contracts/calibrator.js';\nimport type { IOutcomeRecorder, VerificationOutcome } from '../../contracts/interfaces.js';\nimport type { ThoughtType } from '../../contracts/reasoning-types.js';\nimport type { SessionId } from '../../contracts/ids.js';\nimport {\n\tEPSILON,\n\tMIN_OUTCOMES_FOR_TEMPERATURE,\n\tapplyTemperature,\n\tblendWithRawPrior,\n\tfitTemperature,\n} from './calibration-math.js';\nimport { ALL_THOUGHT_TYPES } from './internals.js';\n\n/** Number of bins used by ECE (10-bin → 0.1 increments). */\nconst ECE_BINS = 10;\n\n/**\n * Build per-type empirical sums + counts from a list of outcomes.\n *\n * @param outcomes - Outcomes to aggregate.\n * @returns Map of thought type to `{ sum, count }`. Types with no outcomes\n * are absent from the map.\n */\nfunction aggregatePerType(\n\toutcomes: readonly VerificationOutcome[]\n): Map<ThoughtType, { sum: number; count: number }> {\n\tconst sums = new Map<ThoughtType, { sum: number; count: number }>();\n\tfor (const o of outcomes) {\n\t\tconst prev = sums.get(o.type) ?? { sum: 0, count: 0 };\n\t\tprev.sum += o.actual;\n\t\tprev.count += 1;\n\t\tsums.set(o.type, prev);\n\t}\n\treturn sums;\n}\n\nfunction leaveOneOutBlendedOutcomes(\n\toutcomes: readonly VerificationOutcome[]\n): VerificationOutcome[] {\n\tconst perType = aggregatePerType(outcomes);\n\treturn outcomes.map((outcome) => {\n\t\tconst typeStats = perType.get(outcome.type) ?? { sum: outcome.actual, count: 1 };\n\t\tconst sampleCount = typeStats.count - 1;\n\t\tif (sampleCount === 0) return { ...outcome };\n\t\tconst empiricalMean = (typeStats.sum - outcome.actual) / sampleCount;\n\t\tconst { blended } = blendWithRawPrior(outcome.predicted, empiricalMean, sampleCount);\n\t\treturn { ...outcome, predicted: blended };\n\t});\n}\n\n/**\n * Compute the Brier score over a list of outcomes.\n *\n * @param outcomes - Outcomes to score.\n * @returns Mean squared error between predicted probability and actual label,\n * or `null` if `outcomes` is empty.\n */\nfunction brierScore(outcomes: readonly VerificationOutcome[]): number | null {\n\tif (outcomes.length === 0) return null;\n\tlet sum = 0;\n\tfor (const o of outcomes) {\n\t\tconst diff = o.predicted - o.actual;\n\t\tsum += diff * diff;\n\t}\n\treturn sum / outcomes.length;\n}\n\n/**\n * Compute 10-bin Expected Calibration Error.\n *\n * Buckets predictions in 0.1-wide bins, then sums weighted absolute\n * differences between bin mean confidence and bin accuracy.\n *\n * @param outcomes - Outcomes to evaluate.\n * @returns ECE in `[0, 1]`, or `null` if `outcomes` is empty.\n */\nfunction expectedCalibrationError(outcomes: readonly VerificationOutcome[]): number | null {\n\tif (outcomes.length === 0) return null;\n\tconst binConfSum = new Array<number>(ECE_BINS).fill(0);\n\tconst binAccSum = new Array<number>(ECE_BINS).fill(0);\n\tconst binCount = new Array<number>(ECE_BINS).fill(0);\n\tfor (const o of outcomes) {\n\t\tconst p = Math.min(1 - EPSILON, Math.max(0, o.predicted));\n\t\tconst idx = Math.min(ECE_BINS - 1, Math.floor(p * ECE_BINS));\n\t\tbinConfSum[idx] = (binConfSum[idx] ?? 0) + p;\n\t\tbinAccSum[idx] = (binAccSum[idx] ?? 0) + o.actual;\n\t\tbinCount[idx] = (binCount[idx] ?? 0) + 1;\n\t}\n\tconst total = outcomes.length;\n\tlet ece = 0;\n\tfor (let i = 0; i < ECE_BINS; i++) {\n\t\tconst n = binCount[i] ?? 0;\n\t\tif (n === 0) continue;\n\t\tconst meanConf = (binConfSum[i] ?? 0) / n;\n\t\tconst meanAcc = (binAccSum[i] ?? 0) / n;\n\t\tece += (n / total) * Math.abs(meanConf - meanAcc);\n\t}\n\treturn ece;\n}\n\n/**\n * Build the per-type Brier score breakdown for {@link CalibrationMetrics}.\n *\n * @param outcomes - Outcomes to bucket by type.\n * @returns Record keyed by every {@link ThoughtType}; `null` for types with\n * no recorded outcomes.\n */\nfunction perTypeBrier(\n\toutcomes: readonly VerificationOutcome[]\n): Record<ThoughtType, number | null> {\n\tconst buckets = new Map<string, VerificationOutcome[]>();\n\tfor (const o of outcomes) {\n\t\tconst list = buckets.get(o.type) ?? [];\n\t\tlist.push(o);\n\t\tbuckets.set(o.type, list);\n\t}\n\tconst result = {} as Record<ThoughtType, number | null>;\n\tfor (const t of ALL_THOUGHT_TYPES) {\n\t\tresult[t] = brierScore(buckets.get(t) ?? []);\n\t}\n\treturn result;\n}\n\n/** Empty CalibrationMetrics returned when calibration is disabled. */\nfunction emptyMetrics(): CalibrationMetrics {\n\tconst perType = {} as Record<ThoughtType, number | null>;\n\tfor (const t of ALL_THOUGHT_TYPES) perType[t] = null;\n\treturn {\n\t\tbrierScore: null,\n\t\tece: null,\n\t\tsampleCount: 0,\n\t\tperTypeBrier: perType,\n\t};\n}\n\n/**\n * Confidence calibration service.\n *\n * Stateless w.r.t. outcome storage (delegated to {@link IOutcomeRecorder}),\n * but maintains a small per-session temperature cache that is recomputed\n * via {@link Calibrator.refit}.\n *\n * @example\n * ```typescript\n * const calibrator = new Calibrator(outcomeRecorder, true);\n * const result = calibrator.calibrate(0.9, 'hypothesis', 'session-1');\n * console.log(result.calibrated, result.temperature, result.priorWeight);\n * ```\n */\nexport class Calibrator implements ICalibrator {\n\tpublic readonly enabled: boolean;\n\tprivate readonly _recorder: IOutcomeRecorder;\n\tprivate readonly _temperatures = new Map<SessionId, number>();\n\n\tconstructor(outcomeRecorder: IOutcomeRecorder, enabled: boolean) {\n\t\tthis._recorder = outcomeRecorder;\n\t\tthis.enabled = enabled;\n\t}\n\n\tpublic calibrate(\n\t\trawConfidence: number,\n\t\ttype: ThoughtType,\n\t\tsessionId: SessionId\n\t): CalibrationResult {\n\t\tconst raw = Math.min(1, Math.max(0, rawConfidence));\n\t\tif (!this.enabled) {\n\t\t\treturn { raw, calibrated: raw, temperature: 1.0, priorWeight: 0 };\n\t\t}\n\t\tconst outcomes = this._recorder.getOutcomes(sessionId);\n\t\tconst perType = aggregatePerType(outcomes);\n\t\tconst typeStats = perType.get(type);\n\t\tconst n = typeStats?.count ?? 0;\n\t\tconst observedMean = typeStats === undefined ? 0.5 : typeStats.sum / typeStats.count;\n\t\tconst { blended, priorWeight } = blendWithRawPrior(raw, observedMean, n);\n\t\tconst temperature = this._temperatures.get(sessionId) ?? 1.0;\n\t\tconst calibrated =\n\t\t\toutcomes.length >= MIN_OUTCOMES_FOR_TEMPERATURE\n\t\t\t\t? applyTemperature(blended, temperature)\n\t\t\t\t: blended;\n\t\treturn { raw, calibrated, temperature, priorWeight };\n\t}\n\n\tpublic metrics(sessionId?: SessionId): CalibrationMetrics {\n\t\tif (!this.enabled) return emptyMetrics();\n\t\tconst outcomes =\n\t\t\tsessionId === undefined\n\t\t\t\t? this._recorder.getAllOutcomes()\n\t\t\t\t: this._recorder.getOutcomes(sessionId);\n\t\treturn {\n\t\t\tbrierScore: brierScore(outcomes),\n\t\t\tece: expectedCalibrationError(outcomes),\n\t\t\tsampleCount: outcomes.length,\n\t\t\tperTypeBrier: perTypeBrier(outcomes),\n\t\t};\n\t}\n\n\tpublic refit(sessionId: SessionId): void {\n\t\tif (!this.enabled) return;\n\t\tconst outcomes = this._recorder.getOutcomes(sessionId);\n\t\tthis._temperatures.set(sessionId, fitTemperature(leaveOneOutBlendedOutcomes(outcomes)));\n\t}\n\n\tpublic clearSession(sessionId: SessionId): void {\n\t\tif (!this.enabled) return;\n\t\tthis._temperatures.delete(sessionId);\n\t}\n\n\tpublic clearAll(): void {\n\t\tif (!this.enabled) return;\n\t\tthis._temperatures.clear();\n\t}\n}\n"],"names":["ECE_BINS","aggregatePerType","outcomes","sums","Map","o","prev","leaveOneOutBlendedOutcomes","perType","outcome","typeStats","sampleCount","empiricalMean","blended","blendWithRawPrior","brierScore","sum","diff","expectedCalibrationError","binConfSum","Array","binAccSum","binCount","p","Math","EPSILON","idx","total","ece","i","n","meanConf","meanAcc","perTypeBrier","buckets","list","result","t","ALL_THOUGHT_TYPES","emptyMetrics","Calibrator","outcomeRecorder","enabled","rawConfidence","type","sessionId","raw","observedMean","undefined","priorWeight","temperature","calibrated","MIN_OUTCOMES_FOR_TEMPERATURE","applyTemperature","fitTemperature"],"mappings":";;AA6CA,MAAMA,WAAW;AASjB,SAASC,iBACRC,QAAwC;IAExC,MAAMC,OAAO,IAAIC;IACjB,KAAK,MAAMC,KAAKH,SAAU;QACzB,MAAMI,OAAOH,KAAK,GAAG,CAACE,EAAE,IAAI,KAAK;YAAE,KAAK;YAAG,OAAO;QAAE;QACpDC,KAAK,GAAG,IAAID,EAAE,MAAM;QACpBC,KAAK,KAAK,IAAI;QACdH,KAAK,GAAG,CAACE,EAAE,IAAI,EAAEC;IAClB;IACA,OAAOH;AACR;AAEA,SAASI,2BACRL,QAAwC;IAExC,MAAMM,UAAUP,iBAAiBC;IACjC,OAAOA,SAAS,GAAG,CAAC,CAACO;QACpB,MAAMC,YAAYF,QAAQ,GAAG,CAACC,QAAQ,IAAI,KAAK;YAAE,KAAKA,QAAQ,MAAM;YAAE,OAAO;QAAE;QAC/E,MAAME,cAAcD,UAAU,KAAK,GAAG;QACtC,IAAIC,AAAgB,MAAhBA,aAAmB,OAAO;YAAE,GAAGF,OAAO;QAAC;QAC3C,MAAMG,gBAAiBF,AAAAA,CAAAA,UAAU,GAAG,GAAGD,QAAQ,MAAK,IAAKE;QACzD,MAAM,EAAEE,OAAO,EAAE,GAAGC,kBAAkBL,QAAQ,SAAS,EAAEG,eAAeD;QACxE,OAAO;YAAE,GAAGF,OAAO;YAAE,WAAWI;QAAQ;IACzC;AACD;AASA,SAASE,WAAWb,QAAwC;IAC3D,IAAIA,AAAoB,MAApBA,SAAS,MAAM,EAAQ,OAAO;IAClC,IAAIc,MAAM;IACV,KAAK,MAAMX,KAAKH,SAAU;QACzB,MAAMe,OAAOZ,EAAE,SAAS,GAAGA,EAAE,MAAM;QACnCW,OAAOC,OAAOA;IACf;IACA,OAAOD,MAAMd,SAAS,MAAM;AAC7B;AAWA,SAASgB,yBAAyBhB,QAAwC;IACzE,IAAIA,AAAoB,MAApBA,SAAS,MAAM,EAAQ,OAAO;IAClC,MAAMiB,aAAa,IAAIC,MAAcpB,UAAU,IAAI,CAAC;IACpD,MAAMqB,YAAY,IAAID,MAAcpB,UAAU,IAAI,CAAC;IACnD,MAAMsB,WAAW,IAAIF,MAAcpB,UAAU,IAAI,CAAC;IAClD,KAAK,MAAMK,KAAKH,SAAU;QACzB,MAAMqB,IAAIC,KAAK,GAAG,CAAC,IAAIC,SAASD,KAAK,GAAG,CAAC,GAAGnB,EAAE,SAAS;QACvD,MAAMqB,MAAMF,KAAK,GAAG,CAACxB,WAAW,GAAGwB,KAAK,KAAK,CAACD,IAAIvB;QAClDmB,UAAU,CAACO,IAAI,GAAIP,AAAAA,CAAAA,UAAU,CAACO,IAAI,IAAI,KAAKH;QAC3CF,SAAS,CAACK,IAAI,GAAIL,AAAAA,CAAAA,SAAS,CAACK,IAAI,IAAI,KAAKrB,EAAE,MAAM;QACjDiB,QAAQ,CAACI,IAAI,GAAIJ,AAAAA,CAAAA,QAAQ,CAACI,IAAI,IAAI,KAAK;IACxC;IACA,MAAMC,QAAQzB,SAAS,MAAM;IAC7B,IAAI0B,MAAM;IACV,IAAK,IAAIC,IAAI,GAAGA,IAAI7B,UAAU6B,IAAK;QAClC,MAAMC,IAAIR,QAAQ,CAACO,EAAE,IAAI;QACzB,IAAIC,AAAM,MAANA,GAAS;QACb,MAAMC,WAAYZ,AAAAA,CAAAA,UAAU,CAACU,EAAE,IAAI,KAAKC;QACxC,MAAME,UAAWX,AAAAA,CAAAA,SAAS,CAACQ,EAAE,IAAI,KAAKC;QACtCF,OAAQE,IAAIH,QAASH,KAAK,GAAG,CAACO,WAAWC;IAC1C;IACA,OAAOJ;AACR;AASA,SAASK,aACR/B,QAAwC;IAExC,MAAMgC,UAAU,IAAI9B;IACpB,KAAK,MAAMC,KAAKH,SAAU;QACzB,MAAMiC,OAAOD,QAAQ,GAAG,CAAC7B,EAAE,IAAI,KAAK,EAAE;QACtC8B,KAAK,IAAI,CAAC9B;QACV6B,QAAQ,GAAG,CAAC7B,EAAE,IAAI,EAAE8B;IACrB;IACA,MAAMC,SAAS,CAAC;IAChB,KAAK,MAAMC,KAAKC,kBACfF,MAAM,CAACC,EAAE,GAAGtB,WAAWmB,QAAQ,GAAG,CAACG,MAAM,EAAE;IAE5C,OAAOD;AACR;AAGA,SAASG;IACR,MAAM/B,UAAU,CAAC;IACjB,KAAK,MAAM6B,KAAKC,kBAAmB9B,OAAO,CAAC6B,EAAE,GAAG;IAChD,OAAO;QACN,YAAY;QACZ,KAAK;QACL,aAAa;QACb,cAAc7B;IACf;AACD;AAgBO,MAAMgC;IACI,QAAiB;IAChB,UAA4B;IAC5B,gBAAgB,IAAIpC,MAAyB;IAE9D,YAAYqC,eAAiC,EAAEC,OAAgB,CAAE;QAChE,IAAI,CAAC,SAAS,GAAGD;QACjB,IAAI,CAAC,OAAO,GAAGC;IAChB;IAEO,UACNC,aAAqB,EACrBC,IAAiB,EACjBC,SAAoB,EACA;QACpB,MAAMC,MAAMtB,KAAK,GAAG,CAAC,GAAGA,KAAK,GAAG,CAAC,GAAGmB;QACpC,IAAI,CAAC,IAAI,CAAC,OAAO,EAChB,OAAO;YAAEG;YAAK,YAAYA;YAAK,aAAa;YAAK,aAAa;QAAE;QAEjE,MAAM5C,WAAW,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC2C;QAC5C,MAAMrC,UAAUP,iBAAiBC;QACjC,MAAMQ,YAAYF,QAAQ,GAAG,CAACoC;QAC9B,MAAMd,IAAIpB,WAAW,SAAS;QAC9B,MAAMqC,eAAerC,AAAcsC,WAAdtC,YAA0B,MAAMA,UAAU,GAAG,GAAGA,UAAU,KAAK;QACpF,MAAM,EAAEG,OAAO,EAAEoC,WAAW,EAAE,GAAGnC,kBAAkBgC,KAAKC,cAAcjB;QACtE,MAAMoB,cAAc,IAAI,CAAC,aAAa,CAAC,GAAG,CAACL,cAAc;QACzD,MAAMM,aACLjD,SAAS,MAAM,IAAIkD,+BAChBC,iBAAiBxC,SAASqC,eAC1BrC;QACJ,OAAO;YAAEiC;YAAKK;YAAYD;YAAaD;QAAY;IACpD;IAEO,QAAQJ,SAAqB,EAAsB;QACzD,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,OAAON;QAC1B,MAAMrC,WACL2C,AAAcG,WAAdH,YACG,IAAI,CAAC,SAAS,CAAC,cAAc,KAC7B,IAAI,CAAC,SAAS,CAAC,WAAW,CAACA;QAC/B,OAAO;YACN,YAAY9B,WAAWb;YACvB,KAAKgB,yBAAyBhB;YAC9B,aAAaA,SAAS,MAAM;YAC5B,cAAc+B,aAAa/B;QAC5B;IACD;IAEO,MAAM2C,SAAoB,EAAQ;QACxC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;QACnB,MAAM3C,WAAW,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC2C;QAC5C,IAAI,CAAC,aAAa,CAAC,GAAG,CAACA,WAAWS,eAAe/C,2BAA2BL;IAC7E;IAEO,aAAa2C,SAAoB,EAAQ;QAC/C,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;QACnB,IAAI,CAAC,aAAa,CAAC,MAAM,CAACA;IAC3B;IAEO,WAAiB;QACvB,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;QACnB,IAAI,CAAC,aAAa,CAAC,KAAK;IACzB;AACD"}
|
|
@@ -22,6 +22,7 @@
|
|
|
22
22
|
*/
|
|
23
23
|
import type { PatternSignal } from '../reasoning.js';
|
|
24
24
|
import type { ThoughtData } from '../thought.js';
|
|
25
|
+
import type { VerificationLinks } from './VerificationLinks.js';
|
|
25
26
|
/**
|
|
26
27
|
* Stateless service that detects reasoning patterns from thought history.
|
|
27
28
|
*
|
|
@@ -45,7 +46,7 @@ export declare class PatternDetector {
|
|
|
45
46
|
* @param branches - Map of branch IDs to their thought arrays
|
|
46
47
|
* @returns Array of detected pattern signals (possibly empty)
|
|
47
48
|
*/
|
|
48
|
-
computePatternSignals(history: ThoughtData[], branches: Record<string, ThoughtData[]
|
|
49
|
+
computePatternSignals(history: ThoughtData[], branches: Record<string, ThoughtData[]>, links: VerificationLinks): PatternSignal[];
|
|
49
50
|
/** Detect runs of 3+ consecutive thoughts without verification. */
|
|
50
51
|
private _detectConsecutiveWithoutVerification;
|
|
51
52
|
/** Detect hypothesis thoughts not verified within 3 subsequent thoughts. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PatternDetector.d.ts","sourceRoot":"","sources":["../../../src/core/evaluator/PatternDetector.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"PatternDetector.d.ts","sourceRoot":"","sources":["../../../src/core/evaluator/PatternDetector.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACrD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AACjD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAEhE;;;;;;;;;;;;;;GAcG;AACH,qBAAa,eAAe;IAC3B;;;;;;OAMG;IACI,qBAAqB,CAC3B,OAAO,EAAE,WAAW,EAAE,EACtB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,EAAE,CAAC,EACvC,KAAK,EAAE,iBAAiB,GACtB,aAAa,EAAE;IAalB,mEAAmE;IACnE,OAAO,CAAC,qCAAqC;IAwB7C,4EAA4E;IAC5E,OAAO,CAAC,2BAA2B;IA+BnC,2DAA2D;IAC3D,OAAO,CAAC,6BAA6B;IAmBrC;;;OAGG;IACH,OAAO,CAAC,oBAAoB;IAyB5B,gEAAgE;IAChE,OAAO,CAAC,kBAAkB;IAkB1B,kFAAkF;IAClF,OAAO,CAAC,sBAAsB;IAiC9B,kEAAkE;IAClE,OAAO,CAAC,cAAc;IAuBtB,iFAAiF;IACjF,OAAO,CAAC,0BAA0B;CA6BlC"}
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
class PatternDetector {
|
|
2
|
-
computePatternSignals(history, branches) {
|
|
2
|
+
computePatternSignals(history, branches, links) {
|
|
3
3
|
if (0 === history.length) return [];
|
|
4
4
|
const signals = [];
|
|
5
5
|
signals.push(...this._detectConsecutiveWithoutVerification(history));
|
|
6
|
-
signals.push(...this._detectUnverifiedHypothesis(history));
|
|
6
|
+
signals.push(...this._detectUnverifiedHypothesis(history, links));
|
|
7
7
|
signals.push(...this._detectNoAlternativesExplored(history, branches));
|
|
8
8
|
signals.push(...this._detectMonotonicType(history));
|
|
9
9
|
signals.push(...this._detectConfidenceDrift(history));
|
|
10
|
-
signals.push(...this._detectHealthyVerification(history));
|
|
10
|
+
signals.push(...this._detectHealthyVerification(history, links));
|
|
11
11
|
return signals;
|
|
12
12
|
}
|
|
13
13
|
_detectConsecutiveWithoutVerification(history) {
|
|
@@ -36,16 +36,17 @@ class PatternDetector {
|
|
|
36
36
|
}
|
|
37
37
|
return signals;
|
|
38
38
|
}
|
|
39
|
-
_detectUnverifiedHypothesis(history) {
|
|
39
|
+
_detectUnverifiedHypothesis(history, links) {
|
|
40
40
|
const signals = [];
|
|
41
41
|
for(let i = 0; i < history.length; i++){
|
|
42
|
-
|
|
42
|
+
const hypothesis = history[i];
|
|
43
|
+
if ('hypothesis' !== hypothesis.thought_type || !hypothesis.id || !links.hypothesisIds.has(hypothesis.id)) continue;
|
|
43
44
|
const remaining = history.length - i - 1;
|
|
44
45
|
if (remaining < 3) continue;
|
|
45
46
|
const lookahead = history.slice(i + 1, i + 4);
|
|
46
|
-
const hasVerification = lookahead.some((
|
|
47
|
+
const hasVerification = lookahead.some((verifier)=>links.verifies(verifier, hypothesis));
|
|
47
48
|
if (!hasVerification) {
|
|
48
|
-
const n =
|
|
49
|
+
const n = hypothesis.thought_number ?? i + 1;
|
|
49
50
|
signals.push({
|
|
50
51
|
pattern: 'unverified_hypothesis',
|
|
51
52
|
severity: 'warning',
|
|
@@ -163,15 +164,15 @@ class PatternDetector {
|
|
|
163
164
|
]
|
|
164
165
|
});
|
|
165
166
|
}
|
|
166
|
-
_detectHealthyVerification(history) {
|
|
167
|
+
_detectHealthyVerification(history, links) {
|
|
167
168
|
const signals = [];
|
|
168
169
|
for(let i = 0; i < history.length; i++){
|
|
169
|
-
|
|
170
|
-
|
|
170
|
+
const hypothesis = history[i];
|
|
171
|
+
if ('hypothesis' !== hypothesis.thought_type || !hypothesis.id || !links.hypothesisIds.has(hypothesis.id)) continue;
|
|
171
172
|
const lookahead = history.slice(i + 1, i + 4);
|
|
172
|
-
const verifier = lookahead.find((
|
|
173
|
+
const verifier = lookahead.find((candidate)=>links.verifies(candidate, hypothesis));
|
|
173
174
|
if (verifier) {
|
|
174
|
-
const n =
|
|
175
|
+
const n = hypothesis.thought_number ?? i + 1;
|
|
175
176
|
const m = verifier.thought_number ?? history.indexOf(verifier) + 1;
|
|
176
177
|
signals.push({
|
|
177
178
|
pattern: 'healthy_verification',
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"core/evaluator/PatternDetector.js","sources":["../../../src/core/evaluator/PatternDetector.ts"],"sourcesContent":["/**\n * Pattern detection for sequential thinking reasoning analytics.\n *\n * Provides the {@link PatternDetector} class — a stateless service that detects\n * reasoning patterns (anti-patterns and positive signals) from thought history.\n * Extracted from {@link ThoughtEvaluator} to keep concerns isolated and reduce\n * cyclomatic complexity per function.\n *\n * Detected patterns:\n * - `consecutive_without_verification` (warning) — 3+ consecutive regular thoughts\n * without a verification step\n * - `unverified_hypothesis` (warning) — hypothesis not verified within 3 subsequent\n * thoughts\n * - `no_alternatives_explored` (info) — 5+ thoughts with no critique and no branches\n * - `monotonic_type` (info) — 4+ consecutive thoughts with the same thought_type\n * (requires ≥1 explicit type and ≥5 thoughts)\n * - `confidence_drift` (warning) — 3+ consecutive thoughts with strictly decreasing\n * confidence\n * - `healthy_verification` (info) — hypothesis verified within 3 subsequent thoughts\n *\n * @module core/evaluator/PatternDetector\n */\n\nimport type { PatternSignal } from '../reasoning.js';\nimport type { ThoughtData } from '../thought.js';\n\n/**\n * Stateless service that detects reasoning patterns from thought history.\n *\n * @remarks\n * All methods are pure computations — no side effects, no I/O, no internal state.\n * Designed to be registered as transient in the DI container or composed directly\n * into {@link ThoughtEvaluator}.\n *\n * @example\n * ```typescript\n * const detector = new PatternDetector();\n * const signals = detector.computePatternSignals(history, branches);\n * const warnings = signals.filter((s) => s.severity === 'warning');\n * ```\n */\nexport class PatternDetector {\n\t/**\n\t * Detect all reasoning patterns from history and branches.\n\t *\n\t * @param history - All thoughts in the current session\n\t * @param branches - Map of branch IDs to their thought arrays\n\t * @returns Array of detected pattern signals (possibly empty)\n\t */\n\tpublic computePatternSignals(\n\t\thistory: ThoughtData[],\n\t\tbranches: Record<string, ThoughtData[]>\n\t): PatternSignal[] {\n\t\tif (history.length === 0) return [];\n\n\t\tconst signals: PatternSignal[] = [];\n\t\tsignals.push(...this._detectConsecutiveWithoutVerification(history));\n\t\tsignals.push(...this._detectUnverifiedHypothesis(history));\n\t\tsignals.push(...this._detectNoAlternativesExplored(history, branches));\n\t\tsignals.push(...this._detectMonotonicType(history));\n\t\tsignals.push(...this._detectConfidenceDrift(history));\n\t\tsignals.push(...this._detectHealthyVerification(history));\n\t\treturn signals;\n\t}\n\n\t/** Detect runs of 3+ consecutive thoughts without verification. */\n\tprivate _detectConsecutiveWithoutVerification(history: ThoughtData[]): PatternSignal[] {\n\t\tconst signals: PatternSignal[] = [];\n\t\tlet runStart = 0;\n\t\tfor (let i = 0; i < history.length; i++) {\n\t\t\tconst type = history[i]!.thought_type ?? 'regular';\n\t\t\tif (type === 'verification') {\n\t\t\t\trunStart = i + 1;\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tif (i - runStart + 1 >= 3) {\n\t\t\t\tconst start = history[runStart]!.thought_number ?? runStart + 1;\n\t\t\t\tconst end = history[i]!.thought_number ?? i + 1;\n\t\t\t\tsignals.push({\n\t\t\t\t\tpattern: 'consecutive_without_verification',\n\t\t\t\t\tseverity: 'warning',\n\t\t\t\t\tmessage: `3+ consecutive thoughts (${start}-${end}) without verification`,\n\t\t\t\t\tthought_range: [start, end],\n\t\t\t\t});\n\t\t\t\trunStart = i + 1;\n\t\t\t}\n\t\t}\n\t\treturn signals;\n\t}\n\n\t/** Detect hypothesis thoughts not verified within 3 subsequent thoughts. */\n\tprivate _detectUnverifiedHypothesis(history: ThoughtData[]): PatternSignal[] {\n\t\tconst signals: PatternSignal[] = [];\n\t\tfor (let i = 0; i < history.length; i++) {\n\t\t\tif (history[i]!.thought_type !== 'hypothesis') continue;\n\t\t\tconst remaining = history.length - i - 1;\n\t\t\tif (remaining < 3) continue;\n\t\t\tconst lookahead = history.slice(i + 1, i + 4);\n\t\t\tconst hasVerification = lookahead.some((t) => t.thought_type === 'verification');\n\t\t\tif (!hasVerification) {\n\t\t\t\tconst n = history[i]!.thought_number ?? i + 1;\n\t\t\t\tsignals.push({\n\t\t\t\t\tpattern: 'unverified_hypothesis',\n\t\t\t\t\tseverity: 'warning',\n\t\t\t\t\tmessage: `Hypothesis at thought ${n} has not been verified within 3 thoughts`,\n\t\t\t\t\tthought_range: [n, n],\n\t\t\t\t});\n\t\t\t}\n\t\t}\n\t\treturn signals;\n\t}\n\n\t/** Detect 5+ thoughts with no critique and no branches. */\n\tprivate _detectNoAlternativesExplored(\n\t\thistory: ThoughtData[],\n\t\tbranches: Record<string, ThoughtData[]>\n\t): PatternSignal[] {\n\t\tif (history.length < 5) return [];\n\t\tif (history.some((t) => t.thought_type === 'critique')) return [];\n\t\tif (Object.keys(branches).length > 0) return [];\n\t\tconst start = history[0]!.thought_number ?? 1;\n\t\tconst end = history[history.length - 1]!.thought_number ?? history.length;\n\t\treturn [\n\t\t\t{\n\t\t\t\tpattern: 'no_alternatives_explored',\n\t\t\t\tseverity: 'warning',\n\t\t\t\tmessage: '5+ thoughts with no critique or branching — consider exploring alternatives',\n\t\t\t\tthought_range: [start, end],\n\t\t\t},\n\t\t];\n\t}\n\n\t/**\n\t * Detect runs of 4+ consecutive thoughts with the same thought_type.\n\t * Only fires when history has ≥5 thoughts and at least one explicitly set thought_type.\n\t */\n\tprivate _detectMonotonicType(history: ThoughtData[]): PatternSignal[] {\n\t\tif (history.length < 5) return [];\n\t\tconst hasExplicitType = history.some((t) => t.thought_type !== undefined);\n\t\tif (!hasExplicitType) return [];\n\n\t\tconst signals: PatternSignal[] = [];\n\t\tlet runType = history[0]!.thought_type ?? 'regular';\n\t\tlet runStart = 0;\n\t\tlet runLength = 1;\n\n\t\tfor (let i = 1; i < history.length; i++) {\n\t\t\tconst type = history[i]!.thought_type ?? 'regular';\n\t\t\tif (type === runType) {\n\t\t\t\trunLength++;\n\t\t\t} else {\n\t\t\t\tthis._flushMonotonicRun(history, runStart, runLength, runType, signals);\n\t\t\t\trunType = type;\n\t\t\t\trunStart = i;\n\t\t\t\trunLength = 1;\n\t\t\t}\n\t\t}\n\t\tthis._flushMonotonicRun(history, runStart, runLength, runType, signals);\n\t\treturn signals;\n\t}\n\n\t/** Emit a monotonic_type signal if the run length qualifies. */\n\tprivate _flushMonotonicRun(\n\t\thistory: ThoughtData[],\n\t\trunStart: number,\n\t\trunLength: number,\n\t\trunType: string,\n\t\tsignals: PatternSignal[]\n\t): void {\n\t\tif (runLength < 4) return;\n\t\tconst start = history[runStart]!.thought_number ?? runStart + 1;\n\t\tconst end = history[runStart + runLength - 1]!.thought_number ?? runStart + runLength;\n\t\tsignals.push({\n\t\t\tpattern: 'monotonic_type',\n\t\t\tseverity: 'warning',\n\t\t\tmessage: `4+ consecutive '${runType}' thoughts (${start}-${end}) — consider varying approach`,\n\t\t\tthought_range: [start, end],\n\t\t});\n\t}\n\n\t/** Detect runs of 3+ consecutive thoughts with strictly decreasing confidence. */\n\tprivate _detectConfidenceDrift(history: ThoughtData[]): PatternSignal[] {\n\t\tconst signals: PatternSignal[] = [];\n\t\tlet runStart = -1;\n\t\tlet runLength = 0;\n\t\tlet prevConf = -1;\n\n\t\tfor (let i = 0; i < history.length; i++) {\n\t\t\tconst conf = history[i]!.confidence;\n\t\t\tif (conf === undefined) {\n\t\t\t\tthis._flushDriftRun(history, runStart, runLength, signals);\n\t\t\t\trunStart = -1;\n\t\t\t\trunLength = 0;\n\t\t\t\tprevConf = -1;\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tif (runLength === 0) {\n\t\t\t\trunStart = i;\n\t\t\t\trunLength = 1;\n\t\t\t\tprevConf = conf;\n\t\t\t} else if (conf < prevConf) {\n\t\t\t\trunLength++;\n\t\t\t\tprevConf = conf;\n\t\t\t} else {\n\t\t\t\tthis._flushDriftRun(history, runStart, runLength, signals);\n\t\t\t\trunStart = i;\n\t\t\t\trunLength = 1;\n\t\t\t\tprevConf = conf;\n\t\t\t}\n\t\t}\n\t\tthis._flushDriftRun(history, runStart, runLength, signals);\n\t\treturn signals;\n\t}\n\n\t/** Emit a confidence_drift signal if the run length qualifies. */\n\tprivate _flushDriftRun(\n\t\thistory: ThoughtData[],\n\t\trunStart: number,\n\t\trunLength: number,\n\t\tsignals: PatternSignal[]\n\t): void {\n\t\tif (runLength < 3) return;\n\t\tconst first = history[runStart];\n\t\tconst last = history[runStart + runLength - 1];\n\t\tif (!first || !last) return;\n\t\tif (first.confidence === undefined || last.confidence === undefined) return;\n\t\tconst start = first.thought_number ?? runStart + 1;\n\t\tconst end = last.thought_number ?? runStart + runLength;\n\t\tconst firstConf = first.confidence;\n\t\tconst lastConf = last.confidence;\n\t\tsignals.push({\n\t\t\tpattern: 'confidence_drift',\n\t\t\tseverity: 'warning',\n\t\t\tmessage: `Confidence decreasing across thoughts ${start}-${end} (${firstConf} → ${lastConf})`,\n\t\t\tthought_range: [start, end],\n\t\t});\n\t}\n\n\t/** Detect hypothesis verified within 3 subsequent thoughts — positive signal. */\n\tprivate _detectHealthyVerification(history: ThoughtData[]): PatternSignal[] {\n\t\tconst signals: PatternSignal[] = [];\n\t\tfor (let i = 0; i < history.length; i++) {\n\t\t\tif (history[i]!.thought_type !== 'hypothesis') continue;\n\t\t\tconst hypId = history[i]!.hypothesis_id;\n\t\t\tconst lookahead = history.slice(i + 1, i + 4);\n\t\t\tconst verifier = lookahead.find(\n\t\t\t\t(t) =>\n\t\t\t\t\tt.thought_type === 'verification' &&\n\t\t\t\t\t(t.hypothesis_id === hypId ||\n\t\t\t\t\t\tt.verification_target === (history[i]!.thought_number ?? i + 1))\n\t\t\t);\n\t\t\tif (verifier) {\n\t\t\t\tconst n = history[i]!.thought_number ?? i + 1;\n\t\t\t\tconst m = verifier.thought_number ?? history.indexOf(verifier) + 1;\n\t\t\t\tsignals.push({\n\t\t\t\t\tpattern: 'healthy_verification',\n\t\t\t\t\tseverity: 'info',\n\t\t\t\t\tmessage: `Hypothesis at thought ${n} verified at thought ${m} — good practice`,\n\t\t\t\t\tthought_range: [n, m],\n\t\t\t\t});\n\t\t\t}\n\t\t}\n\t\treturn signals;\n\t}\n}\n"],"names":["PatternDetector","history","branches","signals","runStart","i","type","start","end","remaining","lookahead","hasVerification","t","n","Object","hasExplicitType","undefined","runType","runLength","prevConf","conf","first","last","firstConf","lastConf","hypId","verifier","m"],"mappings":"AAyCO,MAAMA;IAQL,sBACNC,OAAsB,EACtBC,QAAuC,EACrB;QAClB,IAAID,AAAmB,MAAnBA,QAAQ,MAAM,EAAQ,OAAO,EAAE;QAEnC,MAAME,UAA2B,EAAE;QACnCA,QAAQ,IAAI,IAAI,IAAI,CAAC,qCAAqC,CAACF;QAC3DE,QAAQ,IAAI,IAAI,IAAI,CAAC,2BAA2B,CAACF;QACjDE,QAAQ,IAAI,IAAI,IAAI,CAAC,6BAA6B,CAACF,SAASC;QAC5DC,QAAQ,IAAI,IAAI,IAAI,CAAC,oBAAoB,CAACF;QAC1CE,QAAQ,IAAI,IAAI,IAAI,CAAC,sBAAsB,CAACF;QAC5CE,QAAQ,IAAI,IAAI,IAAI,CAAC,0BAA0B,CAACF;QAChD,OAAOE;IACR;IAGQ,sCAAsCF,OAAsB,EAAmB;QACtF,MAAME,UAA2B,EAAE;QACnC,IAAIC,WAAW;QACf,IAAK,IAAIC,IAAI,GAAGA,IAAIJ,QAAQ,MAAM,EAAEI,IAAK;YACxC,MAAMC,OAAOL,OAAO,CAACI,EAAE,CAAE,YAAY,IAAI;YACzC,IAAIC,AAAS,mBAATA,MAAyB;gBAC5BF,WAAWC,IAAI;gBACf;YACD;YACA,IAAIA,IAAID,WAAW,KAAK,GAAG;gBAC1B,MAAMG,QAAQN,OAAO,CAACG,SAAS,CAAE,cAAc,IAAIA,WAAW;gBAC9D,MAAMI,MAAMP,OAAO,CAACI,EAAE,CAAE,cAAc,IAAIA,IAAI;gBAC9CF,QAAQ,IAAI,CAAC;oBACZ,SAAS;oBACT,UAAU;oBACV,SAAS,CAAC,yBAAyB,EAAEI,MAAM,CAAC,EAAEC,IAAI,sBAAsB,CAAC;oBACzE,eAAe;wBAACD;wBAAOC;qBAAI;gBAC5B;gBACAJ,WAAWC,IAAI;YAChB;QACD;QACA,OAAOF;IACR;IAGQ,4BAA4BF,OAAsB,EAAmB;QAC5E,MAAME,UAA2B,EAAE;QACnC,IAAK,IAAIE,IAAI,GAAGA,IAAIJ,QAAQ,MAAM,EAAEI,IAAK;YACxC,IAAIJ,AAA6B,iBAA7BA,OAAO,CAACI,EAAE,CAAE,YAAY,EAAmB;YAC/C,MAAMI,YAAYR,QAAQ,MAAM,GAAGI,IAAI;YACvC,IAAII,YAAY,GAAG;YACnB,MAAMC,YAAYT,QAAQ,KAAK,CAACI,IAAI,GAAGA,IAAI;YAC3C,MAAMM,kBAAkBD,UAAU,IAAI,CAAC,CAACE,IAAMA,AAAmB,mBAAnBA,EAAE,YAAY;YAC5D,IAAI,CAACD,iBAAiB;gBACrB,MAAME,IAAIZ,OAAO,CAACI,EAAE,CAAE,cAAc,IAAIA,IAAI;gBAC5CF,QAAQ,IAAI,CAAC;oBACZ,SAAS;oBACT,UAAU;oBACV,SAAS,CAAC,sBAAsB,EAAEU,EAAE,wCAAwC,CAAC;oBAC7E,eAAe;wBAACA;wBAAGA;qBAAE;gBACtB;YACD;QACD;QACA,OAAOV;IACR;IAGQ,8BACPF,OAAsB,EACtBC,QAAuC,EACrB;QAClB,IAAID,QAAQ,MAAM,GAAG,GAAG,OAAO,EAAE;QACjC,IAAIA,QAAQ,IAAI,CAAC,CAACW,IAAMA,AAAmB,eAAnBA,EAAE,YAAY,GAAkB,OAAO,EAAE;QACjE,IAAIE,OAAO,IAAI,CAACZ,UAAU,MAAM,GAAG,GAAG,OAAO,EAAE;QAC/C,MAAMK,QAAQN,OAAO,CAAC,EAAE,CAAE,cAAc,IAAI;QAC5C,MAAMO,MAAMP,OAAO,CAACA,QAAQ,MAAM,GAAG,EAAE,CAAE,cAAc,IAAIA,QAAQ,MAAM;QACzE,OAAO;YACN;gBACC,SAAS;gBACT,UAAU;gBACV,SAAS;gBACT,eAAe;oBAACM;oBAAOC;iBAAI;YAC5B;SACA;IACF;IAMQ,qBAAqBP,OAAsB,EAAmB;QACrE,IAAIA,QAAQ,MAAM,GAAG,GAAG,OAAO,EAAE;QACjC,MAAMc,kBAAkBd,QAAQ,IAAI,CAAC,CAACW,IAAMA,AAAmBI,WAAnBJ,EAAE,YAAY;QAC1D,IAAI,CAACG,iBAAiB,OAAO,EAAE;QAE/B,MAAMZ,UAA2B,EAAE;QACnC,IAAIc,UAAUhB,OAAO,CAAC,EAAE,CAAE,YAAY,IAAI;QAC1C,IAAIG,WAAW;QACf,IAAIc,YAAY;QAEhB,IAAK,IAAIb,IAAI,GAAGA,IAAIJ,QAAQ,MAAM,EAAEI,IAAK;YACxC,MAAMC,OAAOL,OAAO,CAACI,EAAE,CAAE,YAAY,IAAI;YACzC,IAAIC,SAASW,SACZC;iBACM;gBACN,IAAI,CAAC,kBAAkB,CAACjB,SAASG,UAAUc,WAAWD,SAASd;gBAC/Dc,UAAUX;gBACVF,WAAWC;gBACXa,YAAY;YACb;QACD;QACA,IAAI,CAAC,kBAAkB,CAACjB,SAASG,UAAUc,WAAWD,SAASd;QAC/D,OAAOA;IACR;IAGQ,mBACPF,OAAsB,EACtBG,QAAgB,EAChBc,SAAiB,EACjBD,OAAe,EACfd,OAAwB,EACjB;QACP,IAAIe,YAAY,GAAG;QACnB,MAAMX,QAAQN,OAAO,CAACG,SAAS,CAAE,cAAc,IAAIA,WAAW;QAC9D,MAAMI,MAAMP,OAAO,CAACG,WAAWc,YAAY,EAAE,CAAE,cAAc,IAAId,WAAWc;QAC5Ef,QAAQ,IAAI,CAAC;YACZ,SAAS;YACT,UAAU;YACV,SAAS,CAAC,gBAAgB,EAAEc,QAAQ,YAAY,EAAEV,MAAM,CAAC,EAAEC,IAAI,6BAA6B,CAAC;YAC7F,eAAe;gBAACD;gBAAOC;aAAI;QAC5B;IACD;IAGQ,uBAAuBP,OAAsB,EAAmB;QACvE,MAAME,UAA2B,EAAE;QACnC,IAAIC,WAAW;QACf,IAAIc,YAAY;QAChB,IAAIC,WAAW;QAEf,IAAK,IAAId,IAAI,GAAGA,IAAIJ,QAAQ,MAAM,EAAEI,IAAK;YACxC,MAAMe,OAAOnB,OAAO,CAACI,EAAE,CAAE,UAAU;YACnC,IAAIe,AAASJ,WAATI,MAAoB;gBACvB,IAAI,CAAC,cAAc,CAACnB,SAASG,UAAUc,WAAWf;gBAClDC,WAAW;gBACXc,YAAY;gBACZC,WAAW;gBACX;YACD;YACA,IAAID,AAAc,MAAdA,WAAiB;gBACpBd,WAAWC;gBACXa,YAAY;gBACZC,WAAWC;YACZ,OAAO,IAAIA,OAAOD,UAAU;gBAC3BD;gBACAC,WAAWC;YACZ,OAAO;gBACN,IAAI,CAAC,cAAc,CAACnB,SAASG,UAAUc,WAAWf;gBAClDC,WAAWC;gBACXa,YAAY;gBACZC,WAAWC;YACZ;QACD;QACA,IAAI,CAAC,cAAc,CAACnB,SAASG,UAAUc,WAAWf;QAClD,OAAOA;IACR;IAGQ,eACPF,OAAsB,EACtBG,QAAgB,EAChBc,SAAiB,EACjBf,OAAwB,EACjB;QACP,IAAIe,YAAY,GAAG;QACnB,MAAMG,QAAQpB,OAAO,CAACG,SAAS;QAC/B,MAAMkB,OAAOrB,OAAO,CAACG,WAAWc,YAAY,EAAE;QAC9C,IAAI,CAACG,SAAS,CAACC,MAAM;QACrB,IAAID,AAAqBL,WAArBK,MAAM,UAAU,IAAkBC,AAAoBN,WAApBM,KAAK,UAAU,EAAgB;QACrE,MAAMf,QAAQc,MAAM,cAAc,IAAIjB,WAAW;QACjD,MAAMI,MAAMc,KAAK,cAAc,IAAIlB,WAAWc;QAC9C,MAAMK,YAAYF,MAAM,UAAU;QAClC,MAAMG,WAAWF,KAAK,UAAU;QAChCnB,QAAQ,IAAI,CAAC;YACZ,SAAS;YACT,UAAU;YACV,SAAS,CAAC,sCAAsC,EAAEI,MAAM,CAAC,EAAEC,IAAI,EAAE,EAAEe,UAAU,GAAG,EAAEC,SAAS,CAAC,CAAC;YAC7F,eAAe;gBAACjB;gBAAOC;aAAI;QAC5B;IACD;IAGQ,2BAA2BP,OAAsB,EAAmB;QAC3E,MAAME,UAA2B,EAAE;QACnC,IAAK,IAAIE,IAAI,GAAGA,IAAIJ,QAAQ,MAAM,EAAEI,IAAK;YACxC,IAAIJ,AAA6B,iBAA7BA,OAAO,CAACI,EAAE,CAAE,YAAY,EAAmB;YAC/C,MAAMoB,QAAQxB,OAAO,CAACI,EAAE,CAAE,aAAa;YACvC,MAAMK,YAAYT,QAAQ,KAAK,CAACI,IAAI,GAAGA,IAAI;YAC3C,MAAMqB,WAAWhB,UAAU,IAAI,CAC9B,CAACE,IACAA,AAAmB,mBAAnBA,EAAE,YAAY,IACbA,CAAAA,EAAE,aAAa,KAAKa,SACpBb,EAAE,mBAAmB,KAAMX,CAAAA,OAAO,CAACI,EAAE,CAAE,cAAc,IAAIA,IAAI,EAAC;YAEjE,IAAIqB,UAAU;gBACb,MAAMb,IAAIZ,OAAO,CAACI,EAAE,CAAE,cAAc,IAAIA,IAAI;gBAC5C,MAAMsB,IAAID,SAAS,cAAc,IAAIzB,QAAQ,OAAO,CAACyB,YAAY;gBACjEvB,QAAQ,IAAI,CAAC;oBACZ,SAAS;oBACT,UAAU;oBACV,SAAS,CAAC,sBAAsB,EAAEU,EAAE,qBAAqB,EAAEc,EAAE,gBAAgB,CAAC;oBAC9E,eAAe;wBAACd;wBAAGc;qBAAE;gBACtB;YACD;QACD;QACA,OAAOxB;IACR;AACD"}
|
|
1
|
+
{"version":3,"file":"core/evaluator/PatternDetector.js","sources":["../../../src/core/evaluator/PatternDetector.ts"],"sourcesContent":["/**\n * Pattern detection for sequential thinking reasoning analytics.\n *\n * Provides the {@link PatternDetector} class — a stateless service that detects\n * reasoning patterns (anti-patterns and positive signals) from thought history.\n * Extracted from {@link ThoughtEvaluator} to keep concerns isolated and reduce\n * cyclomatic complexity per function.\n *\n * Detected patterns:\n * - `consecutive_without_verification` (warning) — 3+ consecutive regular thoughts\n * without a verification step\n * - `unverified_hypothesis` (warning) — hypothesis not verified within 3 subsequent\n * thoughts\n * - `no_alternatives_explored` (info) — 5+ thoughts with no critique and no branches\n * - `monotonic_type` (info) — 4+ consecutive thoughts with the same thought_type\n * (requires ≥1 explicit type and ≥5 thoughts)\n * - `confidence_drift` (warning) — 3+ consecutive thoughts with strictly decreasing\n * confidence\n * - `healthy_verification` (info) — hypothesis verified within 3 subsequent thoughts\n *\n * @module core/evaluator/PatternDetector\n */\n\nimport type { PatternSignal } from '../reasoning.js';\nimport type { ThoughtData } from '../thought.js';\nimport type { VerificationLinks } from './VerificationLinks.js';\n\n/**\n * Stateless service that detects reasoning patterns from thought history.\n *\n * @remarks\n * All methods are pure computations — no side effects, no I/O, no internal state.\n * Designed to be registered as transient in the DI container or composed directly\n * into {@link ThoughtEvaluator}.\n *\n * @example\n * ```typescript\n * const detector = new PatternDetector();\n * const signals = detector.computePatternSignals(history, branches);\n * const warnings = signals.filter((s) => s.severity === 'warning');\n * ```\n */\nexport class PatternDetector {\n\t/**\n\t * Detect all reasoning patterns from history and branches.\n\t *\n\t * @param history - All thoughts in the current session\n\t * @param branches - Map of branch IDs to their thought arrays\n\t * @returns Array of detected pattern signals (possibly empty)\n\t */\n\tpublic computePatternSignals(\n\t\thistory: ThoughtData[],\n\t\tbranches: Record<string, ThoughtData[]>,\n\t\tlinks: VerificationLinks\n\t): PatternSignal[] {\n\t\tif (history.length === 0) return [];\n\n\t\tconst signals: PatternSignal[] = [];\n\t\tsignals.push(...this._detectConsecutiveWithoutVerification(history));\n\t\tsignals.push(...this._detectUnverifiedHypothesis(history, links));\n\t\tsignals.push(...this._detectNoAlternativesExplored(history, branches));\n\t\tsignals.push(...this._detectMonotonicType(history));\n\t\tsignals.push(...this._detectConfidenceDrift(history));\n\t\tsignals.push(...this._detectHealthyVerification(history, links));\n\t\treturn signals;\n\t}\n\n\t/** Detect runs of 3+ consecutive thoughts without verification. */\n\tprivate _detectConsecutiveWithoutVerification(history: ThoughtData[]): PatternSignal[] {\n\t\tconst signals: PatternSignal[] = [];\n\t\tlet runStart = 0;\n\t\tfor (let i = 0; i < history.length; i++) {\n\t\t\tconst type = history[i]!.thought_type ?? 'regular';\n\t\t\tif (type === 'verification') {\n\t\t\t\trunStart = i + 1;\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tif (i - runStart + 1 >= 3) {\n\t\t\t\tconst start = history[runStart]!.thought_number ?? runStart + 1;\n\t\t\t\tconst end = history[i]!.thought_number ?? i + 1;\n\t\t\t\tsignals.push({\n\t\t\t\t\tpattern: 'consecutive_without_verification',\n\t\t\t\t\tseverity: 'warning',\n\t\t\t\t\tmessage: `3+ consecutive thoughts (${start}-${end}) without verification`,\n\t\t\t\t\tthought_range: [start, end],\n\t\t\t\t});\n\t\t\t\trunStart = i + 1;\n\t\t\t}\n\t\t}\n\t\treturn signals;\n\t}\n\n\t/** Detect hypothesis thoughts not verified within 3 subsequent thoughts. */\n\tprivate _detectUnverifiedHypothesis(\n\t\thistory: ThoughtData[],\n\t\tlinks: VerificationLinks\n\t): PatternSignal[] {\n\t\tconst signals: PatternSignal[] = [];\n\t\tfor (let i = 0; i < history.length; i++) {\n\t\t\tconst hypothesis = history[i]!;\n\t\t\tif (\n\t\t\t\thypothesis.thought_type !== 'hypothesis' ||\n\t\t\t\t!hypothesis.id ||\n\t\t\t\t!links.hypothesisIds.has(hypothesis.id)\n\t\t\t) {\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tconst remaining = history.length - i - 1;\n\t\t\tif (remaining < 3) continue;\n\t\t\tconst lookahead = history.slice(i + 1, i + 4);\n\t\t\tconst hasVerification = lookahead.some((verifier) => links.verifies(verifier, hypothesis));\n\t\t\tif (!hasVerification) {\n\t\t\t\tconst n = hypothesis.thought_number ?? i + 1;\n\t\t\t\tsignals.push({\n\t\t\t\t\tpattern: 'unverified_hypothesis',\n\t\t\t\t\tseverity: 'warning',\n\t\t\t\t\tmessage: `Hypothesis at thought ${n} has not been verified within 3 thoughts`,\n\t\t\t\t\tthought_range: [n, n],\n\t\t\t\t});\n\t\t\t}\n\t\t}\n\t\treturn signals;\n\t}\n\n\t/** Detect 5+ thoughts with no critique and no branches. */\n\tprivate _detectNoAlternativesExplored(\n\t\thistory: ThoughtData[],\n\t\tbranches: Record<string, ThoughtData[]>\n\t): PatternSignal[] {\n\t\tif (history.length < 5) return [];\n\t\tif (history.some((t) => t.thought_type === 'critique')) return [];\n\t\tif (Object.keys(branches).length > 0) return [];\n\t\tconst start = history[0]!.thought_number ?? 1;\n\t\tconst end = history[history.length - 1]!.thought_number ?? history.length;\n\t\treturn [\n\t\t\t{\n\t\t\t\tpattern: 'no_alternatives_explored',\n\t\t\t\tseverity: 'warning',\n\t\t\t\tmessage: '5+ thoughts with no critique or branching — consider exploring alternatives',\n\t\t\t\tthought_range: [start, end],\n\t\t\t},\n\t\t];\n\t}\n\n\t/**\n\t * Detect runs of 4+ consecutive thoughts with the same thought_type.\n\t * Only fires when history has ≥5 thoughts and at least one explicitly set thought_type.\n\t */\n\tprivate _detectMonotonicType(history: ThoughtData[]): PatternSignal[] {\n\t\tif (history.length < 5) return [];\n\t\tconst hasExplicitType = history.some((t) => t.thought_type !== undefined);\n\t\tif (!hasExplicitType) return [];\n\n\t\tconst signals: PatternSignal[] = [];\n\t\tlet runType = history[0]!.thought_type ?? 'regular';\n\t\tlet runStart = 0;\n\t\tlet runLength = 1;\n\n\t\tfor (let i = 1; i < history.length; i++) {\n\t\t\tconst type = history[i]!.thought_type ?? 'regular';\n\t\t\tif (type === runType) {\n\t\t\t\trunLength++;\n\t\t\t} else {\n\t\t\t\tthis._flushMonotonicRun(history, runStart, runLength, runType, signals);\n\t\t\t\trunType = type;\n\t\t\t\trunStart = i;\n\t\t\t\trunLength = 1;\n\t\t\t}\n\t\t}\n\t\tthis._flushMonotonicRun(history, runStart, runLength, runType, signals);\n\t\treturn signals;\n\t}\n\n\t/** Emit a monotonic_type signal if the run length qualifies. */\n\tprivate _flushMonotonicRun(\n\t\thistory: ThoughtData[],\n\t\trunStart: number,\n\t\trunLength: number,\n\t\trunType: string,\n\t\tsignals: PatternSignal[]\n\t): void {\n\t\tif (runLength < 4) return;\n\t\tconst start = history[runStart]!.thought_number ?? runStart + 1;\n\t\tconst end = history[runStart + runLength - 1]!.thought_number ?? runStart + runLength;\n\t\tsignals.push({\n\t\t\tpattern: 'monotonic_type',\n\t\t\tseverity: 'warning',\n\t\t\tmessage: `4+ consecutive '${runType}' thoughts (${start}-${end}) — consider varying approach`,\n\t\t\tthought_range: [start, end],\n\t\t});\n\t}\n\n\t/** Detect runs of 3+ consecutive thoughts with strictly decreasing confidence. */\n\tprivate _detectConfidenceDrift(history: ThoughtData[]): PatternSignal[] {\n\t\tconst signals: PatternSignal[] = [];\n\t\tlet runStart = -1;\n\t\tlet runLength = 0;\n\t\tlet prevConf = -1;\n\n\t\tfor (let i = 0; i < history.length; i++) {\n\t\t\tconst conf = history[i]!.confidence;\n\t\t\tif (conf === undefined) {\n\t\t\t\tthis._flushDriftRun(history, runStart, runLength, signals);\n\t\t\t\trunStart = -1;\n\t\t\t\trunLength = 0;\n\t\t\t\tprevConf = -1;\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tif (runLength === 0) {\n\t\t\t\trunStart = i;\n\t\t\t\trunLength = 1;\n\t\t\t\tprevConf = conf;\n\t\t\t} else if (conf < prevConf) {\n\t\t\t\trunLength++;\n\t\t\t\tprevConf = conf;\n\t\t\t} else {\n\t\t\t\tthis._flushDriftRun(history, runStart, runLength, signals);\n\t\t\t\trunStart = i;\n\t\t\t\trunLength = 1;\n\t\t\t\tprevConf = conf;\n\t\t\t}\n\t\t}\n\t\tthis._flushDriftRun(history, runStart, runLength, signals);\n\t\treturn signals;\n\t}\n\n\t/** Emit a confidence_drift signal if the run length qualifies. */\n\tprivate _flushDriftRun(\n\t\thistory: ThoughtData[],\n\t\trunStart: number,\n\t\trunLength: number,\n\t\tsignals: PatternSignal[]\n\t): void {\n\t\tif (runLength < 3) return;\n\t\tconst first = history[runStart];\n\t\tconst last = history[runStart + runLength - 1];\n\t\tif (!first || !last) return;\n\t\tif (first.confidence === undefined || last.confidence === undefined) return;\n\t\tconst start = first.thought_number ?? runStart + 1;\n\t\tconst end = last.thought_number ?? runStart + runLength;\n\t\tconst firstConf = first.confidence;\n\t\tconst lastConf = last.confidence;\n\t\tsignals.push({\n\t\t\tpattern: 'confidence_drift',\n\t\t\tseverity: 'warning',\n\t\t\tmessage: `Confidence decreasing across thoughts ${start}-${end} (${firstConf} → ${lastConf})`,\n\t\t\tthought_range: [start, end],\n\t\t});\n\t}\n\n\t/** Detect hypothesis verified within 3 subsequent thoughts — positive signal. */\n\tprivate _detectHealthyVerification(\n\t\thistory: ThoughtData[],\n\t\tlinks: VerificationLinks\n\t): PatternSignal[] {\n\t\tconst signals: PatternSignal[] = [];\n\t\tfor (let i = 0; i < history.length; i++) {\n\t\t\tconst hypothesis = history[i]!;\n\t\t\tif (\n\t\t\t\thypothesis.thought_type !== 'hypothesis' ||\n\t\t\t\t!hypothesis.id ||\n\t\t\t\t!links.hypothesisIds.has(hypothesis.id)\n\t\t\t) {\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tconst lookahead = history.slice(i + 1, i + 4);\n\t\t\tconst verifier = lookahead.find((candidate) => links.verifies(candidate, hypothesis));\n\t\t\tif (verifier) {\n\t\t\t\tconst n = hypothesis.thought_number ?? i + 1;\n\t\t\t\tconst m = verifier.thought_number ?? history.indexOf(verifier) + 1;\n\t\t\t\tsignals.push({\n\t\t\t\t\tpattern: 'healthy_verification',\n\t\t\t\t\tseverity: 'info',\n\t\t\t\t\tmessage: `Hypothesis at thought ${n} verified at thought ${m} — good practice`,\n\t\t\t\t\tthought_range: [n, m],\n\t\t\t\t});\n\t\t\t}\n\t\t}\n\t\treturn signals;\n\t}\n}\n"],"names":["PatternDetector","history","branches","links","signals","runStart","i","type","start","end","hypothesis","remaining","lookahead","hasVerification","verifier","n","t","Object","hasExplicitType","undefined","runType","runLength","prevConf","conf","first","last","firstConf","lastConf","candidate","m"],"mappings":"AA0CO,MAAMA;IAQL,sBACNC,OAAsB,EACtBC,QAAuC,EACvCC,KAAwB,EACN;QAClB,IAAIF,AAAmB,MAAnBA,QAAQ,MAAM,EAAQ,OAAO,EAAE;QAEnC,MAAMG,UAA2B,EAAE;QACnCA,QAAQ,IAAI,IAAI,IAAI,CAAC,qCAAqC,CAACH;QAC3DG,QAAQ,IAAI,IAAI,IAAI,CAAC,2BAA2B,CAACH,SAASE;QAC1DC,QAAQ,IAAI,IAAI,IAAI,CAAC,6BAA6B,CAACH,SAASC;QAC5DE,QAAQ,IAAI,IAAI,IAAI,CAAC,oBAAoB,CAACH;QAC1CG,QAAQ,IAAI,IAAI,IAAI,CAAC,sBAAsB,CAACH;QAC5CG,QAAQ,IAAI,IAAI,IAAI,CAAC,0BAA0B,CAACH,SAASE;QACzD,OAAOC;IACR;IAGQ,sCAAsCH,OAAsB,EAAmB;QACtF,MAAMG,UAA2B,EAAE;QACnC,IAAIC,WAAW;QACf,IAAK,IAAIC,IAAI,GAAGA,IAAIL,QAAQ,MAAM,EAAEK,IAAK;YACxC,MAAMC,OAAON,OAAO,CAACK,EAAE,CAAE,YAAY,IAAI;YACzC,IAAIC,AAAS,mBAATA,MAAyB;gBAC5BF,WAAWC,IAAI;gBACf;YACD;YACA,IAAIA,IAAID,WAAW,KAAK,GAAG;gBAC1B,MAAMG,QAAQP,OAAO,CAACI,SAAS,CAAE,cAAc,IAAIA,WAAW;gBAC9D,MAAMI,MAAMR,OAAO,CAACK,EAAE,CAAE,cAAc,IAAIA,IAAI;gBAC9CF,QAAQ,IAAI,CAAC;oBACZ,SAAS;oBACT,UAAU;oBACV,SAAS,CAAC,yBAAyB,EAAEI,MAAM,CAAC,EAAEC,IAAI,sBAAsB,CAAC;oBACzE,eAAe;wBAACD;wBAAOC;qBAAI;gBAC5B;gBACAJ,WAAWC,IAAI;YAChB;QACD;QACA,OAAOF;IACR;IAGQ,4BACPH,OAAsB,EACtBE,KAAwB,EACN;QAClB,MAAMC,UAA2B,EAAE;QACnC,IAAK,IAAIE,IAAI,GAAGA,IAAIL,QAAQ,MAAM,EAAEK,IAAK;YACxC,MAAMI,aAAaT,OAAO,CAACK,EAAE;YAC7B,IACCI,AAA4B,iBAA5BA,WAAW,YAAY,IACvB,CAACA,WAAW,EAAE,IACd,CAACP,MAAM,aAAa,CAAC,GAAG,CAACO,WAAW,EAAE,GAEtC;YAED,MAAMC,YAAYV,QAAQ,MAAM,GAAGK,IAAI;YACvC,IAAIK,YAAY,GAAG;YACnB,MAAMC,YAAYX,QAAQ,KAAK,CAACK,IAAI,GAAGA,IAAI;YAC3C,MAAMO,kBAAkBD,UAAU,IAAI,CAAC,CAACE,WAAaX,MAAM,QAAQ,CAACW,UAAUJ;YAC9E,IAAI,CAACG,iBAAiB;gBACrB,MAAME,IAAIL,WAAW,cAAc,IAAIJ,IAAI;gBAC3CF,QAAQ,IAAI,CAAC;oBACZ,SAAS;oBACT,UAAU;oBACV,SAAS,CAAC,sBAAsB,EAAEW,EAAE,wCAAwC,CAAC;oBAC7E,eAAe;wBAACA;wBAAGA;qBAAE;gBACtB;YACD;QACD;QACA,OAAOX;IACR;IAGQ,8BACPH,OAAsB,EACtBC,QAAuC,EACrB;QAClB,IAAID,QAAQ,MAAM,GAAG,GAAG,OAAO,EAAE;QACjC,IAAIA,QAAQ,IAAI,CAAC,CAACe,IAAMA,AAAmB,eAAnBA,EAAE,YAAY,GAAkB,OAAO,EAAE;QACjE,IAAIC,OAAO,IAAI,CAACf,UAAU,MAAM,GAAG,GAAG,OAAO,EAAE;QAC/C,MAAMM,QAAQP,OAAO,CAAC,EAAE,CAAE,cAAc,IAAI;QAC5C,MAAMQ,MAAMR,OAAO,CAACA,QAAQ,MAAM,GAAG,EAAE,CAAE,cAAc,IAAIA,QAAQ,MAAM;QACzE,OAAO;YACN;gBACC,SAAS;gBACT,UAAU;gBACV,SAAS;gBACT,eAAe;oBAACO;oBAAOC;iBAAI;YAC5B;SACA;IACF;IAMQ,qBAAqBR,OAAsB,EAAmB;QACrE,IAAIA,QAAQ,MAAM,GAAG,GAAG,OAAO,EAAE;QACjC,MAAMiB,kBAAkBjB,QAAQ,IAAI,CAAC,CAACe,IAAMA,AAAmBG,WAAnBH,EAAE,YAAY;QAC1D,IAAI,CAACE,iBAAiB,OAAO,EAAE;QAE/B,MAAMd,UAA2B,EAAE;QACnC,IAAIgB,UAAUnB,OAAO,CAAC,EAAE,CAAE,YAAY,IAAI;QAC1C,IAAII,WAAW;QACf,IAAIgB,YAAY;QAEhB,IAAK,IAAIf,IAAI,GAAGA,IAAIL,QAAQ,MAAM,EAAEK,IAAK;YACxC,MAAMC,OAAON,OAAO,CAACK,EAAE,CAAE,YAAY,IAAI;YACzC,IAAIC,SAASa,SACZC;iBACM;gBACN,IAAI,CAAC,kBAAkB,CAACpB,SAASI,UAAUgB,WAAWD,SAAShB;gBAC/DgB,UAAUb;gBACVF,WAAWC;gBACXe,YAAY;YACb;QACD;QACA,IAAI,CAAC,kBAAkB,CAACpB,SAASI,UAAUgB,WAAWD,SAAShB;QAC/D,OAAOA;IACR;IAGQ,mBACPH,OAAsB,EACtBI,QAAgB,EAChBgB,SAAiB,EACjBD,OAAe,EACfhB,OAAwB,EACjB;QACP,IAAIiB,YAAY,GAAG;QACnB,MAAMb,QAAQP,OAAO,CAACI,SAAS,CAAE,cAAc,IAAIA,WAAW;QAC9D,MAAMI,MAAMR,OAAO,CAACI,WAAWgB,YAAY,EAAE,CAAE,cAAc,IAAIhB,WAAWgB;QAC5EjB,QAAQ,IAAI,CAAC;YACZ,SAAS;YACT,UAAU;YACV,SAAS,CAAC,gBAAgB,EAAEgB,QAAQ,YAAY,EAAEZ,MAAM,CAAC,EAAEC,IAAI,6BAA6B,CAAC;YAC7F,eAAe;gBAACD;gBAAOC;aAAI;QAC5B;IACD;IAGQ,uBAAuBR,OAAsB,EAAmB;QACvE,MAAMG,UAA2B,EAAE;QACnC,IAAIC,WAAW;QACf,IAAIgB,YAAY;QAChB,IAAIC,WAAW;QAEf,IAAK,IAAIhB,IAAI,GAAGA,IAAIL,QAAQ,MAAM,EAAEK,IAAK;YACxC,MAAMiB,OAAOtB,OAAO,CAACK,EAAE,CAAE,UAAU;YACnC,IAAIiB,AAASJ,WAATI,MAAoB;gBACvB,IAAI,CAAC,cAAc,CAACtB,SAASI,UAAUgB,WAAWjB;gBAClDC,WAAW;gBACXgB,YAAY;gBACZC,WAAW;gBACX;YACD;YACA,IAAID,AAAc,MAAdA,WAAiB;gBACpBhB,WAAWC;gBACXe,YAAY;gBACZC,WAAWC;YACZ,OAAO,IAAIA,OAAOD,UAAU;gBAC3BD;gBACAC,WAAWC;YACZ,OAAO;gBACN,IAAI,CAAC,cAAc,CAACtB,SAASI,UAAUgB,WAAWjB;gBAClDC,WAAWC;gBACXe,YAAY;gBACZC,WAAWC;YACZ;QACD;QACA,IAAI,CAAC,cAAc,CAACtB,SAASI,UAAUgB,WAAWjB;QAClD,OAAOA;IACR;IAGQ,eACPH,OAAsB,EACtBI,QAAgB,EAChBgB,SAAiB,EACjBjB,OAAwB,EACjB;QACP,IAAIiB,YAAY,GAAG;QACnB,MAAMG,QAAQvB,OAAO,CAACI,SAAS;QAC/B,MAAMoB,OAAOxB,OAAO,CAACI,WAAWgB,YAAY,EAAE;QAC9C,IAAI,CAACG,SAAS,CAACC,MAAM;QACrB,IAAID,AAAqBL,WAArBK,MAAM,UAAU,IAAkBC,AAAoBN,WAApBM,KAAK,UAAU,EAAgB;QACrE,MAAMjB,QAAQgB,MAAM,cAAc,IAAInB,WAAW;QACjD,MAAMI,MAAMgB,KAAK,cAAc,IAAIpB,WAAWgB;QAC9C,MAAMK,YAAYF,MAAM,UAAU;QAClC,MAAMG,WAAWF,KAAK,UAAU;QAChCrB,QAAQ,IAAI,CAAC;YACZ,SAAS;YACT,UAAU;YACV,SAAS,CAAC,sCAAsC,EAAEI,MAAM,CAAC,EAAEC,IAAI,EAAE,EAAEiB,UAAU,GAAG,EAAEC,SAAS,CAAC,CAAC;YAC7F,eAAe;gBAACnB;gBAAOC;aAAI;QAC5B;IACD;IAGQ,2BACPR,OAAsB,EACtBE,KAAwB,EACN;QAClB,MAAMC,UAA2B,EAAE;QACnC,IAAK,IAAIE,IAAI,GAAGA,IAAIL,QAAQ,MAAM,EAAEK,IAAK;YACxC,MAAMI,aAAaT,OAAO,CAACK,EAAE;YAC7B,IACCI,AAA4B,iBAA5BA,WAAW,YAAY,IACvB,CAACA,WAAW,EAAE,IACd,CAACP,MAAM,aAAa,CAAC,GAAG,CAACO,WAAW,EAAE,GAEtC;YAED,MAAME,YAAYX,QAAQ,KAAK,CAACK,IAAI,GAAGA,IAAI;YAC3C,MAAMQ,WAAWF,UAAU,IAAI,CAAC,CAACgB,YAAczB,MAAM,QAAQ,CAACyB,WAAWlB;YACzE,IAAII,UAAU;gBACb,MAAMC,IAAIL,WAAW,cAAc,IAAIJ,IAAI;gBAC3C,MAAMuB,IAAIf,SAAS,cAAc,IAAIb,QAAQ,OAAO,CAACa,YAAY;gBACjEV,QAAQ,IAAI,CAAC;oBACZ,SAAS;oBACT,UAAU;oBACV,SAAS,CAAC,sBAAsB,EAAEW,EAAE,qBAAqB,EAAEc,EAAE,gBAAgB,CAAC;oBAC9E,eAAe;wBAACd;wBAAGc;qBAAE;gBACtB;YACD;QACD;QACA,OAAOzB;IACR;AACD"}
|
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
import type { ThoughtType } from '../../contracts/reasoning-types.js';
|
|
14
14
|
import type { ConfidenceSignals } from '../reasoning.js';
|
|
15
15
|
import type { ThoughtData } from '../thought.js';
|
|
16
|
+
import type { VerificationLinks } from './VerificationLinks.js';
|
|
16
17
|
/** Result of {@link SignalComputer.computeStructuralQuality}. */
|
|
17
18
|
interface StructuralQualityResult {
|
|
18
19
|
score: number;
|
|
@@ -78,7 +79,7 @@ export declare class SignalComputer {
|
|
|
78
79
|
* @param branches - Map of branch IDs to their thought arrays
|
|
79
80
|
* @returns Computed confidence signals reflecting reasoning quality
|
|
80
81
|
*/
|
|
81
|
-
computeConfidenceSignals(history: ThoughtData[], branches: Record<string, ThoughtData[]
|
|
82
|
+
computeConfidenceSignals(history: ThoughtData[], branches: Record<string, ThoughtData[]>, links: VerificationLinks): ConfidenceSignals;
|
|
82
83
|
/**
|
|
83
84
|
* Compute the composite structural quality score and its components.
|
|
84
85
|
*
|
|
@@ -95,7 +96,7 @@ export declare class SignalComputer {
|
|
|
95
96
|
*
|
|
96
97
|
* @returns Score + components, or `null` when history is empty
|
|
97
98
|
*/
|
|
98
|
-
computeStructuralQuality(history: ThoughtData[], branches: Record<string, ThoughtData[]>, typeDistribution: Record<ThoughtType, number>, confidences: number[]): StructuralQualityResult | null;
|
|
99
|
+
computeStructuralQuality(history: ThoughtData[], branches: Record<string, ThoughtData[]>, typeDistribution: Record<ThoughtType, number>, confidences: number[], links: VerificationLinks): StructuralQualityResult | null;
|
|
99
100
|
/**
|
|
100
101
|
* Compute confidence stability: 1 - stddev(confidences).
|
|
101
102
|
*
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"SignalComputer.d.ts","sourceRoot":"","sources":["../../../src/core/evaluator/SignalComputer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oCAAoC,CAAC;AACtE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACzD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"SignalComputer.d.ts","sourceRoot":"","sources":["../../../src/core/evaluator/SignalComputer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oCAAoC,CAAC;AACtE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACzD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAEjD,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAehE,iEAAiE;AACjE,UAAU,uBAAuB;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE;QACX,cAAc,EAAE,MAAM,CAAC;QACvB,qBAAqB,EAAE,MAAM,CAAC;QAC9B,gBAAgB,EAAE,MAAM,CAAC;QACzB,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAC;KACpC,CAAC;IACF,cAAc,EAAE;QACf,cAAc,EAAE,MAAM,CAAC;QACvB,qBAAqB,EAAE,MAAM,CAAC;QAC9B,gBAAgB,EAAE,MAAM,CAAC;QACzB,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAC;KACpC,CAAC;CACF;AAED;;;;;;GAMG;AACH,qBAAa,cAAc;IAC1B;;;;;;;;;;;;;;;;;;;;;;;;;;;OA2BG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,eAAe,CAK5B;IAEX;;;OAGG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,qBAAqB,CAIlC;IAEX;;;;;;OAMG;IACI,wBAAwB,CAC9B,OAAO,EAAE,WAAW,EAAE,EACtB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,EAAE,CAAC,EACvC,KAAK,EAAE,iBAAiB,GACtB,iBAAiB;IAkCpB;;;;;;;;;;;;;;;OAeG;IACI,wBAAwB,CAC9B,OAAO,EAAE,WAAW,EAAE,EACtB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,EAAE,CAAC,EACvC,gBAAgB,EAAE,MAAM,CAAC,WAAW,EAAE,MAAM,CAAC,EAC7C,WAAW,EAAE,MAAM,EAAE,EACrB,KAAK,EAAE,iBAAiB,GACtB,uBAAuB,GAAG,IAAI;IAsEjC;;;;;;OAMG;IACI,0BAA0B,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,MAAM,GAAG,IAAI;IAUvE;;;;;;OAMG;IACI,6BAA6B,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,MAAM,GAAG,IAAI;CAS1E"}
|
|
@@ -16,10 +16,10 @@ class SignalComputer {
|
|
|
16
16
|
verificationCoverage: 0.375,
|
|
17
17
|
depthEfficiency: 0.25
|
|
18
18
|
};
|
|
19
|
-
computeConfidenceSignals(history, branches) {
|
|
19
|
+
computeConfidenceSignals(history, branches, links) {
|
|
20
20
|
const typeDistribution = _countByType(history);
|
|
21
21
|
const allConfidences = history.map((t)=>t.confidence).filter((c)=>void 0 !== c);
|
|
22
|
-
const structuralResult = this.computeStructuralQuality(history, branches, typeDistribution, allConfidences);
|
|
22
|
+
const structuralResult = this.computeStructuralQuality(history, branches, typeDistribution, allConfidences, links);
|
|
23
23
|
return {
|
|
24
24
|
reasoning_depth: history.length,
|
|
25
25
|
revision_count: history.filter((t)=>t.is_revision).length,
|
|
@@ -35,7 +35,7 @@ class SignalComputer {
|
|
|
35
35
|
}
|
|
36
36
|
};
|
|
37
37
|
}
|
|
38
|
-
computeStructuralQuality(history, branches, typeDistribution, confidences) {
|
|
38
|
+
computeStructuralQuality(history, branches, typeDistribution, confidences, links) {
|
|
39
39
|
if (0 === history.length) return null;
|
|
40
40
|
const total = history.length;
|
|
41
41
|
let entropy = 0;
|
|
@@ -48,12 +48,7 @@ class SignalComputer {
|
|
|
48
48
|
}
|
|
49
49
|
const rawTypeDiversity = entropy / Math.log2(6);
|
|
50
50
|
const typeDiversity = Math.max(rawTypeDiversity, FLOOR);
|
|
51
|
-
const
|
|
52
|
-
const hypothesisIds = new Set(hypotheses.map((t)=>t.hypothesis_id).filter(Boolean));
|
|
53
|
-
const verifiedIds = new Set(history.filter((t)=>'verification' === t.thought_type && t.hypothesis_id).map((t)=>t.hypothesis_id));
|
|
54
|
-
const rawVerificationCoverage = 0 === hypothesisIds.size ? 1.0 : [
|
|
55
|
-
...hypothesisIds
|
|
56
|
-
].filter((id)=>verifiedIds.has(id)).length / hypothesisIds.size;
|
|
51
|
+
const rawVerificationCoverage = 0 === links.hypotheses.length ? 1.0 : links.verifiedHypothesisCount / links.hypotheses.length;
|
|
57
52
|
const verificationCoverage = Math.max(rawVerificationCoverage, FLOOR);
|
|
58
53
|
const chainDepth = _computeChainDepth(history);
|
|
59
54
|
const branchCount = Object.keys(branches).length;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"core/evaluator/SignalComputer.js","sources":["../../../src/core/evaluator/SignalComputer.ts"],"sourcesContent":["/**\n * Confidence signal computation for sequential thinking.\n *\n * Provides {@link SignalComputer} — a pure stateless service that computes\n * {@link ConfidenceSignals} (including the composite `structural_quality`\n * score and its components) from thought history and branch data.\n *\n * Extracted from `ThoughtEvaluator` as part of the evaluator decomposition.\n * All methods are pure: no side effects, no I/O, no internal state.\n *\n * @module core/evaluator/SignalComputer\n */\n\nimport type { ThoughtType } from '../../contracts/reasoning-types.js';\nimport type { ConfidenceSignals } from '../reasoning.js';\nimport type { ThoughtData } from '../thought.js';\nimport { ALL_THOUGHT_TYPES, _computeChainDepth, _countByType } from './internals.js';\n\n/** Floor value applied to each quality component to prevent geometric mean collapse. */\nconst FLOOR = 0.01;\n\n/**\n * Round a numeric value to a fixed number of decimal places to mitigate\n * IEEE 754 floating-point accumulation errors (e.g. 0.9 + 0.8 averaging to\n * 0.8500000000000001 instead of 0.85).\n */\nfunction roundToPrecision(value: number, decimals: number = 10): number {\n\tconst factor = Math.pow(10, decimals);\n\treturn Math.round(value * factor) / factor;\n}\n\n/** Result of {@link SignalComputer.computeStructuralQuality}. */\ninterface StructuralQualityResult {\n\tscore: number;\n\tcomponents: {\n\t\ttype_diversity: number;\n\t\tverification_coverage: number;\n\t\tdepth_efficiency: number;\n\t\tconfidence_stability: number | null;\n\t};\n\traw_components: {\n\t\ttype_diversity: number;\n\t\tverification_coverage: number;\n\t\tdepth_efficiency: number;\n\t\tconfidence_stability: number | null;\n\t};\n}\n\n/**\n * Stateless service that computes {@link ConfidenceSignals} from thought history.\n *\n * @remarks\n * Pure computation — no side effects, no I/O, no internal state.\n * Safe to register as singleton or transient in DI.\n */\nexport class SignalComputer {\n\t/**\n\t * Structural quality weight configuration.\n\t *\n\t * Weights reflect the relative importance of each quality dimension:\n\t * - type_diversity (0.3): Shannon entropy of thought types — rewards reasoning that uses\n\t * multiple cognitive modes (hypothesis, verification, critique, synthesis) rather than\n\t * monotonic sequences. Based on information theory: higher entropy = more information.\n\t *\n\t * - verification_coverage (0.3): Ratio of verified to total hypotheses — rewards scientific\n\t * rigor. Hypotheses without verification are speculation; verification grounds reasoning.\n\t * Equal weight with diversity because unverified reasoning is a common failure mode.\n\t *\n\t * - depth_efficiency (0.2): Ratio of structural depth to total thoughts — rewards efficient\n\t * reasoning. Deep chains (many revisions/branches) are good; but only if proportionally\n\t * dense. Padding with low-value thoughts penalizes this metric.\n\t *\n\t * - confidence_stability (0.2): Low variance in self-assessed confidence — rewards calibrated\n\t * reasoning. Wildly fluctuating confidence suggests uncertainty about the approach.\n\t * Lower weight because confidence values are LLM self-reports (inherently noisy).\n\t *\n\t * Design rationale:\n\t * - Diversity + Verification (0.6) > Depth + Stability (0.4): Structural properties of the\n\t * reasoning DAG are more important than behavioral signals.\n\t * - All weights are positive (no dimension is penalizing).\n\t * - Weights sum to 1.0 for normalized scoring.\n\t * - When confidence_stability is unavailable (n<2), remaining weights redistribute\n\t * proportionally: td→0.375, vc→0.375, de→0.25 (preserving relative ratios).\n\t */\n\tprivate static readonly QUALITY_WEIGHTS = {\n\t\ttypeDiversity: 0.3,\n\t\tverificationCoverage: 0.3,\n\t\tdepthEfficiency: 0.2,\n\t\tconfidenceStability: 0.2,\n\t} as const;\n\n\t/**\n\t * Redistributed weights when confidence_stability is excluded (n<2).\n\t * Preserves relative ratios of remaining components (sum = 1.0).\n\t */\n\tprivate static readonly QUALITY_WEIGHTS_NO_CS = {\n\t\ttypeDiversity: 0.375,\n\t\tverificationCoverage: 0.375,\n\t\tdepthEfficiency: 0.25,\n\t} as const;\n\n\t/**\n\t * Compute confidence signals from history context.\n\t *\n\t * @param history - All thoughts in the current session\n\t * @param branches - Map of branch IDs to their thought arrays\n\t * @returns Computed confidence signals reflecting reasoning quality\n\t */\n\tpublic computeConfidenceSignals(\n\t\thistory: ThoughtData[],\n\t\tbranches: Record<string, ThoughtData[]>\n\t): ConfidenceSignals {\n\t\tconst typeDistribution = _countByType(history);\n\t\tconst allConfidences = history\n\t\t\t.map((t) => t.confidence)\n\t\t\t.filter((c): c is number => c !== undefined);\n\n\t\t// Compute structural quality components\n\t\tconst structuralResult = this.computeStructuralQuality(\n\t\t\thistory,\n\t\t\tbranches,\n\t\t\ttypeDistribution,\n\t\t\tallConfidences\n\t\t);\n\n\t\treturn {\n\t\t\treasoning_depth: history.length,\n\t\t\trevision_count: history.filter((t) => t.is_revision).length,\n\t\t\tbranch_count: Object.keys(branches).length,\n\t\t\tthought_type_distribution: typeDistribution,\n\t\t\thas_hypothesis: history.some((t) => t.thought_type === 'hypothesis'),\n\t\t\thas_verification: history.some((t) => t.thought_type === 'verification'),\n\t\t\taverage_confidence:\n\t\t\t\tallConfidences.length > 0\n\t\t\t\t\t? roundToPrecision(allConfidences.reduce((a, b) => a + b, 0) / allConfidences.length)\n\t\t\t\t\t: null,\n\t\t\t...(structuralResult !== null && {\n\t\t\t\tstructural_quality: structuralResult.score,\n\t\t\t\tquality_components: structuralResult.components,\n\t\t\t\tquality_components_raw: structuralResult.raw_components,\n\t\t\t}),\n\t\t};\n\t}\n\n\t/**\n\t * Compute the composite structural quality score and its components.\n\t *\n\t * Algorithm (all components floored at {@link FLOOR} = 0.01):\n\t * - `type_diversity` = Shannon entropy of thought_type distribution / log2(6)\n\t * - `verification_coverage` = verified_hypotheses / max(total_hypotheses, 1) (1.0 if none)\n\t * - `depth_efficiency` = max(chain_depth, branch_count + 1) / total_thoughts, clamped to 1.0\n\t * - `confidence_stability` = 1 - stddev(confidences), default 0.5 when empty, null when single value\n\t * - `structural_quality` = td^0.3 * vc^0.3 * de^0.2 * cs^0.2 (weighted geometric mean)\n\t *\n\t * Note: When confidence_stability is null (fewer than 2 confidence values), it is\n\t * excluded from the geometric mean and the remaining weights are redistributed\n\t * proportionally (td→0.375, vc→0.375, de→0.25).\n\t *\n\t * @returns Score + components, or `null` when history is empty\n\t */\n\tpublic computeStructuralQuality(\n\t\thistory: ThoughtData[],\n\t\tbranches: Record<string, ThoughtData[]>,\n\t\ttypeDistribution: Record<ThoughtType, number>,\n\t\tconfidences: number[]\n\t): StructuralQualityResult | null {\n\t\tif (history.length === 0) return null;\n\n\t\t// 1. type_diversity: Shannon entropy / log2(6)\n\t\tconst total = history.length;\n\t\tlet entropy = 0;\n\t\tfor (const type of ALL_THOUGHT_TYPES) {\n\t\t\tconst count = typeDistribution[type];\n\t\t\tif (count > 0) {\n\t\t\t\tconst pk = count / total;\n\t\t\t\tentropy -= pk * Math.log2(pk);\n\t\t\t}\n\t\t}\n\t\tconst rawTypeDiversity = entropy / Math.log2(6);\n\t\tconst typeDiversity = Math.max(rawTypeDiversity, FLOOR);\n\n\t\t// 2. verification_coverage: verified / total hypotheses (1.0 if none)\n\t\tconst hypotheses = history.filter((t) => t.thought_type === 'hypothesis');\n\t\tconst hypothesisIds = new Set(hypotheses.map((t) => t.hypothesis_id).filter(Boolean));\n\t\tconst verifiedIds = new Set(\n\t\t\thistory\n\t\t\t\t.filter((t) => t.thought_type === 'verification' && t.hypothesis_id)\n\t\t\t\t.map((t) => t.hypothesis_id)\n\t\t);\n\t\tconst rawVerificationCoverage =\n\t\t\thypothesisIds.size === 0\n\t\t\t\t? 1.0\n\t\t\t\t: [...hypothesisIds].filter((id) => verifiedIds.has(id)).length / hypothesisIds.size;\n\t\tconst verificationCoverage = Math.max(rawVerificationCoverage, FLOOR);\n\n\t\t// 3. depth_efficiency: max(chain_depth, branch_count + 1) / total_thoughts, clamped to 1.0\n\t\t// NOTE (Metis H3): Branching is desirable — treat branches as depth-equivalent.\n\t\tconst chainDepth = _computeChainDepth(history);\n\t\tconst branchCount = Object.keys(branches).length;\n\t\tconst effectiveDepth = Math.max(chainDepth, branchCount + 1);\n\t\tconst rawDepthEfficiency = Math.min(effectiveDepth / total, 1.0);\n\t\tconst depthEfficiency = Math.max(rawDepthEfficiency, FLOOR);\n\n\t\t// 4. confidence_stability: 1 - stddev(confidences), default 0.5\n\t\tconst confidenceStability = this.computeConfidenceStability(confidences);\n\t\tconst rawConfidenceStability = this.computeRawConfidenceStability(confidences);\n\n\t\tconst components: StructuralQualityResult['components'] = {\n\t\t\ttype_diversity: typeDiversity,\n\t\t\tverification_coverage: verificationCoverage,\n\t\t\tdepth_efficiency: depthEfficiency,\n\t\t\tconfidence_stability: confidenceStability,\n\t\t};\n\n\t\tconst rawComponents: StructuralQualityResult['raw_components'] = {\n\t\t\ttype_diversity: rawTypeDiversity,\n\t\t\tverification_coverage: rawVerificationCoverage,\n\t\t\tdepth_efficiency: rawDepthEfficiency,\n\t\t\tconfidence_stability: rawConfidenceStability,\n\t\t};\n\n\t\t// Weighted geometric mean: td^0.3 * vc^0.3 * de^0.2 * cs^0.2\n\t\t// When confidence_stability is null (n<2), exclude it from the geomean\n\t\t// and redistribute its weight proportionally to remaining components\n\t\t// (normalized td=0.375, vc=0.375, de=0.25)\n\t\tlet score: number;\n\t\tif (confidenceStability !== null) {\n\t\t\tconst w = SignalComputer.QUALITY_WEIGHTS;\n\t\t\tscore =\n\t\t\t\tMath.pow(typeDiversity, w.typeDiversity) *\n\t\t\t\tMath.pow(verificationCoverage, w.verificationCoverage) *\n\t\t\t\tMath.pow(depthEfficiency, w.depthEfficiency) *\n\t\t\t\tMath.pow(confidenceStability, w.confidenceStability);\n\t\t} else {\n\t\t\tconst w3 = SignalComputer.QUALITY_WEIGHTS_NO_CS;\n\t\t\tscore =\n\t\t\t\tMath.pow(typeDiversity, w3.typeDiversity) *\n\t\t\t\tMath.pow(verificationCoverage, w3.verificationCoverage) *\n\t\t\t\tMath.pow(depthEfficiency, w3.depthEfficiency);\n\t\t}\n\n\t\treturn { score, components, raw_components: rawComponents };\n\t}\n\n\t/**\n\t * Compute confidence stability: 1 - stddev(confidences).\n\t *\n\t * - Empty input → 0.5 (neutral default)\n\t * - Single value → null (insufficient data, excluded from structural quality)\n\t * - Otherwise → max(1 - stddev, FLOOR)\n\t */\n\tpublic computeConfidenceStability(confidences: number[]): number | null {\n\t\tif (confidences.length === 0) return 0.5;\n\t\tif (confidences.length === 1) return null;\n\n\t\tconst mean = confidences.reduce((a, b) => a + b, 0) / confidences.length;\n\t\tconst variance = confidences.reduce((sum, c) => sum + (c - mean) ** 2, 0) / confidences.length;\n\t\tconst stddev = Math.sqrt(variance);\n\t\treturn Math.max(1 - stddev, FLOOR);\n\t}\n\n\t/**\n\t * Compute raw (unfloored) confidence stability: 1 - stddev(confidences).\n\t *\n\t * - Empty input → 0.5 (neutral default)\n\t * - Single value → null (insufficient data)\n\t * - Otherwise → 1 - stddev (may be below FLOOR)\n\t */\n\tpublic computeRawConfidenceStability(confidences: number[]): number | null {\n\t\tif (confidences.length === 0) return 0.5;\n\t\tif (confidences.length === 1) return null;\n\n\t\tconst mean = confidences.reduce((a, b) => a + b, 0) / confidences.length;\n\t\tconst variance = confidences.reduce((sum, c) => sum + (c - mean) ** 2, 0) / confidences.length;\n\t\tconst stddev = Math.sqrt(variance);\n\t\treturn 1 - stddev;\n\t}\n}\n"],"names":["FLOOR","roundToPrecision","value","decimals","factor","Math","SignalComputer","history","branches","typeDistribution","_countByType","allConfidences","t","c","undefined","structuralResult","Object","a","b","confidences","total","entropy","type","ALL_THOUGHT_TYPES","count","pk","rawTypeDiversity","typeDiversity","hypotheses","hypothesisIds","Set","Boolean","verifiedIds","rawVerificationCoverage","id","verificationCoverage","chainDepth","_computeChainDepth","branchCount","effectiveDepth","rawDepthEfficiency","depthEfficiency","confidenceStability","rawConfidenceStability","components","rawComponents","score","w","w3","mean","variance","sum","stddev"],"mappings":";AAmBA,MAAMA,QAAQ;AAOd,SAASC,iBAAiBC,KAAa,EAAEC,WAAmB,EAAE;IAC7D,MAAMC,SAASC,KAAK,GAAG,CAAC,IAAIF;IAC5B,OAAOE,KAAK,KAAK,CAACH,QAAQE,UAAUA;AACrC;AA0BO,MAAME;IA6BZ,OAAwB,kBAAkB;QACzC,eAAe;QACf,sBAAsB;QACtB,iBAAiB;QACjB,qBAAqB;IACtB,EAAW;IAMX,OAAwB,wBAAwB;QAC/C,eAAe;QACf,sBAAsB;QACtB,iBAAiB;IAClB,EAAW;IASJ,yBACNC,OAAsB,EACtBC,QAAuC,EACnB;QACpB,MAAMC,mBAAmBC,aAAaH;QACtC,MAAMI,iBAAiBJ,QACrB,GAAG,CAAC,CAACK,IAAMA,EAAE,UAAU,EACvB,MAAM,CAAC,CAACC,IAAmBA,AAAMC,WAAND;QAG7B,MAAME,mBAAmB,IAAI,CAAC,wBAAwB,CACrDR,SACAC,UACAC,kBACAE;QAGD,OAAO;YACN,iBAAiBJ,QAAQ,MAAM;YAC/B,gBAAgBA,QAAQ,MAAM,CAAC,CAACK,IAAMA,EAAE,WAAW,EAAE,MAAM;YAC3D,cAAcI,OAAO,IAAI,CAACR,UAAU,MAAM;YAC1C,2BAA2BC;YAC3B,gBAAgBF,QAAQ,IAAI,CAAC,CAACK,IAAMA,AAAmB,iBAAnBA,EAAE,YAAY;YAClD,kBAAkBL,QAAQ,IAAI,CAAC,CAACK,IAAMA,AAAmB,mBAAnBA,EAAE,YAAY;YACpD,oBACCD,eAAe,MAAM,GAAG,IACrBV,iBAAiBU,eAAe,MAAM,CAAC,CAACM,GAAGC,IAAMD,IAAIC,GAAG,KAAKP,eAAe,MAAM,IAClF;YACJ,GAAII,AAAqB,SAArBA,oBAA6B;gBAChC,oBAAoBA,iBAAiB,KAAK;gBAC1C,oBAAoBA,iBAAiB,UAAU;gBAC/C,wBAAwBA,iBAAiB,cAAc;YACxD,CAAC;QACF;IACD;IAkBO,yBACNR,OAAsB,EACtBC,QAAuC,EACvCC,gBAA6C,EAC7CU,WAAqB,EACY;QACjC,IAAIZ,AAAmB,MAAnBA,QAAQ,MAAM,EAAQ,OAAO;QAGjC,MAAMa,QAAQb,QAAQ,MAAM;QAC5B,IAAIc,UAAU;QACd,KAAK,MAAMC,QAAQC,kBAAmB;YACrC,MAAMC,QAAQf,gBAAgB,CAACa,KAAK;YACpC,IAAIE,QAAQ,GAAG;gBACd,MAAMC,KAAKD,QAAQJ;gBACnBC,WAAWI,KAAKpB,KAAK,IAAI,CAACoB;YAC3B;QACD;QACA,MAAMC,mBAAmBL,UAAUhB,KAAK,IAAI,CAAC;QAC7C,MAAMsB,gBAAgBtB,KAAK,GAAG,CAACqB,kBAAkB1B;QAGjD,MAAM4B,aAAarB,QAAQ,MAAM,CAAC,CAACK,IAAMA,AAAmB,iBAAnBA,EAAE,YAAY;QACvD,MAAMiB,gBAAgB,IAAIC,IAAIF,WAAW,GAAG,CAAC,CAAChB,IAAMA,EAAE,aAAa,EAAE,MAAM,CAACmB;QAC5E,MAAMC,cAAc,IAAIF,IACvBvB,QACE,MAAM,CAAC,CAACK,IAAMA,AAAmB,mBAAnBA,EAAE,YAAY,IAAuBA,EAAE,aAAa,EAClE,GAAG,CAAC,CAACA,IAAMA,EAAE,aAAa;QAE7B,MAAMqB,0BACLJ,AAAuB,MAAvBA,cAAc,IAAI,GACf,MACA;eAAIA;SAAc,CAAC,MAAM,CAAC,CAACK,KAAOF,YAAY,GAAG,CAACE,KAAK,MAAM,GAAGL,cAAc,IAAI;QACtF,MAAMM,uBAAuB9B,KAAK,GAAG,CAAC4B,yBAAyBjC;QAI/D,MAAMoC,aAAaC,mBAAmB9B;QACtC,MAAM+B,cAActB,OAAO,IAAI,CAACR,UAAU,MAAM;QAChD,MAAM+B,iBAAiBlC,KAAK,GAAG,CAAC+B,YAAYE,cAAc;QAC1D,MAAME,qBAAqBnC,KAAK,GAAG,CAACkC,iBAAiBnB,OAAO;QAC5D,MAAMqB,kBAAkBpC,KAAK,GAAG,CAACmC,oBAAoBxC;QAGrD,MAAM0C,sBAAsB,IAAI,CAAC,0BAA0B,CAACvB;QAC5D,MAAMwB,yBAAyB,IAAI,CAAC,6BAA6B,CAACxB;QAElE,MAAMyB,aAAoD;YACzD,gBAAgBjB;YAChB,uBAAuBQ;YACvB,kBAAkBM;YAClB,sBAAsBC;QACvB;QAEA,MAAMG,gBAA2D;YAChE,gBAAgBnB;YAChB,uBAAuBO;YACvB,kBAAkBO;YAClB,sBAAsBG;QACvB;QAMA,IAAIG;QACJ,IAAIJ,AAAwB,SAAxBA,qBAA8B;YACjC,MAAMK,IAAIzC,eAAe,eAAe;YACxCwC,QACCzC,KAAK,GAAG,CAACsB,eAAeoB,EAAE,aAAa,IACvC1C,KAAK,GAAG,CAAC8B,sBAAsBY,EAAE,oBAAoB,IACrD1C,KAAK,GAAG,CAACoC,iBAAiBM,EAAE,eAAe,IAC3C1C,KAAK,GAAG,CAACqC,qBAAqBK,EAAE,mBAAmB;QACrD,OAAO;YACN,MAAMC,KAAK1C,eAAe,qBAAqB;YAC/CwC,QACCzC,KAAK,GAAG,CAACsB,eAAeqB,GAAG,aAAa,IACxC3C,KAAK,GAAG,CAAC8B,sBAAsBa,GAAG,oBAAoB,IACtD3C,KAAK,GAAG,CAACoC,iBAAiBO,GAAG,eAAe;QAC9C;QAEA,OAAO;YAAEF;YAAOF;YAAY,gBAAgBC;QAAc;IAC3D;IASO,2BAA2B1B,WAAqB,EAAiB;QACvE,IAAIA,AAAuB,MAAvBA,YAAY,MAAM,EAAQ,OAAO;QACrC,IAAIA,AAAuB,MAAvBA,YAAY,MAAM,EAAQ,OAAO;QAErC,MAAM8B,OAAO9B,YAAY,MAAM,CAAC,CAACF,GAAGC,IAAMD,IAAIC,GAAG,KAAKC,YAAY,MAAM;QACxE,MAAM+B,WAAW/B,YAAY,MAAM,CAAC,CAACgC,KAAKtC,IAAMsC,MAAOtC,AAAAA,CAAAA,IAAIoC,IAAG,KAAM,GAAG,KAAK9B,YAAY,MAAM;QAC9F,MAAMiC,SAAS/C,KAAK,IAAI,CAAC6C;QACzB,OAAO7C,KAAK,GAAG,CAAC,IAAI+C,QAAQpD;IAC7B;IASO,8BAA8BmB,WAAqB,EAAiB;QAC1E,IAAIA,AAAuB,MAAvBA,YAAY,MAAM,EAAQ,OAAO;QACrC,IAAIA,AAAuB,MAAvBA,YAAY,MAAM,EAAQ,OAAO;QAErC,MAAM8B,OAAO9B,YAAY,MAAM,CAAC,CAACF,GAAGC,IAAMD,IAAIC,GAAG,KAAKC,YAAY,MAAM;QACxE,MAAM+B,WAAW/B,YAAY,MAAM,CAAC,CAACgC,KAAKtC,IAAMsC,MAAOtC,AAAAA,CAAAA,IAAIoC,IAAG,KAAM,GAAG,KAAK9B,YAAY,MAAM;QAC9F,MAAMiC,SAAS/C,KAAK,IAAI,CAAC6C;QACzB,OAAO,IAAIE;IACZ;AACD"}
|
|
1
|
+
{"version":3,"file":"core/evaluator/SignalComputer.js","sources":["../../../src/core/evaluator/SignalComputer.ts"],"sourcesContent":["/**\n * Confidence signal computation for sequential thinking.\n *\n * Provides {@link SignalComputer} — a pure stateless service that computes\n * {@link ConfidenceSignals} (including the composite `structural_quality`\n * score and its components) from thought history and branch data.\n *\n * Extracted from `ThoughtEvaluator` as part of the evaluator decomposition.\n * All methods are pure: no side effects, no I/O, no internal state.\n *\n * @module core/evaluator/SignalComputer\n */\n\nimport type { ThoughtType } from '../../contracts/reasoning-types.js';\nimport type { ConfidenceSignals } from '../reasoning.js';\nimport type { ThoughtData } from '../thought.js';\nimport { ALL_THOUGHT_TYPES, _computeChainDepth, _countByType } from './internals.js';\nimport type { VerificationLinks } from './VerificationLinks.js';\n\n/** Floor value applied to each quality component to prevent geometric mean collapse. */\nconst FLOOR = 0.01;\n\n/**\n * Round a numeric value to a fixed number of decimal places to mitigate\n * IEEE 754 floating-point accumulation errors (e.g. 0.9 + 0.8 averaging to\n * 0.8500000000000001 instead of 0.85).\n */\nfunction roundToPrecision(value: number, decimals: number = 10): number {\n\tconst factor = Math.pow(10, decimals);\n\treturn Math.round(value * factor) / factor;\n}\n\n/** Result of {@link SignalComputer.computeStructuralQuality}. */\ninterface StructuralQualityResult {\n\tscore: number;\n\tcomponents: {\n\t\ttype_diversity: number;\n\t\tverification_coverage: number;\n\t\tdepth_efficiency: number;\n\t\tconfidence_stability: number | null;\n\t};\n\traw_components: {\n\t\ttype_diversity: number;\n\t\tverification_coverage: number;\n\t\tdepth_efficiency: number;\n\t\tconfidence_stability: number | null;\n\t};\n}\n\n/**\n * Stateless service that computes {@link ConfidenceSignals} from thought history.\n *\n * @remarks\n * Pure computation — no side effects, no I/O, no internal state.\n * Safe to register as singleton or transient in DI.\n */\nexport class SignalComputer {\n\t/**\n\t * Structural quality weight configuration.\n\t *\n\t * Weights reflect the relative importance of each quality dimension:\n\t * - type_diversity (0.3): Shannon entropy of thought types — rewards reasoning that uses\n\t * multiple cognitive modes (hypothesis, verification, critique, synthesis) rather than\n\t * monotonic sequences. Based on information theory: higher entropy = more information.\n\t *\n\t * - verification_coverage (0.3): Ratio of verified to total hypotheses — rewards scientific\n\t * rigor. Hypotheses without verification are speculation; verification grounds reasoning.\n\t * Equal weight with diversity because unverified reasoning is a common failure mode.\n\t *\n\t * - depth_efficiency (0.2): Ratio of structural depth to total thoughts — rewards efficient\n\t * reasoning. Deep chains (many revisions/branches) are good; but only if proportionally\n\t * dense. Padding with low-value thoughts penalizes this metric.\n\t *\n\t * - confidence_stability (0.2): Low variance in self-assessed confidence — rewards calibrated\n\t * reasoning. Wildly fluctuating confidence suggests uncertainty about the approach.\n\t * Lower weight because confidence values are LLM self-reports (inherently noisy).\n\t *\n\t * Design rationale:\n\t * - Diversity + Verification (0.6) > Depth + Stability (0.4): Structural properties of the\n\t * reasoning DAG are more important than behavioral signals.\n\t * - All weights are positive (no dimension is penalizing).\n\t * - Weights sum to 1.0 for normalized scoring.\n\t * - When confidence_stability is unavailable (n<2), remaining weights redistribute\n\t * proportionally: td→0.375, vc→0.375, de→0.25 (preserving relative ratios).\n\t */\n\tprivate static readonly QUALITY_WEIGHTS = {\n\t\ttypeDiversity: 0.3,\n\t\tverificationCoverage: 0.3,\n\t\tdepthEfficiency: 0.2,\n\t\tconfidenceStability: 0.2,\n\t} as const;\n\n\t/**\n\t * Redistributed weights when confidence_stability is excluded (n<2).\n\t * Preserves relative ratios of remaining components (sum = 1.0).\n\t */\n\tprivate static readonly QUALITY_WEIGHTS_NO_CS = {\n\t\ttypeDiversity: 0.375,\n\t\tverificationCoverage: 0.375,\n\t\tdepthEfficiency: 0.25,\n\t} as const;\n\n\t/**\n\t * Compute confidence signals from history context.\n\t *\n\t * @param history - All thoughts in the current session\n\t * @param branches - Map of branch IDs to their thought arrays\n\t * @returns Computed confidence signals reflecting reasoning quality\n\t */\n\tpublic computeConfidenceSignals(\n\t\thistory: ThoughtData[],\n\t\tbranches: Record<string, ThoughtData[]>,\n\t\tlinks: VerificationLinks\n\t): ConfidenceSignals {\n\t\tconst typeDistribution = _countByType(history);\n\t\tconst allConfidences = history\n\t\t\t.map((t) => t.confidence)\n\t\t\t.filter((c): c is number => c !== undefined);\n\n\t\t// Compute structural quality components\n\t\tconst structuralResult = this.computeStructuralQuality(\n\t\t\thistory,\n\t\t\tbranches,\n\t\t\ttypeDistribution,\n\t\t\tallConfidences,\n\t\t\tlinks\n\t\t);\n\n\t\treturn {\n\t\t\treasoning_depth: history.length,\n\t\t\trevision_count: history.filter((t) => t.is_revision).length,\n\t\t\tbranch_count: Object.keys(branches).length,\n\t\t\tthought_type_distribution: typeDistribution,\n\t\t\thas_hypothesis: history.some((t) => t.thought_type === 'hypothesis'),\n\t\t\thas_verification: history.some((t) => t.thought_type === 'verification'),\n\t\t\taverage_confidence:\n\t\t\t\tallConfidences.length > 0\n\t\t\t\t\t? roundToPrecision(allConfidences.reduce((a, b) => a + b, 0) / allConfidences.length)\n\t\t\t\t\t: null,\n\t\t\t...(structuralResult !== null && {\n\t\t\t\tstructural_quality: structuralResult.score,\n\t\t\t\tquality_components: structuralResult.components,\n\t\t\t\tquality_components_raw: structuralResult.raw_components,\n\t\t\t}),\n\t\t};\n\t}\n\n\t/**\n\t * Compute the composite structural quality score and its components.\n\t *\n\t * Algorithm (all components floored at {@link FLOOR} = 0.01):\n\t * - `type_diversity` = Shannon entropy of thought_type distribution / log2(6)\n\t * - `verification_coverage` = verified_hypotheses / max(total_hypotheses, 1) (1.0 if none)\n\t * - `depth_efficiency` = max(chain_depth, branch_count + 1) / total_thoughts, clamped to 1.0\n\t * - `confidence_stability` = 1 - stddev(confidences), default 0.5 when empty, null when single value\n\t * - `structural_quality` = td^0.3 * vc^0.3 * de^0.2 * cs^0.2 (weighted geometric mean)\n\t *\n\t * Note: When confidence_stability is null (fewer than 2 confidence values), it is\n\t * excluded from the geometric mean and the remaining weights are redistributed\n\t * proportionally (td→0.375, vc→0.375, de→0.25).\n\t *\n\t * @returns Score + components, or `null` when history is empty\n\t */\n\tpublic computeStructuralQuality(\n\t\thistory: ThoughtData[],\n\t\tbranches: Record<string, ThoughtData[]>,\n\t\ttypeDistribution: Record<ThoughtType, number>,\n\t\tconfidences: number[],\n\t\tlinks: VerificationLinks\n\t): StructuralQualityResult | null {\n\t\tif (history.length === 0) return null;\n\n\t\t// 1. type_diversity: Shannon entropy / log2(6)\n\t\tconst total = history.length;\n\t\tlet entropy = 0;\n\t\tfor (const type of ALL_THOUGHT_TYPES) {\n\t\t\tconst count = typeDistribution[type];\n\t\t\tif (count > 0) {\n\t\t\t\tconst pk = count / total;\n\t\t\t\tentropy -= pk * Math.log2(pk);\n\t\t\t}\n\t\t}\n\t\tconst rawTypeDiversity = entropy / Math.log2(6);\n\t\tconst typeDiversity = Math.max(rawTypeDiversity, FLOOR);\n\n\t\t// 2. verification_coverage: verified / total hypotheses (1.0 if none)\n\t\tconst rawVerificationCoverage =\n\t\t\tlinks.hypotheses.length === 0 ? 1.0 : links.verifiedHypothesisCount / links.hypotheses.length;\n\t\tconst verificationCoverage = Math.max(rawVerificationCoverage, FLOOR);\n\n\t\t// 3. depth_efficiency: max(chain_depth, branch_count + 1) / total_thoughts, clamped to 1.0\n\t\t// NOTE (Metis H3): Branching is desirable — treat branches as depth-equivalent.\n\t\tconst chainDepth = _computeChainDepth(history);\n\t\tconst branchCount = Object.keys(branches).length;\n\t\tconst effectiveDepth = Math.max(chainDepth, branchCount + 1);\n\t\tconst rawDepthEfficiency = Math.min(effectiveDepth / total, 1.0);\n\t\tconst depthEfficiency = Math.max(rawDepthEfficiency, FLOOR);\n\n\t\t// 4. confidence_stability: 1 - stddev(confidences), default 0.5\n\t\tconst confidenceStability = this.computeConfidenceStability(confidences);\n\t\tconst rawConfidenceStability = this.computeRawConfidenceStability(confidences);\n\n\t\tconst components: StructuralQualityResult['components'] = {\n\t\t\ttype_diversity: typeDiversity,\n\t\t\tverification_coverage: verificationCoverage,\n\t\t\tdepth_efficiency: depthEfficiency,\n\t\t\tconfidence_stability: confidenceStability,\n\t\t};\n\n\t\tconst rawComponents: StructuralQualityResult['raw_components'] = {\n\t\t\ttype_diversity: rawTypeDiversity,\n\t\t\tverification_coverage: rawVerificationCoverage,\n\t\t\tdepth_efficiency: rawDepthEfficiency,\n\t\t\tconfidence_stability: rawConfidenceStability,\n\t\t};\n\n\t\t// Weighted geometric mean: td^0.3 * vc^0.3 * de^0.2 * cs^0.2\n\t\t// When confidence_stability is null (n<2), exclude it from the geomean\n\t\t// and redistribute its weight proportionally to remaining components\n\t\t// (normalized td=0.375, vc=0.375, de=0.25)\n\t\tlet score: number;\n\t\tif (confidenceStability !== null) {\n\t\t\tconst w = SignalComputer.QUALITY_WEIGHTS;\n\t\t\tscore =\n\t\t\t\tMath.pow(typeDiversity, w.typeDiversity) *\n\t\t\t\tMath.pow(verificationCoverage, w.verificationCoverage) *\n\t\t\t\tMath.pow(depthEfficiency, w.depthEfficiency) *\n\t\t\t\tMath.pow(confidenceStability, w.confidenceStability);\n\t\t} else {\n\t\t\tconst w3 = SignalComputer.QUALITY_WEIGHTS_NO_CS;\n\t\t\tscore =\n\t\t\t\tMath.pow(typeDiversity, w3.typeDiversity) *\n\t\t\t\tMath.pow(verificationCoverage, w3.verificationCoverage) *\n\t\t\t\tMath.pow(depthEfficiency, w3.depthEfficiency);\n\t\t}\n\n\t\treturn { score, components, raw_components: rawComponents };\n\t}\n\n\t/**\n\t * Compute confidence stability: 1 - stddev(confidences).\n\t *\n\t * - Empty input → 0.5 (neutral default)\n\t * - Single value → null (insufficient data, excluded from structural quality)\n\t * - Otherwise → max(1 - stddev, FLOOR)\n\t */\n\tpublic computeConfidenceStability(confidences: number[]): number | null {\n\t\tif (confidences.length === 0) return 0.5;\n\t\tif (confidences.length === 1) return null;\n\n\t\tconst mean = confidences.reduce((a, b) => a + b, 0) / confidences.length;\n\t\tconst variance = confidences.reduce((sum, c) => sum + (c - mean) ** 2, 0) / confidences.length;\n\t\tconst stddev = Math.sqrt(variance);\n\t\treturn Math.max(1 - stddev, FLOOR);\n\t}\n\n\t/**\n\t * Compute raw (unfloored) confidence stability: 1 - stddev(confidences).\n\t *\n\t * - Empty input → 0.5 (neutral default)\n\t * - Single value → null (insufficient data)\n\t * - Otherwise → 1 - stddev (may be below FLOOR)\n\t */\n\tpublic computeRawConfidenceStability(confidences: number[]): number | null {\n\t\tif (confidences.length === 0) return 0.5;\n\t\tif (confidences.length === 1) return null;\n\n\t\tconst mean = confidences.reduce((a, b) => a + b, 0) / confidences.length;\n\t\tconst variance = confidences.reduce((sum, c) => sum + (c - mean) ** 2, 0) / confidences.length;\n\t\tconst stddev = Math.sqrt(variance);\n\t\treturn 1 - stddev;\n\t}\n}\n"],"names":["FLOOR","roundToPrecision","value","decimals","factor","Math","SignalComputer","history","branches","links","typeDistribution","_countByType","allConfidences","t","c","undefined","structuralResult","Object","a","b","confidences","total","entropy","type","ALL_THOUGHT_TYPES","count","pk","rawTypeDiversity","typeDiversity","rawVerificationCoverage","verificationCoverage","chainDepth","_computeChainDepth","branchCount","effectiveDepth","rawDepthEfficiency","depthEfficiency","confidenceStability","rawConfidenceStability","components","rawComponents","score","w","w3","mean","variance","sum","stddev"],"mappings":";AAoBA,MAAMA,QAAQ;AAOd,SAASC,iBAAiBC,KAAa,EAAEC,WAAmB,EAAE;IAC7D,MAAMC,SAASC,KAAK,GAAG,CAAC,IAAIF;IAC5B,OAAOE,KAAK,KAAK,CAACH,QAAQE,UAAUA;AACrC;AA0BO,MAAME;IA6BZ,OAAwB,kBAAkB;QACzC,eAAe;QACf,sBAAsB;QACtB,iBAAiB;QACjB,qBAAqB;IACtB,EAAW;IAMX,OAAwB,wBAAwB;QAC/C,eAAe;QACf,sBAAsB;QACtB,iBAAiB;IAClB,EAAW;IASJ,yBACNC,OAAsB,EACtBC,QAAuC,EACvCC,KAAwB,EACJ;QACpB,MAAMC,mBAAmBC,aAAaJ;QACtC,MAAMK,iBAAiBL,QACrB,GAAG,CAAC,CAACM,IAAMA,EAAE,UAAU,EACvB,MAAM,CAAC,CAACC,IAAmBA,AAAMC,WAAND;QAG7B,MAAME,mBAAmB,IAAI,CAAC,wBAAwB,CACrDT,SACAC,UACAE,kBACAE,gBACAH;QAGD,OAAO;YACN,iBAAiBF,QAAQ,MAAM;YAC/B,gBAAgBA,QAAQ,MAAM,CAAC,CAACM,IAAMA,EAAE,WAAW,EAAE,MAAM;YAC3D,cAAcI,OAAO,IAAI,CAACT,UAAU,MAAM;YAC1C,2BAA2BE;YAC3B,gBAAgBH,QAAQ,IAAI,CAAC,CAACM,IAAMA,AAAmB,iBAAnBA,EAAE,YAAY;YAClD,kBAAkBN,QAAQ,IAAI,CAAC,CAACM,IAAMA,AAAmB,mBAAnBA,EAAE,YAAY;YACpD,oBACCD,eAAe,MAAM,GAAG,IACrBX,iBAAiBW,eAAe,MAAM,CAAC,CAACM,GAAGC,IAAMD,IAAIC,GAAG,KAAKP,eAAe,MAAM,IAClF;YACJ,GAAII,AAAqB,SAArBA,oBAA6B;gBAChC,oBAAoBA,iBAAiB,KAAK;gBAC1C,oBAAoBA,iBAAiB,UAAU;gBAC/C,wBAAwBA,iBAAiB,cAAc;YACxD,CAAC;QACF;IACD;IAkBO,yBACNT,OAAsB,EACtBC,QAAuC,EACvCE,gBAA6C,EAC7CU,WAAqB,EACrBX,KAAwB,EACS;QACjC,IAAIF,AAAmB,MAAnBA,QAAQ,MAAM,EAAQ,OAAO;QAGjC,MAAMc,QAAQd,QAAQ,MAAM;QAC5B,IAAIe,UAAU;QACd,KAAK,MAAMC,QAAQC,kBAAmB;YACrC,MAAMC,QAAQf,gBAAgB,CAACa,KAAK;YACpC,IAAIE,QAAQ,GAAG;gBACd,MAAMC,KAAKD,QAAQJ;gBACnBC,WAAWI,KAAKrB,KAAK,IAAI,CAACqB;YAC3B;QACD;QACA,MAAMC,mBAAmBL,UAAUjB,KAAK,IAAI,CAAC;QAC7C,MAAMuB,gBAAgBvB,KAAK,GAAG,CAACsB,kBAAkB3B;QAGjD,MAAM6B,0BACLpB,AAA4B,MAA5BA,MAAM,UAAU,CAAC,MAAM,GAAS,MAAMA,MAAM,uBAAuB,GAAGA,MAAM,UAAU,CAAC,MAAM;QAC9F,MAAMqB,uBAAuBzB,KAAK,GAAG,CAACwB,yBAAyB7B;QAI/D,MAAM+B,aAAaC,mBAAmBzB;QACtC,MAAM0B,cAAchB,OAAO,IAAI,CAACT,UAAU,MAAM;QAChD,MAAM0B,iBAAiB7B,KAAK,GAAG,CAAC0B,YAAYE,cAAc;QAC1D,MAAME,qBAAqB9B,KAAK,GAAG,CAAC6B,iBAAiBb,OAAO;QAC5D,MAAMe,kBAAkB/B,KAAK,GAAG,CAAC8B,oBAAoBnC;QAGrD,MAAMqC,sBAAsB,IAAI,CAAC,0BAA0B,CAACjB;QAC5D,MAAMkB,yBAAyB,IAAI,CAAC,6BAA6B,CAAClB;QAElE,MAAMmB,aAAoD;YACzD,gBAAgBX;YAChB,uBAAuBE;YACvB,kBAAkBM;YAClB,sBAAsBC;QACvB;QAEA,MAAMG,gBAA2D;YAChE,gBAAgBb;YAChB,uBAAuBE;YACvB,kBAAkBM;YAClB,sBAAsBG;QACvB;QAMA,IAAIG;QACJ,IAAIJ,AAAwB,SAAxBA,qBAA8B;YACjC,MAAMK,IAAIpC,eAAe,eAAe;YACxCmC,QACCpC,KAAK,GAAG,CAACuB,eAAec,EAAE,aAAa,IACvCrC,KAAK,GAAG,CAACyB,sBAAsBY,EAAE,oBAAoB,IACrDrC,KAAK,GAAG,CAAC+B,iBAAiBM,EAAE,eAAe,IAC3CrC,KAAK,GAAG,CAACgC,qBAAqBK,EAAE,mBAAmB;QACrD,OAAO;YACN,MAAMC,KAAKrC,eAAe,qBAAqB;YAC/CmC,QACCpC,KAAK,GAAG,CAACuB,eAAee,GAAG,aAAa,IACxCtC,KAAK,GAAG,CAACyB,sBAAsBa,GAAG,oBAAoB,IACtDtC,KAAK,GAAG,CAAC+B,iBAAiBO,GAAG,eAAe;QAC9C;QAEA,OAAO;YAAEF;YAAOF;YAAY,gBAAgBC;QAAc;IAC3D;IASO,2BAA2BpB,WAAqB,EAAiB;QACvE,IAAIA,AAAuB,MAAvBA,YAAY,MAAM,EAAQ,OAAO;QACrC,IAAIA,AAAuB,MAAvBA,YAAY,MAAM,EAAQ,OAAO;QAErC,MAAMwB,OAAOxB,YAAY,MAAM,CAAC,CAACF,GAAGC,IAAMD,IAAIC,GAAG,KAAKC,YAAY,MAAM;QACxE,MAAMyB,WAAWzB,YAAY,MAAM,CAAC,CAAC0B,KAAKhC,IAAMgC,MAAOhC,AAAAA,CAAAA,IAAI8B,IAAG,KAAM,GAAG,KAAKxB,YAAY,MAAM;QAC9F,MAAM2B,SAAS1C,KAAK,IAAI,CAACwC;QACzB,OAAOxC,KAAK,GAAG,CAAC,IAAI0C,QAAQ/C;IAC7B;IASO,8BAA8BoB,WAAqB,EAAiB;QAC1E,IAAIA,AAAuB,MAAvBA,YAAY,MAAM,EAAQ,OAAO;QACrC,IAAIA,AAAuB,MAAvBA,YAAY,MAAM,EAAQ,OAAO;QAErC,MAAMwB,OAAOxB,YAAY,MAAM,CAAC,CAACF,GAAGC,IAAMD,IAAIC,GAAG,KAAKC,YAAY,MAAM;QACxE,MAAMyB,WAAWzB,YAAY,MAAM,CAAC,CAAC0B,KAAKhC,IAAMgC,MAAOhC,AAAAA,CAAAA,IAAI8B,IAAG,KAAM,GAAG,KAAKxB,YAAY,MAAM;QAC9F,MAAM2B,SAAS1C,KAAK,IAAI,CAACwC;QACzB,OAAO,IAAIE;IACZ;AACD"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { ThoughtId } from '../../contracts/ids.js';
|
|
2
|
+
import type { ResolvedThoughtReferences } from '../IHistoryManager.js';
|
|
3
|
+
import type { ThoughtData } from '../thought.js';
|
|
4
|
+
export type ActiveThoughtBranches = Readonly<Record<string, readonly ThoughtData[]>>;
|
|
5
|
+
export interface VerificationLinks {
|
|
6
|
+
readonly thoughts: readonly ThoughtData[];
|
|
7
|
+
readonly hypotheses: readonly ThoughtData[];
|
|
8
|
+
readonly hypothesisIds: ReadonlySet<ThoughtId>;
|
|
9
|
+
readonly verifiedHypothesisIds: ReadonlySet<ThoughtId>;
|
|
10
|
+
readonly verifiedHypothesisCount: number;
|
|
11
|
+
targetFor(verifierId: ThoughtId): ThoughtId | undefined;
|
|
12
|
+
isVerified(hypothesis: ThoughtData): boolean;
|
|
13
|
+
verifies(verifier: ThoughtData, hypothesis: ThoughtData): boolean;
|
|
14
|
+
}
|
|
15
|
+
export declare function resolvedVerificationTarget(thought: ThoughtData, references: ResolvedThoughtReferences): ThoughtId | undefined;
|
|
16
|
+
export declare function resolveVerificationLinks(history: readonly ThoughtData[], branches: ActiveThoughtBranches, admissionTargets?: ReadonlyMap<ThoughtId, ThoughtId>): VerificationLinks;
|
|
17
|
+
export declare function collectActiveMainThoughts(history: readonly ThoughtData[], branches: ActiveThoughtBranches): ThoughtData[];
|
|
18
|
+
export declare function collectActiveThoughts(history: readonly ThoughtData[], branches: ActiveThoughtBranches): ThoughtData[];
|
|
19
|
+
//# sourceMappingURL=VerificationLinks.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"VerificationLinks.d.ts","sourceRoot":"","sources":["../../../src/core/evaluator/VerificationLinks.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,uBAAuB,CAAC;AACvE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAEjD,MAAM,MAAM,qBAAqB,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,SAAS,WAAW,EAAE,CAAC,CAAC,CAAC;AAErF,MAAM,WAAW,iBAAiB;IACjC,QAAQ,CAAC,QAAQ,EAAE,SAAS,WAAW,EAAE,CAAC;IAC1C,QAAQ,CAAC,UAAU,EAAE,SAAS,WAAW,EAAE,CAAC;IAC5C,QAAQ,CAAC,aAAa,EAAE,WAAW,CAAC,SAAS,CAAC,CAAC;IAC/C,QAAQ,CAAC,qBAAqB,EAAE,WAAW,CAAC,SAAS,CAAC,CAAC;IACvD,QAAQ,CAAC,uBAAuB,EAAE,MAAM,CAAC;IACzC,SAAS,CAAC,UAAU,EAAE,SAAS,GAAG,SAAS,GAAG,SAAS,CAAC;IACxD,UAAU,CAAC,UAAU,EAAE,WAAW,GAAG,OAAO,CAAC;IAC7C,QAAQ,CAAC,QAAQ,EAAE,WAAW,EAAE,UAAU,EAAE,WAAW,GAAG,OAAO,CAAC;CAClE;AAED,wBAAgB,0BAA0B,CACzC,OAAO,EAAE,WAAW,EACpB,UAAU,EAAE,yBAAyB,GACnC,SAAS,GAAG,SAAS,CAIvB;AAED,wBAAgB,wBAAwB,CACvC,OAAO,EAAE,SAAS,WAAW,EAAE,EAC/B,QAAQ,EAAE,qBAAqB,EAC/B,gBAAgB,GAAE,WAAW,CAAC,SAAS,EAAE,SAAS,CAAa,GAC7D,iBAAiB,CA0CnB;AAED,wBAAgB,yBAAyB,CACxC,OAAO,EAAE,SAAS,WAAW,EAAE,EAC/B,QAAQ,EAAE,qBAAqB,GAC7B,WAAW,EAAE,CAmBf;AAED,wBAAgB,qBAAqB,CACpC,OAAO,EAAE,SAAS,WAAW,EAAE,EAC/B,QAAQ,EAAE,qBAAqB,GAC7B,WAAW,EAAE,CAef"}
|