@akemi-mio/creativity 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ConceptMixer.d.ts +17 -1
- package/dist/ConceptMixer.js +36 -3
- package/dist/CreativityPrompt.d.ts +5 -1
- package/dist/CreativityPrompt.js +36 -1
- package/dist/DrizzleFeedbackEventStore.d.ts +22 -0
- package/dist/DrizzleFeedbackEventStore.js +101 -0
- package/dist/ExperimentRunStore.d.ts +106 -0
- package/dist/ExperimentRunStore.js +160 -0
- package/dist/FeedbackEvents.d.ts +86 -0
- package/dist/FeedbackEvents.js +133 -0
- package/dist/FeedbackLearning.d.ts +195 -0
- package/dist/FeedbackLearning.js +456 -0
- package/dist/FeedbackRollout.d.ts +72 -0
- package/dist/FeedbackRollout.js +136 -0
- package/dist/FeedbackRuntime.d.ts +37 -0
- package/dist/FeedbackRuntime.js +72 -0
- package/dist/HypothesisGenerator.d.ts +4 -1
- package/dist/HypothesisGenerator.js +9 -7
- package/dist/IdeaGenerator.d.ts +7 -1
- package/dist/IdeaGenerator.js +8 -4
- package/dist/InsightSamplingExperiment.d.ts +82 -0
- package/dist/InsightSamplingExperiment.js +309 -0
- package/dist/JsonlFeedbackEventStore.d.ts +32 -0
- package/dist/JsonlFeedbackEventStore.js +79 -0
- package/dist/MechanismGraph.d.ts +46 -0
- package/dist/MechanismGraph.js +76 -0
- package/dist/MechanismRegistry.d.ts +118 -0
- package/dist/MechanismRegistry.js +233 -0
- package/dist/WhenThenRunner.d.ts +135 -0
- package/dist/WhenThenRunner.js +358 -0
- package/dist/ingestFeedbackEvents.d.ts +32 -0
- package/dist/ingestFeedbackEvents.js +42 -0
- package/dist/types.d.ts +26 -1
- package/dist/types.js +15 -0
- package/package.json +67 -3
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* FeedbackRuntime — 正向反馈闭环受控启用(设计 v1.0,P3)
|
|
3
|
+
*
|
|
4
|
+
* 把"是否/如何消费反馈"从代码常量提升为可配置、可校验的运行时状态(R6 纪律):
|
|
5
|
+
* - off 关闭:生成行为与无反馈时逐字节一致(默认)。
|
|
6
|
+
* - shadow 影子运行:计算反馈调整并记录 delta,但不应用到排序/prompt。
|
|
7
|
+
* - canary 小流量:按比例应用反馈(λ 与探索预算生效)。
|
|
8
|
+
* - full 正式启用:全部应用。
|
|
9
|
+
*
|
|
10
|
+
* 每阶段预固定基线、指标与回滚条件;回滚 = 切回 off(配置热切换,无需发版)。
|
|
11
|
+
* 反馈永不进入 checkPromotionGate/checkSubmitGate 输入(spec §二.2)。
|
|
12
|
+
*/
|
|
13
|
+
export type FeedbackMode = 'off' | 'shadow' | 'canary' | 'full';
|
|
14
|
+
export interface FeedbackRuntimeConfig {
|
|
15
|
+
mode: FeedbackMode;
|
|
16
|
+
/** λ clamp:单对调整占总评分范围的比例上限 */
|
|
17
|
+
lambdaClampPct: number;
|
|
18
|
+
/** 探索预算:保留给低频/新组合的槽位比例 */
|
|
19
|
+
explorationBudget: number;
|
|
20
|
+
/** 证据时间衰减半衰期(天) */
|
|
21
|
+
halfLifeDays: number;
|
|
22
|
+
/** canary 模式应用反馈的比例 0..1 */
|
|
23
|
+
canaryFraction: number;
|
|
24
|
+
}
|
|
25
|
+
export declare const DEFAULT_FEEDBACK_RUNTIME: FeedbackRuntimeConfig;
|
|
26
|
+
export declare const FEEDBACK_MODES: readonly FeedbackMode[];
|
|
27
|
+
/**
|
|
28
|
+
* 解析并校验运行时配置;失败返回 null(fail-closed,调用方回退默认 off)。
|
|
29
|
+
* 未知字段忽略。
|
|
30
|
+
*/
|
|
31
|
+
export declare function parseFeedbackRuntime(raw: unknown): FeedbackRuntimeConfig | null;
|
|
32
|
+
/** 从环境变量解析(缺省全 off)。 */
|
|
33
|
+
export declare function resolveFeedbackRuntime(env?: Record<string, string | undefined>): FeedbackRuntimeConfig;
|
|
34
|
+
/** 该模式下是否实际应用反馈到生成(shadow 只观测)。 */
|
|
35
|
+
export declare function isFeedbackApplied(mode: FeedbackMode): boolean;
|
|
36
|
+
/** 该模式下是否观测(计算 delta 并记录)。 */
|
|
37
|
+
export declare function isFeedbackObserved(mode: FeedbackMode): boolean;
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* FeedbackRuntime — 正向反馈闭环受控启用(设计 v1.0,P3)
|
|
4
|
+
*
|
|
5
|
+
* 把"是否/如何消费反馈"从代码常量提升为可配置、可校验的运行时状态(R6 纪律):
|
|
6
|
+
* - off 关闭:生成行为与无反馈时逐字节一致(默认)。
|
|
7
|
+
* - shadow 影子运行:计算反馈调整并记录 delta,但不应用到排序/prompt。
|
|
8
|
+
* - canary 小流量:按比例应用反馈(λ 与探索预算生效)。
|
|
9
|
+
* - full 正式启用:全部应用。
|
|
10
|
+
*
|
|
11
|
+
* 每阶段预固定基线、指标与回滚条件;回滚 = 切回 off(配置热切换,无需发版)。
|
|
12
|
+
* 反馈永不进入 checkPromotionGate/checkSubmitGate 输入(spec §二.2)。
|
|
13
|
+
*/
|
|
14
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
+
exports.FEEDBACK_MODES = exports.DEFAULT_FEEDBACK_RUNTIME = void 0;
|
|
16
|
+
exports.parseFeedbackRuntime = parseFeedbackRuntime;
|
|
17
|
+
exports.resolveFeedbackRuntime = resolveFeedbackRuntime;
|
|
18
|
+
exports.isFeedbackApplied = isFeedbackApplied;
|
|
19
|
+
exports.isFeedbackObserved = isFeedbackObserved;
|
|
20
|
+
exports.DEFAULT_FEEDBACK_RUNTIME = {
|
|
21
|
+
mode: 'off',
|
|
22
|
+
lambdaClampPct: 0.1,
|
|
23
|
+
explorationBudget: 0.2,
|
|
24
|
+
halfLifeDays: 30,
|
|
25
|
+
canaryFraction: 0.1,
|
|
26
|
+
};
|
|
27
|
+
exports.FEEDBACK_MODES = ['off', 'shadow', 'canary', 'full'];
|
|
28
|
+
function clampNumber(value, min, max, fallback) {
|
|
29
|
+
const num = typeof value === 'string' ? Number(value) : value;
|
|
30
|
+
if (typeof num !== 'number' || !Number.isFinite(num))
|
|
31
|
+
return fallback;
|
|
32
|
+
return Math.max(min, Math.min(max, num));
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* 解析并校验运行时配置;失败返回 null(fail-closed,调用方回退默认 off)。
|
|
36
|
+
* 未知字段忽略。
|
|
37
|
+
*/
|
|
38
|
+
function parseFeedbackRuntime(raw) {
|
|
39
|
+
if (raw === null || typeof raw !== 'object')
|
|
40
|
+
return null;
|
|
41
|
+
const obj = raw;
|
|
42
|
+
const mode = obj.mode;
|
|
43
|
+
if (!exports.FEEDBACK_MODES.includes(mode))
|
|
44
|
+
return null;
|
|
45
|
+
return {
|
|
46
|
+
mode: mode,
|
|
47
|
+
lambdaClampPct: clampNumber(obj.lambdaClampPct, 0, 0.5, exports.DEFAULT_FEEDBACK_RUNTIME.lambdaClampPct),
|
|
48
|
+
explorationBudget: clampNumber(obj.explorationBudget, 0, 1, exports.DEFAULT_FEEDBACK_RUNTIME.explorationBudget),
|
|
49
|
+
halfLifeDays: clampNumber(obj.halfLifeDays, 1, 3650, exports.DEFAULT_FEEDBACK_RUNTIME.halfLifeDays),
|
|
50
|
+
canaryFraction: clampNumber(obj.canaryFraction, 0, 1, exports.DEFAULT_FEEDBACK_RUNTIME.canaryFraction),
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
/** 从环境变量解析(缺省全 off)。 */
|
|
54
|
+
function resolveFeedbackRuntime(env = process.env) {
|
|
55
|
+
const parsed = parseFeedbackRuntime({
|
|
56
|
+
mode: env.MIO_FEEDBACK_MODE,
|
|
57
|
+
lambdaClampPct: env.MIO_FEEDBACK_LAMBDA_PCT,
|
|
58
|
+
explorationBudget: env.MIO_FEEDBACK_EXPLORATION,
|
|
59
|
+
halfLifeDays: env.MIO_FEEDBACK_HALF_LIFE_DAYS,
|
|
60
|
+
canaryFraction: env.MIO_FEEDBACK_CANARY_FRACTION,
|
|
61
|
+
});
|
|
62
|
+
// env 全缺省时 parse 返回 null(mode undefined)→ 回退默认 off
|
|
63
|
+
return parsed ?? exports.DEFAULT_FEEDBACK_RUNTIME;
|
|
64
|
+
}
|
|
65
|
+
/** 该模式下是否实际应用反馈到生成(shadow 只观测)。 */
|
|
66
|
+
function isFeedbackApplied(mode) {
|
|
67
|
+
return mode === 'canary' || mode === 'full';
|
|
68
|
+
}
|
|
69
|
+
/** 该模式下是否观测(计算 delta 并记录)。 */
|
|
70
|
+
function isFeedbackObserved(mode) {
|
|
71
|
+
return mode === 'shadow' || mode === 'canary' || mode === 'full';
|
|
72
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { Hypothesis, ConceptCombo, CreativitySource, ExternalSignal } from './types';
|
|
2
|
+
import type { FeedbackPromptBlock } from './FeedbackLearning';
|
|
2
3
|
/**
|
|
3
4
|
* HypothesisGenerator — 将概念组合转化为可验证的假设
|
|
4
5
|
*
|
|
@@ -38,7 +39,9 @@ export declare class HypothesisGenerator {
|
|
|
38
39
|
/** 外部信号 — 不参与配对,作为审视视角注入 LLM */
|
|
39
40
|
externalSignals?: ExternalSignal[],
|
|
40
41
|
/** 显式技法;缺省走五技法轮换(rotationSeed + 调用次数) */
|
|
41
|
-
method?: string
|
|
42
|
+
method?: string,
|
|
43
|
+
/** 正向反馈闭环 P2:不可变快照的结构化提示块。缺省 = 提示词逐字节不变。 */
|
|
44
|
+
feedback?: FeedbackPromptBlock): Promise<Hypothesis[]>;
|
|
42
45
|
/**
|
|
43
46
|
* 调 LLM 生成创意
|
|
44
47
|
*/
|
|
@@ -42,13 +42,15 @@ class HypothesisGenerator {
|
|
|
42
42
|
/** 外部信号 — 不参与配对,作为审视视角注入 LLM */
|
|
43
43
|
externalSignals = [],
|
|
44
44
|
/** 显式技法;缺省走五技法轮换(rotationSeed + 调用次数) */
|
|
45
|
-
method
|
|
45
|
+
method,
|
|
46
|
+
/** 正向反馈闭环 P2:不可变快照的结构化提示块。缺省 = 提示词逐字节不变。 */
|
|
47
|
+
feedback) {
|
|
46
48
|
if (combos.length === 0)
|
|
47
49
|
return [];
|
|
48
50
|
const technique = method ?? (0, CreativityPrompt_1.pickTechnique)(this.rotationSeed + this.callCount);
|
|
49
51
|
this.callCount += 1;
|
|
50
52
|
// 第一级:远程 LLM 驱动生成
|
|
51
|
-
const llmResults = await this.tryLLM(sources, combos, dreamMode, externalSignals, technique);
|
|
53
|
+
const llmResults = await this.tryLLM(sources, combos, dreamMode, externalSignals, technique, feedback);
|
|
52
54
|
if (llmResults.length > 0) {
|
|
53
55
|
return this.tag(llmResults.map((r) => ({
|
|
54
56
|
id: `hyp_${Date.now()}_${++this.idCounter}_${this.rng().toString(36).slice(2, 4)}`,
|
|
@@ -68,7 +70,7 @@ class HypothesisGenerator {
|
|
|
68
70
|
// 第二级:本地 transformers.js 模型
|
|
69
71
|
if (this.localModel.isEnabled) {
|
|
70
72
|
(0, Logger_1.log)('INFO', 'hypothesis_fallback_local', { count: combos.length });
|
|
71
|
-
const localResults = await this.tryLocalModel(sources, combos, dreamMode, technique);
|
|
73
|
+
const localResults = await this.tryLocalModel(sources, combos, dreamMode, technique, feedback);
|
|
72
74
|
if (localResults.length > 0) {
|
|
73
75
|
return this.tag(localResults.map((r) => ({
|
|
74
76
|
id: `hyp_local_${Date.now()}_${++this.idCounter}_${this.rng().toString(36).slice(2, 4)}`,
|
|
@@ -105,14 +107,14 @@ class HypothesisGenerator {
|
|
|
105
107
|
/**
|
|
106
108
|
* 调 LLM 生成创意
|
|
107
109
|
*/
|
|
108
|
-
async tryLLM(sources, combos, dreamMode, externalSignals = [], technique = '') {
|
|
110
|
+
async tryLLM(sources, combos, dreamMode, externalSignals = [], technique = '', feedback) {
|
|
109
111
|
// 限制输入量,避免超 token
|
|
110
112
|
const topCombos = combos.slice(0, dreamMode ? 5 : 3);
|
|
111
113
|
const comboInfo = topCombos.map((c) => ({
|
|
112
114
|
sources: c.sources,
|
|
113
115
|
description: c.description,
|
|
114
116
|
}));
|
|
115
|
-
const prompt = (0, CreativityPrompt_1.buildCreativityPrompt)(sources, comboInfo, externalSignals);
|
|
117
|
+
const prompt = (0, CreativityPrompt_1.buildCreativityPrompt)(sources, comboInfo, externalSignals, feedback);
|
|
116
118
|
try {
|
|
117
119
|
const result = await this.chatJson(prompt, {
|
|
118
120
|
system: (0, CreativityPrompt_1.buildSystemPrompt)(technique),
|
|
@@ -141,13 +143,13 @@ class HypothesisGenerator {
|
|
|
141
143
|
/**
|
|
142
144
|
* 调本地 transformers.js 模型生成创意
|
|
143
145
|
*/
|
|
144
|
-
async tryLocalModel(sources, combos, dreamMode, technique = '') {
|
|
146
|
+
async tryLocalModel(sources, combos, dreamMode, technique = '', feedback) {
|
|
145
147
|
const topCombos = combos.slice(0, dreamMode ? 3 : 2);
|
|
146
148
|
const comboInfo = topCombos.map((c) => ({
|
|
147
149
|
sources: c.sources,
|
|
148
150
|
description: c.description,
|
|
149
151
|
}));
|
|
150
|
-
const prompt = (0, CreativityPrompt_1.buildCreativityPrompt)(sources, comboInfo);
|
|
152
|
+
const prompt = (0, CreativityPrompt_1.buildCreativityPrompt)(sources, comboInfo, [], feedback);
|
|
151
153
|
const result = await this.localModel.generate(prompt, {
|
|
152
154
|
system: (0, CreativityPrompt_1.buildSystemPrompt)(technique),
|
|
153
155
|
temperature: dreamMode ? 0.8 : 0.6,
|
package/dist/IdeaGenerator.d.ts
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import type { CreativitySource, ConceptCombo, Hypothesis, CreativeIdea, Strategy, ExternalSignal } from './types';
|
|
2
|
+
import { type FeedbackMixerInput } from './ConceptMixer';
|
|
3
|
+
import type { FeedbackPromptBlock } from './FeedbackLearning';
|
|
2
4
|
/**
|
|
3
5
|
* IdeaGenerator — 创造力引擎的核心编排器
|
|
4
6
|
*
|
|
@@ -29,7 +31,11 @@ export declare class IdeaGenerator {
|
|
|
29
31
|
* 完整一轮"灵感涌现"流程
|
|
30
32
|
* @param strategy 生成策略,约束 ConceptMixer 配对空间
|
|
31
33
|
*/
|
|
32
|
-
generateIdeas(sources: CreativitySource[], maxIdeas?: number, strategy?: Strategy, externalSignals?: ExternalSignal[], method?: string
|
|
34
|
+
generateIdeas(sources: CreativitySource[], maxIdeas?: number, strategy?: Strategy, externalSignals?: ExternalSignal[], method?: string,
|
|
35
|
+
/** 正向反馈闭环 P2:结构化提示块(缺省 = 提示词逐字节不变)。 */
|
|
36
|
+
feedbackPrompt?: FeedbackPromptBlock,
|
|
37
|
+
/** 正向反馈闭环 P2:mixer 有界修正 + 探索预算(缺省 = 配对排序逐字节不变)。 */
|
|
38
|
+
mixerFeedback?: FeedbackMixerInput): Promise<CreativeIdea[]>;
|
|
33
39
|
/**
|
|
34
40
|
* Dream Mode — 高随机性、跨时间跨度的"梦境"模式
|
|
35
41
|
*/
|
package/dist/IdeaGenerator.js
CHANGED
|
@@ -34,18 +34,22 @@ class IdeaGenerator {
|
|
|
34
34
|
* 完整一轮"灵感涌现"流程
|
|
35
35
|
* @param strategy 生成策略,约束 ConceptMixer 配对空间
|
|
36
36
|
*/
|
|
37
|
-
async generateIdeas(sources, maxIdeas = 5, strategy = 'explore', externalSignals = [], method
|
|
37
|
+
async generateIdeas(sources, maxIdeas = 5, strategy = 'explore', externalSignals = [], method,
|
|
38
|
+
/** 正向反馈闭环 P2:结构化提示块(缺省 = 提示词逐字节不变)。 */
|
|
39
|
+
feedbackPrompt,
|
|
40
|
+
/** 正向反馈闭环 P2:mixer 有界修正 + 探索预算(缺省 = 配对排序逐字节不变)。 */
|
|
41
|
+
mixerFeedback) {
|
|
38
42
|
if (sources.length < 2)
|
|
39
43
|
return [];
|
|
40
44
|
// 1. 随机扰动:温度越高,来源选择越随机
|
|
41
45
|
const activeSources = this.applyTemperature(sources);
|
|
42
|
-
// 2. 概念重组:按策略约束配对 +
|
|
43
|
-
const scoredCombos = this.mixer.mix(activeSources, maxIdeas * 3, this.exploredPairs, strategy);
|
|
46
|
+
// 2. 概念重组:按策略约束配对 + 打分(可选反馈有界修正)
|
|
47
|
+
const scoredCombos = this.mixer.mix(activeSources, maxIdeas * 3, this.exploredPairs, strategy, mixerFeedback);
|
|
44
48
|
const combos = scoredCombos.map((c) => c.combo);
|
|
45
49
|
if (combos.length === 0)
|
|
46
50
|
return [];
|
|
47
51
|
// 3. 用 LLM 生成假设(外部信号作为审视视角注入,不参与配对)
|
|
48
|
-
const hypotheses = await this.hypothesisGen.generate(combos, activeSources, false, externalSignals, method);
|
|
52
|
+
const hypotheses = await this.hypothesisGen.generate(combos, activeSources, false, externalSignals, method, feedbackPrompt);
|
|
49
53
|
// 4. 可行性/质量门禁(与 dream 模式同一套门槛)——此前只有梦境路径
|
|
50
54
|
// 过 feasibilityGate,主路径仅按 novelty 筛,可行性 0 分也能入库。
|
|
51
55
|
const feasible = this.feasibilityGate(hypotheses);
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* InsightSamplingExperiment — insight 抽样策略 dogfood 实验(机制涌现 P3,定案 D4/R4)
|
|
3
|
+
*
|
|
4
|
+
* 首域 = insight 抽样策略(issue-6 P4 sourceQuota 有现成 before/after + 历史 observations 可重放)。
|
|
5
|
+
* 三组对照:现有 quota 抽样 / 变体 A / 变体 B + 消融(移除配额/移除时间序保持)+ 未见任务泛化。
|
|
6
|
+
* 产出 CapabilityDelta 报告;允许 inconclusive(不逼结论)。
|
|
7
|
+
* 硬前置 A1:MIO EXPERIMENT 标志路由 experiment-memory.jsonl(由 ExperimentRunStore 提供)。
|
|
8
|
+
*/
|
|
9
|
+
import type { ExperimentRun } from './ExperimentRunStore';
|
|
10
|
+
/** Observation 最小形状(内联,避免跨包依赖) */
|
|
11
|
+
interface ObservationLike {
|
|
12
|
+
id: string;
|
|
13
|
+
timestamp: string;
|
|
14
|
+
source: string;
|
|
15
|
+
content: string;
|
|
16
|
+
}
|
|
17
|
+
export type SamplingVariant = 'baseline' | 'variant_a' | 'variant_b' | 'ablation_no_quota' | 'ablation_no_time_order';
|
|
18
|
+
/**
|
|
19
|
+
* baseline: 现有 quota 抽样(issue-6 P4 sampleBySourceQuota)。
|
|
20
|
+
* variant_a: 更高 minPerSource(保底提升,家族优先更强)。
|
|
21
|
+
* variant_b: 只用 Phase A 保底,不做 Phase B 余量补齐(纯家族公平)。
|
|
22
|
+
* ablation_no_quota: 移除配额 → 退化为 slice(0, limit)(issue-6 修复前)。
|
|
23
|
+
* ablation_no_time_order: 移除时间序保持 → 输出按 weight 降序。
|
|
24
|
+
*/
|
|
25
|
+
export declare function sampleVariant(observations: readonly ObservationLike[], variant: SamplingVariant, limit?: number): ObservationLike[];
|
|
26
|
+
export interface SamplingMetrics {
|
|
27
|
+
/** 家族覆盖数(sourceFamily 去重) */
|
|
28
|
+
familyCoverage: number;
|
|
29
|
+
/** 最大家族占比(0-1;越低越公平) */
|
|
30
|
+
maxFamilyShare: number;
|
|
31
|
+
/** 时间序保持度:spearman 相关(0-1;1=完全保持) */
|
|
32
|
+
timeOrderPreservation: number;
|
|
33
|
+
/** 小源(len<quota)是否全部入选 */
|
|
34
|
+
smallSourceIncluded: boolean;
|
|
35
|
+
/** 样本量 */
|
|
36
|
+
sampleSize: number;
|
|
37
|
+
}
|
|
38
|
+
/** 计算抽样指标 */
|
|
39
|
+
export declare function computeSamplingMetrics(original: readonly ObservationLike[], sampled: readonly ObservationLike[]): SamplingMetrics;
|
|
40
|
+
export interface VariantResult {
|
|
41
|
+
variant: SamplingVariant;
|
|
42
|
+
metrics: SamplingMetrics;
|
|
43
|
+
runIds: string[];
|
|
44
|
+
}
|
|
45
|
+
export interface CapabilityDeltaReport {
|
|
46
|
+
experimentId: string;
|
|
47
|
+
hypothesisId?: string;
|
|
48
|
+
baselineVariant: SamplingVariant;
|
|
49
|
+
variants: VariantResult[];
|
|
50
|
+
/** 相对 baseline 的 delta(每指标:variant - baseline) */
|
|
51
|
+
deltas: Array<{
|
|
52
|
+
variant: SamplingVariant;
|
|
53
|
+
familyCoverageDelta: number;
|
|
54
|
+
maxFamilyShareDelta: number;
|
|
55
|
+
timeOrderPreservationDelta: number;
|
|
56
|
+
smallSourceIncludedDelta: number;
|
|
57
|
+
}>;
|
|
58
|
+
/** 泛化:在未见任务(held-out observations)上的表现 */
|
|
59
|
+
generalization?: {
|
|
60
|
+
heldOutSize: number;
|
|
61
|
+
baselineMetrics: SamplingMetrics;
|
|
62
|
+
bestVariant: SamplingVariant;
|
|
63
|
+
bestMetrics: SamplingMetrics;
|
|
64
|
+
};
|
|
65
|
+
/** 结论(允许 inconclusive) */
|
|
66
|
+
conclusion: 'variant_a_wins' | 'variant_b_wins' | 'baseline_wins' | 'inconclusive';
|
|
67
|
+
reason: string;
|
|
68
|
+
createdAt: number;
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* 运行 dogfood 实验:三组对照 + 消融,产出 CapabilityDelta 报告。
|
|
72
|
+
* observations 从历史重放;splitTrainTest 按时间切分(前 70% train,后 30% held-out)。
|
|
73
|
+
*/
|
|
74
|
+
export declare function runInsightSamplingExperiment(experimentId: string, observations: readonly ObservationLike[], options?: {
|
|
75
|
+
hypothesisId?: string;
|
|
76
|
+
limit?: number;
|
|
77
|
+
/** 每变体重复次数(取指标均值,降噪) */
|
|
78
|
+
repeats?: number;
|
|
79
|
+
}): CapabilityDeltaReport;
|
|
80
|
+
/** 把 CapabilityDelta 报告转成 ExperimentRun 列表(供 ExperimentRunStore.checkPromotionGate 使用) */
|
|
81
|
+
export declare function reportToExperimentRuns(report: CapabilityDeltaReport, feedbackSnapshotVersion?: string): ExperimentRun[];
|
|
82
|
+
export {};
|
|
@@ -0,0 +1,309 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* InsightSamplingExperiment — insight 抽样策略 dogfood 实验(机制涌现 P3,定案 D4/R4)
|
|
4
|
+
*
|
|
5
|
+
* 首域 = insight 抽样策略(issue-6 P4 sourceQuota 有现成 before/after + 历史 observations 可重放)。
|
|
6
|
+
* 三组对照:现有 quota 抽样 / 变体 A / 变体 B + 消融(移除配额/移除时间序保持)+ 未见任务泛化。
|
|
7
|
+
* 产出 CapabilityDelta 报告;允许 inconclusive(不逼结论)。
|
|
8
|
+
* 硬前置 A1:MIO EXPERIMENT 标志路由 experiment-memory.jsonl(由 ExperimentRunStore 提供)。
|
|
9
|
+
*/
|
|
10
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
11
|
+
exports.sampleVariant = sampleVariant;
|
|
12
|
+
exports.computeSamplingMetrics = computeSamplingMetrics;
|
|
13
|
+
exports.runInsightSamplingExperiment = runInsightSamplingExperiment;
|
|
14
|
+
exports.reportToExperimentRuns = reportToExperimentRuns;
|
|
15
|
+
/** RSS feedUrl 折叠(与 observer sourceQuota.sourceFamily 同语义) */
|
|
16
|
+
function sourceFamily(source) {
|
|
17
|
+
return /^https?:\/\//.test(source) ? 'rss' : source;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* baseline: 现有 quota 抽样(issue-6 P4 sampleBySourceQuota)。
|
|
21
|
+
* variant_a: 更高 minPerSource(保底提升,家族优先更强)。
|
|
22
|
+
* variant_b: 只用 Phase A 保底,不做 Phase B 余量补齐(纯家族公平)。
|
|
23
|
+
* ablation_no_quota: 移除配额 → 退化为 slice(0, limit)(issue-6 修复前)。
|
|
24
|
+
* ablation_no_time_order: 移除时间序保持 → 输出按 weight 降序。
|
|
25
|
+
*/
|
|
26
|
+
function sampleVariant(observations, variant, limit = 30) {
|
|
27
|
+
const obs = [...observations];
|
|
28
|
+
switch (variant) {
|
|
29
|
+
case 'baseline':
|
|
30
|
+
return sampleByQuota(obs, limit, 5);
|
|
31
|
+
case 'variant_a':
|
|
32
|
+
return sampleByQuota(obs, limit, 8);
|
|
33
|
+
case 'variant_b': {
|
|
34
|
+
// Phase A only: 每家族取 min(quota, len),quota = floor(limit/家族数) 或 5
|
|
35
|
+
const byFamily = new Map();
|
|
36
|
+
for (const o of obs) {
|
|
37
|
+
const f = sourceFamily(o.source);
|
|
38
|
+
const g = byFamily.get(f) ?? [];
|
|
39
|
+
g.push(o);
|
|
40
|
+
byFamily.set(f, g);
|
|
41
|
+
}
|
|
42
|
+
const families = [...byFamily.keys()];
|
|
43
|
+
let quota = 5;
|
|
44
|
+
if (families.length * quota > limit)
|
|
45
|
+
quota = Math.max(1, Math.floor(limit / families.length));
|
|
46
|
+
const out = [];
|
|
47
|
+
for (const f of families)
|
|
48
|
+
out.push(...byFamily.get(f).slice(0, quota));
|
|
49
|
+
// 保持时间序
|
|
50
|
+
out.sort((a, b) => a.timestamp.localeCompare(b.timestamp));
|
|
51
|
+
return out.slice(0, limit);
|
|
52
|
+
}
|
|
53
|
+
case 'ablation_no_quota':
|
|
54
|
+
return obs.slice(0, limit);
|
|
55
|
+
case 'ablation_no_time_order': {
|
|
56
|
+
const sampled = sampleByQuota(obs, limit, 5);
|
|
57
|
+
return sampled.sort((a, b) => sourceFamily(a.source).localeCompare(sourceFamily(b.source)));
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* 本地 quota 抽样(与 observer sourceQuota.sampleBySourceQuota 同语义的精简版)。
|
|
63
|
+
* Phase A 每家族保底 + Phase B 余量最大余数法 + 时间序保持。
|
|
64
|
+
*/
|
|
65
|
+
function sampleByQuota(observations, limit, minPerSource) {
|
|
66
|
+
if (limit <= 0)
|
|
67
|
+
return [];
|
|
68
|
+
if (observations.length <= limit)
|
|
69
|
+
return observations.slice();
|
|
70
|
+
const groups = new Map();
|
|
71
|
+
observations.forEach((obs, idx) => {
|
|
72
|
+
const family = sourceFamily(obs.source);
|
|
73
|
+
let g = groups.get(family);
|
|
74
|
+
if (!g) {
|
|
75
|
+
g = [];
|
|
76
|
+
groups.set(family, g);
|
|
77
|
+
}
|
|
78
|
+
g.push({ idx, obs });
|
|
79
|
+
});
|
|
80
|
+
const families = [...groups.keys()];
|
|
81
|
+
let quota = minPerSource;
|
|
82
|
+
if (families.length * quota > limit)
|
|
83
|
+
quota = Math.max(1, Math.floor(limit / families.length));
|
|
84
|
+
const picked = new Set();
|
|
85
|
+
const taken = new Map();
|
|
86
|
+
for (const family of families) {
|
|
87
|
+
const g = groups.get(family);
|
|
88
|
+
const take = Math.min(quota, g.length);
|
|
89
|
+
for (let i = 0; i < take; i++)
|
|
90
|
+
picked.add(g[i].idx);
|
|
91
|
+
taken.set(family, take);
|
|
92
|
+
}
|
|
93
|
+
const remaining = limit - picked.size;
|
|
94
|
+
if (remaining > 0) {
|
|
95
|
+
const shares = families.map((family) => {
|
|
96
|
+
const group = groups.get(family);
|
|
97
|
+
return { family, group, pool: group.length - taken.get(family), base: 0, frac: 0 };
|
|
98
|
+
});
|
|
99
|
+
const totalPool = shares.reduce((s, x) => s + x.pool, 0);
|
|
100
|
+
if (totalPool > 0) {
|
|
101
|
+
for (const s of shares) {
|
|
102
|
+
const exact = (remaining * s.pool) / totalPool;
|
|
103
|
+
s.base = Math.floor(exact);
|
|
104
|
+
s.frac = exact - s.base;
|
|
105
|
+
}
|
|
106
|
+
let leftover = remaining - shares.reduce((s, x) => s + x.base, 0);
|
|
107
|
+
const byFrac = [...shares].sort((a, b) => b.frac - a.frac);
|
|
108
|
+
for (const s of byFrac) {
|
|
109
|
+
if (leftover <= 0)
|
|
110
|
+
break;
|
|
111
|
+
if (s.pool > s.base) {
|
|
112
|
+
s.base++;
|
|
113
|
+
leftover--;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
for (const s of shares) {
|
|
117
|
+
const start = taken.get(s.family);
|
|
118
|
+
for (let i = start; i < start + s.base; i++)
|
|
119
|
+
picked.add(s.group[i].idx);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
return observations.filter((_, i) => picked.has(i));
|
|
124
|
+
}
|
|
125
|
+
/** 计算抽样指标 */
|
|
126
|
+
function computeSamplingMetrics(original, sampled) {
|
|
127
|
+
const families = new Set(original.map((o) => sourceFamily(o.source)));
|
|
128
|
+
const sampledFamilies = new Set(sampled.map((o) => sourceFamily(o.source)));
|
|
129
|
+
const byFamily = new Map();
|
|
130
|
+
for (const s of sampled) {
|
|
131
|
+
const f = sourceFamily(s.source);
|
|
132
|
+
byFamily.set(f, (byFamily.get(f) ?? 0) + 1);
|
|
133
|
+
}
|
|
134
|
+
const maxCount = Math.max(0, ...byFamily.values());
|
|
135
|
+
const maxFamilyShare = sampled.length > 0 ? maxCount / sampled.length : 0;
|
|
136
|
+
// 时间序保持度:采样序列在原序列中的排名相关(简化为顺序单调比)
|
|
137
|
+
const idxMap = new Map();
|
|
138
|
+
original.forEach((o, i) => idxMap.set(o.id, i));
|
|
139
|
+
const ranks = sampled.map((s) => idxMap.get(s.id) ?? -1);
|
|
140
|
+
let concordant = 0;
|
|
141
|
+
let total = 0;
|
|
142
|
+
for (let i = 0; i < ranks.length; i++) {
|
|
143
|
+
for (let j = i + 1; j < ranks.length; j++) {
|
|
144
|
+
if (ranks[i] < 0 || ranks[j] < 0)
|
|
145
|
+
continue;
|
|
146
|
+
total++;
|
|
147
|
+
if (ranks[i] < ranks[j])
|
|
148
|
+
concordant++;
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
const timeOrderPreservation = total > 0 ? concordant / total : 1;
|
|
152
|
+
// 小源:len < 5 的家族是否全部入选
|
|
153
|
+
const origByFamily = new Map();
|
|
154
|
+
for (const o of original) {
|
|
155
|
+
const f = sourceFamily(o.source);
|
|
156
|
+
const g = origByFamily.get(f) ?? [];
|
|
157
|
+
g.push(o);
|
|
158
|
+
origByFamily.set(f, g);
|
|
159
|
+
}
|
|
160
|
+
const smallSourceIncluded = [...origByFamily.entries()].every(([f, g]) => {
|
|
161
|
+
if (g.length >= 5)
|
|
162
|
+
return true;
|
|
163
|
+
const inSample = sampled.filter((s) => sourceFamily(s.source) === f);
|
|
164
|
+
return inSample.length >= g.length;
|
|
165
|
+
});
|
|
166
|
+
return {
|
|
167
|
+
familyCoverage: sampledFamilies.size,
|
|
168
|
+
maxFamilyShare,
|
|
169
|
+
timeOrderPreservation,
|
|
170
|
+
smallSourceIncluded,
|
|
171
|
+
sampleSize: sampled.length,
|
|
172
|
+
};
|
|
173
|
+
}
|
|
174
|
+
/**
|
|
175
|
+
* 运行 dogfood 实验:三组对照 + 消融,产出 CapabilityDelta 报告。
|
|
176
|
+
* observations 从历史重放;splitTrainTest 按时间切分(前 70% train,后 30% held-out)。
|
|
177
|
+
*/
|
|
178
|
+
function runInsightSamplingExperiment(experimentId, observations, options) {
|
|
179
|
+
const limit = options?.limit ?? 30;
|
|
180
|
+
const repeats = options?.repeats ?? 1;
|
|
181
|
+
const sorted = [...observations].sort((a, b) => a.timestamp.localeCompare(b.timestamp));
|
|
182
|
+
const split = Math.max(1, Math.floor(sorted.length * 0.7));
|
|
183
|
+
const train = sorted.slice(0, split);
|
|
184
|
+
const heldOut = sorted.slice(split);
|
|
185
|
+
const variants = ['baseline', 'variant_a', 'variant_b', 'ablation_no_quota', 'ablation_no_time_order'];
|
|
186
|
+
const variantResults = [];
|
|
187
|
+
for (const v of variants) {
|
|
188
|
+
let metricsAcc = null;
|
|
189
|
+
const runIds = [];
|
|
190
|
+
for (let r = 0; r < repeats; r++) {
|
|
191
|
+
const sampled = sampleVariant(train, v, limit);
|
|
192
|
+
const m = computeSamplingMetrics(train, sampled);
|
|
193
|
+
metricsAcc = metricsAcc
|
|
194
|
+
? {
|
|
195
|
+
familyCoverage: metricsAcc.familyCoverage + m.familyCoverage,
|
|
196
|
+
maxFamilyShare: metricsAcc.maxFamilyShare + m.maxFamilyShare,
|
|
197
|
+
timeOrderPreservation: metricsAcc.timeOrderPreservation + m.timeOrderPreservation,
|
|
198
|
+
smallSourceIncluded: metricsAcc.smallSourceIncluded && m.smallSourceIncluded,
|
|
199
|
+
sampleSize: metricsAcc.sampleSize + m.sampleSize,
|
|
200
|
+
}
|
|
201
|
+
: m;
|
|
202
|
+
runIds.push(`${experimentId}:${v}:${r}`);
|
|
203
|
+
}
|
|
204
|
+
const avg = metricsAcc
|
|
205
|
+
? {
|
|
206
|
+
familyCoverage: Math.round(metricsAcc.familyCoverage / repeats),
|
|
207
|
+
maxFamilyShare: metricsAcc.maxFamilyShare / repeats,
|
|
208
|
+
timeOrderPreservation: metricsAcc.timeOrderPreservation / repeats,
|
|
209
|
+
smallSourceIncluded: metricsAcc.smallSourceIncluded,
|
|
210
|
+
sampleSize: Math.round(metricsAcc.sampleSize / repeats),
|
|
211
|
+
}
|
|
212
|
+
: { familyCoverage: 0, maxFamilyShare: 1, timeOrderPreservation: 0, smallSourceIncluded: false, sampleSize: 0 };
|
|
213
|
+
variantResults.push({ variant: v, metrics: avg, runIds });
|
|
214
|
+
}
|
|
215
|
+
const baseline = variantResults.find((r) => r.variant === 'baseline');
|
|
216
|
+
const deltas = variantResults
|
|
217
|
+
.filter((r) => r.variant !== 'baseline')
|
|
218
|
+
.map((r) => ({
|
|
219
|
+
variant: r.variant,
|
|
220
|
+
familyCoverageDelta: r.metrics.familyCoverage - baseline.metrics.familyCoverage,
|
|
221
|
+
maxFamilyShareDelta: r.metrics.maxFamilyShare - baseline.metrics.maxFamilyShare,
|
|
222
|
+
timeOrderPreservationDelta: r.metrics.timeOrderPreservation - baseline.metrics.timeOrderPreservation,
|
|
223
|
+
smallSourceIncludedDelta: (r.metrics.smallSourceIncluded ? 1 : 0) - (baseline.metrics.smallSourceIncluded ? 1 : 0),
|
|
224
|
+
}));
|
|
225
|
+
// 泛化:held-out 上 baseline vs best variant(按 familyCoverage + smallSourceIncluded 排序)
|
|
226
|
+
let generalization;
|
|
227
|
+
if (heldOut.length > 0) {
|
|
228
|
+
const baseSampled = sampleVariant(heldOut, 'baseline', limit);
|
|
229
|
+
const baseMetrics = computeSamplingMetrics(heldOut, baseSampled);
|
|
230
|
+
const scored = ['variant_a', 'variant_b'].map((v) => {
|
|
231
|
+
const s = sampleVariant(heldOut, v, limit);
|
|
232
|
+
return { v, m: computeSamplingMetrics(heldOut, s) };
|
|
233
|
+
});
|
|
234
|
+
const best = scored.sort((a, b) => {
|
|
235
|
+
const sc = (m) => m.familyCoverage * 10 + (m.smallSourceIncluded ? 5 : 0) - m.maxFamilyShare;
|
|
236
|
+
return sc(b.m) - sc(a.m);
|
|
237
|
+
})[0];
|
|
238
|
+
generalization = {
|
|
239
|
+
heldOutSize: heldOut.length,
|
|
240
|
+
baselineMetrics: baseMetrics,
|
|
241
|
+
bestVariant: best?.v ?? 'baseline',
|
|
242
|
+
bestMetrics: best?.m ?? baseMetrics,
|
|
243
|
+
};
|
|
244
|
+
}
|
|
245
|
+
// 结论:variant_a / variant_b 优于 baseline → 对应 wins;否则 baseline_wins;样本太少 → inconclusive
|
|
246
|
+
const aDelta = deltas.find((d) => d.variant === 'variant_a');
|
|
247
|
+
const bDelta = deltas.find((d) => d.variant === 'variant_b');
|
|
248
|
+
const aBetter = aDelta && (aDelta.familyCoverageDelta > 0 || aDelta.smallSourceIncludedDelta > 0);
|
|
249
|
+
const bBetter = bDelta && (bDelta.familyCoverageDelta > 0 || bDelta.smallSourceIncludedDelta > 0);
|
|
250
|
+
let conclusion;
|
|
251
|
+
let reason;
|
|
252
|
+
if (train.length < 10) {
|
|
253
|
+
conclusion = 'inconclusive';
|
|
254
|
+
reason = 'too_few_observations';
|
|
255
|
+
}
|
|
256
|
+
else if (aBetter && bBetter) {
|
|
257
|
+
// 双优 → inconclusive(需人工判断)
|
|
258
|
+
conclusion = 'inconclusive';
|
|
259
|
+
reason = 'both_variants_improve';
|
|
260
|
+
}
|
|
261
|
+
else if (aBetter) {
|
|
262
|
+
conclusion = 'variant_a_wins';
|
|
263
|
+
reason = 'higher_family_coverage_or_small_source';
|
|
264
|
+
}
|
|
265
|
+
else if (bBetter) {
|
|
266
|
+
conclusion = 'variant_b_wins';
|
|
267
|
+
reason = 'higher_family_coverage_or_small_source';
|
|
268
|
+
}
|
|
269
|
+
else {
|
|
270
|
+
conclusion = 'baseline_wins';
|
|
271
|
+
reason = 'no_improvement_over_baseline';
|
|
272
|
+
}
|
|
273
|
+
return {
|
|
274
|
+
experimentId,
|
|
275
|
+
hypothesisId: options?.hypothesisId,
|
|
276
|
+
baselineVariant: 'baseline',
|
|
277
|
+
variants: variantResults,
|
|
278
|
+
deltas,
|
|
279
|
+
generalization,
|
|
280
|
+
conclusion,
|
|
281
|
+
reason,
|
|
282
|
+
createdAt: Date.now(),
|
|
283
|
+
};
|
|
284
|
+
}
|
|
285
|
+
/** 把 CapabilityDelta 报告转成 ExperimentRun 列表(供 ExperimentRunStore.checkPromotionGate 使用) */
|
|
286
|
+
function reportToExperimentRuns(report, feedbackSnapshotVersion) {
|
|
287
|
+
return report.variants.map((v) => ({
|
|
288
|
+
id: `${report.experimentId}:${v.variant}`,
|
|
289
|
+
hypothesisId: report.hypothesisId,
|
|
290
|
+
candidateId: v.variant,
|
|
291
|
+
baselineId: report.baselineVariant,
|
|
292
|
+
...(feedbackSnapshotVersion ? { feedbackSnapshotVersion } : {}),
|
|
293
|
+
assignment: (v.variant === 'baseline'
|
|
294
|
+
? 'baseline'
|
|
295
|
+
: v.variant.startsWith('ablation')
|
|
296
|
+
? 'ablation'
|
|
297
|
+
: 'candidate'),
|
|
298
|
+
outcome: report.conclusion === 'inconclusive' ? 'inconclusive' : v.variant === report.conclusion.replace('_wins', '') ? 'success' : 'failure',
|
|
299
|
+
metrics: {
|
|
300
|
+
familyCoverage: v.metrics.familyCoverage,
|
|
301
|
+
maxFamilyShare: v.metrics.maxFamilyShare,
|
|
302
|
+
timeOrderPreservation: v.metrics.timeOrderPreservation,
|
|
303
|
+
smallSourceIncluded: v.metrics.smallSourceIncluded,
|
|
304
|
+
sampleSize: v.metrics.sampleSize,
|
|
305
|
+
},
|
|
306
|
+
evidenceIds: v.runIds,
|
|
307
|
+
createdAt: report.createdAt,
|
|
308
|
+
}));
|
|
309
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* JsonlFeedbackEventStore — CLI/MCP 侧证据事件追加存储(P0)
|
|
3
|
+
*
|
|
4
|
+
* 落盘:`<dataDir>/feedback-events.jsonl`(append-only)。
|
|
5
|
+
* 幂等:加载时建 idempotencyKey 集合,同键追加直接跳过(不抛错、不重复生效)。
|
|
6
|
+
* 这是交换/重放格式,不是权威真相源——主进程 ingest 单向导入 SQLite。
|
|
7
|
+
*/
|
|
8
|
+
import { type HypothesisFeedbackEvent, type HypothesisFeedbackEventInput, type NormalizeQuarantineReason } from './FeedbackEvents';
|
|
9
|
+
export interface JsonlAppendResult {
|
|
10
|
+
/** 事件已追加并生效 */
|
|
11
|
+
appended: boolean;
|
|
12
|
+
/** 因 idempotencyKey 已存在而跳过 */
|
|
13
|
+
deduped: boolean;
|
|
14
|
+
event?: HypothesisFeedbackEvent;
|
|
15
|
+
/** 规范化失败(进隔离队列由调用方处理) */
|
|
16
|
+
quarantine?: {
|
|
17
|
+
reason: NormalizeQuarantineReason;
|
|
18
|
+
raw: unknown;
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
export declare class JsonlFeedbackEventStore {
|
|
22
|
+
readonly eventsPath: string;
|
|
23
|
+
private seenKeys;
|
|
24
|
+
constructor(dataDir: string);
|
|
25
|
+
/** 重读文件(长活 MCP 进程需看到外部追加;幂等键集合整体重建,成本可接受)。 */
|
|
26
|
+
private ensureLoaded;
|
|
27
|
+
/** 供长活进程在外部写入后强制重读。 */
|
|
28
|
+
invalidate(): void;
|
|
29
|
+
append(input: HypothesisFeedbackEventInput, now?: number): JsonlAppendResult;
|
|
30
|
+
/** 读取全部可解析事件(损坏行跳过,供 ingest/回放使用)。 */
|
|
31
|
+
list(): HypothesisFeedbackEvent[];
|
|
32
|
+
}
|