@akemi-mio/creativity 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ConceptMixer.d.ts +41 -0
- package/dist/ConceptMixer.js +207 -0
- package/dist/CreativityPrompt.d.ts +24 -0
- package/dist/CreativityPrompt.js +206 -0
- package/dist/ExperimentPlanner.d.ts +14 -0
- package/dist/ExperimentPlanner.js +60 -0
- package/dist/ExperimentRunStore.d.ts +97 -0
- package/dist/ExperimentRunStore.js +160 -0
- package/dist/HypothesisGenerator.d.ts +60 -0
- package/dist/HypothesisGenerator.js +246 -0
- package/dist/IdeaGenerator.d.ts +51 -0
- package/dist/IdeaGenerator.js +145 -0
- package/dist/InsightSamplingExperiment.d.ts +82 -0
- package/dist/InsightSamplingExperiment.js +308 -0
- package/dist/LocalModelService.d.ts +38 -0
- package/dist/LocalModelService.js +162 -0
- package/dist/MechanismGraph.d.ts +46 -0
- package/dist/MechanismGraph.js +76 -0
- package/dist/MechanismRegistry.d.ts +118 -0
- package/dist/MechanismRegistry.js +233 -0
- package/dist/NoveltyScorer.d.ts +40 -0
- package/dist/NoveltyScorer.js +93 -0
- package/dist/SourceAggregator.d.ts +21 -0
- package/dist/SourceAggregator.js +59 -0
- package/dist/TemplateLibrary.d.ts +29 -0
- package/dist/TemplateLibrary.js +484 -0
- package/dist/WhenThenRunner.d.ts +135 -0
- package/dist/WhenThenRunner.js +358 -0
- package/dist/types.d.ts +213 -0
- package/dist/types.js +9 -0
- package/package.json +67 -4
- package/README.md +0 -3
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* InsightSamplingExperiment — insight 抽样策略 dogfood 实验(机制涌现 P3,定案 D4/R4)
|
|
3
|
+
*
|
|
4
|
+
* 首域 = insight 抽样策略(issue-6 P4 sourceQuota 有现成 before/after + 历史 observations 可重放)。
|
|
5
|
+
* 三组对照:现有 quota 抽样 / 变体 A / 变体 B + 消融(移除配额/移除时间序保持)+ 未见任务泛化。
|
|
6
|
+
* 产出 CapabilityDelta 报告;允许 inconclusive(不逼结论)。
|
|
7
|
+
* 硬前置 A1:MIO EXPERIMENT 标志路由 experiment-memory.jsonl(由 ExperimentRunStore 提供)。
|
|
8
|
+
*/
|
|
9
|
+
import type { ExperimentRun } from './ExperimentRunStore';
|
|
10
|
+
/** Observation 最小形状(内联,避免跨包依赖) */
|
|
11
|
+
interface ObservationLike {
|
|
12
|
+
id: string;
|
|
13
|
+
timestamp: string;
|
|
14
|
+
source: string;
|
|
15
|
+
content: string;
|
|
16
|
+
}
|
|
17
|
+
export type SamplingVariant = 'baseline' | 'variant_a' | 'variant_b' | 'ablation_no_quota' | 'ablation_no_time_order';
|
|
18
|
+
/**
|
|
19
|
+
* baseline: 现有 quota 抽样(issue-6 P4 sampleBySourceQuota)。
|
|
20
|
+
* variant_a: 更高 minPerSource(保底提升,家族优先更强)。
|
|
21
|
+
* variant_b: 只用 Phase A 保底,不做 Phase B 余量补齐(纯家族公平)。
|
|
22
|
+
* ablation_no_quota: 移除配额 → 退化为 slice(0, limit)(issue-6 修复前)。
|
|
23
|
+
* ablation_no_time_order: 移除时间序保持 → 输出按 weight 降序。
|
|
24
|
+
*/
|
|
25
|
+
export declare function sampleVariant(observations: readonly ObservationLike[], variant: SamplingVariant, limit?: number): ObservationLike[];
|
|
26
|
+
export interface SamplingMetrics {
|
|
27
|
+
/** 家族覆盖数(sourceFamily 去重) */
|
|
28
|
+
familyCoverage: number;
|
|
29
|
+
/** 最大家族占比(0-1;越低越公平) */
|
|
30
|
+
maxFamilyShare: number;
|
|
31
|
+
/** 时间序保持度:spearman 相关(0-1;1=完全保持) */
|
|
32
|
+
timeOrderPreservation: number;
|
|
33
|
+
/** 小源(len<quota)是否全部入选 */
|
|
34
|
+
smallSourceIncluded: boolean;
|
|
35
|
+
/** 样本量 */
|
|
36
|
+
sampleSize: number;
|
|
37
|
+
}
|
|
38
|
+
/** 计算抽样指标 */
|
|
39
|
+
export declare function computeSamplingMetrics(original: readonly ObservationLike[], sampled: readonly ObservationLike[]): SamplingMetrics;
|
|
40
|
+
export interface VariantResult {
|
|
41
|
+
variant: SamplingVariant;
|
|
42
|
+
metrics: SamplingMetrics;
|
|
43
|
+
runIds: string[];
|
|
44
|
+
}
|
|
45
|
+
export interface CapabilityDeltaReport {
|
|
46
|
+
experimentId: string;
|
|
47
|
+
hypothesisId?: string;
|
|
48
|
+
baselineVariant: SamplingVariant;
|
|
49
|
+
variants: VariantResult[];
|
|
50
|
+
/** 相对 baseline 的 delta(每指标:variant - baseline) */
|
|
51
|
+
deltas: Array<{
|
|
52
|
+
variant: SamplingVariant;
|
|
53
|
+
familyCoverageDelta: number;
|
|
54
|
+
maxFamilyShareDelta: number;
|
|
55
|
+
timeOrderPreservationDelta: number;
|
|
56
|
+
smallSourceIncludedDelta: number;
|
|
57
|
+
}>;
|
|
58
|
+
/** 泛化:在未见任务(held-out observations)上的表现 */
|
|
59
|
+
generalization?: {
|
|
60
|
+
heldOutSize: number;
|
|
61
|
+
baselineMetrics: SamplingMetrics;
|
|
62
|
+
bestVariant: SamplingVariant;
|
|
63
|
+
bestMetrics: SamplingMetrics;
|
|
64
|
+
};
|
|
65
|
+
/** 结论(允许 inconclusive) */
|
|
66
|
+
conclusion: 'variant_a_wins' | 'variant_b_wins' | 'baseline_wins' | 'inconclusive';
|
|
67
|
+
reason: string;
|
|
68
|
+
createdAt: number;
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* 运行 dogfood 实验:三组对照 + 消融,产出 CapabilityDelta 报告。
|
|
72
|
+
* observations 从历史重放;splitTrainTest 按时间切分(前 70% train,后 30% held-out)。
|
|
73
|
+
*/
|
|
74
|
+
export declare function runInsightSamplingExperiment(experimentId: string, observations: readonly ObservationLike[], options?: {
|
|
75
|
+
hypothesisId?: string;
|
|
76
|
+
limit?: number;
|
|
77
|
+
/** 每变体重复次数(取指标均值,降噪) */
|
|
78
|
+
repeats?: number;
|
|
79
|
+
}): CapabilityDeltaReport;
|
|
80
|
+
/** 把 CapabilityDelta 报告转成 ExperimentRun 列表(供 ExperimentRunStore.checkPromotionGate 使用) */
|
|
81
|
+
export declare function reportToExperimentRuns(report: CapabilityDeltaReport): ExperimentRun[];
|
|
82
|
+
export {};
|
|
@@ -0,0 +1,308 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
/**
|
|
3
|
+
* InsightSamplingExperiment — insight 抽样策略 dogfood 实验(机制涌现 P3,定案 D4/R4)
|
|
4
|
+
*
|
|
5
|
+
* 首域 = insight 抽样策略(issue-6 P4 sourceQuota 有现成 before/after + 历史 observations 可重放)。
|
|
6
|
+
* 三组对照:现有 quota 抽样 / 变体 A / 变体 B + 消融(移除配额/移除时间序保持)+ 未见任务泛化。
|
|
7
|
+
* 产出 CapabilityDelta 报告;允许 inconclusive(不逼结论)。
|
|
8
|
+
* 硬前置 A1:MIO EXPERIMENT 标志路由 experiment-memory.jsonl(由 ExperimentRunStore 提供)。
|
|
9
|
+
*/
|
|
10
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
11
|
+
exports.sampleVariant = sampleVariant;
|
|
12
|
+
exports.computeSamplingMetrics = computeSamplingMetrics;
|
|
13
|
+
exports.runInsightSamplingExperiment = runInsightSamplingExperiment;
|
|
14
|
+
exports.reportToExperimentRuns = reportToExperimentRuns;
|
|
15
|
+
/** RSS feedUrl 折叠(与 observer sourceQuota.sourceFamily 同语义) */
|
|
16
|
+
function sourceFamily(source) {
|
|
17
|
+
return /^https?:\/\//.test(source) ? 'rss' : source;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* baseline: 现有 quota 抽样(issue-6 P4 sampleBySourceQuota)。
|
|
21
|
+
* variant_a: 更高 minPerSource(保底提升,家族优先更强)。
|
|
22
|
+
* variant_b: 只用 Phase A 保底,不做 Phase B 余量补齐(纯家族公平)。
|
|
23
|
+
* ablation_no_quota: 移除配额 → 退化为 slice(0, limit)(issue-6 修复前)。
|
|
24
|
+
* ablation_no_time_order: 移除时间序保持 → 输出按 weight 降序。
|
|
25
|
+
*/
|
|
26
|
+
function sampleVariant(observations, variant, limit = 30) {
|
|
27
|
+
const obs = [...observations];
|
|
28
|
+
switch (variant) {
|
|
29
|
+
case 'baseline':
|
|
30
|
+
return sampleByQuota(obs, limit, 5);
|
|
31
|
+
case 'variant_a':
|
|
32
|
+
return sampleByQuota(obs, limit, 8);
|
|
33
|
+
case 'variant_b': {
|
|
34
|
+
// Phase A only: 每家族取 min(quota, len),quota = floor(limit/家族数) 或 5
|
|
35
|
+
const byFamily = new Map();
|
|
36
|
+
for (const o of obs) {
|
|
37
|
+
const f = sourceFamily(o.source);
|
|
38
|
+
const g = byFamily.get(f) ?? [];
|
|
39
|
+
g.push(o);
|
|
40
|
+
byFamily.set(f, g);
|
|
41
|
+
}
|
|
42
|
+
const families = [...byFamily.keys()];
|
|
43
|
+
let quota = 5;
|
|
44
|
+
if (families.length * quota > limit)
|
|
45
|
+
quota = Math.max(1, Math.floor(limit / families.length));
|
|
46
|
+
const out = [];
|
|
47
|
+
for (const f of families)
|
|
48
|
+
out.push(...byFamily.get(f).slice(0, quota));
|
|
49
|
+
// 保持时间序
|
|
50
|
+
out.sort((a, b) => a.timestamp.localeCompare(b.timestamp));
|
|
51
|
+
return out.slice(0, limit);
|
|
52
|
+
}
|
|
53
|
+
case 'ablation_no_quota':
|
|
54
|
+
return obs.slice(0, limit);
|
|
55
|
+
case 'ablation_no_time_order': {
|
|
56
|
+
const sampled = sampleByQuota(obs, limit, 5);
|
|
57
|
+
return sampled.sort((a, b) => sourceFamily(a.source).localeCompare(sourceFamily(b.source)));
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* 本地 quota 抽样(与 observer sourceQuota.sampleBySourceQuota 同语义的精简版)。
|
|
63
|
+
* Phase A 每家族保底 + Phase B 余量最大余数法 + 时间序保持。
|
|
64
|
+
*/
|
|
65
|
+
function sampleByQuota(observations, limit, minPerSource) {
|
|
66
|
+
if (limit <= 0)
|
|
67
|
+
return [];
|
|
68
|
+
if (observations.length <= limit)
|
|
69
|
+
return observations.slice();
|
|
70
|
+
const groups = new Map();
|
|
71
|
+
observations.forEach((obs, idx) => {
|
|
72
|
+
const family = sourceFamily(obs.source);
|
|
73
|
+
let g = groups.get(family);
|
|
74
|
+
if (!g) {
|
|
75
|
+
g = [];
|
|
76
|
+
groups.set(family, g);
|
|
77
|
+
}
|
|
78
|
+
g.push({ idx, obs });
|
|
79
|
+
});
|
|
80
|
+
const families = [...groups.keys()];
|
|
81
|
+
let quota = minPerSource;
|
|
82
|
+
if (families.length * quota > limit)
|
|
83
|
+
quota = Math.max(1, Math.floor(limit / families.length));
|
|
84
|
+
const picked = new Set();
|
|
85
|
+
const taken = new Map();
|
|
86
|
+
for (const family of families) {
|
|
87
|
+
const g = groups.get(family);
|
|
88
|
+
const take = Math.min(quota, g.length);
|
|
89
|
+
for (let i = 0; i < take; i++)
|
|
90
|
+
picked.add(g[i].idx);
|
|
91
|
+
taken.set(family, take);
|
|
92
|
+
}
|
|
93
|
+
const remaining = limit - picked.size;
|
|
94
|
+
if (remaining > 0) {
|
|
95
|
+
const shares = families.map((family) => {
|
|
96
|
+
const group = groups.get(family);
|
|
97
|
+
return { family, group, pool: group.length - taken.get(family), base: 0, frac: 0 };
|
|
98
|
+
});
|
|
99
|
+
const totalPool = shares.reduce((s, x) => s + x.pool, 0);
|
|
100
|
+
if (totalPool > 0) {
|
|
101
|
+
for (const s of shares) {
|
|
102
|
+
const exact = (remaining * s.pool) / totalPool;
|
|
103
|
+
s.base = Math.floor(exact);
|
|
104
|
+
s.frac = exact - s.base;
|
|
105
|
+
}
|
|
106
|
+
let leftover = remaining - shares.reduce((s, x) => s + x.base, 0);
|
|
107
|
+
const byFrac = [...shares].sort((a, b) => b.frac - a.frac);
|
|
108
|
+
for (const s of byFrac) {
|
|
109
|
+
if (leftover <= 0)
|
|
110
|
+
break;
|
|
111
|
+
if (s.pool > s.base) {
|
|
112
|
+
s.base++;
|
|
113
|
+
leftover--;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
for (const s of shares) {
|
|
117
|
+
const start = taken.get(s.family);
|
|
118
|
+
for (let i = start; i < start + s.base; i++)
|
|
119
|
+
picked.add(s.group[i].idx);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
return observations.filter((_, i) => picked.has(i));
|
|
124
|
+
}
|
|
125
|
+
/** 计算抽样指标 */
|
|
126
|
+
function computeSamplingMetrics(original, sampled) {
|
|
127
|
+
const families = new Set(original.map((o) => sourceFamily(o.source)));
|
|
128
|
+
const sampledFamilies = new Set(sampled.map((o) => sourceFamily(o.source)));
|
|
129
|
+
const byFamily = new Map();
|
|
130
|
+
for (const s of sampled) {
|
|
131
|
+
const f = sourceFamily(s.source);
|
|
132
|
+
byFamily.set(f, (byFamily.get(f) ?? 0) + 1);
|
|
133
|
+
}
|
|
134
|
+
const maxCount = Math.max(0, ...byFamily.values());
|
|
135
|
+
const maxFamilyShare = sampled.length > 0 ? maxCount / sampled.length : 0;
|
|
136
|
+
// 时间序保持度:采样序列在原序列中的排名相关(简化为顺序单调比)
|
|
137
|
+
const idxMap = new Map();
|
|
138
|
+
original.forEach((o, i) => idxMap.set(o.id, i));
|
|
139
|
+
const ranks = sampled.map((s) => idxMap.get(s.id) ?? -1);
|
|
140
|
+
let concordant = 0;
|
|
141
|
+
let total = 0;
|
|
142
|
+
for (let i = 0; i < ranks.length; i++) {
|
|
143
|
+
for (let j = i + 1; j < ranks.length; j++) {
|
|
144
|
+
if (ranks[i] < 0 || ranks[j] < 0)
|
|
145
|
+
continue;
|
|
146
|
+
total++;
|
|
147
|
+
if (ranks[i] < ranks[j])
|
|
148
|
+
concordant++;
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
const timeOrderPreservation = total > 0 ? concordant / total : 1;
|
|
152
|
+
// 小源:len < 5 的家族是否全部入选
|
|
153
|
+
const origByFamily = new Map();
|
|
154
|
+
for (const o of original) {
|
|
155
|
+
const f = sourceFamily(o.source);
|
|
156
|
+
const g = origByFamily.get(f) ?? [];
|
|
157
|
+
g.push(o);
|
|
158
|
+
origByFamily.set(f, g);
|
|
159
|
+
}
|
|
160
|
+
const smallSourceIncluded = [...origByFamily.entries()].every(([f, g]) => {
|
|
161
|
+
if (g.length >= 5)
|
|
162
|
+
return true;
|
|
163
|
+
const inSample = sampled.filter((s) => sourceFamily(s.source) === f);
|
|
164
|
+
return inSample.length >= g.length;
|
|
165
|
+
});
|
|
166
|
+
return {
|
|
167
|
+
familyCoverage: sampledFamilies.size,
|
|
168
|
+
maxFamilyShare,
|
|
169
|
+
timeOrderPreservation,
|
|
170
|
+
smallSourceIncluded,
|
|
171
|
+
sampleSize: sampled.length,
|
|
172
|
+
};
|
|
173
|
+
}
|
|
174
|
+
/**
|
|
175
|
+
* 运行 dogfood 实验:三组对照 + 消融,产出 CapabilityDelta 报告。
|
|
176
|
+
* observations 从历史重放;splitTrainTest 按时间切分(前 70% train,后 30% held-out)。
|
|
177
|
+
*/
|
|
178
|
+
function runInsightSamplingExperiment(experimentId, observations, options) {
|
|
179
|
+
const limit = options?.limit ?? 30;
|
|
180
|
+
const repeats = options?.repeats ?? 1;
|
|
181
|
+
const sorted = [...observations].sort((a, b) => a.timestamp.localeCompare(b.timestamp));
|
|
182
|
+
const split = Math.max(1, Math.floor(sorted.length * 0.7));
|
|
183
|
+
const train = sorted.slice(0, split);
|
|
184
|
+
const heldOut = sorted.slice(split);
|
|
185
|
+
const variants = ['baseline', 'variant_a', 'variant_b', 'ablation_no_quota', 'ablation_no_time_order'];
|
|
186
|
+
const variantResults = [];
|
|
187
|
+
for (const v of variants) {
|
|
188
|
+
let metricsAcc = null;
|
|
189
|
+
const runIds = [];
|
|
190
|
+
for (let r = 0; r < repeats; r++) {
|
|
191
|
+
const sampled = sampleVariant(train, v, limit);
|
|
192
|
+
const m = computeSamplingMetrics(train, sampled);
|
|
193
|
+
metricsAcc = metricsAcc
|
|
194
|
+
? {
|
|
195
|
+
familyCoverage: metricsAcc.familyCoverage + m.familyCoverage,
|
|
196
|
+
maxFamilyShare: metricsAcc.maxFamilyShare + m.maxFamilyShare,
|
|
197
|
+
timeOrderPreservation: metricsAcc.timeOrderPreservation + m.timeOrderPreservation,
|
|
198
|
+
smallSourceIncluded: metricsAcc.smallSourceIncluded && m.smallSourceIncluded,
|
|
199
|
+
sampleSize: metricsAcc.sampleSize + m.sampleSize,
|
|
200
|
+
}
|
|
201
|
+
: m;
|
|
202
|
+
runIds.push(`${experimentId}:${v}:${r}`);
|
|
203
|
+
}
|
|
204
|
+
const avg = metricsAcc
|
|
205
|
+
? {
|
|
206
|
+
familyCoverage: Math.round(metricsAcc.familyCoverage / repeats),
|
|
207
|
+
maxFamilyShare: metricsAcc.maxFamilyShare / repeats,
|
|
208
|
+
timeOrderPreservation: metricsAcc.timeOrderPreservation / repeats,
|
|
209
|
+
smallSourceIncluded: metricsAcc.smallSourceIncluded,
|
|
210
|
+
sampleSize: Math.round(metricsAcc.sampleSize / repeats),
|
|
211
|
+
}
|
|
212
|
+
: { familyCoverage: 0, maxFamilyShare: 1, timeOrderPreservation: 0, smallSourceIncluded: false, sampleSize: 0 };
|
|
213
|
+
variantResults.push({ variant: v, metrics: avg, runIds });
|
|
214
|
+
}
|
|
215
|
+
const baseline = variantResults.find((r) => r.variant === 'baseline');
|
|
216
|
+
const deltas = variantResults
|
|
217
|
+
.filter((r) => r.variant !== 'baseline')
|
|
218
|
+
.map((r) => ({
|
|
219
|
+
variant: r.variant,
|
|
220
|
+
familyCoverageDelta: r.metrics.familyCoverage - baseline.metrics.familyCoverage,
|
|
221
|
+
maxFamilyShareDelta: r.metrics.maxFamilyShare - baseline.metrics.maxFamilyShare,
|
|
222
|
+
timeOrderPreservationDelta: r.metrics.timeOrderPreservation - baseline.metrics.timeOrderPreservation,
|
|
223
|
+
smallSourceIncludedDelta: (r.metrics.smallSourceIncluded ? 1 : 0) - (baseline.metrics.smallSourceIncluded ? 1 : 0),
|
|
224
|
+
}));
|
|
225
|
+
// 泛化:held-out 上 baseline vs best variant(按 familyCoverage + smallSourceIncluded 排序)
|
|
226
|
+
let generalization;
|
|
227
|
+
if (heldOut.length > 0) {
|
|
228
|
+
const baseSampled = sampleVariant(heldOut, 'baseline', limit);
|
|
229
|
+
const baseMetrics = computeSamplingMetrics(heldOut, baseSampled);
|
|
230
|
+
const scored = ['variant_a', 'variant_b'].map((v) => {
|
|
231
|
+
const s = sampleVariant(heldOut, v, limit);
|
|
232
|
+
return { v, m: computeSamplingMetrics(heldOut, s) };
|
|
233
|
+
});
|
|
234
|
+
const best = scored.sort((a, b) => {
|
|
235
|
+
const sc = (m) => m.familyCoverage * 10 + (m.smallSourceIncluded ? 5 : 0) - m.maxFamilyShare;
|
|
236
|
+
return sc(b.m) - sc(a.m);
|
|
237
|
+
})[0];
|
|
238
|
+
generalization = {
|
|
239
|
+
heldOutSize: heldOut.length,
|
|
240
|
+
baselineMetrics: baseMetrics,
|
|
241
|
+
bestVariant: best?.v ?? 'baseline',
|
|
242
|
+
bestMetrics: best?.m ?? baseMetrics,
|
|
243
|
+
};
|
|
244
|
+
}
|
|
245
|
+
// 结论:variant_a / variant_b 优于 baseline → 对应 wins;否则 baseline_wins;样本太少 → inconclusive
|
|
246
|
+
const aDelta = deltas.find((d) => d.variant === 'variant_a');
|
|
247
|
+
const bDelta = deltas.find((d) => d.variant === 'variant_b');
|
|
248
|
+
const aBetter = aDelta && (aDelta.familyCoverageDelta > 0 || aDelta.smallSourceIncludedDelta > 0);
|
|
249
|
+
const bBetter = bDelta && (bDelta.familyCoverageDelta > 0 || bDelta.smallSourceIncludedDelta > 0);
|
|
250
|
+
let conclusion;
|
|
251
|
+
let reason;
|
|
252
|
+
if (train.length < 10) {
|
|
253
|
+
conclusion = 'inconclusive';
|
|
254
|
+
reason = 'too_few_observations';
|
|
255
|
+
}
|
|
256
|
+
else if (aBetter && bBetter) {
|
|
257
|
+
// 双优 → inconclusive(需人工判断)
|
|
258
|
+
conclusion = 'inconclusive';
|
|
259
|
+
reason = 'both_variants_improve';
|
|
260
|
+
}
|
|
261
|
+
else if (aBetter) {
|
|
262
|
+
conclusion = 'variant_a_wins';
|
|
263
|
+
reason = 'higher_family_coverage_or_small_source';
|
|
264
|
+
}
|
|
265
|
+
else if (bBetter) {
|
|
266
|
+
conclusion = 'variant_b_wins';
|
|
267
|
+
reason = 'higher_family_coverage_or_small_source';
|
|
268
|
+
}
|
|
269
|
+
else {
|
|
270
|
+
conclusion = 'baseline_wins';
|
|
271
|
+
reason = 'no_improvement_over_baseline';
|
|
272
|
+
}
|
|
273
|
+
return {
|
|
274
|
+
experimentId,
|
|
275
|
+
hypothesisId: options?.hypothesisId,
|
|
276
|
+
baselineVariant: 'baseline',
|
|
277
|
+
variants: variantResults,
|
|
278
|
+
deltas,
|
|
279
|
+
generalization,
|
|
280
|
+
conclusion,
|
|
281
|
+
reason,
|
|
282
|
+
createdAt: Date.now(),
|
|
283
|
+
};
|
|
284
|
+
}
|
|
285
|
+
/** 把 CapabilityDelta 报告转成 ExperimentRun 列表(供 ExperimentRunStore.checkPromotionGate 使用) */
|
|
286
|
+
function reportToExperimentRuns(report) {
|
|
287
|
+
return report.variants.map((v) => ({
|
|
288
|
+
id: `${report.experimentId}:${v.variant}`,
|
|
289
|
+
hypothesisId: report.hypothesisId,
|
|
290
|
+
candidateId: v.variant,
|
|
291
|
+
baselineId: report.baselineVariant,
|
|
292
|
+
assignment: (v.variant === 'baseline'
|
|
293
|
+
? 'baseline'
|
|
294
|
+
: v.variant.startsWith('ablation')
|
|
295
|
+
? 'ablation'
|
|
296
|
+
: 'candidate'),
|
|
297
|
+
outcome: report.conclusion === 'inconclusive' ? 'inconclusive' : v.variant === report.conclusion.replace('_wins', '') ? 'success' : 'failure',
|
|
298
|
+
metrics: {
|
|
299
|
+
familyCoverage: v.metrics.familyCoverage,
|
|
300
|
+
maxFamilyShare: v.metrics.maxFamilyShare,
|
|
301
|
+
timeOrderPreservation: v.metrics.timeOrderPreservation,
|
|
302
|
+
smallSourceIncluded: v.metrics.smallSourceIncluded,
|
|
303
|
+
sampleSize: v.metrics.sampleSize,
|
|
304
|
+
},
|
|
305
|
+
evidenceIds: v.runIds,
|
|
306
|
+
createdAt: report.createdAt,
|
|
307
|
+
}));
|
|
308
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LocalModelService — 用 transformers.js 在进程内跑小模型
|
|
3
|
+
*
|
|
4
|
+
* 零外部依赖,进程内运行,不需要 Ollama/Python/独立服务。
|
|
5
|
+
* 懒加载,优先加载项目目录下的模型文件(models/Xenova/...),
|
|
6
|
+
* 不存在则从 HuggingFace 下载。
|
|
7
|
+
*
|
|
8
|
+
* 已下载模型位置:
|
|
9
|
+
* models/Xenova/Qwen2.5-0.5B-Instruct/
|
|
10
|
+
*
|
|
11
|
+
* 环境变量:
|
|
12
|
+
* LOCAL_HYPOTHESIS_MODEL — 模型 ID (默认 Xenova/Qwen2.5-0.5B-Instruct)
|
|
13
|
+
* LOCAL_HYPOTHESIS_DISABLED — 设为 'true' 禁用本地模型
|
|
14
|
+
* MODEL_CACHE_DIR — 缓存目录,缺省自动探测
|
|
15
|
+
*/
|
|
16
|
+
export declare class LocalModelService {
|
|
17
|
+
private generator;
|
|
18
|
+
private loadAttempted;
|
|
19
|
+
private modelId;
|
|
20
|
+
constructor(modelId?: string);
|
|
21
|
+
get isEnabled(): boolean;
|
|
22
|
+
get isLoaded(): boolean;
|
|
23
|
+
/**
|
|
24
|
+
* 生成文本 — 兼容 chatJson 接口格式
|
|
25
|
+
*/
|
|
26
|
+
generate(userText: string, options?: {
|
|
27
|
+
system?: string;
|
|
28
|
+
temperature?: number;
|
|
29
|
+
maxTokens?: number;
|
|
30
|
+
}): Promise<{
|
|
31
|
+
data?: any;
|
|
32
|
+
error?: string;
|
|
33
|
+
}>;
|
|
34
|
+
/**
|
|
35
|
+
* 尝试从生成文本中提取 JSON
|
|
36
|
+
*/
|
|
37
|
+
private tryParse;
|
|
38
|
+
}
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.LocalModelService = void 0;
|
|
4
|
+
const path_1 = require("path");
|
|
5
|
+
const fs_1 = require("fs");
|
|
6
|
+
const Logger_1 = require("@akemi-mio/core/logger/Logger");
|
|
7
|
+
/**
|
|
8
|
+
* LocalModelService — 用 transformers.js 在进程内跑小模型
|
|
9
|
+
*
|
|
10
|
+
* 零外部依赖,进程内运行,不需要 Ollama/Python/独立服务。
|
|
11
|
+
* 懒加载,优先加载项目目录下的模型文件(models/Xenova/...),
|
|
12
|
+
* 不存在则从 HuggingFace 下载。
|
|
13
|
+
*
|
|
14
|
+
* 已下载模型位置:
|
|
15
|
+
* models/Xenova/Qwen2.5-0.5B-Instruct/
|
|
16
|
+
*
|
|
17
|
+
* 环境变量:
|
|
18
|
+
* LOCAL_HYPOTHESIS_MODEL — 模型 ID (默认 Xenova/Qwen2.5-0.5B-Instruct)
|
|
19
|
+
* LOCAL_HYPOTHESIS_DISABLED — 设为 'true' 禁用本地模型
|
|
20
|
+
* MODEL_CACHE_DIR — 缓存目录,缺省自动探测
|
|
21
|
+
*/
|
|
22
|
+
class LocalModelService {
|
|
23
|
+
generator = null;
|
|
24
|
+
loadAttempted = false;
|
|
25
|
+
modelId;
|
|
26
|
+
constructor(modelId) {
|
|
27
|
+
this.modelId = modelId || process.env.LOCAL_HYPOTHESIS_MODEL || 'Xenova/Qwen2.5-0.5B-Instruct';
|
|
28
|
+
}
|
|
29
|
+
get isEnabled() {
|
|
30
|
+
return process.env.LOCAL_HYPOTHESIS_DISABLED !== 'true';
|
|
31
|
+
}
|
|
32
|
+
get isLoaded() {
|
|
33
|
+
return this.generator !== null;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* 生成文本 — 兼容 chatJson 接口格式
|
|
37
|
+
*/
|
|
38
|
+
async generate(userText, options) {
|
|
39
|
+
if (!this.isEnabled) {
|
|
40
|
+
return { error: 'local model disabled by env' };
|
|
41
|
+
}
|
|
42
|
+
// 测试环境下跳过实际模型加载
|
|
43
|
+
if (process.env.VITEST || process.env.NODE_ENV === 'test') {
|
|
44
|
+
return { error: 'local model disabled in test environment' };
|
|
45
|
+
}
|
|
46
|
+
// 懒加载模型
|
|
47
|
+
if (!this.generator) {
|
|
48
|
+
if (this.loadAttempted)
|
|
49
|
+
return { error: 'model previously failed to load' };
|
|
50
|
+
this.loadAttempted = true;
|
|
51
|
+
try {
|
|
52
|
+
(0, Logger_1.log)('INFO', 'local_model_loading', { model: this.modelId });
|
|
53
|
+
const { pipeline, env } = await import('@xenova/transformers');
|
|
54
|
+
// 优先使用项目目录下的本地模型
|
|
55
|
+
const modelsDir = await resolveModelsDir();
|
|
56
|
+
const localModelPath = (0, path_1.resolve)(modelsDir, this.modelId);
|
|
57
|
+
const hasLocalFiles = (0, fs_1.existsSync)(localModelPath);
|
|
58
|
+
if (hasLocalFiles) {
|
|
59
|
+
env.localModelPath = modelsDir;
|
|
60
|
+
(0, Logger_1.log)('INFO', 'local_model_use_cache', { path: localModelPath });
|
|
61
|
+
}
|
|
62
|
+
else {
|
|
63
|
+
(0, Logger_1.log)('INFO', 'local_model_no_cache', { path: localModelPath });
|
|
64
|
+
}
|
|
65
|
+
this.generator = await pipeline('text-generation', this.modelId, {
|
|
66
|
+
cache_dir: process.env.MODEL_CACHE_DIR,
|
|
67
|
+
});
|
|
68
|
+
(0, Logger_1.log)('INFO', 'local_model_loaded', { model: this.modelId, local: hasLocalFiles });
|
|
69
|
+
}
|
|
70
|
+
catch (err) {
|
|
71
|
+
(0, Logger_1.log)('WARN', 'local_model_load_failed', { model: this.modelId, error: err.message });
|
|
72
|
+
this.generator = null;
|
|
73
|
+
return { error: `local model load failed: ${err.message}` };
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
// 构造 prompts
|
|
77
|
+
const systemMsg = options?.system || '';
|
|
78
|
+
const chatTemplate = systemMsg
|
|
79
|
+
? `<|im_start|>system\n${systemMsg}<|im_end|>\n<|im_start|>user\n${userText}<|im_end|>\n<|im_start|>assistant\n`
|
|
80
|
+
: `<|im_start|>user\n${userText}<|im_end|>\n<|im_start|>assistant\n`;
|
|
81
|
+
try {
|
|
82
|
+
const result = await this.generator(chatTemplate, {
|
|
83
|
+
max_new_tokens: options?.maxTokens || 768,
|
|
84
|
+
temperature: options?.temperature ?? 0.7,
|
|
85
|
+
do_sample: true,
|
|
86
|
+
});
|
|
87
|
+
const rawText = Array.isArray(result) ? result[0]?.generated_text : result?.generated_text;
|
|
88
|
+
if (!rawText)
|
|
89
|
+
return { error: 'empty generation' };
|
|
90
|
+
// 去掉输入部分,只取生成的文本
|
|
91
|
+
const generated = rawText.slice(chatTemplate.length).trim();
|
|
92
|
+
// 尝试解析 JSON
|
|
93
|
+
const parsed = this.tryParse(generated);
|
|
94
|
+
if (parsed) {
|
|
95
|
+
return { data: parsed };
|
|
96
|
+
}
|
|
97
|
+
return { error: `generated non-JSON: ${generated.slice(0, 100)}` };
|
|
98
|
+
}
|
|
99
|
+
catch (err) {
|
|
100
|
+
(0, Logger_1.log)('WARN', 'local_model_generation_error', { error: err.message });
|
|
101
|
+
return { error: `generation failed: ${err.message}` };
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* 尝试从生成文本中提取 JSON
|
|
106
|
+
*/
|
|
107
|
+
tryParse(text) {
|
|
108
|
+
try {
|
|
109
|
+
return JSON.parse(text);
|
|
110
|
+
}
|
|
111
|
+
catch {
|
|
112
|
+
const match = text.match(/```(?:json)?\s*([\s\S]*?)```/);
|
|
113
|
+
if (match) {
|
|
114
|
+
try {
|
|
115
|
+
return JSON.parse(match[1].trim());
|
|
116
|
+
}
|
|
117
|
+
catch {
|
|
118
|
+
// ignore
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
const arrMatch = text.match(/\[[\s\S]*\]/);
|
|
122
|
+
if (arrMatch) {
|
|
123
|
+
try {
|
|
124
|
+
return JSON.parse(arrMatch[0]);
|
|
125
|
+
}
|
|
126
|
+
catch {
|
|
127
|
+
// ignore
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
return null;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
exports.LocalModelService = LocalModelService;
|
|
135
|
+
/** 自动探测项目 models 目录 */
|
|
136
|
+
async function resolveModelsDir() {
|
|
137
|
+
// Electron 环境
|
|
138
|
+
try {
|
|
139
|
+
const { app } = await import('electron');
|
|
140
|
+
// 优先项目目录(dev),回退到 userData/models(安装版)
|
|
141
|
+
if ((0, fs_1.existsSync)((0, path_1.resolve)(app.getAppPath(), 'models'))) {
|
|
142
|
+
return (0, path_1.resolve)(app.getAppPath(), 'models');
|
|
143
|
+
}
|
|
144
|
+
return (0, path_1.resolve)(app.getPath('userData'), 'models');
|
|
145
|
+
}
|
|
146
|
+
catch {
|
|
147
|
+
// not in Electron
|
|
148
|
+
}
|
|
149
|
+
// Node.js 开发环境:逐级向上找 models/
|
|
150
|
+
let dir = process.cwd();
|
|
151
|
+
for (let i = 0; i < 5; i++) {
|
|
152
|
+
const p = (0, path_1.resolve)(dir, 'models');
|
|
153
|
+
if ((0, fs_1.existsSync)(p))
|
|
154
|
+
return p;
|
|
155
|
+
const parent = (0, path_1.resolve)(dir, '@akemi-mio/main');
|
|
156
|
+
if (parent === dir)
|
|
157
|
+
break;
|
|
158
|
+
dir = parent;
|
|
159
|
+
}
|
|
160
|
+
// fallback: CWD/models
|
|
161
|
+
return (0, path_1.resolve)(process.cwd(), 'models');
|
|
162
|
+
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import type { ProposedRelation } from './types';
|
|
2
|
+
/**
|
|
3
|
+
* Mechanism Graph v0 — traces.jsonl 派生读模型(非独立图存储)。
|
|
4
|
+
*
|
|
5
|
+
* 定案 A6:confidence 由边的推断来源映射——identity=verified → high,
|
|
6
|
+
* inferred → low。聚合键为 (agent, capabilityId, outcome)。
|
|
7
|
+
* 持久化图后置到 P3 验证有产出之后。
|
|
8
|
+
*/
|
|
9
|
+
export interface TraceEventLike {
|
|
10
|
+
event_type?: string;
|
|
11
|
+
outcome?: string;
|
|
12
|
+
agent?: string;
|
|
13
|
+
payload?: Record<string, unknown>;
|
|
14
|
+
}
|
|
15
|
+
export interface MechanismEdge {
|
|
16
|
+
agent: string;
|
|
17
|
+
capabilityId: string;
|
|
18
|
+
outcome: string;
|
|
19
|
+
count: number;
|
|
20
|
+
/** A6: identity=verified→high / inferred→low */
|
|
21
|
+
confidence: 'high' | 'low';
|
|
22
|
+
/** 聚合中出现过的 inference 种类(identity 优先) */
|
|
23
|
+
inferences: Array<'identity' | 'inferred'>;
|
|
24
|
+
}
|
|
25
|
+
export interface MechanismGraph {
|
|
26
|
+
edges: MechanismEdge[];
|
|
27
|
+
/** 参与聚合的事件总数(仅计入含 agent+capabilityId 的 tool_call/task 类事件) */
|
|
28
|
+
eventCount: number;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* 从 traces.jsonl 行事件聚合机制边。
|
|
32
|
+
*
|
|
33
|
+
* 规则:
|
|
34
|
+
* - 仅计入 payload.capabilityId 存在的事件(能力绑定相关)。
|
|
35
|
+
* - 每条边键 = `${agent}|${capabilityId}|${outcome}`,count 累加。
|
|
36
|
+
* - inference:payload.inference === 'identity' 或 event_type 含 identity
|
|
37
|
+
* 时记 identity;否则记 inferred。
|
|
38
|
+
* - confidence:边内出现过任一 identity → high,否则 low。
|
|
39
|
+
*/
|
|
40
|
+
export declare function deriveMechanismGraph(events: readonly TraceEventLike[]): MechanismGraph;
|
|
41
|
+
/** 把机制假设的 proposedRelations 投影为与派生图同形状的边(供对照,confidence 走 inference 字段)。 */
|
|
42
|
+
export declare function relationsToEdges(agent: string, relations: readonly ProposedRelation[]): MechanismEdge[];
|
|
43
|
+
/** 是否为机制假设:带非空 proposedRelations 即机制假设,否则普通创意(一个 store 两种粒度)。 */
|
|
44
|
+
export declare function isMechanismHypothesis(h: {
|
|
45
|
+
proposedRelations?: ProposedRelation[];
|
|
46
|
+
}): boolean;
|