oh-my-knowledge 0.35.0 → 0.36.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/README.zh.md +1 -1
- package/dist/assets/agent-skills/omk/SKILL.md +2 -2
- package/dist/assets/agent-skills/omk/references/commands.md +36 -0
- package/dist/authoring/evolver.js +1 -1
- package/dist/cli/commands/doctor.js +2 -1
- package/dist/cli/commands/eval/index.js +7 -6
- package/dist/cli/commands/list.d.ts +46 -0
- package/dist/cli/commands/list.js +243 -0
- package/dist/cli/lib/i18n-dict/list.d.ts +3 -0
- package/dist/cli/lib/i18n-dict/list.js +32 -0
- package/dist/cli/lib/i18n-dict.d.ts +2 -1
- package/dist/cli/lib/i18n-dict.js +2 -0
- package/dist/cli/lib/run-tally.js +1 -1
- package/dist/cli/lib/shared.js +1 -1
- package/dist/doctor/index.js +1 -1
- package/dist/eval-core/evaluation-reporting.d.ts +1 -0
- package/dist/eval-core/evaluation-reporting.js +6 -4
- package/dist/eval-workflows/batch-evaluation-workflow.js +3 -2
- package/dist/eval-workflows/run-evaluation.js +2 -2
- package/dist/inputs/skill-loader.js +6 -3
- package/dist/managed/index.d.ts +1 -0
- package/dist/managed/index.js +1 -0
- package/dist/managed/list-view.d.ts +55 -0
- package/dist/managed/list-view.js +66 -0
- package/dist/managed/store.js +45 -8
- package/dist/observability/experience.d.ts +2 -0
- package/dist/observability/experience.js +31 -2
- package/dist/observability/inbox.js +31 -5
- package/dist/observability/review-state.js +22 -11
- package/dist/observability/soft-standards/llm-extractor.js +3 -3
- package/dist/observability/soft-standards/skill-standards-store.d.ts +1 -0
- package/dist/observability/soft-standards/skill-standards-store.js +77 -14
- package/dist/observability/soft-standards/types.d.ts +2 -2
- package/dist/renderer/html-renderer.js +3 -3
- package/dist/server/report-server.js +1 -1
- package/dist/server/report-store.d.ts +1 -1
- package/dist/server/report-store.js +16 -15
- package/dist/server/skill-index.js +5 -4
- package/dist/types/doctor.d.ts +2 -2
- package/dist/types/doctor.js +1 -1
- package/dist/types/observability.d.ts +6 -6
- package/dist/types/report.d.ts +9 -4
- package/package.json +1 -1
|
@@ -20,7 +20,7 @@ export declare function queryJobList(jobStore: JobStore, query?: JobQuery): Prom
|
|
|
20
20
|
export declare function queryJob(jobStore: JobStore, id: string): Promise<EvaluationJob | null>;
|
|
21
21
|
export interface RunListItem {
|
|
22
22
|
id: string;
|
|
23
|
-
|
|
23
|
+
kind: ReportDocument['kind'];
|
|
24
24
|
meta: ReportDocument['meta'];
|
|
25
25
|
summary?: EvaluationReport['summary'];
|
|
26
26
|
items?: BatchEvaluationReport['items'];
|
|
@@ -39,32 +39,33 @@ export function createFileStore(dir) {
|
|
|
39
39
|
if (!data || typeof data !== 'object')
|
|
40
40
|
return null;
|
|
41
41
|
const record = data;
|
|
42
|
-
|
|
42
|
+
const kind = record.kind === 'evaluation' || record.kind === 'batch-evaluation'
|
|
43
|
+
? record.kind
|
|
44
|
+
: undefined;
|
|
45
|
+
if (kind === 'evaluation') {
|
|
43
46
|
if (!record.meta || !record.summary || !Array.isArray(record.results))
|
|
44
47
|
return null;
|
|
45
|
-
return {
|
|
48
|
+
return {
|
|
49
|
+
...record,
|
|
50
|
+
kind,
|
|
51
|
+
id: typeof record.id === 'string' && record.id ? record.id : fallbackId,
|
|
52
|
+
};
|
|
46
53
|
}
|
|
47
|
-
if (
|
|
54
|
+
if (kind === 'batch-evaluation') {
|
|
48
55
|
if (!record.meta || !Array.isArray(record.items))
|
|
49
56
|
return null;
|
|
50
|
-
return { ...record, id: typeof record.id === 'string' && record.id ? record.id : fallbackId };
|
|
51
|
-
}
|
|
52
|
-
if (record.kind === undefined
|
|
53
|
-
&& record.overview === undefined
|
|
54
|
-
&& record.artifacts === undefined
|
|
55
|
-
&& record.meta
|
|
56
|
-
&& record.summary
|
|
57
|
-
&& Array.isArray(record.results)) {
|
|
58
57
|
return {
|
|
59
58
|
...record,
|
|
60
|
-
|
|
59
|
+
kind,
|
|
61
60
|
id: typeof record.id === 'string' && record.id ? record.id : fallbackId,
|
|
62
61
|
};
|
|
63
62
|
}
|
|
63
|
+
// 只认 canonical 顶层 `kind`(evaluation / batch-evaluation)。不再为旧格式(顶层无该判别字段的
|
|
64
|
+
// 历史文件)做读兼容 —— 顶层 kind cutover 是硬切换,旧文件直接判脏丢弃。
|
|
64
65
|
return null;
|
|
65
66
|
}
|
|
66
67
|
function isEvaluationReport(report) {
|
|
67
|
-
return report.
|
|
68
|
+
return report.kind === 'evaluation';
|
|
68
69
|
}
|
|
69
70
|
// Studio 每个 / 和 /skills/<name> 请求都调 list(),里面对每个 .json 同步 readFile +
|
|
70
71
|
// JSON.parse。报告数上来后这是主性能瓶颈。缓存策略:fingerprint = dir mtime + 文件名
|
|
@@ -217,9 +218,9 @@ export async function queryJob(jobStore, id) {
|
|
|
217
218
|
export async function queryRunList(reportStore) {
|
|
218
219
|
return (await reportStore.list()).map((report) => ({
|
|
219
220
|
id: report.id,
|
|
220
|
-
|
|
221
|
+
kind: report.kind,
|
|
221
222
|
meta: report.meta,
|
|
222
|
-
...(report.
|
|
223
|
+
...(report.kind === 'evaluation' ? { summary: report.summary } : { items: report.items }),
|
|
223
224
|
}));
|
|
224
225
|
}
|
|
225
226
|
export async function queryRun(reportStore, id) {
|
|
@@ -69,7 +69,7 @@ function buildIndexFingerprint(reports, analysesDir, doctorsDir, observationsDir
|
|
|
69
69
|
// 每一段的 right-hand-side 从旧的 "{dir-mtime}-{file-count}" 双标量升级成
|
|
70
70
|
// safeDirJsonContentFingerprint 返回的 "{dir-mtime}|{file1}:{m}:{s},..."
|
|
71
71
|
// content-aware 字符串。
|
|
72
|
-
const reportIds = reports.map((r) => `${r.id}:${r.meta?.timestamp ?? ''}:${r.
|
|
72
|
+
const reportIds = reports.map((r) => `${r.id}:${r.meta?.timestamp ?? ''}:${r.kind === 'evaluation' ? r.meta.evolve?.skillName ?? '' : ''}`).join(',');
|
|
73
73
|
const doctorsFp = safeDirJsonContentFingerprint(doctorsDir);
|
|
74
74
|
const analysesFp = safeDirJsonContentFingerprint(analysesDir);
|
|
75
75
|
const observationsFp = safeDirJsonContentFingerprint(observationsDir);
|
|
@@ -223,7 +223,8 @@ function scanDoctorReports(dir) {
|
|
|
223
223
|
continue;
|
|
224
224
|
try {
|
|
225
225
|
const data = JSON.parse(readFileSync(join(dir, file), 'utf-8'));
|
|
226
|
-
|
|
226
|
+
const kind = data?.kind === 'doctor' ? data.kind : null;
|
|
227
|
+
if (!kind || !Array.isArray(data.skills))
|
|
227
228
|
continue;
|
|
228
229
|
const ts = data.timestamp;
|
|
229
230
|
for (const sr of data.skills) {
|
|
@@ -260,7 +261,7 @@ export function buildSkillIndex(reports, analysesDir, doctorsDir, observationsDi
|
|
|
260
261
|
// ── eval 聚合(历史 list)─────────────────────────────────
|
|
261
262
|
const evalBy = {};
|
|
262
263
|
for (const r of reports) {
|
|
263
|
-
if (r.
|
|
264
|
+
if (r.kind !== 'evaluation')
|
|
264
265
|
continue;
|
|
265
266
|
const variants = r.meta.variants || [];
|
|
266
267
|
for (const v of variants) {
|
|
@@ -347,7 +348,7 @@ export function buildSkillIndex(reports, analysesDir, doctorsDir, observationsDi
|
|
|
347
348
|
// list 页对每个 entry 跑 detectInsights 的 CPU 开销迁移到这里,只 miss 时算一次。
|
|
348
349
|
const insightsBySkill = new Map();
|
|
349
350
|
for (const ent of entries) {
|
|
350
|
-
const evalReport = ent.eval ? reports.find((r) => r.id === ent.eval.reportId && r.
|
|
351
|
+
const evalReport = ent.eval ? reports.find((r) => r.id === ent.eval.reportId && r.kind === 'evaluation') : undefined;
|
|
351
352
|
insightsBySkill.set(ent.skillName, detectInsights(ent, evalReport ?? null, {
|
|
352
353
|
diagnostics: diagnosisBundle.bySkill[ent.skillName] ?? [],
|
|
353
354
|
}));
|
package/dist/types/doctor.d.ts
CHANGED
|
@@ -10,7 +10,7 @@ export type DoctorSkillStatus = 'pass' | 'warn' | 'fail';
|
|
|
10
10
|
export type DoctorOutcome = 'passed' | 'warnings_only' | 'failed';
|
|
11
11
|
/** Bumped whenever DoctorReport schema changes in a way CI consumers should
|
|
12
12
|
* be able to detect. CI can pin/check this when parsing the JSON. */
|
|
13
|
-
export declare const DOCTOR_REPORT_SCHEMA_VERSION = "
|
|
13
|
+
export declare const DOCTOR_REPORT_SCHEMA_VERSION = "3.0.0";
|
|
14
14
|
export interface DoctorRuleResult {
|
|
15
15
|
ruleId: string;
|
|
16
16
|
severity: DoctorSeverity;
|
|
@@ -94,7 +94,7 @@ export interface DoctorSkillReport {
|
|
|
94
94
|
status: DoctorSkillStatus;
|
|
95
95
|
}
|
|
96
96
|
export interface DoctorReport {
|
|
97
|
-
|
|
97
|
+
kind: 'doctor';
|
|
98
98
|
/** Schema version the JSON consumer can pin/check. Bumped on any
|
|
99
99
|
* user-visible change to this report's shape. See DOCTOR_REPORT_SCHEMA_VERSION. */
|
|
100
100
|
schemaVersion: string;
|
package/dist/types/doctor.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/** Bumped whenever DoctorReport schema changes in a way CI consumers should
|
|
2
2
|
* be able to detect. CI can pin/check this when parsing the JSON. */
|
|
3
|
-
export const DOCTOR_REPORT_SCHEMA_VERSION = '
|
|
3
|
+
export const DOCTOR_REPORT_SCHEMA_VERSION = '3.0.0';
|
|
4
4
|
export function isComposerRule(r) {
|
|
5
5
|
return r.ruleKind === 'composer';
|
|
6
6
|
}
|
|
@@ -43,8 +43,8 @@ export interface ObservationReviewStateEntry {
|
|
|
43
43
|
snippet?: string;
|
|
44
44
|
}
|
|
45
45
|
export interface ObservationReviewState {
|
|
46
|
-
|
|
47
|
-
schemaVersion:
|
|
46
|
+
kind: 'observe-review-state';
|
|
47
|
+
schemaVersion: 2;
|
|
48
48
|
updatedAt: string;
|
|
49
49
|
entries: Record<string, ObservationReviewStateEntry>;
|
|
50
50
|
}
|
|
@@ -184,8 +184,8 @@ export interface ObservationSessionTimeRange {
|
|
|
184
184
|
durationMs?: number;
|
|
185
185
|
}
|
|
186
186
|
export interface ObservationInboxReport {
|
|
187
|
-
|
|
188
|
-
schemaVersion:
|
|
187
|
+
kind: 'observe-inbox';
|
|
188
|
+
schemaVersion: 2;
|
|
189
189
|
meta: {
|
|
190
190
|
tracePath: string;
|
|
191
191
|
generatedAt: string;
|
|
@@ -735,8 +735,8 @@ export interface ExperienceSkillSummary {
|
|
|
735
735
|
relatedObservationIds: string[];
|
|
736
736
|
}
|
|
737
737
|
export interface ObservationExperienceReport {
|
|
738
|
-
|
|
739
|
-
schemaVersion:
|
|
738
|
+
kind: 'observe-experience';
|
|
739
|
+
schemaVersion: 2;
|
|
740
740
|
scope: 'evidence-only';
|
|
741
741
|
generatedAt: string;
|
|
742
742
|
meta: {
|
package/dist/types/report.d.ts
CHANGED
|
@@ -253,8 +253,10 @@ export interface ReportMeta {
|
|
|
253
253
|
* for eval-side reports). Value 1 was specified for v0.21+ but never emitted in practice.
|
|
254
254
|
* Value 2 marked the artifactHashes tree-hash era for local dir-skills (git dir-skills still
|
|
255
255
|
* SKILL.md-only). Value 3 extends whole-tree hashing to git dir-skills via isolated copies
|
|
256
|
-
* (all dir-skills bind).
|
|
257
|
-
*
|
|
256
|
+
* (all dir-skills bind). Value 4 keeps the v3 hash/binding semantics but marks the canonical
|
|
257
|
+
* top-level discriminant era, so external consumers can version-gate the JSON shape. Drift /
|
|
258
|
+
* lineage consumers gate on `>= 2` (tree-hash era); git-dir-skill binding additionally
|
|
259
|
+
* requires `>= 3`. */
|
|
258
260
|
schemaVersion?: number;
|
|
259
261
|
/** SHA256-12 of every sample's content (sample_id → hash). Same hash = same sample. */
|
|
260
262
|
sampleHashes?: Record<string, string>;
|
|
@@ -355,7 +357,7 @@ export interface SampleSnapshot {
|
|
|
355
357
|
tripwire?: boolean;
|
|
356
358
|
}
|
|
357
359
|
export interface EvaluationReport {
|
|
358
|
-
|
|
360
|
+
kind: 'evaluation';
|
|
359
361
|
id: string;
|
|
360
362
|
meta: ReportMeta;
|
|
361
363
|
summary: Record<string, VariantSummary>;
|
|
@@ -369,6 +371,9 @@ export interface EvaluationReport {
|
|
|
369
371
|
export type Report = EvaluationReport;
|
|
370
372
|
export interface BatchEvaluationMeta {
|
|
371
373
|
mode: 'skill';
|
|
374
|
+
/** Batch report JSON schema version. Kept in lockstep with child EvaluationReport top-level
|
|
375
|
+
* shape so external consumers can identify the canonical discriminant era. */
|
|
376
|
+
schemaVersion: number;
|
|
372
377
|
model: string;
|
|
373
378
|
executor: string;
|
|
374
379
|
skillDir: string;
|
|
@@ -408,7 +413,7 @@ export interface BatchEvaluationItem {
|
|
|
408
413
|
variance?: VarianceData;
|
|
409
414
|
}
|
|
410
415
|
export interface BatchEvaluationReport {
|
|
411
|
-
|
|
416
|
+
kind: 'batch-evaluation';
|
|
412
417
|
id: string;
|
|
413
418
|
mode: 'skill';
|
|
414
419
|
meta: BatchEvaluationMeta;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "oh-my-knowledge",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.36.0",
|
|
4
4
|
"packageManager": "yarn@4.16.0",
|
|
5
5
|
"description": "Evaluation framework for LLM knowledge inputs — prompts, RAG corpora, skills, agent workflows. Fix the model, vary the artifact. Built-in statistical rigor: bootstrap CI, Krippendorff α, length-debias, saturation curves.",
|
|
6
6
|
"type": "module",
|