@wowok/agent-mcp 2.3.13 → 2.3.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/harness/checkpoint.d.ts +8 -0
- package/dist/harness/checkpoint.js +129 -0
- package/dist/harness/index.d.ts +33 -0
- package/dist/harness/index.js +75 -0
- package/dist/harness/plan.d.ts +18 -0
- package/dist/harness/plan.js +252 -0
- package/dist/harness/recover.d.ts +17 -0
- package/dist/harness/recover.js +139 -0
- package/dist/harness/types.d.ts +137 -0
- package/dist/harness/types.js +1 -0
- package/dist/harness/verify.d.ts +42 -0
- package/dist/harness/verify.js +237 -0
- package/dist/index.js +134 -52
- package/dist/loop-engineering/aggregate.d.ts +50 -0
- package/dist/loop-engineering/aggregate.js +132 -0
- package/dist/loop-engineering/diagnose.d.ts +21 -0
- package/dist/loop-engineering/diagnose.js +179 -0
- package/dist/loop-engineering/improve.d.ts +26 -0
- package/dist/loop-engineering/improve.js +178 -0
- package/dist/loop-engineering/index.d.ts +4 -0
- package/dist/loop-engineering/index.js +4 -0
- package/dist/loop-engineering/pipeline.d.ts +17 -0
- package/dist/loop-engineering/pipeline.js +56 -0
- package/dist/mode-market/index.d.ts +3 -0
- package/dist/mode-market/index.js +3 -0
- package/dist/mode-market/registry.d.ts +53 -0
- package/dist/mode-market/registry.js +124 -0
- package/dist/mode-market/review.d.ts +27 -0
- package/dist/mode-market/review.js +214 -0
- package/dist/mode-market/submission.d.ts +25 -0
- package/dist/mode-market/submission.js +85 -0
- package/dist/schema/call/allocation.d.ts +40 -40
- package/dist/schema/call/arbitration.d.ts +90 -90
- package/dist/schema/call/base.d.ts +1054 -0
- package/dist/schema/call/base.js +100 -22
- package/dist/schema/call/bridge-handler.js +36 -18
- package/dist/schema/call/bridge.d.ts +115 -115
- package/dist/schema/call/handler.d.ts +2 -1
- package/dist/schema/call/handler.js +94 -3
- package/dist/schema/call/index.d.ts +1 -0
- package/dist/schema/call/index.js +1 -0
- package/dist/schema/call/machine.d.ts +144 -144
- package/dist/schema/call/order.d.ts +12 -12
- package/dist/schema/call/progress.d.ts +6 -6
- package/dist/schema/call/reward.d.ts +6 -6
- package/dist/schema/call/semantic.d.ts +23 -0
- package/dist/schema/call/semantic.js +764 -0
- package/dist/schema/call/service.d.ts +142 -142
- package/dist/schema/call/treasury.d.ts +204 -204
- package/dist/schema/local/index.d.ts +113 -2
- package/dist/schema/local/index.js +11 -1
- package/dist/schema/messenger/index.d.ts +416 -92
- package/dist/schema/messenger/index.js +9 -0
- package/dist/schema/operations.d.ts +988 -495
- package/dist/schema/operations.js +14 -3
- package/dist/schema/query/index.d.ts +346 -342
- package/dist/schema/query/index.js +4 -1
- package/dist/schemas/account_operation.output.json +3 -0
- package/dist/schemas/account_operation.schema.json +1 -1
- package/dist/schemas/index.json +1 -1
- package/dist/schemas/local_info_operation.output.json +3 -0
- package/dist/schemas/local_mark_operation.output.json +3 -0
- package/dist/schemas/messenger_operation.output.json +21 -0
- package/dist/schemas/messenger_operation.schema.json +21 -0
- package/dist/schemas/onchain_events.output.json +3 -0
- package/dist/schemas/onchain_operations.output.json +464 -21
- package/dist/schemas/onchain_operations.schema.json +21 -17
- package/dist/schemas/onchain_operations_allocation.schema.json +11 -9
- package/dist/schemas/onchain_operations_arbitration.schema.json +11 -9
- package/dist/schemas/onchain_operations_contact.schema.json +11 -9
- package/dist/schemas/onchain_operations_demand.schema.json +11 -9
- package/dist/schemas/onchain_operations_gen_passport.schema.json +21 -17
- package/dist/schemas/onchain_operations_guard.schema.json +11 -9
- package/dist/schemas/onchain_operations_machine.schema.json +11 -9
- package/dist/schemas/onchain_operations_order.schema.json +11 -9
- package/dist/schemas/onchain_operations_payment.schema.json +11 -9
- package/dist/schemas/onchain_operations_permission.schema.json +11 -9
- package/dist/schemas/onchain_operations_personal.schema.json +11 -9
- package/dist/schemas/onchain_operations_progress.schema.json +11 -9
- package/dist/schemas/onchain_operations_repository.schema.json +11 -9
- package/dist/schemas/onchain_operations_reward.schema.json +11 -9
- package/dist/schemas/onchain_operations_service.schema.json +11 -9
- package/dist/schemas/onchain_operations_treasury.schema.json +11 -9
- package/dist/schemas/onchain_table_data.output.json +3 -0
- package/dist/schemas/query_toolkit.schema.json +5 -1
- package/dist/telemetry/index.d.ts +19 -0
- package/dist/telemetry/index.js +112 -0
- package/dist/telemetry/redact.d.ts +2 -0
- package/dist/telemetry/redact.js +23 -0
- package/dist/telemetry/storage.d.ts +8 -0
- package/dist/telemetry/storage.js +80 -0
- package/package.json +6 -3
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
function percentile(sortedValues, p) {
|
|
2
|
+
if (sortedValues.length === 0)
|
|
3
|
+
return 0;
|
|
4
|
+
const idx = Math.ceil((p / 100) * sortedValues.length) - 1;
|
|
5
|
+
return sortedValues[Math.max(0, idx)];
|
|
6
|
+
}
|
|
7
|
+
function topMessages(messages, limit = 5) {
|
|
8
|
+
const counts = new Map();
|
|
9
|
+
for (const msg of messages) {
|
|
10
|
+
if (!msg)
|
|
11
|
+
continue;
|
|
12
|
+
counts.set(msg, (counts.get(msg) || 0) + 1);
|
|
13
|
+
}
|
|
14
|
+
return Array.from(counts.entries())
|
|
15
|
+
.map(([message, count]) => ({ message, count }))
|
|
16
|
+
.sort((a, b) => b.count - a.count)
|
|
17
|
+
.slice(0, limit);
|
|
18
|
+
}
|
|
19
|
+
export function aggregateByTool(events) {
|
|
20
|
+
const groups = new Map();
|
|
21
|
+
for (const e of events) {
|
|
22
|
+
const key = e.tool_name || "unknown";
|
|
23
|
+
if (!groups.has(key))
|
|
24
|
+
groups.set(key, []);
|
|
25
|
+
groups.get(key).push(e);
|
|
26
|
+
}
|
|
27
|
+
const results = [];
|
|
28
|
+
for (const [tool_name, group] of groups) {
|
|
29
|
+
const total = group.length;
|
|
30
|
+
const success = group.filter((e) => e.status === "success").length;
|
|
31
|
+
const failed = group.filter((e) => e.status === "failed").length;
|
|
32
|
+
const pending = group.filter((e) => e.status === "pending_input").length;
|
|
33
|
+
const semanticHits = group.filter((e) => e.semantic_hit).length;
|
|
34
|
+
const durations = group.map((e) => e.duration_ms).sort((a, b) => a - b);
|
|
35
|
+
const errorDist = {};
|
|
36
|
+
const errorMsgs = [];
|
|
37
|
+
for (const e of group) {
|
|
38
|
+
if (e.error_code) {
|
|
39
|
+
errorDist[e.error_code] = (errorDist[e.error_code] || 0) + 1;
|
|
40
|
+
}
|
|
41
|
+
if (e.error_message)
|
|
42
|
+
errorMsgs.push(e.error_message);
|
|
43
|
+
}
|
|
44
|
+
results.push({
|
|
45
|
+
tool_name,
|
|
46
|
+
total_calls: total,
|
|
47
|
+
success_count: success,
|
|
48
|
+
failed_count: failed,
|
|
49
|
+
pending_input_count: pending,
|
|
50
|
+
success_rate: total > 0 ? success / total : 0,
|
|
51
|
+
avg_duration_ms: durations.length > 0 ? durations.reduce((a, b) => a + b, 0) / durations.length : 0,
|
|
52
|
+
p50_duration_ms: percentile(durations, 50),
|
|
53
|
+
p95_duration_ms: percentile(durations, 95),
|
|
54
|
+
semantic_hit_rate: total > 0 ? semanticHits / total : 0,
|
|
55
|
+
error_distribution: errorDist,
|
|
56
|
+
top_error_messages: topMessages(errorMsgs),
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
return results.sort((a, b) => b.total_calls - a.total_calls);
|
|
60
|
+
}
|
|
61
|
+
export function aggregateByOperation(events) {
|
|
62
|
+
const groups = new Map();
|
|
63
|
+
for (const e of events) {
|
|
64
|
+
const key = e.operation_type || "unknown";
|
|
65
|
+
if (!groups.has(key))
|
|
66
|
+
groups.set(key, []);
|
|
67
|
+
groups.get(key).push(e);
|
|
68
|
+
}
|
|
69
|
+
const results = [];
|
|
70
|
+
for (const [operation_type, group] of groups) {
|
|
71
|
+
const total = group.length;
|
|
72
|
+
const success = group.filter((e) => e.status === "success").length;
|
|
73
|
+
const semanticHits = group.filter((e) => e.semantic_hit).length;
|
|
74
|
+
const errorDist = {};
|
|
75
|
+
for (const e of group) {
|
|
76
|
+
if (e.error_code) {
|
|
77
|
+
errorDist[e.error_code] = (errorDist[e.error_code] || 0) + 1;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
results.push({
|
|
81
|
+
operation_type,
|
|
82
|
+
total_calls: total,
|
|
83
|
+
success_rate: total > 0 ? success / total : 0,
|
|
84
|
+
semantic_hit_rate: total > 0 ? semanticHits / total : 0,
|
|
85
|
+
error_distribution: errorDist,
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
return results.sort((a, b) => b.total_calls - a.total_calls);
|
|
89
|
+
}
|
|
90
|
+
export function aggregate(events) {
|
|
91
|
+
const total = events.length;
|
|
92
|
+
const success = events.filter((e) => e.status === "success").length;
|
|
93
|
+
const semanticHits = events.filter((e) => e.semantic_hit).length;
|
|
94
|
+
const unknownErrors = events.filter((e) => e.error_code === "unknown").length;
|
|
95
|
+
const durations = events.map((e) => e.duration_ms);
|
|
96
|
+
const errorDist = {};
|
|
97
|
+
const intentDist = {};
|
|
98
|
+
for (const e of events) {
|
|
99
|
+
if (e.error_code) {
|
|
100
|
+
errorDist[e.error_code] = (errorDist[e.error_code] || 0) + 1;
|
|
101
|
+
}
|
|
102
|
+
if (e.intent) {
|
|
103
|
+
intentDist[e.intent] = (intentDist[e.intent] || 0) + 1;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
const timestamps = events
|
|
107
|
+
.map((e) => e.timestamp)
|
|
108
|
+
.filter(Boolean)
|
|
109
|
+
.sort();
|
|
110
|
+
return {
|
|
111
|
+
aggregation_date: new Date().toISOString(),
|
|
112
|
+
total_events: total,
|
|
113
|
+
date_range: {
|
|
114
|
+
earliest: timestamps[0] || "",
|
|
115
|
+
latest: timestamps[timestamps.length - 1] || "",
|
|
116
|
+
},
|
|
117
|
+
per_tool: aggregateByTool(events),
|
|
118
|
+
per_operation: aggregateByOperation(events),
|
|
119
|
+
overall: {
|
|
120
|
+
total_calls: total,
|
|
121
|
+
success_count: success,
|
|
122
|
+
failed_count: events.filter((e) => e.status === "failed").length,
|
|
123
|
+
success_rate: total > 0 ? success / total : 0,
|
|
124
|
+
semantic_hit_rate: total > 0 ? semanticHits / total : 0,
|
|
125
|
+
unknown_error_count: unknownErrors,
|
|
126
|
+
unknown_error_rate: total > 0 ? unknownErrors / total : 0,
|
|
127
|
+
avg_duration_ms: durations.length > 0 ? durations.reduce((a, b) => a + b, 0) / durations.length : 0,
|
|
128
|
+
},
|
|
129
|
+
error_code_distribution: errorDist,
|
|
130
|
+
intent_distribution: intentDist,
|
|
131
|
+
};
|
|
132
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { AggregatedStats } from "./aggregate.js";
|
|
2
|
+
export type DiagnosisType = "rule_gap" | "description_quality" | "skill_need" | "recovery_ineffectiveness" | "latency_anomaly";
|
|
3
|
+
export type DiagnosisSeverity = "P0" | "P1" | "P2" | "P3";
|
|
4
|
+
export interface Diagnosis {
|
|
5
|
+
diagnosis_id: string;
|
|
6
|
+
type: DiagnosisType;
|
|
7
|
+
severity: DiagnosisSeverity;
|
|
8
|
+
title: string;
|
|
9
|
+
description: string;
|
|
10
|
+
evidence: Record<string, unknown>;
|
|
11
|
+
target: string;
|
|
12
|
+
proposed_improvement_type: "rule_addition" | "description_rewrite" | "skill_creation" | "recovery_priority_adjustment" | "investigation";
|
|
13
|
+
}
|
|
14
|
+
export interface DiagnosisReport {
|
|
15
|
+
report_id: string;
|
|
16
|
+
diagnosis_date: string;
|
|
17
|
+
total_diagnoses: number;
|
|
18
|
+
diagnoses: Diagnosis[];
|
|
19
|
+
summary: string;
|
|
20
|
+
}
|
|
21
|
+
export declare function diagnose(stats: AggregatedStats): DiagnosisReport;
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
const THRESHOLDS = {
|
|
2
|
+
UNKNOWN_ERROR_RATE_P0: 0.15,
|
|
3
|
+
UNKNOWN_ERROR_RATE_P1: 0.08,
|
|
4
|
+
LOW_SEMANTIC_HIT_RATE: 0.50,
|
|
5
|
+
LOW_SUCCESS_RATE: 0.70,
|
|
6
|
+
HIGH_LATENCY_MS: 5000,
|
|
7
|
+
MIN_SAMPLE_SIZE: 10,
|
|
8
|
+
MIN_UNKNOWN_ERRORS_FOR_RULE: 3,
|
|
9
|
+
};
|
|
10
|
+
function diagnoseRuleGaps(stats) {
|
|
11
|
+
const diagnoses = [];
|
|
12
|
+
if (stats.overall.unknown_error_rate >= THRESHOLDS.UNKNOWN_ERROR_RATE_P1 &&
|
|
13
|
+
stats.overall.unknown_error_count >= THRESHOLDS.MIN_UNKNOWN_ERRORS_FOR_RULE) {
|
|
14
|
+
const severity = stats.overall.unknown_error_rate >= THRESHOLDS.UNKNOWN_ERROR_RATE_P0 ? "P0" : "P1";
|
|
15
|
+
const unknownMessages = [];
|
|
16
|
+
for (const tool of stats.per_tool) {
|
|
17
|
+
for (const msg of tool.top_error_messages) {
|
|
18
|
+
unknownMessages.push(msg);
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
diagnoses.push({
|
|
22
|
+
diagnosis_id: `DIAG-RULE-${Date.now()}-001`,
|
|
23
|
+
type: "rule_gap",
|
|
24
|
+
severity,
|
|
25
|
+
title: `High unknown error rate (${(stats.overall.unknown_error_rate * 100).toFixed(1)}%)`,
|
|
26
|
+
description: `${stats.overall.unknown_error_count} errors classified as 'unknown' out of ${stats.overall.total_calls} total calls. The ERROR_RULES table does not cover these error patterns.`,
|
|
27
|
+
evidence: {
|
|
28
|
+
unknown_error_count: stats.overall.unknown_error_count,
|
|
29
|
+
unknown_error_rate: stats.overall.unknown_error_rate,
|
|
30
|
+
top_error_messages: unknownMessages.slice(0, 5),
|
|
31
|
+
},
|
|
32
|
+
target: "ERROR_RULES",
|
|
33
|
+
proposed_improvement_type: "rule_addition",
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
for (const tool of stats.per_tool) {
|
|
37
|
+
const unknownCount = tool.error_distribution["unknown"] || 0;
|
|
38
|
+
if (unknownCount >= THRESHOLDS.MIN_UNKNOWN_ERRORS_FOR_RULE) {
|
|
39
|
+
diagnoses.push({
|
|
40
|
+
diagnosis_id: `DIAG-RULE-${Date.now()}-${tool.tool_name}`,
|
|
41
|
+
type: "rule_gap",
|
|
42
|
+
severity: "P2",
|
|
43
|
+
title: `Tool '${tool.tool_name}' has ${unknownCount} unknown errors`,
|
|
44
|
+
description: `Tool-specific error patterns not covered by ERROR_RULES.`,
|
|
45
|
+
evidence: {
|
|
46
|
+
tool_name: tool.tool_name,
|
|
47
|
+
unknown_error_count: unknownCount,
|
|
48
|
+
top_error_messages: tool.top_error_messages.slice(0, 3),
|
|
49
|
+
},
|
|
50
|
+
target: `ERROR_RULES (for ${tool.tool_name})`,
|
|
51
|
+
proposed_improvement_type: "rule_addition",
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
return diagnoses;
|
|
56
|
+
}
|
|
57
|
+
function diagnoseDescriptionQuality(stats) {
|
|
58
|
+
const diagnoses = [];
|
|
59
|
+
for (const tool of stats.per_tool) {
|
|
60
|
+
if (tool.total_calls < THRESHOLDS.MIN_SAMPLE_SIZE)
|
|
61
|
+
continue;
|
|
62
|
+
if (tool.semantic_hit_rate < THRESHOLDS.LOW_SEMANTIC_HIT_RATE) {
|
|
63
|
+
const severity = tool.semantic_hit_rate < 0.25 ? "P1" : "P2";
|
|
64
|
+
diagnoses.push({
|
|
65
|
+
diagnosis_id: `DIAG-DESC-${Date.now()}-${tool.tool_name}`,
|
|
66
|
+
type: "description_quality",
|
|
67
|
+
severity,
|
|
68
|
+
title: `Low semantic hit rate for '${tool.tool_name}' (${(tool.semantic_hit_rate * 100).toFixed(1)}%)`,
|
|
69
|
+
description: `The semantic layer is not producing summaries for most calls to this tool. This may indicate description gaps or missing INTENT_RULES for this operation type.`,
|
|
70
|
+
evidence: {
|
|
71
|
+
tool_name: tool.tool_name,
|
|
72
|
+
semantic_hit_rate: tool.semantic_hit_rate,
|
|
73
|
+
total_calls: tool.total_calls,
|
|
74
|
+
},
|
|
75
|
+
target: `${tool.tool_name} description / INTENT_RULES`,
|
|
76
|
+
proposed_improvement_type: "description_rewrite",
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
return diagnoses;
|
|
81
|
+
}
|
|
82
|
+
function diagnoseSkillNeeds(stats) {
|
|
83
|
+
const diagnoses = [];
|
|
84
|
+
for (const op of stats.per_operation) {
|
|
85
|
+
if (op.total_calls < THRESHOLDS.MIN_SAMPLE_SIZE)
|
|
86
|
+
continue;
|
|
87
|
+
if (op.success_rate < THRESHOLDS.LOW_SUCCESS_RATE && op.success_rate > 0) {
|
|
88
|
+
diagnoses.push({
|
|
89
|
+
diagnosis_id: `DIAG-SKILL-${Date.now()}-${op.operation_type}`,
|
|
90
|
+
type: "skill_need",
|
|
91
|
+
severity: "P2",
|
|
92
|
+
title: `Operation '${op.operation_type}' has low success rate (${(op.success_rate * 100).toFixed(1)}%)`,
|
|
93
|
+
description: `This operation type has a high failure rate, suggesting users may lack guidance. A Skill with step-by-step dialogue and failure playbooks could help.`,
|
|
94
|
+
evidence: {
|
|
95
|
+
operation_type: op.operation_type,
|
|
96
|
+
success_rate: op.success_rate,
|
|
97
|
+
total_calls: op.total_calls,
|
|
98
|
+
error_distribution: op.error_distribution,
|
|
99
|
+
},
|
|
100
|
+
target: `new Skill for ${op.operation_type}`,
|
|
101
|
+
proposed_improvement_type: "skill_creation",
|
|
102
|
+
});
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
return diagnoses;
|
|
106
|
+
}
|
|
107
|
+
function diagnoseRecoveryIneffectiveness(stats) {
|
|
108
|
+
const diagnoses = [];
|
|
109
|
+
const errorEntries = Object.entries(stats.error_code_distribution)
|
|
110
|
+
.filter(([code]) => code !== "unknown")
|
|
111
|
+
.sort((a, b) => b[1] - a[1]);
|
|
112
|
+
for (const [errorCode, count] of errorEntries) {
|
|
113
|
+
const total = stats.overall.total_calls;
|
|
114
|
+
if (total > 0 && count / total > 0.10 && count >= THRESHOLDS.MIN_SAMPLE_SIZE) {
|
|
115
|
+
diagnoses.push({
|
|
116
|
+
diagnosis_id: `DIAG-RECOV-${Date.now()}-${errorCode}`,
|
|
117
|
+
type: "recovery_ineffectiveness",
|
|
118
|
+
severity: "P2",
|
|
119
|
+
title: `Error code '${errorCode}' accounts for ${count} failures (${((count / total) * 100).toFixed(1)}%)`,
|
|
120
|
+
description: `This error code is frequent. The recovery strategy may need priority adjustment or the root cause should be addressed at the source.`,
|
|
121
|
+
evidence: {
|
|
122
|
+
error_code: errorCode,
|
|
123
|
+
count,
|
|
124
|
+
percentage: count / total,
|
|
125
|
+
},
|
|
126
|
+
target: `RECOVERY_RULES['${errorCode}']`,
|
|
127
|
+
proposed_improvement_type: "recovery_priority_adjustment",
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
return diagnoses;
|
|
132
|
+
}
|
|
133
|
+
function diagnoseLatencyAnomalies(stats) {
|
|
134
|
+
const diagnoses = [];
|
|
135
|
+
for (const tool of stats.per_tool) {
|
|
136
|
+
if (tool.total_calls < THRESHOLDS.MIN_SAMPLE_SIZE)
|
|
137
|
+
continue;
|
|
138
|
+
if (tool.p95_duration_ms > THRESHOLDS.HIGH_LATENCY_MS) {
|
|
139
|
+
diagnoses.push({
|
|
140
|
+
diagnosis_id: `DIAG-LATENCY-${Date.now()}-${tool.tool_name}`,
|
|
141
|
+
type: "latency_anomaly",
|
|
142
|
+
severity: "P3",
|
|
143
|
+
title: `Tool '${tool.tool_name}' has high p95 latency (${tool.p95_duration_ms}ms)`,
|
|
144
|
+
description: `The 95th percentile latency exceeds the threshold. This may indicate performance issues or RPC bottlenecks.`,
|
|
145
|
+
evidence: {
|
|
146
|
+
tool_name: tool.tool_name,
|
|
147
|
+
p95_duration_ms: tool.p95_duration_ms,
|
|
148
|
+
avg_duration_ms: tool.avg_duration_ms,
|
|
149
|
+
},
|
|
150
|
+
target: tool.tool_name,
|
|
151
|
+
proposed_improvement_type: "investigation",
|
|
152
|
+
});
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
return diagnoses;
|
|
156
|
+
}
|
|
157
|
+
export function diagnose(stats) {
|
|
158
|
+
const all = [
|
|
159
|
+
...diagnoseRuleGaps(stats),
|
|
160
|
+
...diagnoseDescriptionQuality(stats),
|
|
161
|
+
...diagnoseSkillNeeds(stats),
|
|
162
|
+
...diagnoseRecoveryIneffectiveness(stats),
|
|
163
|
+
...diagnoseLatencyAnomalies(stats),
|
|
164
|
+
];
|
|
165
|
+
const severityOrder = { P0: 0, P1: 1, P2: 2, P3: 3 };
|
|
166
|
+
all.sort((a, b) => severityOrder[a.severity] - severityOrder[b.severity]);
|
|
167
|
+
const p0Count = all.filter((d) => d.severity === "P0").length;
|
|
168
|
+
const p1Count = all.filter((d) => d.severity === "P1").length;
|
|
169
|
+
const summary = `${all.length} diagnoses: ${p0Count} P0, ${p1Count} P1, ${all.length - p0Count - p1Count} P2/P3. ` +
|
|
170
|
+
`Overall success rate: ${(stats.overall.success_rate * 100).toFixed(1)}%. ` +
|
|
171
|
+
`Unknown error rate: ${(stats.overall.unknown_error_rate * 100).toFixed(1)}%.`;
|
|
172
|
+
return {
|
|
173
|
+
report_id: `DIAG-REPORT-${Date.now()}`,
|
|
174
|
+
diagnosis_date: new Date().toISOString(),
|
|
175
|
+
total_diagnoses: all.length,
|
|
176
|
+
diagnoses: all,
|
|
177
|
+
summary,
|
|
178
|
+
};
|
|
179
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { DiagnosisReport } from "./diagnose.js";
|
|
2
|
+
export type ProposalType = "error_rule_addition" | "intent_rule_addition" | "description_rewrite" | "skill_creation" | "recovery_priority_adjustment" | "investigation";
|
|
3
|
+
export type AutomationLevel = "full" | "semi" | "manual";
|
|
4
|
+
export interface ImprovementProposal {
|
|
5
|
+
proposal_id: string;
|
|
6
|
+
source_diagnosis_id: string;
|
|
7
|
+
type: ProposalType;
|
|
8
|
+
priority: string;
|
|
9
|
+
automation: AutomationLevel;
|
|
10
|
+
title: string;
|
|
11
|
+
description: string;
|
|
12
|
+
target: string;
|
|
13
|
+
proposed_change: Record<string, unknown>;
|
|
14
|
+
evidence: Record<string, unknown>;
|
|
15
|
+
review_status: "pending" | "approved" | "rejected" | "applied";
|
|
16
|
+
ab_test_recommended: boolean;
|
|
17
|
+
}
|
|
18
|
+
export interface ImprovementReport {
|
|
19
|
+
report_id: string;
|
|
20
|
+
generated_date: string;
|
|
21
|
+
source_diagnosis_report_id: string;
|
|
22
|
+
total_proposals: number;
|
|
23
|
+
proposals: ImprovementProposal[];
|
|
24
|
+
summary: string;
|
|
25
|
+
}
|
|
26
|
+
export declare function generateImprovements(report: DiagnosisReport): ImprovementReport;
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
function proposeErrorRuleAddition(diagnosis) {
|
|
2
|
+
const evidence = diagnosis.evidence;
|
|
3
|
+
const topMessages = evidence.top_error_messages || [];
|
|
4
|
+
const candidatePatterns = topMessages.slice(0, 3).map((m) => {
|
|
5
|
+
const msg = m.message.toLowerCase();
|
|
6
|
+
let pattern = msg
|
|
7
|
+
.replace(/0x[0-9a-f]+/gi, "0xADDR")
|
|
8
|
+
.replace(/\d+/g, "\\d+")
|
|
9
|
+
.replace(/[^\w\s\\]/g, " ")
|
|
10
|
+
.replace(/\s+/g, " ")
|
|
11
|
+
.trim();
|
|
12
|
+
if (pattern.length > 60)
|
|
13
|
+
pattern = pattern.slice(0, 60);
|
|
14
|
+
return { pattern, sample_count: m.count };
|
|
15
|
+
});
|
|
16
|
+
return {
|
|
17
|
+
proposal_id: `PROP-RULE-${Date.now()}-${Math.random().toString(36).slice(2, 6)}`,
|
|
18
|
+
source_diagnosis_id: diagnosis.diagnosis_id,
|
|
19
|
+
type: "error_rule_addition",
|
|
20
|
+
priority: diagnosis.severity,
|
|
21
|
+
automation: "semi",
|
|
22
|
+
title: `Add ERROR_RULES entries for ${diagnosis.target}`,
|
|
23
|
+
description: `Generate new error classification rules to cover patterns currently falling into 'unknown'. AI should draft regex patterns; human reviewer validates accuracy and assigns correct error_code.`,
|
|
24
|
+
target: "ERROR_RULES",
|
|
25
|
+
proposed_change: {
|
|
26
|
+
action: "append",
|
|
27
|
+
rules: candidatePatterns.map((p) => ({
|
|
28
|
+
pattern: p.pattern,
|
|
29
|
+
error_code: "unknown",
|
|
30
|
+
retryable: false,
|
|
31
|
+
recovery_hint: "TODO: assign by reviewer",
|
|
32
|
+
sample_message: p.sample_count,
|
|
33
|
+
})),
|
|
34
|
+
note: "error_code must be assigned by human reviewer based on semantic analysis",
|
|
35
|
+
},
|
|
36
|
+
evidence: diagnosis.evidence,
|
|
37
|
+
review_status: "pending",
|
|
38
|
+
ab_test_recommended: true,
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
function proposeDescriptionRewrite(diagnosis) {
|
|
42
|
+
const evidence = diagnosis.evidence;
|
|
43
|
+
return {
|
|
44
|
+
proposal_id: `PROP-DESC-${Date.now()}-${Math.random().toString(36).slice(2, 6)}`,
|
|
45
|
+
source_diagnosis_id: diagnosis.diagnosis_id,
|
|
46
|
+
type: "description_rewrite",
|
|
47
|
+
priority: diagnosis.severity,
|
|
48
|
+
automation: "semi",
|
|
49
|
+
title: `Rewrite description for ${evidence.tool_name || diagnosis.target}`,
|
|
50
|
+
description: `The semantic layer produces no summary for ${((1 - (evidence.semantic_hit_rate || 0)) * 100).toFixed(1)}% of calls to this tool. AI should rewrite the tool description and INTENT_RULES to improve coverage. A/B test before merging.`,
|
|
51
|
+
target: diagnosis.target,
|
|
52
|
+
proposed_change: {
|
|
53
|
+
action: "rewrite",
|
|
54
|
+
current_semantic_hit_rate: evidence.semantic_hit_rate,
|
|
55
|
+
target_semantic_hit_rate: 0.80,
|
|
56
|
+
rewrite_guidelines: [
|
|
57
|
+
"Follow 4-part format: 含义 + Constraint + Relation + Example",
|
|
58
|
+
"Add explicit intent signals to INTENT_RULES",
|
|
59
|
+
"Include concrete examples for common operation types",
|
|
60
|
+
],
|
|
61
|
+
},
|
|
62
|
+
evidence: diagnosis.evidence,
|
|
63
|
+
review_status: "pending",
|
|
64
|
+
ab_test_recommended: true,
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
function proposeSkillCreation(diagnosis) {
|
|
68
|
+
const evidence = diagnosis.evidence;
|
|
69
|
+
return {
|
|
70
|
+
proposal_id: `PROP-SKILL-${Date.now()}-${Math.random().toString(36).slice(2, 6)}`,
|
|
71
|
+
source_diagnosis_id: diagnosis.diagnosis_id,
|
|
72
|
+
type: "skill_creation",
|
|
73
|
+
priority: diagnosis.severity,
|
|
74
|
+
automation: "semi",
|
|
75
|
+
title: `Create new Skill for '${evidence.operation_type}' operations`,
|
|
76
|
+
description: `This operation type has a ${(evidence.success_rate ? (1 - evidence.success_rate) * 100 : 0).toFixed(1)}% failure rate across ${evidence.total_calls} calls. A Skill with R1-R10 dialogue scripts, decision trees, and failure playbooks could guide users to success.`,
|
|
77
|
+
target: `wowok-${evidence.operation_type}-guide`,
|
|
78
|
+
proposed_change: {
|
|
79
|
+
action: "create",
|
|
80
|
+
skill_name: `wowok-${evidence.operation_type}-guide`,
|
|
81
|
+
skill_structure: {
|
|
82
|
+
sections: ["R1-R10 dialogues", "Decision trees", "Failure playbooks", "Tier layering"],
|
|
83
|
+
target_role: "provider",
|
|
84
|
+
loading: "on-demand",
|
|
85
|
+
},
|
|
86
|
+
top_errors: evidence.error_distribution,
|
|
87
|
+
},
|
|
88
|
+
evidence: diagnosis.evidence,
|
|
89
|
+
review_status: "pending",
|
|
90
|
+
ab_test_recommended: false,
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
function proposeRecoveryAdjustment(diagnosis) {
|
|
94
|
+
const evidence = diagnosis.evidence;
|
|
95
|
+
return {
|
|
96
|
+
proposal_id: `PROP-RECOV-${Date.now()}-${Math.random().toString(36).slice(2, 6)}`,
|
|
97
|
+
source_diagnosis_id: diagnosis.diagnosis_id,
|
|
98
|
+
type: "recovery_priority_adjustment",
|
|
99
|
+
priority: diagnosis.severity,
|
|
100
|
+
automation: "semi",
|
|
101
|
+
title: `Review recovery strategy for '${evidence.error_code}' errors`,
|
|
102
|
+
description: `Error code '${evidence.error_code}' accounts for ${evidence.count} failures (${((evidence.percentage || 0) * 100).toFixed(1)}%). The current recovery strategy may need tuning — consider increasing max_attempts, adjusting automation level, or addressing the root cause.`,
|
|
103
|
+
target: `RECOVERY_RULES['${evidence.error_code}']`,
|
|
104
|
+
proposed_change: {
|
|
105
|
+
action: "review_and_adjust",
|
|
106
|
+
current_strategy: "see RECOVERY_RULES table",
|
|
107
|
+
suggestions: [
|
|
108
|
+
"Increase max_attempts if failures are transient",
|
|
109
|
+
"Reduce max_attempts if failures are persistent (wastes user time)",
|
|
110
|
+
"Upgrade automation from 'semi' to 'full' if recovery is deterministic",
|
|
111
|
+
"Consider root-cause fix in description/rules to prevent the error",
|
|
112
|
+
],
|
|
113
|
+
},
|
|
114
|
+
evidence: diagnosis.evidence,
|
|
115
|
+
review_status: "pending",
|
|
116
|
+
ab_test_recommended: false,
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
function proposeInvestigation(diagnosis) {
|
|
120
|
+
return {
|
|
121
|
+
proposal_id: `PROP-INV-${Date.now()}-${Math.random().toString(36).slice(2, 6)}`,
|
|
122
|
+
source_diagnosis_id: diagnosis.diagnosis_id,
|
|
123
|
+
type: "investigation",
|
|
124
|
+
priority: diagnosis.severity,
|
|
125
|
+
automation: "manual",
|
|
126
|
+
title: diagnosis.title,
|
|
127
|
+
description: `Manual investigation required. ${diagnosis.description}`,
|
|
128
|
+
target: diagnosis.target,
|
|
129
|
+
proposed_change: {
|
|
130
|
+
action: "investigate",
|
|
131
|
+
steps: [
|
|
132
|
+
"Collect additional telemetry around the anomaly",
|
|
133
|
+
"Check if the issue is reproducible",
|
|
134
|
+
"Determine if the root cause is in the SDK, MCP layer, or user behavior",
|
|
135
|
+
],
|
|
136
|
+
},
|
|
137
|
+
evidence: diagnosis.evidence,
|
|
138
|
+
review_status: "pending",
|
|
139
|
+
ab_test_recommended: false,
|
|
140
|
+
};
|
|
141
|
+
}
|
|
142
|
+
export function generateImprovements(report) {
|
|
143
|
+
const proposals = [];
|
|
144
|
+
for (const diagnosis of report.diagnoses) {
|
|
145
|
+
let proposal;
|
|
146
|
+
switch (diagnosis.proposed_improvement_type) {
|
|
147
|
+
case "rule_addition":
|
|
148
|
+
proposal = proposeErrorRuleAddition(diagnosis);
|
|
149
|
+
break;
|
|
150
|
+
case "description_rewrite":
|
|
151
|
+
proposal = proposeDescriptionRewrite(diagnosis);
|
|
152
|
+
break;
|
|
153
|
+
case "skill_creation":
|
|
154
|
+
proposal = proposeSkillCreation(diagnosis);
|
|
155
|
+
break;
|
|
156
|
+
case "recovery_priority_adjustment":
|
|
157
|
+
proposal = proposeRecoveryAdjustment(diagnosis);
|
|
158
|
+
break;
|
|
159
|
+
default:
|
|
160
|
+
proposal = proposeInvestigation(diagnosis);
|
|
161
|
+
}
|
|
162
|
+
proposals.push(proposal);
|
|
163
|
+
}
|
|
164
|
+
const p0Count = proposals.filter((p) => p.priority === "P0").length;
|
|
165
|
+
const semiCount = proposals.filter((p) => p.automation === "semi").length;
|
|
166
|
+
const abTestCount = proposals.filter((p) => p.ab_test_recommended).length;
|
|
167
|
+
const summary = `${proposals.length} proposals generated from ${report.total_diagnoses} diagnoses. ` +
|
|
168
|
+
`${p0Count} P0, ${semiCount} semi-automatic, ${abTestCount} recommend A/B testing. ` +
|
|
169
|
+
`All proposals require human review before application.`;
|
|
170
|
+
return {
|
|
171
|
+
report_id: `IMPROVE-REPORT-${Date.now()}`,
|
|
172
|
+
generated_date: new Date().toISOString(),
|
|
173
|
+
source_diagnosis_report_id: report.report_id,
|
|
174
|
+
total_proposals: proposals.length,
|
|
175
|
+
proposals,
|
|
176
|
+
summary,
|
|
177
|
+
};
|
|
178
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { TelemetryEvent } from "../telemetry/index.js";
|
|
2
|
+
import { type AggregatedStats } from "./aggregate.js";
|
|
3
|
+
import { type DiagnosisReport } from "./diagnose.js";
|
|
4
|
+
import { type ImprovementReport } from "./improve.js";
|
|
5
|
+
export interface LoopEngineeringReport {
|
|
6
|
+
pipeline_run_id: string;
|
|
7
|
+
run_date: string;
|
|
8
|
+
stage: "aggregate" | "diagnose" | "improve" | "complete";
|
|
9
|
+
aggregated_stats: AggregatedStats | null;
|
|
10
|
+
diagnosis_report: DiagnosisReport | null;
|
|
11
|
+
improvement_report: ImprovementReport | null;
|
|
12
|
+
error?: string;
|
|
13
|
+
}
|
|
14
|
+
export declare function runLoopEngineeringPipeline(): Promise<LoopEngineeringReport>;
|
|
15
|
+
export declare function runDiagnoseFromStats(stats: AggregatedStats): DiagnosisReport;
|
|
16
|
+
export declare function runImproveFromDiagnoses(report: DiagnosisReport): ImprovementReport;
|
|
17
|
+
export declare function runAggregateFromEvents(events: TelemetryEvent[]): AggregatedStats;
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import { getStorage } from "../telemetry/storage.js";
|
|
2
|
+
import { aggregate } from "./aggregate.js";
|
|
3
|
+
import { diagnose } from "./diagnose.js";
|
|
4
|
+
import { generateImprovements } from "./improve.js";
|
|
5
|
+
async function loadTelemetryEvents() {
|
|
6
|
+
const storage = getStorage();
|
|
7
|
+
const rawEvents = await storage.query({});
|
|
8
|
+
return rawEvents;
|
|
9
|
+
}
|
|
10
|
+
async function runAggregateStage() {
|
|
11
|
+
const events = await loadTelemetryEvents();
|
|
12
|
+
return aggregate(events);
|
|
13
|
+
}
|
|
14
|
+
function runDiagnoseStage(stats) {
|
|
15
|
+
return diagnose(stats);
|
|
16
|
+
}
|
|
17
|
+
function runImproveStage(diagnosisReport) {
|
|
18
|
+
return generateImprovements(diagnosisReport);
|
|
19
|
+
}
|
|
20
|
+
export async function runLoopEngineeringPipeline() {
|
|
21
|
+
const runId = `LE-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
|
|
22
|
+
const runDate = new Date().toISOString();
|
|
23
|
+
try {
|
|
24
|
+
const aggregatedStats = await runAggregateStage();
|
|
25
|
+
const diagnosisReport = runDiagnoseStage(aggregatedStats);
|
|
26
|
+
const improvementReport = runImproveStage(diagnosisReport);
|
|
27
|
+
return {
|
|
28
|
+
pipeline_run_id: runId,
|
|
29
|
+
run_date: runDate,
|
|
30
|
+
stage: "complete",
|
|
31
|
+
aggregated_stats: aggregatedStats,
|
|
32
|
+
diagnosis_report: diagnosisReport,
|
|
33
|
+
improvement_report: improvementReport,
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
catch (error) {
|
|
37
|
+
return {
|
|
38
|
+
pipeline_run_id: runId,
|
|
39
|
+
run_date: runDate,
|
|
40
|
+
stage: "complete",
|
|
41
|
+
aggregated_stats: null,
|
|
42
|
+
diagnosis_report: null,
|
|
43
|
+
improvement_report: null,
|
|
44
|
+
error: error instanceof Error ? error.message : String(error),
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
export function runDiagnoseFromStats(stats) {
|
|
49
|
+
return runDiagnoseStage(stats);
|
|
50
|
+
}
|
|
51
|
+
export function runImproveFromDiagnoses(report) {
|
|
52
|
+
return runImproveStage(report);
|
|
53
|
+
}
|
|
54
|
+
export function runAggregateFromEvents(events) {
|
|
55
|
+
return aggregate(events);
|
|
56
|
+
}
|