@agentguard-run/spend 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +5 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/outcomes/index.d.ts +1 -0
- package/dist/outcomes/index.d.ts.map +1 -1
- package/dist/outcomes/index.js +1 -0
- package/dist/outcomes/index.js.map +1 -1
- package/dist/outcomes/metadata-learning.d.ts +95 -0
- package/dist/outcomes/metadata-learning.d.ts.map +1 -0
- package/dist/outcomes/metadata-learning.js +131 -0
- package/dist/outcomes/metadata-learning.js.map +1 -0
- package/dist/telemetry.js +1 -1
- package/dist/telemetry.js.map +1 -1
- package/package.json +1 -1
- package/src/outcomes/index.ts +1 -0
- package/src/outcomes/metadata-learning.ts +182 -0
package/CHANGELOG.md
CHANGED
package/dist/index.d.ts
CHANGED
|
@@ -32,7 +32,7 @@ export { withSpendGuardAnthropic, type AnthropicBindingOptions } from './binding
|
|
|
32
32
|
export { withSpendGuardBedrock, type BedrockBindingOptions } from './bindings/bedrock';
|
|
33
33
|
export { recommend, defaultKey, type ModelRouteRecommendation, type RecommendOptions } from './router';
|
|
34
34
|
export { DEFAULT_LOCALE, SUPPORTED_LOCALES, TRANSLATIONS, type SupportedLocale, resolveLocale, t, formatBlockedTrace, type BlockedTraceArgs, } from './i18n';
|
|
35
|
-
export declare const AGENTGUARD_SPEND_VERSION = "0.
|
|
35
|
+
export declare const AGENTGUARD_SPEND_VERSION = "0.11.0";
|
|
36
36
|
export declare const agentguard: {
|
|
37
37
|
workflow: typeof runWorkflow;
|
|
38
38
|
};
|
package/dist/index.js
CHANGED
|
@@ -147,7 +147,7 @@ Object.defineProperty(exports, "TRANSLATIONS", { enumerable: true, get: function
|
|
|
147
147
|
Object.defineProperty(exports, "resolveLocale", { enumerable: true, get: function () { return i18n_1.resolveLocale; } });
|
|
148
148
|
Object.defineProperty(exports, "t", { enumerable: true, get: function () { return i18n_1.t; } });
|
|
149
149
|
Object.defineProperty(exports, "formatBlockedTrace", { enumerable: true, get: function () { return i18n_1.formatBlockedTrace; } });
|
|
150
|
-
exports.AGENTGUARD_SPEND_VERSION = '0.
|
|
150
|
+
exports.AGENTGUARD_SPEND_VERSION = '0.11.0';
|
|
151
151
|
exports.agentguard = {
|
|
152
152
|
workflow: context_1.workflow,
|
|
153
153
|
};
|
package/dist/outcomes/index.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/outcomes/index.ts"],"names":[],"mappings":"AAAA,cAAc,SAAS,CAAC;AACxB,cAAc,UAAU,CAAC;AACzB,cAAc,gBAAgB,CAAC;AAC/B,cAAc,WAAW,CAAC;AAC1B,cAAc,YAAY,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/outcomes/index.ts"],"names":[],"mappings":"AAAA,cAAc,SAAS,CAAC;AACxB,cAAc,UAAU,CAAC;AACzB,cAAc,gBAAgB,CAAC;AAC/B,cAAc,WAAW,CAAC;AAC1B,cAAc,YAAY,CAAC;AAC3B,cAAc,qBAAqB,CAAC"}
|
package/dist/outcomes/index.js
CHANGED
|
@@ -19,4 +19,5 @@ __exportStar(require("./ledger"), exports);
|
|
|
19
19
|
__exportStar(require("./quality-gate"), exports);
|
|
20
20
|
__exportStar(require("./runtime"), exports);
|
|
21
21
|
__exportStar(require("./learning"), exports);
|
|
22
|
+
__exportStar(require("./metadata-learning"), exports);
|
|
22
23
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/outcomes/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,0CAAwB;AACxB,2CAAyB;AACzB,iDAA+B;AAC/B,4CAA0B;AAC1B,6CAA2B"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/outcomes/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,0CAAwB;AACxB,2CAAyB;AACzB,iDAA+B;AAC/B,4CAA0B;AAC1B,6CAA2B;AAC3B,sDAAoC"}
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
export type LearningFeedbackVerdict = 'accept' | 'edit' | 'reject' | null;
|
|
2
|
+
export type LearningFeedbackReason = 'missing_citation' | 'wrong_format' | 'wrong_category' | 'too_long' | 'missed_clause' | 'needs_review' | 'great' | null;
|
|
3
|
+
export interface LearningMetadataRow {
|
|
4
|
+
tenant_id: string;
|
|
5
|
+
vertical: string;
|
|
6
|
+
outcome_slug: string;
|
|
7
|
+
prompt_version: string;
|
|
8
|
+
model_id: string;
|
|
9
|
+
effort: 'low' | 'medium' | 'high' | 'xhigh' | 'unknown';
|
|
10
|
+
quality_score: number;
|
|
11
|
+
passed: boolean;
|
|
12
|
+
escalated: boolean;
|
|
13
|
+
cost_cents: number;
|
|
14
|
+
input_tokens: number;
|
|
15
|
+
output_tokens: number;
|
|
16
|
+
duration_ms: number;
|
|
17
|
+
guardrail_tier: string;
|
|
18
|
+
feedback_verdict: LearningFeedbackVerdict;
|
|
19
|
+
feedback_reason: LearningFeedbackReason;
|
|
20
|
+
timestamp: string;
|
|
21
|
+
}
|
|
22
|
+
export interface LearningAggregate {
|
|
23
|
+
key: string;
|
|
24
|
+
vertical: string;
|
|
25
|
+
outcome_slug: string;
|
|
26
|
+
prompt_version: string;
|
|
27
|
+
model_id: string;
|
|
28
|
+
effort: string;
|
|
29
|
+
sample_count: number;
|
|
30
|
+
distinct_tenant_count: number;
|
|
31
|
+
pass_rate: number;
|
|
32
|
+
mean_quality_score: number;
|
|
33
|
+
escalation_rate: number;
|
|
34
|
+
mean_cost_cents: number;
|
|
35
|
+
mean_latency_ms: number;
|
|
36
|
+
feedback_distribution: {
|
|
37
|
+
accept: number;
|
|
38
|
+
edit: number;
|
|
39
|
+
reject: number;
|
|
40
|
+
accept_rate: number;
|
|
41
|
+
reject_rate: number;
|
|
42
|
+
};
|
|
43
|
+
reason_counts: Record<string, number>;
|
|
44
|
+
revision_suggestion: string;
|
|
45
|
+
}
|
|
46
|
+
export interface LearningProposal {
|
|
47
|
+
proposal_id: string;
|
|
48
|
+
status: 'shadow';
|
|
49
|
+
vertical: string;
|
|
50
|
+
outcome_slug: string;
|
|
51
|
+
proposed: {
|
|
52
|
+
prompt_version: string;
|
|
53
|
+
model_id: string;
|
|
54
|
+
effort: string;
|
|
55
|
+
fit_score: number;
|
|
56
|
+
};
|
|
57
|
+
current: {
|
|
58
|
+
prompt_version: string;
|
|
59
|
+
model_id: string;
|
|
60
|
+
effort: string;
|
|
61
|
+
fit_score: number;
|
|
62
|
+
} | null;
|
|
63
|
+
sample_count: number;
|
|
64
|
+
distinct_tenant_count: number;
|
|
65
|
+
revision_suggestion: string;
|
|
66
|
+
human_promotion_required: true;
|
|
67
|
+
}
|
|
68
|
+
export declare function assertLearningMetadataOnly(value: Record<string, unknown>): void;
|
|
69
|
+
export declare function normalizeLearningMetadataRow(input: Record<string, unknown>): LearningMetadataRow;
|
|
70
|
+
export declare function aggregateLearningMetadata(rows: Record<string, unknown>[]): LearningAggregate[];
|
|
71
|
+
export declare function proposeLearningMetadataImprovements(aggregates: LearningAggregate[], current?: Record<string, LearningAggregate>, opts?: {
|
|
72
|
+
minRuns?: number;
|
|
73
|
+
minTenants?: number;
|
|
74
|
+
}): LearningProposal[];
|
|
75
|
+
export declare function revisionSuggestion(reasons: Record<string, number>): string;
|
|
76
|
+
export interface PortableAiTeamOutcome {
|
|
77
|
+
vertical: string;
|
|
78
|
+
name: string;
|
|
79
|
+
outcome_slug: string;
|
|
80
|
+
prompt_version: string;
|
|
81
|
+
caps?: Record<string, number>;
|
|
82
|
+
effort?: string;
|
|
83
|
+
guardrail_tier?: string;
|
|
84
|
+
enabled?: boolean;
|
|
85
|
+
}
|
|
86
|
+
export interface PortableAiTeamExport {
|
|
87
|
+
schema_version: 1;
|
|
88
|
+
export_type: 'agentguard_ai_team';
|
|
89
|
+
exported_at: string;
|
|
90
|
+
zero_data_plane: true;
|
|
91
|
+
no_lock_in: true;
|
|
92
|
+
outcomes: PortableAiTeamOutcome[];
|
|
93
|
+
}
|
|
94
|
+
export declare function exportAiTeamConfig(outcomes: PortableAiTeamOutcome[]): PortableAiTeamExport;
|
|
95
|
+
//# sourceMappingURL=metadata-learning.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"metadata-learning.d.ts","sourceRoot":"","sources":["../../src/outcomes/metadata-learning.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,uBAAuB,GAAG,QAAQ,GAAG,MAAM,GAAG,QAAQ,GAAG,IAAI,CAAC;AAC1E,MAAM,MAAM,sBAAsB,GAAG,kBAAkB,GAAG,cAAc,GAAG,gBAAgB,GAAG,UAAU,GAAG,eAAe,GAAG,cAAc,GAAG,OAAO,GAAG,IAAI,CAAC;AAE7J,MAAM,WAAW,mBAAmB;IAClC,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,KAAK,GAAG,QAAQ,GAAG,MAAM,GAAG,OAAO,GAAG,SAAS,CAAC;IACxD,aAAa,EAAE,MAAM,CAAC;IACtB,MAAM,EAAE,OAAO,CAAC;IAChB,SAAS,EAAE,OAAO,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,gBAAgB,EAAE,uBAAuB,CAAC;IAC1C,eAAe,EAAE,sBAAsB,CAAC;IACxC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,iBAAiB;IAChC,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;IACrB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,EAAE,MAAM,CAAC;IACxB,eAAe,EAAE,MAAM,CAAC;IACxB,eAAe,EAAE,MAAM,CAAC;IACxB,qBAAqB,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAA;KAAE,CAAC;IAClH,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACtC,mBAAmB,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,gBAAgB;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE,QAAQ,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,CAAC;IACrB,QAAQ,EAAE;QAAE,cAAc,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;IAC1F,OAAO,EAAE;QAAE,cAAc,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IAChG,YAAY,EAAE,MAAM,CAAC;IACrB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,mBAAmB,EAAE,MAAM,CAAC;IAC5B,wBAAwB,EAAE,IAAI,CAAC;CAChC;AAOD,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAQ/E;AAED,wBAAgB,4BAA4B,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,mBAAmB,CA0BhG;AAED,wBAAgB,yBAAyB,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,GAAG,iBAAiB,EAAE,CAe9F;AAED,wBAAgB,mCAAmC,CAAC,UAAU,EAAE,iBAAiB,EAAE,EAAE,OAAO,GAAE,MAAM,CAAC,MAAM,EAAE,iBAAiB,CAAM,EAAE,IAAI,GAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,gBAAgB,EAAE,CAa9M;AAED,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,MAAM,CAG1E;AAUD,MAAM,WAAW,qBAAqB;IACpC,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,oBAAoB;IACnC,cAAc,EAAE,CAAC,CAAC;IAClB,WAAW,EAAE,oBAAoB,CAAC;IAClC,WAAW,EAAE,MAAM,CAAC;IACpB,eAAe,EAAE,IAAI,CAAC;IACtB,UAAU,EAAE,IAAI,CAAC;IACjB,QAAQ,EAAE,qBAAqB,EAAE,CAAC;CACnC;AAED,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,qBAAqB,EAAE,GAAG,oBAAoB,CAkB1F"}
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.assertLearningMetadataOnly = assertLearningMetadataOnly;
|
|
4
|
+
exports.normalizeLearningMetadataRow = normalizeLearningMetadataRow;
|
|
5
|
+
exports.aggregateLearningMetadata = aggregateLearningMetadata;
|
|
6
|
+
exports.proposeLearningMetadataImprovements = proposeLearningMetadataImprovements;
|
|
7
|
+
exports.revisionSuggestion = revisionSuggestion;
|
|
8
|
+
exports.exportAiTeamConfig = exportAiTeamConfig;
|
|
9
|
+
const ALLOWED = new Set(['tenant_id', 'vertical', 'outcome_slug', 'prompt_version', 'model_id', 'effort', 'quality_score', 'passed', 'escalated', 'cost_cents', 'input_tokens', 'output_tokens', 'duration_ms', 'guardrail_tier', 'feedback_verdict', 'feedback_reason', 'timestamp']);
|
|
10
|
+
const REASONS = new Set(['missing_citation', 'wrong_format', 'wrong_category', 'too_long', 'missed_clause', 'needs_review', 'great', null]);
|
|
11
|
+
const VERDICTS = new Set(['accept', 'edit', 'reject', null]);
|
|
12
|
+
const CONTENT = /^(prompt|completion|content|messages|input|inputs|output|outputs|text|raw|transcript|document|free_text|edited_text|email|phone|address|api_key|secret)$/i;
|
|
13
|
+
function assertLearningMetadataOnly(value) {
|
|
14
|
+
for (const key of Object.keys(value)) {
|
|
15
|
+
if (!ALLOWED.has(key))
|
|
16
|
+
throw new Error('learning field not allowed: ' + key);
|
|
17
|
+
if (key !== 'prompt_version' && CONTENT.test(key))
|
|
18
|
+
throw new Error('learning data-plane field not allowed: ' + key);
|
|
19
|
+
const child = value[key];
|
|
20
|
+
if (child && typeof child === 'object')
|
|
21
|
+
throw new Error('learning nested value not allowed: ' + key);
|
|
22
|
+
if (typeof child === 'string' && (child.length > 160 || child.includes('\n')))
|
|
23
|
+
throw new Error('learning value looks like content: ' + key);
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
function normalizeLearningMetadataRow(input) {
|
|
27
|
+
assertLearningMetadataOnly(input);
|
|
28
|
+
const verdict = clean(input.feedback_verdict, null, 20);
|
|
29
|
+
const reason = clean(input.feedback_reason, null, 40);
|
|
30
|
+
if (!VERDICTS.has(verdict))
|
|
31
|
+
throw new Error('invalid learning verdict');
|
|
32
|
+
if (!REASONS.has(reason))
|
|
33
|
+
throw new Error('invalid learning reason');
|
|
34
|
+
const effort = clean(input.effort, 'unknown', 20);
|
|
35
|
+
return {
|
|
36
|
+
tenant_id: clean(input.tenant_id, 'anonymous', 96),
|
|
37
|
+
vertical: clean(input.vertical, 'unknown', 48),
|
|
38
|
+
outcome_slug: clean(input.outcome_slug, 'unknown', 96),
|
|
39
|
+
prompt_version: clean(input.prompt_version, 'unknown', 48),
|
|
40
|
+
model_id: clean(input.model_id, 'unknown', 120),
|
|
41
|
+
effort: ['low', 'medium', 'high', 'xhigh'].includes(effort) ? effort : 'unknown',
|
|
42
|
+
quality_score: Math.min(1, number(input.quality_score, 0)),
|
|
43
|
+
passed: input.passed !== false,
|
|
44
|
+
escalated: input.escalated === true,
|
|
45
|
+
cost_cents: Math.round(number(input.cost_cents, 0)),
|
|
46
|
+
input_tokens: Math.round(number(input.input_tokens, 0)),
|
|
47
|
+
output_tokens: Math.round(number(input.output_tokens, 0)),
|
|
48
|
+
duration_ms: Math.round(number(input.duration_ms, 0)),
|
|
49
|
+
guardrail_tier: clean(input.guardrail_tier, 'unknown', 64),
|
|
50
|
+
feedback_verdict: verdict,
|
|
51
|
+
feedback_reason: reason,
|
|
52
|
+
timestamp: new Date(String(input.timestamp || Date.now())).toISOString(),
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
function aggregateLearningMetadata(rows) {
|
|
56
|
+
const groups = new Map();
|
|
57
|
+
for (const raw of rows) {
|
|
58
|
+
const row = normalizeLearningMetadataRow(raw);
|
|
59
|
+
const key = [row.vertical, row.outcome_slug, row.prompt_version, row.model_id, row.effort].join('|');
|
|
60
|
+
const group = groups.get(key) ?? { key, vertical: row.vertical, outcome_slug: row.outcome_slug, prompt_version: row.prompt_version, model_id: row.model_id, effort: row.effort, sample_count: 0, tenants: new Set(), passed: 0, escalated: 0, cost: 0, latency: 0, quality: 0, verdicts: { accept: 0, edit: 0, reject: 0 }, reasons: {} };
|
|
61
|
+
group.sample_count += 1;
|
|
62
|
+
group.tenants.add(row.tenant_id);
|
|
63
|
+
if (row.passed)
|
|
64
|
+
group.passed += 1;
|
|
65
|
+
if (row.escalated)
|
|
66
|
+
group.escalated += 1;
|
|
67
|
+
group.cost += row.cost_cents;
|
|
68
|
+
group.latency += row.duration_ms;
|
|
69
|
+
group.quality += row.quality_score;
|
|
70
|
+
if (row.feedback_verdict)
|
|
71
|
+
group.verdicts[row.feedback_verdict] += 1;
|
|
72
|
+
if (row.feedback_reason)
|
|
73
|
+
group.reasons[row.feedback_reason] = (group.reasons[row.feedback_reason] || 0) + 1;
|
|
74
|
+
groups.set(key, group);
|
|
75
|
+
}
|
|
76
|
+
return [...groups.values()].map((g) => {
|
|
77
|
+
const totalVerdicts = Math.max(1, g.verdicts.accept + g.verdicts.edit + g.verdicts.reject);
|
|
78
|
+
return { key: g.key, vertical: g.vertical, outcome_slug: g.outcome_slug, prompt_version: g.prompt_version, model_id: g.model_id, effort: g.effort, sample_count: g.sample_count, distinct_tenant_count: g.tenants.size, pass_rate: round(g.passed / g.sample_count), mean_quality_score: round(g.quality / g.sample_count), escalation_rate: round(g.escalated / g.sample_count), mean_cost_cents: round(g.cost / g.sample_count), mean_latency_ms: round(g.latency / g.sample_count), feedback_distribution: { ...g.verdicts, accept_rate: round(g.verdicts.accept / totalVerdicts), reject_rate: round(g.verdicts.reject / totalVerdicts) }, reason_counts: g.reasons, revision_suggestion: revisionSuggestion(g.reasons) };
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
function proposeLearningMetadataImprovements(aggregates, current = {}, opts = {}) {
|
|
82
|
+
const minRuns = opts.minRuns ?? 20, minTenants = opts.minTenants ?? 5;
|
|
83
|
+
const byOutcome = new Map();
|
|
84
|
+
for (const aggregate of aggregates) {
|
|
85
|
+
const key = aggregate.vertical + '|' + aggregate.outcome_slug;
|
|
86
|
+
byOutcome.set(key, [...(byOutcome.get(key) || []), aggregate]);
|
|
87
|
+
}
|
|
88
|
+
const proposals = [];
|
|
89
|
+
for (const [key, rows] of byOutcome.entries()) {
|
|
90
|
+
const eligible = rows.filter((r) => r.sample_count >= minRuns && r.distinct_tenant_count >= minTenants).sort((a, b) => fitScore(b) - fitScore(a));
|
|
91
|
+
if (!eligible.length)
|
|
92
|
+
continue;
|
|
93
|
+
const best = eligible[0];
|
|
94
|
+
const baseline = current[key] || rows[0];
|
|
95
|
+
if (best.key === baseline.key)
|
|
96
|
+
continue;
|
|
97
|
+
proposals.push({ proposal_id: 'ag_lp_' + Math.abs(hash(best.key)).toString(16), status: 'shadow', vertical: best.vertical, outcome_slug: best.outcome_slug, proposed: pick(best), current: baseline ? pick(baseline) : null, sample_count: best.sample_count, distinct_tenant_count: best.distinct_tenant_count, revision_suggestion: best.revision_suggestion, human_promotion_required: true });
|
|
98
|
+
}
|
|
99
|
+
return proposals;
|
|
100
|
+
}
|
|
101
|
+
function revisionSuggestion(reasons) {
|
|
102
|
+
const top = Object.entries(reasons).sort((a, b) => b[1] - a[1])[0]?.[0];
|
|
103
|
+
return { missing_citation: 'tighten cite-or-refuse instruction', wrong_format: 'make output schema stricter', wrong_category: 'add category examples', too_long: 'shorten answer format', missed_clause: 'expand clause watchlist', needs_review: 'lower reviewer cascade threshold', great: 'preserve current template' }[top || ''] || 'keep collecting metadata';
|
|
104
|
+
}
|
|
105
|
+
function pick(a) { return { prompt_version: a.prompt_version, model_id: a.model_id, effort: a.effort, fit_score: fitScore(a) }; }
|
|
106
|
+
function fitScore(a) { return round(a.pass_rate * 0.32 + a.mean_quality_score * 0.28 + a.feedback_distribution.accept_rate * 0.24 - a.feedback_distribution.reject_rate * 0.18 - a.escalation_rate * 0.08 - Math.min(0.12, a.mean_cost_cents / 2000) - Math.min(0.08, a.mean_latency_ms / 120000)); }
|
|
107
|
+
function clean(value, fallback, max) { const raw = value == null ? fallback : String(value).toLowerCase().replace(/[^a-z0-9_./:-]/g, '_').slice(0, max); return raw || fallback; }
|
|
108
|
+
function number(value, fallback) { const n = Number(value); return Number.isFinite(n) && n >= 0 ? n : fallback; }
|
|
109
|
+
function round(value) { return Math.round(value * 10000) / 10000; }
|
|
110
|
+
function hash(value) { let h = 0; for (let i = 0; i < value.length; i++)
|
|
111
|
+
h = Math.imul(31, h) + value.charCodeAt(i) | 0; return h; }
|
|
112
|
+
function exportAiTeamConfig(outcomes) {
|
|
113
|
+
return {
|
|
114
|
+
schema_version: 1,
|
|
115
|
+
export_type: 'agentguard_ai_team',
|
|
116
|
+
exported_at: new Date().toISOString(),
|
|
117
|
+
zero_data_plane: true,
|
|
118
|
+
no_lock_in: true,
|
|
119
|
+
outcomes: outcomes.map((outcome) => ({
|
|
120
|
+
vertical: clean(outcome.vertical, 'unknown', 48),
|
|
121
|
+
name: String(outcome.name || 'AI teammate').slice(0, 120),
|
|
122
|
+
outcome_slug: clean(outcome.outcome_slug, 'unknown', 96),
|
|
123
|
+
prompt_version: clean(outcome.prompt_version, 'unknown', 48),
|
|
124
|
+
caps: outcome.caps,
|
|
125
|
+
effort: outcome.effort,
|
|
126
|
+
guardrail_tier: outcome.guardrail_tier,
|
|
127
|
+
enabled: outcome.enabled !== false,
|
|
128
|
+
})),
|
|
129
|
+
};
|
|
130
|
+
}
|
|
131
|
+
//# sourceMappingURL=metadata-learning.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"metadata-learning.js","sourceRoot":"","sources":["../../src/outcomes/metadata-learning.ts"],"names":[],"mappings":";;AA4DA,gEAQC;AAED,oEA0BC;AAED,8DAeC;AAED,kFAaC;AAED,gDAGC;AA8BD,gDAkBC;AA9HD,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,WAAW,EAAC,UAAU,EAAC,cAAc,EAAC,gBAAgB,EAAC,UAAU,EAAC,QAAQ,EAAC,eAAe,EAAC,QAAQ,EAAC,WAAW,EAAC,YAAY,EAAC,cAAc,EAAC,eAAe,EAAC,aAAa,EAAC,gBAAgB,EAAC,kBAAkB,EAAC,iBAAiB,EAAC,WAAW,CAAC,CAAC,CAAC;AACvQ,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,kBAAkB,EAAC,cAAc,EAAC,gBAAgB,EAAC,UAAU,EAAC,eAAe,EAAC,cAAc,EAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;AACtI,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAC,MAAM,EAAC,QAAQ,EAAE,IAAI,CAAC,CAAC,CAAC;AAC3D,MAAM,OAAO,GAAG,2JAA2J,CAAC;AAE5K,SAAgB,0BAA0B,CAAC,KAA8B;IACvE,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACrC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,GAAG,GAAG,CAAC,CAAC;QAC7E,IAAI,GAAG,KAAK,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,yCAAyC,GAAG,GAAG,CAAC,CAAC;QACpH,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;QACzB,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ;YAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,GAAG,GAAG,CAAC,CAAC;QACrG,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,GAAG,GAAG,CAAC,CAAC;IAC9I,CAAC;AACH,CAAC;AAED,SAAgB,4BAA4B,CAAC,KAA8B;IACzE,0BAA0B,CAAC,KAAK,CAAC,CAAC;IAClC,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,gBAAgB,EAAE,IAAI,EAAE,EAAE,CAA4B,CAAC;IACnF,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,eAAe,EAAE,IAAI,EAAE,EAAE,CAA2B,CAAC;IAChF,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;IACxE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;IACrE,MAAM,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;IAClD,OAAO;QACL,SAAS,EAAE,KAAK,CAAC,KAAK,CAAC,SAAS,EAAE,WAAW,EAAE,EAAE,CAAC;QAClD,QAAQ,EAAE,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,EAAE,CAAC;QAC9C,YAAY,EAAE,KAAK,CAAC,KAAK,CAAC,YAAY,EAAE,SAAS,EAAE,EAAE,CAAC;QACtD,cAAc,EAAE,KAAK,CAAC,KAAK,CAAC,cAAc,EAAE,SAAS,EAAE,EAAE,CAAC;QAC1D,QAAQ,EAAE,KAAK,CAAC,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,GAAG,CAAC;QAC/C,MAAM,EAAE,CAAC,KAAK,EAAC,QAAQ,EAAC,MAAM,EAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAuC,CAAC,CAAC,CAAC,SAAS;QAC9G,aAAa,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC;QAC1D,MAAM,EAAE,KAAK,CAAC,MAAM,KAAK,KAAK;QAC9B,SAAS,EAAE,KAAK,CAAC,SAAS,KAAK,IAAI;QACnC,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;QACnD,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC;QACvD,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC;QACzD,WAAW,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QACrD,cAAc,EAAE,KAAK,CAAC,KAAK,CAAC,cAAc,EAAE,SAAS,EAAE,EAAE,CAAC;QAC1D,gBAAgB,EAAE,OAAO;QACzB,eAAe,EAAE,MAAM;QACvB,SAAS,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE;KACzE,CAAC;AACJ,CAAC;AAED,SAAgB,yBAAyB,CAAC,IAA+B;IACvE,MAAM,MAAM,GAAG,IAAI,GAAG,EAAe,CAAC;IACtC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,4BAA4B,CAAC,GAAG,CAAC,CAAC;QAC9C,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,QAAQ,EAAC,GAAG,CAAC,YAAY,EAAC,GAAG,CAAC,cAAc,EAAC,GAAG,CAAC,QAAQ,EAAC,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjG,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,YAAY,EAAE,GAAG,CAAC,YAAY,EAAE,cAAc,EAAE,GAAG,CAAC,cAAc,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,YAAY,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,GAAG,EAAU,EAAE,MAAM,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,OAAO,EAAE,EAA4B,EAAE,CAAC;QAC5W,KAAK,CAAC,YAAY,IAAI,CAAC,CAAC;QAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAAC,IAAI,GAAG,CAAC,MAAM;YAAE,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC;QAAC,IAAI,GAAG,CAAC,SAAS;YAAE,KAAK,CAAC,SAAS,IAAI,CAAC,CAAC;QAAC,KAAK,CAAC,IAAI,IAAI,GAAG,CAAC,UAAU,CAAC;QAAC,KAAK,CAAC,OAAO,IAAI,GAAG,CAAC,WAAW,CAAC;QAAC,KAAK,CAAC,OAAO,IAAI,GAAG,CAAC,aAAa,CAAC;QAC1O,IAAI,GAAG,CAAC,gBAAgB;YAAE,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC;QACpE,IAAI,GAAG,CAAC,eAAe;YAAE,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QAC5G,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACzB,CAAC;IACD,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACpC,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC3F,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,YAAY,EAAE,CAAC,CAAC,YAAY,EAAE,cAAc,EAAE,CAAC,CAAC,cAAc,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC,YAAY,EAAE,qBAAqB,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,YAAY,CAAC,EAAE,kBAAkB,EAAE,KAAK,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,YAAY,CAAC,EAAE,eAAe,EAAE,KAAK,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,YAAY,CAAC,EAAE,eAAe,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,YAAY,CAAC,EAAE,eAAe,EAAE,KAAK,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,YAAY,CAAC,EAAE,qBAAqB,EAAE,EAAE,GAAG,CAAC,CAAC,QAAQ,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,aAAa,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,aAAa,CAAC,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC,OAAO,EAAE,mBAAmB,EAAE,kBAAkB,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;IAChsB,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAgB,mCAAmC,CAAC,UAA+B,EAAE,UAA6C,EAAE,EAAE,OAAkD,EAAE;IACxL,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,EAAE,EAAE,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;IACtE,MAAM,SAAS,GAAG,IAAI,GAAG,EAA+B,CAAC;IACzD,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QAAC,MAAM,GAAG,GAAG,SAAS,CAAC,QAAQ,GAAG,GAAG,GAAG,SAAS,CAAC,YAAY,CAAC;QAAC,SAAS,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;IAAC,CAAC;IACtK,MAAM,SAAS,GAAuB,EAAE,CAAC;IACzC,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,SAAS,CAAC,OAAO,EAAE,EAAE,CAAC;QAC9C,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,IAAI,OAAO,IAAI,CAAC,CAAC,qBAAqB,IAAI,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;QAClJ,IAAI,CAAC,QAAQ,CAAC,MAAM;YAAE,SAAS;QAC/B,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAE,CAAC;QAAC,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,CAAE,CAAC;QACrE,IAAI,IAAI,CAAC,GAAG,KAAK,QAAQ,CAAC,GAAG;YAAE,SAAS;QACxC,SAAS,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE,qBAAqB,EAAE,IAAI,CAAC,qBAAqB,EAAE,mBAAmB,EAAE,IAAI,CAAC,mBAAmB,EAAE,wBAAwB,EAAE,IAAI,EAAE,CAAC,CAAC;IACpY,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAgB,kBAAkB,CAAC,OAA+B;IAChE,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IACxE,OAAQ,EAAE,gBAAgB,EAAE,oCAAoC,EAAE,YAAY,EAAE,6BAA6B,EAAE,cAAc,EAAE,uBAAuB,EAAE,QAAQ,EAAE,uBAAuB,EAAE,aAAa,EAAE,yBAAyB,EAAE,YAAY,EAAE,kCAAkC,EAAE,KAAK,EAAE,2BAA2B,EAA6B,CAAC,GAAG,IAAI,EAAE,CAAC,IAAI,0BAA0B,CAAC;AAClY,CAAC;AAED,SAAS,IAAI,CAAC,CAAoB,IAAI,OAAO,EAAE,cAAc,EAAE,CAAC,CAAC,cAAc,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACpJ,SAAS,QAAQ,CAAC,CAAoB,IAAI,OAAO,KAAK,CAAC,CAAC,CAAC,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,qBAAqB,CAAC,WAAW,GAAG,IAAI,GAAG,CAAC,CAAC,qBAAqB,CAAC,WAAW,GAAG,IAAI,GAAG,CAAC,CAAC,eAAe,GAAG,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,eAAe,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,eAAe,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AACxT,SAAS,KAAK,CAAC,KAAc,EAAE,QAAa,EAAE,GAAW,IAAI,MAAM,GAAG,GAAG,KAAK,IAAI,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,iBAAiB,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,GAAG,IAAI,QAAQ,CAAC,CAAC,CAAC;AACxM,SAAS,MAAM,CAAC,KAAc,EAAE,QAAgB,IAAI,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;AAClI,SAAS,KAAK,CAAC,KAAa,IAAI,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC;AAC3E,SAAS,IAAI,CAAC,KAAa,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE;IAAE,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AAuB5I,SAAgB,kBAAkB,CAAC,QAAiC;IAClE,OAAO;QACL,cAAc,EAAE,CAAC;QACjB,WAAW,EAAE,oBAAoB;QACjC,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACrC,eAAe,EAAE,IAAI;QACrB,UAAU,EAAE,IAAI;QAChB,QAAQ,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;YACnC,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,SAAS,EAAE,EAAE,CAAW;YAC1D,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,IAAI,IAAI,aAAa,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;YACzD,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,YAAY,EAAE,SAAS,EAAE,EAAE,CAAW;YAClE,cAAc,EAAE,KAAK,CAAC,OAAO,CAAC,cAAc,EAAE,SAAS,EAAE,EAAE,CAAW;YACtE,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,cAAc,EAAE,OAAO,CAAC,cAAc;YACtC,OAAO,EAAE,OAAO,CAAC,OAAO,KAAK,KAAK;SACnC,CAAC,CAAC;KACJ,CAAC;AACJ,CAAC"}
|
package/dist/telemetry.js
CHANGED
|
@@ -43,7 +43,7 @@ const https = __importStar(require("https"));
|
|
|
43
43
|
const os = __importStar(require("os"));
|
|
44
44
|
const path = __importStar(require("path"));
|
|
45
45
|
const crypto_1 = require("crypto");
|
|
46
|
-
const AGENTGUARD_SPEND_VERSION = '0.
|
|
46
|
+
const AGENTGUARD_SPEND_VERSION = '0.11.0';
|
|
47
47
|
const STATE_DIR = path.join(os.homedir(), '.agentguard');
|
|
48
48
|
const INSTALL_PATH = path.join(STATE_DIR, 'install.json');
|
|
49
49
|
const BEACON_URL = 'https://agentguard.run/api/beacon';
|
package/dist/telemetry.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"telemetry.js","sourceRoot":"","sources":["../src/telemetry.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAwBA,0CAOC;AAED,0CAOC;AAED,4CAIC;AAED,0DAIC;AAED,oDAwCC;AA9FD,uCAAyB;AACzB,6CAA+B;AAC/B,uCAAyB;AACzB,2CAA6B;AAC7B,mCAAoC;AACpC,MAAM,wBAAwB,GAAG,
|
|
1
|
+
{"version":3,"file":"telemetry.js","sourceRoot":"","sources":["../src/telemetry.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAwBA,0CAOC;AAED,0CAOC;AAED,4CAIC;AAED,0DAIC;AAED,oDAwCC;AA9FD,uCAAyB;AACzB,6CAA+B;AAC/B,uCAAyB;AACzB,2CAA6B;AAC7B,mCAAoC;AACpC,MAAM,wBAAwB,GAAG,QAAQ,CAAC;AAE1C,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,aAAa,CAAC,CAAC;AACzD,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;AAC1D,MAAM,UAAU,GAAG,mCAAmC,CAAC;AACvD,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;AAcrC,SAAgB,eAAe;IAC7B,MAAM,KAAK,GAAG,SAAS,EAAE,CAAC;IAC1B,OAAO;QACL,OAAO,EAAE,gBAAgB,CAAC,KAAK,CAAC;QAChC,kBAAkB,EAAE,mBAAmB,CAAC,KAAK,CAAC,IAAI,YAAY,EAAE;QAChE,SAAS,EAAE,YAAY;KACxB,CAAC;AACJ,CAAC;AAED,SAAgB,eAAe;IAC7B,MAAM,KAAK,GAAG,SAAS,EAAE,CAAC;IAC1B,MAAM,SAAS,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;IAC9C,MAAM,IAAI,GAAmB,EAAE,GAAG,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IACzD,IAAI,SAAS;QAAE,IAAI,CAAC,kBAAkB,GAAG,SAAS,CAAC;IACnD,SAAS,CAAC,IAAI,CAAC,CAAC;IAChB,OAAO,eAAe,EAAE,CAAC;AAC3B,CAAC;AAED,SAAgB,gBAAgB;IAC9B,MAAM,KAAK,GAAG,SAAS,EAAE,CAAC;IAC1B,SAAS,CAAC,EAAE,GAAG,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;IACxC,OAAO,eAAe,EAAE,CAAC;AAC3B,CAAC;AAED,SAAgB,uBAAuB;IACrC,MAAM,KAAK,GAAG,SAAS,EAAE,CAAC;IAC1B,SAAS,CAAC,EAAE,GAAG,KAAK,EAAE,kBAAkB,EAAE,IAAA,mBAAU,GAAE,EAAE,CAAC,CAAC;IAC1D,OAAO,eAAe,EAAE,CAAC;AAC3B,CAAC;AAED,SAAgB,oBAAoB,CAAC,KAAa;IAChD,IAAI,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC;QAAE,OAAO;IAClC,MAAM,KAAK,GAAG,SAAS,EAAE,CAAC;IAC1B,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC;QAAE,OAAO;IACrC,MAAM,SAAS,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;IAC9C,IAAI,CAAC,SAAS;QAAE,OAAO;IACvB,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAEtB,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC;QAC7B,KAAK;QACL,QAAQ,EAAE,uBAAuB;QACjC,WAAW,EAAE,wBAAwB;QACrC,OAAO,EAAE,QAAQ,OAAO,CAAC,OAAO,EAAE;QAClC,SAAS,EAAE,OAAO,CAAC,QAAQ;QAC3B,oBAAoB,EAAE,SAAS;QAC/B,eAAe,EAAE,IAAI,EAAE;QACvB,GAAG,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC;KACpC,CAAC,CAAC;IAEH,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC;QAChC,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CACvB;YACE,MAAM,EAAE,MAAM;YACd,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,IAAI,EAAE,GAAG,CAAC,QAAQ;YAClB,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,gBAAgB,EAAE,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC;aAC7C;YACD,OAAO,EAAE,IAAI;SACd,EACD,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,EAAE,CACtB,CAAC;QACF,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QACjC,GAAG,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;QACvC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IACnB,CAAC;IAAC,MAAM,CAAC;QACP,2DAA2D;IAC7D,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAqB;IAC7C,IAAI,OAAO,CAAC,GAAG,CAAC,oBAAoB,KAAK,GAAG;QAAE,OAAO,KAAK,CAAC;IAC3D,IAAI,OAAO,CAAC,GAAG,CAAC,oBAAoB,KAAK,GAAG;QAAE,OAAO,IAAI,CAAC;IAC1D,OAAO,KAAK,CAAC,OAAO,KAAK,IAAI,CAAC;AAChC,CAAC;AAED,SAAS,SAAS;IAChB,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAmB,CAAC;IAC9E,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,SAAS,SAAS,CAAC,KAAqB;IACtC,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,mBAAmB,CAAC,KAAK,CAAC,CAAC;QAC7C,MAAM,IAAI,GAAmB,EAAE,GAAG,KAAK,EAAE,CAAC;QAC1C,IAAI,SAAS;YAAE,IAAI,CAAC,kBAAkB,GAAG,SAAS,CAAC;QACnD,OAAO,IAAI,CAAC,oBAAoB,CAAC;QACjC,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7C,EAAE,CAAC,aAAa,CAAC,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QACtF,EAAE,CAAC,SAAS,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC;QAClC,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAqB;IACjD,MAAM,QAAQ,GAAG,mBAAmB,CAAC,KAAK,CAAC,CAAC;IAC5C,IAAI,QAAQ,EAAE,CAAC;QACb,IAAI,KAAK,CAAC,kBAAkB,KAAK,QAAQ,IAAI,KAAK,CAAC,oBAAoB,EAAE,CAAC;YACxE,SAAS,CAAC,EAAE,GAAG,KAAK,EAAE,kBAAkB,EAAE,QAAQ,EAAE,CAAC,CAAC;QACxD,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,MAAM,KAAK,GAAG,YAAY,EAAE,CAAC;IAC7B,IAAI,KAAK;QAAE,OAAO,KAAK,CAAC;IAExB,MAAM,SAAS,GAAG,IAAA,mBAAU,GAAE,CAAC;IAC/B,OAAO,SAAS,CAAC,EAAE,GAAG,KAAK,EAAE,kBAAkB,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC;AACnF,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAqB;IAChD,MAAM,EAAE,GAAG,KAAK,CAAC,kBAAkB,IAAI,KAAK,CAAC,oBAAoB,CAAC;IAClE,OAAO,OAAO,EAAE,KAAK,QAAQ,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7D,CAAC;AAED,SAAS,YAAY;IACnB,MAAM,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,qBAAqB,IAAI,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC;IAC5F,OAAO,OAAO,EAAE,KAAK,QAAQ,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7D,CAAC;AAED,SAAS,IAAI;IACX,OAAO,OAAO,CACZ,OAAO,CAAC,GAAG,CAAC,EAAE;QACZ,OAAO,CAAC,GAAG,CAAC,cAAc;QAC1B,OAAO,CAAC,GAAG,CAAC,SAAS;QACrB,OAAO,CAAC,GAAG,CAAC,QAAQ;QACpB,OAAO,CAAC,GAAG,CAAC,SAAS;QACrB,OAAO,CAAC,GAAG,CAAC,MAAM;QAClB,OAAO,CAAC,GAAG,CAAC,OAAO,CACtB,CAAC;AACJ,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@agentguard-run/spend",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.11.0",
|
|
4
4
|
"description": "All terminology and labels used in AgentGuard materials are descriptive of software functionality only, not legal definitions or guarantees of compliance. Terms like receipt, audit log, evidence, audit trail, and attestation refer solely to cryptographically-signed records produced by the software. Full functional-use disclaimer in README.",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
package/src/outcomes/index.ts
CHANGED
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
export type LearningFeedbackVerdict = 'accept' | 'edit' | 'reject' | null;
|
|
2
|
+
export type LearningFeedbackReason = 'missing_citation' | 'wrong_format' | 'wrong_category' | 'too_long' | 'missed_clause' | 'needs_review' | 'great' | null;
|
|
3
|
+
|
|
4
|
+
export interface LearningMetadataRow {
|
|
5
|
+
tenant_id: string;
|
|
6
|
+
vertical: string;
|
|
7
|
+
outcome_slug: string;
|
|
8
|
+
prompt_version: string;
|
|
9
|
+
model_id: string;
|
|
10
|
+
effort: 'low' | 'medium' | 'high' | 'xhigh' | 'unknown';
|
|
11
|
+
quality_score: number;
|
|
12
|
+
passed: boolean;
|
|
13
|
+
escalated: boolean;
|
|
14
|
+
cost_cents: number;
|
|
15
|
+
input_tokens: number;
|
|
16
|
+
output_tokens: number;
|
|
17
|
+
duration_ms: number;
|
|
18
|
+
guardrail_tier: string;
|
|
19
|
+
feedback_verdict: LearningFeedbackVerdict;
|
|
20
|
+
feedback_reason: LearningFeedbackReason;
|
|
21
|
+
timestamp: string;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export interface LearningAggregate {
|
|
25
|
+
key: string;
|
|
26
|
+
vertical: string;
|
|
27
|
+
outcome_slug: string;
|
|
28
|
+
prompt_version: string;
|
|
29
|
+
model_id: string;
|
|
30
|
+
effort: string;
|
|
31
|
+
sample_count: number;
|
|
32
|
+
distinct_tenant_count: number;
|
|
33
|
+
pass_rate: number;
|
|
34
|
+
mean_quality_score: number;
|
|
35
|
+
escalation_rate: number;
|
|
36
|
+
mean_cost_cents: number;
|
|
37
|
+
mean_latency_ms: number;
|
|
38
|
+
feedback_distribution: { accept: number; edit: number; reject: number; accept_rate: number; reject_rate: number };
|
|
39
|
+
reason_counts: Record<string, number>;
|
|
40
|
+
revision_suggestion: string;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export interface LearningProposal {
|
|
44
|
+
proposal_id: string;
|
|
45
|
+
status: 'shadow';
|
|
46
|
+
vertical: string;
|
|
47
|
+
outcome_slug: string;
|
|
48
|
+
proposed: { prompt_version: string; model_id: string; effort: string; fit_score: number };
|
|
49
|
+
current: { prompt_version: string; model_id: string; effort: string; fit_score: number } | null;
|
|
50
|
+
sample_count: number;
|
|
51
|
+
distinct_tenant_count: number;
|
|
52
|
+
revision_suggestion: string;
|
|
53
|
+
human_promotion_required: true;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
const ALLOWED = new Set(['tenant_id','vertical','outcome_slug','prompt_version','model_id','effort','quality_score','passed','escalated','cost_cents','input_tokens','output_tokens','duration_ms','guardrail_tier','feedback_verdict','feedback_reason','timestamp']);
|
|
57
|
+
const REASONS = new Set(['missing_citation','wrong_format','wrong_category','too_long','missed_clause','needs_review','great', null]);
|
|
58
|
+
const VERDICTS = new Set(['accept','edit','reject', null]);
|
|
59
|
+
const CONTENT = /^(prompt|completion|content|messages|input|inputs|output|outputs|text|raw|transcript|document|free_text|edited_text|email|phone|address|api_key|secret)$/i;
|
|
60
|
+
|
|
61
|
+
export function assertLearningMetadataOnly(value: Record<string, unknown>): void {
|
|
62
|
+
for (const key of Object.keys(value)) {
|
|
63
|
+
if (!ALLOWED.has(key)) throw new Error('learning field not allowed: ' + key);
|
|
64
|
+
if (key !== 'prompt_version' && CONTENT.test(key)) throw new Error('learning data-plane field not allowed: ' + key);
|
|
65
|
+
const child = value[key];
|
|
66
|
+
if (child && typeof child === 'object') throw new Error('learning nested value not allowed: ' + key);
|
|
67
|
+
if (typeof child === 'string' && (child.length > 160 || child.includes('\n'))) throw new Error('learning value looks like content: ' + key);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export function normalizeLearningMetadataRow(input: Record<string, unknown>): LearningMetadataRow {
|
|
72
|
+
assertLearningMetadataOnly(input);
|
|
73
|
+
const verdict = clean(input.feedback_verdict, null, 20) as LearningFeedbackVerdict;
|
|
74
|
+
const reason = clean(input.feedback_reason, null, 40) as LearningFeedbackReason;
|
|
75
|
+
if (!VERDICTS.has(verdict)) throw new Error('invalid learning verdict');
|
|
76
|
+
if (!REASONS.has(reason)) throw new Error('invalid learning reason');
|
|
77
|
+
const effort = clean(input.effort, 'unknown', 20);
|
|
78
|
+
return {
|
|
79
|
+
tenant_id: clean(input.tenant_id, 'anonymous', 96),
|
|
80
|
+
vertical: clean(input.vertical, 'unknown', 48),
|
|
81
|
+
outcome_slug: clean(input.outcome_slug, 'unknown', 96),
|
|
82
|
+
prompt_version: clean(input.prompt_version, 'unknown', 48),
|
|
83
|
+
model_id: clean(input.model_id, 'unknown', 120),
|
|
84
|
+
effort: ['low','medium','high','xhigh'].includes(effort) ? effort as LearningMetadataRow['effort'] : 'unknown',
|
|
85
|
+
quality_score: Math.min(1, number(input.quality_score, 0)),
|
|
86
|
+
passed: input.passed !== false,
|
|
87
|
+
escalated: input.escalated === true,
|
|
88
|
+
cost_cents: Math.round(number(input.cost_cents, 0)),
|
|
89
|
+
input_tokens: Math.round(number(input.input_tokens, 0)),
|
|
90
|
+
output_tokens: Math.round(number(input.output_tokens, 0)),
|
|
91
|
+
duration_ms: Math.round(number(input.duration_ms, 0)),
|
|
92
|
+
guardrail_tier: clean(input.guardrail_tier, 'unknown', 64),
|
|
93
|
+
feedback_verdict: verdict,
|
|
94
|
+
feedback_reason: reason,
|
|
95
|
+
timestamp: new Date(String(input.timestamp || Date.now())).toISOString(),
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
export function aggregateLearningMetadata(rows: Record<string, unknown>[]): LearningAggregate[] {
|
|
100
|
+
const groups = new Map<string, any>();
|
|
101
|
+
for (const raw of rows) {
|
|
102
|
+
const row = normalizeLearningMetadataRow(raw);
|
|
103
|
+
const key = [row.vertical,row.outcome_slug,row.prompt_version,row.model_id,row.effort].join('|');
|
|
104
|
+
const group = groups.get(key) ?? { key, vertical: row.vertical, outcome_slug: row.outcome_slug, prompt_version: row.prompt_version, model_id: row.model_id, effort: row.effort, sample_count: 0, tenants: new Set<string>(), passed: 0, escalated: 0, cost: 0, latency: 0, quality: 0, verdicts: { accept: 0, edit: 0, reject: 0 }, reasons: {} as Record<string, number> };
|
|
105
|
+
group.sample_count += 1; group.tenants.add(row.tenant_id); if (row.passed) group.passed += 1; if (row.escalated) group.escalated += 1; group.cost += row.cost_cents; group.latency += row.duration_ms; group.quality += row.quality_score;
|
|
106
|
+
if (row.feedback_verdict) group.verdicts[row.feedback_verdict] += 1;
|
|
107
|
+
if (row.feedback_reason) group.reasons[row.feedback_reason] = (group.reasons[row.feedback_reason] || 0) + 1;
|
|
108
|
+
groups.set(key, group);
|
|
109
|
+
}
|
|
110
|
+
return [...groups.values()].map((g) => {
|
|
111
|
+
const totalVerdicts = Math.max(1, g.verdicts.accept + g.verdicts.edit + g.verdicts.reject);
|
|
112
|
+
return { key: g.key, vertical: g.vertical, outcome_slug: g.outcome_slug, prompt_version: g.prompt_version, model_id: g.model_id, effort: g.effort, sample_count: g.sample_count, distinct_tenant_count: g.tenants.size, pass_rate: round(g.passed / g.sample_count), mean_quality_score: round(g.quality / g.sample_count), escalation_rate: round(g.escalated / g.sample_count), mean_cost_cents: round(g.cost / g.sample_count), mean_latency_ms: round(g.latency / g.sample_count), feedback_distribution: { ...g.verdicts, accept_rate: round(g.verdicts.accept / totalVerdicts), reject_rate: round(g.verdicts.reject / totalVerdicts) }, reason_counts: g.reasons, revision_suggestion: revisionSuggestion(g.reasons) };
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
export function proposeLearningMetadataImprovements(aggregates: LearningAggregate[], current: Record<string, LearningAggregate> = {}, opts: { minRuns?: number; minTenants?: number } = {}): LearningProposal[] {
|
|
117
|
+
const minRuns = opts.minRuns ?? 20, minTenants = opts.minTenants ?? 5;
|
|
118
|
+
const byOutcome = new Map<string, LearningAggregate[]>();
|
|
119
|
+
for (const aggregate of aggregates) { const key = aggregate.vertical + '|' + aggregate.outcome_slug; byOutcome.set(key, [...(byOutcome.get(key) || []), aggregate]); }
|
|
120
|
+
const proposals: LearningProposal[] = [];
|
|
121
|
+
for (const [key, rows] of byOutcome.entries()) {
|
|
122
|
+
const eligible = rows.filter((r) => r.sample_count >= minRuns && r.distinct_tenant_count >= minTenants).sort((a, b) => fitScore(b) - fitScore(a));
|
|
123
|
+
if (!eligible.length) continue;
|
|
124
|
+
const best = eligible[0]!; const baseline = current[key] || rows[0]!;
|
|
125
|
+
if (best.key === baseline.key) continue;
|
|
126
|
+
proposals.push({ proposal_id: 'ag_lp_' + Math.abs(hash(best.key)).toString(16), status: 'shadow', vertical: best.vertical, outcome_slug: best.outcome_slug, proposed: pick(best), current: baseline ? pick(baseline) : null, sample_count: best.sample_count, distinct_tenant_count: best.distinct_tenant_count, revision_suggestion: best.revision_suggestion, human_promotion_required: true });
|
|
127
|
+
}
|
|
128
|
+
return proposals;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
export function revisionSuggestion(reasons: Record<string, number>): string {
|
|
132
|
+
const top = Object.entries(reasons).sort((a, b) => b[1] - a[1])[0]?.[0];
|
|
133
|
+
return ({ missing_citation: 'tighten cite-or-refuse instruction', wrong_format: 'make output schema stricter', wrong_category: 'add category examples', too_long: 'shorten answer format', missed_clause: 'expand clause watchlist', needs_review: 'lower reviewer cascade threshold', great: 'preserve current template' } as Record<string, string>)[top || ''] || 'keep collecting metadata';
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
function pick(a: LearningAggregate) { return { prompt_version: a.prompt_version, model_id: a.model_id, effort: a.effort, fit_score: fitScore(a) }; }
|
|
137
|
+
function fitScore(a: LearningAggregate) { return round(a.pass_rate * 0.32 + a.mean_quality_score * 0.28 + a.feedback_distribution.accept_rate * 0.24 - a.feedback_distribution.reject_rate * 0.18 - a.escalation_rate * 0.08 - Math.min(0.12, a.mean_cost_cents / 2000) - Math.min(0.08, a.mean_latency_ms / 120000)); }
|
|
138
|
+
function clean(value: unknown, fallback: any, max: number) { const raw = value == null ? fallback : String(value).toLowerCase().replace(/[^a-z0-9_./:-]/g, '_').slice(0, max); return raw || fallback; }
|
|
139
|
+
function number(value: unknown, fallback: number) { const n = Number(value); return Number.isFinite(n) && n >= 0 ? n : fallback; }
|
|
140
|
+
function round(value: number) { return Math.round(value * 10000) / 10000; }
|
|
141
|
+
function hash(value: string) { let h = 0; for (let i = 0; i < value.length; i++) h = Math.imul(31, h) + value.charCodeAt(i) | 0; return h; }
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
export interface PortableAiTeamOutcome {
|
|
145
|
+
vertical: string;
|
|
146
|
+
name: string;
|
|
147
|
+
outcome_slug: string;
|
|
148
|
+
prompt_version: string;
|
|
149
|
+
caps?: Record<string, number>;
|
|
150
|
+
effort?: string;
|
|
151
|
+
guardrail_tier?: string;
|
|
152
|
+
enabled?: boolean;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
export interface PortableAiTeamExport {
|
|
156
|
+
schema_version: 1;
|
|
157
|
+
export_type: 'agentguard_ai_team';
|
|
158
|
+
exported_at: string;
|
|
159
|
+
zero_data_plane: true;
|
|
160
|
+
no_lock_in: true;
|
|
161
|
+
outcomes: PortableAiTeamOutcome[];
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
export function exportAiTeamConfig(outcomes: PortableAiTeamOutcome[]): PortableAiTeamExport {
|
|
165
|
+
return {
|
|
166
|
+
schema_version: 1,
|
|
167
|
+
export_type: 'agentguard_ai_team',
|
|
168
|
+
exported_at: new Date().toISOString(),
|
|
169
|
+
zero_data_plane: true,
|
|
170
|
+
no_lock_in: true,
|
|
171
|
+
outcomes: outcomes.map((outcome) => ({
|
|
172
|
+
vertical: clean(outcome.vertical, 'unknown', 48) as string,
|
|
173
|
+
name: String(outcome.name || 'AI teammate').slice(0, 120),
|
|
174
|
+
outcome_slug: clean(outcome.outcome_slug, 'unknown', 96) as string,
|
|
175
|
+
prompt_version: clean(outcome.prompt_version, 'unknown', 48) as string,
|
|
176
|
+
caps: outcome.caps,
|
|
177
|
+
effort: outcome.effort,
|
|
178
|
+
guardrail_tier: outcome.guardrail_tier,
|
|
179
|
+
enabled: outcome.enabled !== false,
|
|
180
|
+
})),
|
|
181
|
+
};
|
|
182
|
+
}
|