@axiom-lattice/protocols 5.0.0 → 6.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +10 -10
- package/CHANGELOG.md +32 -0
- package/dist/index.d.mts +175 -1
- package/dist/index.d.ts +175 -1
- package/dist/index.js +23 -1
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +20 -1
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
- package/src/EvalStoreProtocol.ts +4 -0
- package/src/McpUiProtocol.ts +22 -2
- package/src/MenuProtocol.ts +2 -0
- package/src/ObservabilityDashboardProtocol.ts +167 -0
- package/src/ScheduleLatticeProtocol.ts +7 -0
- package/src/TaskStoreProtocol.ts +8 -0
- package/src/__tests__/McpUiProtocol.test.ts +48 -0
- package/src/__tests__/MenuProtocol.test.ts +4 -0
- package/src/__tests__/ObservabilityDashboardProtocol.test.ts +17 -0
- package/src/index.ts +1 -0
package/package.json
CHANGED
package/src/EvalStoreProtocol.ts
CHANGED
|
@@ -104,6 +104,8 @@ export interface EvalRun {
|
|
|
104
104
|
interruptedCases?: number;
|
|
105
105
|
avgScore: number;
|
|
106
106
|
error?: string;
|
|
107
|
+
/** A client requested cancellation; the executing worker polls this flag. */
|
|
108
|
+
cancelRequested?: boolean;
|
|
107
109
|
/**
|
|
108
110
|
* When true, this run executed ONLY the hold-out validation suite.
|
|
109
111
|
* Per-case results of holdout runs are withheld from agents (aggregate
|
|
@@ -179,6 +181,8 @@ export interface EvalStore {
|
|
|
179
181
|
getRunById(tenantId: string, id: string): Promise<EvalRun | null>;
|
|
180
182
|
createRun(tenantId: string, projectId: string, id: string, data: CreateEvalRunRequest): Promise<EvalRun>;
|
|
181
183
|
updateRunStatus(tenantId: string, id: string, updates: { status?: EvalRun['status']; passedCases?: number; failedCases?: number; interruptedCases?: number; avgScore?: number; error?: string; completedAt?: Date }): Promise<EvalRun | null>;
|
|
184
|
+
/** Request cooperative cancellation for a running evaluation. */
|
|
185
|
+
requestCancel?(tenantId: string, id: string): Promise<boolean>;
|
|
182
186
|
deleteRun(tenantId: string, id: string): Promise<boolean>;
|
|
183
187
|
|
|
184
188
|
getResultsByRun(tenantId: string, runId: string): Promise<EvalRunResult[]>;
|
package/src/McpUiProtocol.ts
CHANGED
|
@@ -32,7 +32,15 @@ export type McpUiRef =
|
|
|
32
32
|
*/
|
|
33
33
|
structuredContent?: unknown;
|
|
34
34
|
}
|
|
35
|
-
| {
|
|
35
|
+
| {
|
|
36
|
+
kind: "plugin";
|
|
37
|
+
pluginType: string;
|
|
38
|
+
resource: string;
|
|
39
|
+
displayMode: McpUiDisplayMode;
|
|
40
|
+
title?: string;
|
|
41
|
+
/** Opaque HMAC-signed run/connection scope (1..4096 chars); never contains secrets. */
|
|
42
|
+
scope?: string;
|
|
43
|
+
};
|
|
36
44
|
|
|
37
45
|
/** Resolved UI resource body returned by the gateway. */
|
|
38
46
|
export interface McpUiResourcePayload {
|
|
@@ -45,6 +53,7 @@ export interface McpUiResourcePayload {
|
|
|
45
53
|
}
|
|
46
54
|
|
|
47
55
|
const MODES: McpUiDisplayMode[] = ["inline", "fullscreen", "pip"];
|
|
56
|
+
const MAX_MCP_UI_SCOPE_LENGTH = 4096;
|
|
48
57
|
const isObj = (v: unknown): v is Record<string, unknown> => typeof v === "object" && v !== null;
|
|
49
58
|
|
|
50
59
|
/** Type guard for a parsed `McpUiRef`. */
|
|
@@ -70,7 +79,18 @@ export function parseMcpUiRef(value: unknown): McpUiRef | null {
|
|
|
70
79
|
};
|
|
71
80
|
}
|
|
72
81
|
if (value.kind === "plugin" && typeof value.pluginType === "string" && typeof value.resource === "string") {
|
|
73
|
-
|
|
82
|
+
const scope =
|
|
83
|
+
typeof value.scope === "string" && value.scope.length > 0 && value.scope.length <= MAX_MCP_UI_SCOPE_LENGTH
|
|
84
|
+
? value.scope
|
|
85
|
+
: undefined;
|
|
86
|
+
return {
|
|
87
|
+
kind: "plugin",
|
|
88
|
+
pluginType: value.pluginType,
|
|
89
|
+
resource: value.resource,
|
|
90
|
+
displayMode,
|
|
91
|
+
...(title ? { title } : {}),
|
|
92
|
+
...(scope ? { scope } : {}),
|
|
93
|
+
};
|
|
74
94
|
}
|
|
75
95
|
return null;
|
|
76
96
|
}
|
package/src/MenuProtocol.ts
CHANGED
|
@@ -54,6 +54,8 @@ export type McpAppMenuConfig =
|
|
|
54
54
|
resource: string;
|
|
55
55
|
displayMode?: McpUiDisplayMode;
|
|
56
56
|
title?: string;
|
|
57
|
+
/** Signed scope copied from the ref so a pinned App can keep calling. */
|
|
58
|
+
scope?: string;
|
|
57
59
|
toolInput?: Record<string, unknown>;
|
|
58
60
|
toolResult?: unknown;
|
|
59
61
|
ui?: ToolCallUi;
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Time range accepted by an observability dashboard.
|
|
3
|
+
*/
|
|
4
|
+
export type ObservabilityDashboardRange = "1h" | "24h" | "7d" | "30d";
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Input passed by the gateway to the selected observability provider.
|
|
8
|
+
*/
|
|
9
|
+
export interface ObservabilityDashboardInput {
|
|
10
|
+
/** Time window to aggregate. */
|
|
11
|
+
range: ObservabilityDashboardRange;
|
|
12
|
+
/** Tenant resolved from the authenticated request, never from the client. */
|
|
13
|
+
tenantId: string;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Provider-owned dashboard result.
|
|
18
|
+
*
|
|
19
|
+
* The `payload` shape belongs to the provider; consumers must check `version`
|
|
20
|
+
* before rendering it.
|
|
21
|
+
*/
|
|
22
|
+
export interface ObservabilityDashboardResult {
|
|
23
|
+
/** Provider payload version. */
|
|
24
|
+
version: number;
|
|
25
|
+
/** Section keys that could not be loaded. */
|
|
26
|
+
warnings?: string[];
|
|
27
|
+
/** Provider-specific dashboard data. */
|
|
28
|
+
payload: unknown;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Cross-provider response envelope returned by the gateway.
|
|
33
|
+
*/
|
|
34
|
+
export interface ObservabilityDashboardEnvelope {
|
|
35
|
+
/** Provider type, also used as the UI element key suffix. */
|
|
36
|
+
provider: string;
|
|
37
|
+
/** Requested time window. */
|
|
38
|
+
range: ObservabilityDashboardRange;
|
|
39
|
+
/** Server generation timestamp. */
|
|
40
|
+
generatedAt: string;
|
|
41
|
+
/** Provider payload version. */
|
|
42
|
+
version: number;
|
|
43
|
+
/** Section keys that could not be loaded. */
|
|
44
|
+
warnings?: string[];
|
|
45
|
+
/** Provider-specific dashboard data. */
|
|
46
|
+
payload: unknown;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** Prefix used to register a dashboard renderer in the GenUI element registry. */
|
|
50
|
+
export const OBSERVABILITY_DASHBOARD_ELEMENT_KEY_PREFIX =
|
|
51
|
+
"observability_dashboard_";
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Returns the GenUI element key for a provider dashboard.
|
|
55
|
+
*
|
|
56
|
+
* @param provider - Provider type such as `"langfuse"`.
|
|
57
|
+
* @returns The registry key used by both the provider renderer and the panel.
|
|
58
|
+
*/
|
|
59
|
+
export function observabilityDashboardElementKey(provider: string): string {
|
|
60
|
+
return `${OBSERVABILITY_DASHBOARD_ELEMENT_KEY_PREFIX}${provider}`;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** Version of the built-in Langfuse dashboard payload. */
|
|
64
|
+
export const LANGFUSE_OBSERVABILITY_DASHBOARD_VERSION = 3;
|
|
65
|
+
|
|
66
|
+
/** One point in a Langfuse dashboard time series. */
|
|
67
|
+
export interface LangfuseObservabilitySeriesPoint {
|
|
68
|
+
/** ISO timestamp of the bucket start. */
|
|
69
|
+
bucket: string;
|
|
70
|
+
/** Aggregated value, or `null` when the bucket has no value. */
|
|
71
|
+
value: number | null;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** Agent breakdown row, derived from the `assistant:<id>` trace tag. */
|
|
75
|
+
export interface LangfuseAgentBreakdownRow {
|
|
76
|
+
/** Assistant id from the trace tag. */
|
|
77
|
+
assistantId: string;
|
|
78
|
+
/** Display name from the trace name, when Langfuse captured one. */
|
|
79
|
+
name?: string;
|
|
80
|
+
/** Root-observation count, i.e. invocations started by this agent. */
|
|
81
|
+
calls: number;
|
|
82
|
+
/** All observations emitted by this agent. */
|
|
83
|
+
observations: number;
|
|
84
|
+
/** Total cost in USD. */
|
|
85
|
+
costUsd: number;
|
|
86
|
+
/** Input tokens consumed. */
|
|
87
|
+
inputTokens: number;
|
|
88
|
+
/** Output tokens produced. */
|
|
89
|
+
outputTokens: number;
|
|
90
|
+
/** Error-level observation count. */
|
|
91
|
+
errorCount: number;
|
|
92
|
+
/** Error ratio in `[0, 1]` over observations. */
|
|
93
|
+
errorRate: number;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** One model's cost over time. */
|
|
97
|
+
export interface LangfuseModelCostSeries {
|
|
98
|
+
/** Model name. */
|
|
99
|
+
model: string;
|
|
100
|
+
/** Cost points for this model. */
|
|
101
|
+
points: LangfuseObservabilitySeriesPoint[];
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/** Model breakdown row. */
|
|
105
|
+
export interface LangfuseModelBreakdownRow {
|
|
106
|
+
/** Model name reported by Langfuse. */
|
|
107
|
+
model: string;
|
|
108
|
+
/** Observation count. */
|
|
109
|
+
count: number;
|
|
110
|
+
/** Total cost in USD. */
|
|
111
|
+
costUsd: number;
|
|
112
|
+
/** Input tokens consumed. */
|
|
113
|
+
inputTokens: number;
|
|
114
|
+
/** Output tokens produced. */
|
|
115
|
+
outputTokens: number;
|
|
116
|
+
/** p95 generation latency in milliseconds, or `null` when unknown. */
|
|
117
|
+
p95LatencyMs: number | null;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/** Tool breakdown row. */
|
|
121
|
+
export interface LangfuseToolBreakdownRow {
|
|
122
|
+
/** Tool observation name. */
|
|
123
|
+
tool: string;
|
|
124
|
+
/** Observation count. */
|
|
125
|
+
count: number;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* Built-in Langfuse dashboard payload.
|
|
130
|
+
*
|
|
131
|
+
* Aggregated for one tenant and time range; it never contains prompts,
|
|
132
|
+
* completions, user text, or credentials.
|
|
133
|
+
*/
|
|
134
|
+
export interface LangfuseObservabilityDashboardPayload {
|
|
135
|
+
/** Provider sample rate; values below 1 mean the numbers are sampled. */
|
|
136
|
+
samplingRate: number;
|
|
137
|
+
/** Tracing environment used for the query, when configured. */
|
|
138
|
+
environment?: string;
|
|
139
|
+
kpis: {
|
|
140
|
+
traces: number;
|
|
141
|
+
costUsd: number;
|
|
142
|
+
inputTokens: number;
|
|
143
|
+
outputTokens: number;
|
|
144
|
+
/** Error observation ratio in `[0, 1]`. */
|
|
145
|
+
errorRate: number;
|
|
146
|
+
};
|
|
147
|
+
series: {
|
|
148
|
+
traces: LangfuseObservabilitySeriesPoint[];
|
|
149
|
+
cost: LangfuseObservabilitySeriesPoint[];
|
|
150
|
+
tokens: {
|
|
151
|
+
input: LangfuseObservabilitySeriesPoint[];
|
|
152
|
+
output: LangfuseObservabilitySeriesPoint[];
|
|
153
|
+
};
|
|
154
|
+
latency: {
|
|
155
|
+
p50: LangfuseObservabilitySeriesPoint[];
|
|
156
|
+
p95: LangfuseObservabilitySeriesPoint[];
|
|
157
|
+
p99: LangfuseObservabilitySeriesPoint[];
|
|
158
|
+
};
|
|
159
|
+
errorRate: LangfuseObservabilitySeriesPoint[];
|
|
160
|
+
toolCalls: LangfuseObservabilitySeriesPoint[];
|
|
161
|
+
/** Per-model cost trend. Optional so older payloads keep working. */
|
|
162
|
+
costByModel?: LangfuseModelCostSeries[];
|
|
163
|
+
};
|
|
164
|
+
agents: LangfuseAgentBreakdownRow[];
|
|
165
|
+
models: LangfuseModelBreakdownRow[];
|
|
166
|
+
tools: LangfuseToolBreakdownRow[];
|
|
167
|
+
}
|
|
@@ -361,6 +361,12 @@ export interface ScheduleClient {
|
|
|
361
361
|
*/
|
|
362
362
|
restore(): Promise<number>;
|
|
363
363
|
|
|
364
|
+
/**
|
|
365
|
+
* Reconcile this replica's timers with the shared store. Implementations
|
|
366
|
+
* that support multi-replica scheduling should expose this.
|
|
367
|
+
*/
|
|
368
|
+
reconcile?(): Promise<number>;
|
|
369
|
+
|
|
364
370
|
// ===== Storage =====
|
|
365
371
|
|
|
366
372
|
/**
|
|
@@ -414,4 +420,5 @@ export interface ScheduleLatticeProtocol
|
|
|
414
420
|
|
|
415
421
|
// Recovery
|
|
416
422
|
restore: () => Promise<number>;
|
|
423
|
+
reconcile?: () => Promise<number>;
|
|
417
424
|
}
|
package/src/TaskStoreProtocol.ts
CHANGED
|
@@ -379,6 +379,14 @@ export interface TaskListFilter {
|
|
|
379
379
|
*/
|
|
380
380
|
priority?: string;
|
|
381
381
|
|
|
382
|
+
/**
|
|
383
|
+
* Filter by the isolated run thread linked from `context.thread_id`.
|
|
384
|
+
*
|
|
385
|
+
* @remarks Used to authorize read-only views of task execution threads that
|
|
386
|
+
* do not have a durable Thread row or AgentRun row.
|
|
387
|
+
*/
|
|
388
|
+
threadId?: string;
|
|
389
|
+
|
|
382
390
|
/**
|
|
383
391
|
* Filter by workspace ID
|
|
384
392
|
*/
|
|
@@ -41,3 +41,51 @@ describe("parseMcpUiRef", () => {
|
|
|
41
41
|
expect(isMcpUiRef("x")).toBe(false);
|
|
42
42
|
});
|
|
43
43
|
});
|
|
44
|
+
|
|
45
|
+
describe("plugin scope", () => {
|
|
46
|
+
const plugin = {
|
|
47
|
+
kind: "plugin" as const,
|
|
48
|
+
pluginType: "p",
|
|
49
|
+
resource: "ui/x.html",
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
it("preserves a scope between 1 and 4096 characters", () => {
|
|
53
|
+
const scope = "a".repeat(64);
|
|
54
|
+
expect(parseMcpUiRef({ ...plugin, scope })).toEqual({
|
|
55
|
+
...plugin,
|
|
56
|
+
displayMode: "inline",
|
|
57
|
+
scope,
|
|
58
|
+
});
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
it("preserves boundary scopes of 1 and 4096 characters", () => {
|
|
62
|
+
for (const scope of ["a", "b".repeat(4096)]) {
|
|
63
|
+
expect(parseMcpUiRef({ ...plugin, scope })).toEqual({
|
|
64
|
+
...plugin,
|
|
65
|
+
displayMode: "inline",
|
|
66
|
+
scope,
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
it("drops empty, non-string, and over-long scopes", () => {
|
|
72
|
+
for (const scope of ["", 42, "x".repeat(4097)]) {
|
|
73
|
+
const ref = parseMcpUiRef({ ...plugin, scope });
|
|
74
|
+
expect(ref && "scope" in ref).toBe(false);
|
|
75
|
+
}
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
it("does not carry scope on an mcp ref", () => {
|
|
79
|
+
const ref = parseMcpUiRef({
|
|
80
|
+
kind: "mcp",
|
|
81
|
+
serverKey: "weather",
|
|
82
|
+
resourceUri: "ui://weather/dashboard",
|
|
83
|
+
scope: "signed-scope-token",
|
|
84
|
+
});
|
|
85
|
+
expect(ref && "scope" in ref).toBe(false);
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
it("keeps a legacy plugin ref unchanged", () => {
|
|
89
|
+
expect(parseMcpUiRef(plugin)).toEqual({ ...plugin, displayMode: "inline" });
|
|
90
|
+
});
|
|
91
|
+
});
|
|
@@ -15,10 +15,14 @@ describe("MenuProtocol mcpApp content type", () => {
|
|
|
15
15
|
pluginType: "my-plugin",
|
|
16
16
|
resource: "ui/report.html",
|
|
17
17
|
title: "Report",
|
|
18
|
+
scope: "signed-scope-token",
|
|
18
19
|
};
|
|
19
20
|
for (const cfg of [mcp, plugin] as MenuContentConfig[]) {
|
|
20
21
|
expect(JSON.parse(JSON.stringify(cfg))).toEqual(cfg);
|
|
21
22
|
}
|
|
23
|
+
const parsed = JSON.parse(JSON.stringify(plugin)) as McpAppMenuConfig;
|
|
24
|
+
expect(parsed.kind).toBe("plugin");
|
|
25
|
+
expect(parsed.kind === "plugin" ? parsed.scope : undefined).toBe("signed-scope-token");
|
|
22
26
|
});
|
|
23
27
|
|
|
24
28
|
it("includes file and mcpApp in MenuContentType", () => {
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import {
|
|
2
|
+
LANGFUSE_OBSERVABILITY_DASHBOARD_VERSION,
|
|
3
|
+
OBSERVABILITY_DASHBOARD_ELEMENT_KEY_PREFIX,
|
|
4
|
+
observabilityDashboardElementKey,
|
|
5
|
+
} from "../ObservabilityDashboardProtocol";
|
|
6
|
+
|
|
7
|
+
describe("observabilityDashboardElementKey", () => {
|
|
8
|
+
it("namespaces the provider key", () => {
|
|
9
|
+
expect(observabilityDashboardElementKey("langfuse")).toBe(
|
|
10
|
+
"observability_dashboard_langfuse",
|
|
11
|
+
);
|
|
12
|
+
expect(OBSERVABILITY_DASHBOARD_ELEMENT_KEY_PREFIX).toBe(
|
|
13
|
+
"observability_dashboard_",
|
|
14
|
+
);
|
|
15
|
+
expect(LANGFUSE_OBSERVABILITY_DASHBOARD_VERSION).toBe(3);
|
|
16
|
+
});
|
|
17
|
+
});
|
package/src/index.ts
CHANGED