llm-cli-gateway 2.11.1 → 2.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/async-job-orchestration/SKILL.md +288 -0
- package/.agents/skills/implement-review-fix/SKILL.md +154 -0
- package/.agents/skills/multi-llm-review/SKILL.md +174 -0
- package/.agents/skills/public-demo-session/SKILL.md +100 -0
- package/.agents/skills/secure-orchestration/SKILL.md +227 -0
- package/.agents/skills/session-workflow/SKILL.md +271 -0
- package/CHANGELOG.md +40 -0
- package/README.md +48 -19
- package/dist/acp/provider-registry.js +5 -5
- package/dist/api-provider.d.ts +7 -0
- package/dist/api-provider.js +7 -0
- package/dist/api-request.js +1 -0
- package/dist/async-job-manager.d.ts +11 -1
- package/dist/async-job-manager.js +44 -3
- package/dist/config.d.ts +2 -0
- package/dist/config.js +3 -0
- package/dist/index.d.ts +38 -2
- package/dist/index.js +609 -156
- package/dist/job-store.d.ts +48 -1
- package/dist/job-store.js +184 -0
- package/dist/provider-codegen.js +3 -0
- package/dist/provider-tool-capabilities.js +7 -7
- package/dist/upstream-contracts.d.ts +1 -0
- package/dist/upstream-contracts.js +128 -21
- package/dist/validation-orchestrator.d.ts +5 -2
- package/dist/validation-orchestrator.js +71 -5
- package/dist/validation-receipt.d.ts +68 -0
- package/dist/validation-receipt.js +245 -0
- package/dist/validation-report.d.ts +4 -2
- package/dist/validation-report.js +18 -1
- package/dist/validation-tools.js +58 -9
- package/npm-shrinkwrap.json +5 -5
- package/package.json +10 -4
- package/dist/xai-api-provider.d.ts +0 -43
- package/dist/xai-api-provider.js +0 -191
|
@@ -3,7 +3,8 @@ import { getProviderRuntimeStatus } from "./provider-status.js";
|
|
|
3
3
|
import { createApiProvider } from "./api-provider.js";
|
|
4
4
|
import { prepareApiRequest } from "./api-request.js";
|
|
5
5
|
import { normalizeJobResult, normalizeSkippedProvider, normalizeStartedJob, } from "./validation-normalizer.js";
|
|
6
|
-
import { buildValidationReport } from "./validation-report.js";
|
|
6
|
+
import { buildValidationReport, deriveValidationRunStatus, } from "./validation-report.js";
|
|
7
|
+
import { getRequestContext, principalCanAccess, resolveOwnerPrincipal } from "./request-context.js";
|
|
7
8
|
import { buildJudgePrompt, buildValidationPrompt, } from "./validation-prompts.js";
|
|
8
9
|
function findApiReviewer(deps, provider) {
|
|
9
10
|
return deps.apiProviders?.find(p => p.name === provider) ?? null;
|
|
@@ -21,8 +22,13 @@ function dispatchProviderJob(deps, provider, prompt, correlationId) {
|
|
|
21
22
|
if (api) {
|
|
22
23
|
const apiProvider = createApiProvider(api.name, api.kind);
|
|
23
24
|
const apiRequest = prepareApiRequest(api, { prompt });
|
|
24
|
-
return deps.asyncJobManager.startHttpJob({
|
|
25
|
-
|
|
25
|
+
return deps.asyncJobManager.startHttpJob({
|
|
26
|
+
provider: apiProvider,
|
|
27
|
+
apiRequest,
|
|
28
|
+
correlationId,
|
|
29
|
+
writeFlightStart: true,
|
|
30
|
+
flightRecorderEntry: { model: apiRequest.model, prompt },
|
|
31
|
+
}).snapshot;
|
|
26
32
|
}
|
|
27
33
|
return deps.asyncJobManager.startJob(provider, buildProviderArgs(provider, prompt), correlationId);
|
|
28
34
|
}
|
|
@@ -39,9 +45,16 @@ export function startValidationRun(deps, input) {
|
|
|
39
45
|
const providers = uniqueProviders(input.providers);
|
|
40
46
|
const results = providers.map(provider => startProviderJob(deps, provider, prompt, validationId));
|
|
41
47
|
const runningCount = results.filter(result => result.status === "running").length;
|
|
42
|
-
const skippedCount = results.filter(result => result.status === "skipped").length;
|
|
43
48
|
const synthesis = plannedJudgeSynthesis(input);
|
|
44
|
-
const status =
|
|
49
|
+
const status = deriveValidationRunStatus(results, synthesis.status);
|
|
50
|
+
persistValidationRun(deps, {
|
|
51
|
+
validationId,
|
|
52
|
+
startedAt,
|
|
53
|
+
intent: input.intent,
|
|
54
|
+
input,
|
|
55
|
+
providers,
|
|
56
|
+
results,
|
|
57
|
+
});
|
|
45
58
|
const reportInput = {
|
|
46
59
|
validationId,
|
|
47
60
|
status,
|
|
@@ -103,6 +116,7 @@ export function startJudgeSynthesis(deps, input) {
|
|
|
103
116
|
question: input.question,
|
|
104
117
|
providerResults: completedResults,
|
|
105
118
|
}), `validation-judge-${randomUUID()}-${input.judgeProvider}`);
|
|
119
|
+
linkJudgeJob(deps, input.validationId, input.judgeProvider, snapshot);
|
|
106
120
|
return {
|
|
107
121
|
status: "running",
|
|
108
122
|
judgeModel: input.judgeProvider,
|
|
@@ -150,6 +164,58 @@ function plannedJudgeSynthesis(input) {
|
|
|
150
164
|
note: "Collect provider results first, then call synthesize_validation with those results.",
|
|
151
165
|
};
|
|
152
166
|
}
|
|
167
|
+
function linkJudgeJob(deps, validationId, provider, snapshot) {
|
|
168
|
+
const store = deps.validationRunStore;
|
|
169
|
+
if (!store || !validationId)
|
|
170
|
+
return;
|
|
171
|
+
try {
|
|
172
|
+
const run = store.getValidationRun(validationId);
|
|
173
|
+
if (!run)
|
|
174
|
+
return;
|
|
175
|
+
if (!principalCanAccess(run.ownerPrincipal, resolveOwnerPrincipal(getRequestContext())))
|
|
176
|
+
return;
|
|
177
|
+
store.setValidationJudgeLink(validationId, {
|
|
178
|
+
provider: String(provider),
|
|
179
|
+
jobId: snapshot.id,
|
|
180
|
+
correlationId: snapshot.correlationId,
|
|
181
|
+
});
|
|
182
|
+
}
|
|
183
|
+
catch {
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
function persistValidationRun(deps, args) {
|
|
187
|
+
const store = deps.validationRunStore;
|
|
188
|
+
if (!store)
|
|
189
|
+
return;
|
|
190
|
+
try {
|
|
191
|
+
const providerLinks = args.results
|
|
192
|
+
.filter(result => result.rawJobReference !== null)
|
|
193
|
+
.map(result => ({
|
|
194
|
+
provider: String(result.provider),
|
|
195
|
+
jobId: result.rawJobReference.jobId,
|
|
196
|
+
correlationId: result.rawJobReference.correlationId,
|
|
197
|
+
}));
|
|
198
|
+
store.recordValidationRun({
|
|
199
|
+
validationId: args.validationId,
|
|
200
|
+
ownerPrincipal: resolveOwnerPrincipal(getRequestContext()),
|
|
201
|
+
intent: args.intent,
|
|
202
|
+
createdAt: args.startedAt,
|
|
203
|
+
requestJson: JSON.stringify({
|
|
204
|
+
question: args.input.question,
|
|
205
|
+
content: args.input.content,
|
|
206
|
+
focus: args.input.focus,
|
|
207
|
+
riskLevel: args.input.riskLevel,
|
|
208
|
+
modelList: args.providers,
|
|
209
|
+
judgeProvider: args.input.judgeProvider ?? null,
|
|
210
|
+
}),
|
|
211
|
+
providerLinks,
|
|
212
|
+
judgeLink: null,
|
|
213
|
+
status: "running",
|
|
214
|
+
});
|
|
215
|
+
}
|
|
216
|
+
catch {
|
|
217
|
+
}
|
|
218
|
+
}
|
|
153
219
|
function buildProviderArgs(provider, prompt) {
|
|
154
220
|
if (provider === "claude" || provider === "grok" || provider === "mistral") {
|
|
155
221
|
return ["-p", prompt];
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import type { AsyncJobManager } from "./async-job-manager.js";
|
|
2
|
+
import type { ValidationRunRecord, ValidationRunStore } from "./job-store.js";
|
|
3
|
+
import { type ValidationReport } from "./validation-report.js";
|
|
4
|
+
export declare const VALIDATION_RECEIPT_SCHEMA_VERSION = "validation-receipt.v1";
|
|
5
|
+
export interface ReceiptDeps {
|
|
6
|
+
asyncJobManager: AsyncJobManager;
|
|
7
|
+
validationRunStore?: ValidationRunStore | null;
|
|
8
|
+
}
|
|
9
|
+
export type ValidationReportV1Content = ValidationReport["structuredContent"];
|
|
10
|
+
export interface ValidationReceipt {
|
|
11
|
+
schemaVersion: string;
|
|
12
|
+
validationId: string;
|
|
13
|
+
ownerPrincipal: string;
|
|
14
|
+
mintedAt: string;
|
|
15
|
+
intent: string;
|
|
16
|
+
models: string[];
|
|
17
|
+
report: ValidationReportV1Content;
|
|
18
|
+
humanReadable: string;
|
|
19
|
+
canonicalSha256: string;
|
|
20
|
+
prevSha256?: string | null;
|
|
21
|
+
seq?: number | null;
|
|
22
|
+
signature?: string | null;
|
|
23
|
+
}
|
|
24
|
+
export interface ValidationRunState {
|
|
25
|
+
validationId: string;
|
|
26
|
+
status: ValidationRunRecord["status"];
|
|
27
|
+
providers: Array<{
|
|
28
|
+
provider: string;
|
|
29
|
+
jobId: string;
|
|
30
|
+
status: string;
|
|
31
|
+
}>;
|
|
32
|
+
judge: {
|
|
33
|
+
provider: string;
|
|
34
|
+
jobId: string;
|
|
35
|
+
status: string;
|
|
36
|
+
} | null;
|
|
37
|
+
}
|
|
38
|
+
export interface RawResponse {
|
|
39
|
+
provider: string;
|
|
40
|
+
jobId: string;
|
|
41
|
+
text: string;
|
|
42
|
+
}
|
|
43
|
+
export type ValidationReceiptResult = {
|
|
44
|
+
status: "minted";
|
|
45
|
+
validationId: string;
|
|
46
|
+
receipt: ValidationReceipt;
|
|
47
|
+
mintedAt: string;
|
|
48
|
+
rawResponses?: RawResponse[];
|
|
49
|
+
} | {
|
|
50
|
+
status: "pending";
|
|
51
|
+
validationId: string;
|
|
52
|
+
run: ValidationRunState;
|
|
53
|
+
} | {
|
|
54
|
+
status: "expired_unminted";
|
|
55
|
+
validationId: string;
|
|
56
|
+
} | {
|
|
57
|
+
status: "not_found";
|
|
58
|
+
validationId: string;
|
|
59
|
+
};
|
|
60
|
+
export declare function canonicalJson(value: unknown): string;
|
|
61
|
+
export declare function computeCanonicalSha256(structuredContent: ValidationReportV1Content): string;
|
|
62
|
+
export declare function resolveValidationReceipt(deps: ReceiptDeps, validationId: string, opts: {
|
|
63
|
+
caller: string;
|
|
64
|
+
includeRawResponses?: boolean;
|
|
65
|
+
}): ValidationReceiptResult;
|
|
66
|
+
export declare function eagerMintFromValidationId(deps: ReceiptDeps, validationId: string): void;
|
|
67
|
+
export declare function eagerMintFromJobId(deps: ReceiptDeps, jobId: string): void;
|
|
68
|
+
export declare function currentCaller(): string;
|
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { normalizeJobResult, normalizeSkippedProvider } from "./validation-normalizer.js";
|
|
3
|
+
import { buildValidationReport, deriveValidationRunStatus, renderHumanReport, } from "./validation-report.js";
|
|
4
|
+
import { getRequestContext, principalCanAccess, resolveOwnerPrincipal } from "./request-context.js";
|
|
5
|
+
export const VALIDATION_RECEIPT_SCHEMA_VERSION = "validation-receipt.v1";
|
|
6
|
+
const TERMINAL_JOB_STATUSES = new Set(["completed", "failed", "canceled", "orphaned"]);
|
|
7
|
+
export function canonicalJson(value) {
|
|
8
|
+
return JSON.stringify(sortDeep(value));
|
|
9
|
+
}
|
|
10
|
+
function sortDeep(value) {
|
|
11
|
+
if (Array.isArray(value))
|
|
12
|
+
return value.map(sortDeep);
|
|
13
|
+
if (value && typeof value === "object") {
|
|
14
|
+
const out = {};
|
|
15
|
+
for (const key of Object.keys(value).sort()) {
|
|
16
|
+
out[key] = sortDeep(value[key]);
|
|
17
|
+
}
|
|
18
|
+
return out;
|
|
19
|
+
}
|
|
20
|
+
return value;
|
|
21
|
+
}
|
|
22
|
+
export function computeCanonicalSha256(structuredContent) {
|
|
23
|
+
return createHash("sha256").update(canonicalJson(structuredContent), "utf8").digest("hex");
|
|
24
|
+
}
|
|
25
|
+
function isTerminal(status) {
|
|
26
|
+
return TERMINAL_JOB_STATUSES.has(status);
|
|
27
|
+
}
|
|
28
|
+
function parseRequest(requestJson) {
|
|
29
|
+
try {
|
|
30
|
+
const parsed = JSON.parse(requestJson);
|
|
31
|
+
return parsed && typeof parsed === "object" ? parsed : {};
|
|
32
|
+
}
|
|
33
|
+
catch {
|
|
34
|
+
return {};
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
function tryMint(deps, run) {
|
|
38
|
+
const store = deps.validationRunStore;
|
|
39
|
+
if (!store)
|
|
40
|
+
return { kind: "pending" };
|
|
41
|
+
const providerResults = [];
|
|
42
|
+
for (const link of run.providerLinks) {
|
|
43
|
+
const jobResult = deps.asyncJobManager.getJobResult(link.jobId);
|
|
44
|
+
if (!jobResult)
|
|
45
|
+
return { kind: "evicted" };
|
|
46
|
+
if (!isTerminal(jobResult.status))
|
|
47
|
+
return { kind: "pending" };
|
|
48
|
+
providerResults.push({ link, result: normalizeJobResult(link.provider, null, jobResult) });
|
|
49
|
+
}
|
|
50
|
+
let judgeStatus = null;
|
|
51
|
+
if (run.judgeLink) {
|
|
52
|
+
const judgeResult = deps.asyncJobManager.getJobResult(run.judgeLink.jobId);
|
|
53
|
+
if (!judgeResult)
|
|
54
|
+
return { kind: "evicted" };
|
|
55
|
+
if (!isTerminal(judgeResult.status))
|
|
56
|
+
return { kind: "pending" };
|
|
57
|
+
judgeStatus = judgeResult.status;
|
|
58
|
+
}
|
|
59
|
+
const request = parseRequest(run.requestJson);
|
|
60
|
+
const requested = Array.isArray(request.modelList) ? request.modelList : [];
|
|
61
|
+
const dispatched = new Set(run.providerLinks.map(link => link.provider));
|
|
62
|
+
const results = providerResults.map(entry => entry.result);
|
|
63
|
+
for (const provider of requested) {
|
|
64
|
+
if (!dispatched.has(provider)) {
|
|
65
|
+
results.push(normalizeSkippedProvider(provider, "Provider was not dispatched for this run."));
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
const judgeCompleted = judgeStatus === "completed";
|
|
69
|
+
const synthesis = run.judgeLink
|
|
70
|
+
? {
|
|
71
|
+
status: judgeCompleted ? "completed" : "skipped",
|
|
72
|
+
judgeModel: run.judgeLink.provider,
|
|
73
|
+
rawJobReference: {
|
|
74
|
+
jobId: run.judgeLink.jobId,
|
|
75
|
+
correlationId: run.judgeLink.correlationId,
|
|
76
|
+
statusTool: "job_status",
|
|
77
|
+
resultTool: "job_result",
|
|
78
|
+
},
|
|
79
|
+
note: judgeCompleted
|
|
80
|
+
? "Judge synthesis completed."
|
|
81
|
+
: `Judge job ended in '${judgeStatus}' without a synthesis result.`,
|
|
82
|
+
}
|
|
83
|
+
: {
|
|
84
|
+
status: "not_requested",
|
|
85
|
+
judgeModel: null,
|
|
86
|
+
rawJobReference: null,
|
|
87
|
+
note: "No judge synthesis was requested.",
|
|
88
|
+
};
|
|
89
|
+
const modelList = (requested.length > 0 ? requested : Array.from(dispatched));
|
|
90
|
+
const report = buildValidationReport({
|
|
91
|
+
validationId: run.validationId,
|
|
92
|
+
status: deriveValidationRunStatus(results, synthesis.status),
|
|
93
|
+
startedAt: run.createdAt,
|
|
94
|
+
intent: run.intent,
|
|
95
|
+
originalRequest: {
|
|
96
|
+
question: request.question,
|
|
97
|
+
content: request.content,
|
|
98
|
+
focus: request.focus,
|
|
99
|
+
},
|
|
100
|
+
modelList,
|
|
101
|
+
results,
|
|
102
|
+
synthesis,
|
|
103
|
+
});
|
|
104
|
+
const structuredContent = report.structuredContent;
|
|
105
|
+
const record = {
|
|
106
|
+
validationId: run.validationId,
|
|
107
|
+
ownerPrincipal: run.ownerPrincipal,
|
|
108
|
+
mintedAt: new Date().toISOString(),
|
|
109
|
+
schemaVersion: VALIDATION_RECEIPT_SCHEMA_VERSION,
|
|
110
|
+
reportJson: JSON.stringify(structuredContent),
|
|
111
|
+
canonicalSha256: computeCanonicalSha256(structuredContent),
|
|
112
|
+
prevSha256: null,
|
|
113
|
+
seq: null,
|
|
114
|
+
signature: null,
|
|
115
|
+
models: structuredContent.modelList,
|
|
116
|
+
hasMaterialDisagreement: structuredContent.disagreements.hasMaterialDisagreement,
|
|
117
|
+
confidence: structuredContent.confidence,
|
|
118
|
+
};
|
|
119
|
+
store.recordValidationReceipt(record);
|
|
120
|
+
store.setValidationRunStatus(run.validationId, "finalized");
|
|
121
|
+
const stored = store.getValidationReceipt(run.validationId);
|
|
122
|
+
return { kind: "minted", record: stored ?? record };
|
|
123
|
+
}
|
|
124
|
+
function receiptEnvelope(record) {
|
|
125
|
+
const report = JSON.parse(record.reportJson);
|
|
126
|
+
return {
|
|
127
|
+
schemaVersion: record.schemaVersion,
|
|
128
|
+
validationId: record.validationId,
|
|
129
|
+
ownerPrincipal: record.ownerPrincipal,
|
|
130
|
+
mintedAt: record.mintedAt,
|
|
131
|
+
intent: report.intent,
|
|
132
|
+
models: record.models,
|
|
133
|
+
report,
|
|
134
|
+
humanReadable: renderHumanReport(report),
|
|
135
|
+
canonicalSha256: record.canonicalSha256,
|
|
136
|
+
prevSha256: record.prevSha256,
|
|
137
|
+
seq: record.seq,
|
|
138
|
+
signature: record.signature,
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
function collectRawResponses(deps, receipt, caller) {
|
|
142
|
+
const out = [];
|
|
143
|
+
const refs = [];
|
|
144
|
+
for (const output of receipt.report.perModelOutputs) {
|
|
145
|
+
if (output.jobId)
|
|
146
|
+
refs.push({ provider: output.provider, jobId: output.jobId });
|
|
147
|
+
}
|
|
148
|
+
const judgeRef = receipt.report.synthesis.rawJobReference;
|
|
149
|
+
if (judgeRef?.jobId) {
|
|
150
|
+
refs.push({ provider: receipt.report.synthesis.judgeModel ?? "judge", jobId: judgeRef.jobId });
|
|
151
|
+
}
|
|
152
|
+
for (const ref of refs) {
|
|
153
|
+
if (!principalCanAccess(deps.asyncJobManager.getJobOwner(ref.jobId), caller))
|
|
154
|
+
continue;
|
|
155
|
+
const jobResult = deps.asyncJobManager.getJobResult(ref.jobId);
|
|
156
|
+
if (jobResult)
|
|
157
|
+
out.push({ provider: ref.provider, jobId: ref.jobId, text: jobResult.stdout });
|
|
158
|
+
}
|
|
159
|
+
return out;
|
|
160
|
+
}
|
|
161
|
+
function mintedResult(deps, record, includeRawResponses, caller) {
|
|
162
|
+
const receipt = receiptEnvelope(record);
|
|
163
|
+
return {
|
|
164
|
+
status: "minted",
|
|
165
|
+
validationId: record.validationId,
|
|
166
|
+
receipt,
|
|
167
|
+
mintedAt: record.mintedAt,
|
|
168
|
+
...(includeRawResponses ? { rawResponses: collectRawResponses(deps, receipt, caller) } : {}),
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
function runStateOf(deps, run) {
|
|
172
|
+
const snapStatus = (jobId) => deps.asyncJobManager.getJobSnapshot(jobId)?.status ?? "evicted";
|
|
173
|
+
return {
|
|
174
|
+
validationId: run.validationId,
|
|
175
|
+
status: run.status,
|
|
176
|
+
providers: run.providerLinks.map(link => ({
|
|
177
|
+
provider: link.provider,
|
|
178
|
+
jobId: link.jobId,
|
|
179
|
+
status: snapStatus(link.jobId),
|
|
180
|
+
})),
|
|
181
|
+
judge: run.judgeLink
|
|
182
|
+
? {
|
|
183
|
+
provider: run.judgeLink.provider,
|
|
184
|
+
jobId: run.judgeLink.jobId,
|
|
185
|
+
status: snapStatus(run.judgeLink.jobId),
|
|
186
|
+
}
|
|
187
|
+
: null,
|
|
188
|
+
};
|
|
189
|
+
}
|
|
190
|
+
export function resolveValidationReceipt(deps, validationId, opts) {
|
|
191
|
+
const store = deps.validationRunStore;
|
|
192
|
+
if (!store)
|
|
193
|
+
return { status: "not_found", validationId };
|
|
194
|
+
const existing = store.getValidationReceipt(validationId);
|
|
195
|
+
if (existing) {
|
|
196
|
+
if (!principalCanAccess(existing.ownerPrincipal, opts.caller)) {
|
|
197
|
+
return { status: "not_found", validationId };
|
|
198
|
+
}
|
|
199
|
+
return mintedResult(deps, existing, opts.includeRawResponses ?? false, opts.caller);
|
|
200
|
+
}
|
|
201
|
+
const run = store.getValidationRun(validationId);
|
|
202
|
+
if (!run || !principalCanAccess(run.ownerPrincipal, opts.caller)) {
|
|
203
|
+
return { status: "not_found", validationId };
|
|
204
|
+
}
|
|
205
|
+
const outcome = tryMint(deps, run);
|
|
206
|
+
if (outcome.kind === "minted") {
|
|
207
|
+
return mintedResult(deps, outcome.record, opts.includeRawResponses ?? false, opts.caller);
|
|
208
|
+
}
|
|
209
|
+
if (outcome.kind === "evicted") {
|
|
210
|
+
return { status: "expired_unminted", validationId };
|
|
211
|
+
}
|
|
212
|
+
return { status: "pending", validationId, run: runStateOf(deps, run) };
|
|
213
|
+
}
|
|
214
|
+
export function eagerMintFromValidationId(deps, validationId) {
|
|
215
|
+
const store = deps.validationRunStore;
|
|
216
|
+
if (!store)
|
|
217
|
+
return;
|
|
218
|
+
try {
|
|
219
|
+
if (store.getValidationReceipt(validationId))
|
|
220
|
+
return;
|
|
221
|
+
const run = store.getValidationRun(validationId);
|
|
222
|
+
if (!run)
|
|
223
|
+
return;
|
|
224
|
+
tryMint(deps, run);
|
|
225
|
+
}
|
|
226
|
+
catch {
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
export function eagerMintFromJobId(deps, jobId) {
|
|
230
|
+
const store = deps.validationRunStore;
|
|
231
|
+
if (!store)
|
|
232
|
+
return;
|
|
233
|
+
let validationId = null;
|
|
234
|
+
try {
|
|
235
|
+
validationId = store.getValidationRunIdByJobId(jobId);
|
|
236
|
+
}
|
|
237
|
+
catch {
|
|
238
|
+
return;
|
|
239
|
+
}
|
|
240
|
+
if (validationId)
|
|
241
|
+
eagerMintFromValidationId(deps, validationId);
|
|
242
|
+
}
|
|
243
|
+
export function currentCaller() {
|
|
244
|
+
return resolveOwnerPrincipal(getRequestContext());
|
|
245
|
+
}
|
|
@@ -3,7 +3,7 @@ import type { ValidationIntent } from "./validation-prompts.js";
|
|
|
3
3
|
export type ValidationReportConfidence = "none" | "low" | "medium" | "high";
|
|
4
4
|
export interface ValidationReportInput {
|
|
5
5
|
validationId: string;
|
|
6
|
-
status: "running" | "partial" | "not_started";
|
|
6
|
+
status: "running" | "partial" | "not_started" | "completed";
|
|
7
7
|
startedAt: string;
|
|
8
8
|
intent: ValidationIntent;
|
|
9
9
|
originalRequest: {
|
|
@@ -14,7 +14,7 @@ export interface ValidationReportInput {
|
|
|
14
14
|
modelList: ValidationProvider[];
|
|
15
15
|
results: NormalizedValidationResult[];
|
|
16
16
|
synthesis: {
|
|
17
|
-
status: "not_requested" | "waiting_for_provider_results" | "running" | "skipped";
|
|
17
|
+
status: "not_requested" | "waiting_for_provider_results" | "running" | "skipped" | "completed";
|
|
18
18
|
judgeModel: ValidationProvider | null;
|
|
19
19
|
rawJobReference: NormalizedValidationResult["rawJobReference"];
|
|
20
20
|
note: string;
|
|
@@ -55,3 +55,5 @@ export interface ValidationReport {
|
|
|
55
55
|
};
|
|
56
56
|
}
|
|
57
57
|
export declare function buildValidationReport(input: ValidationReportInput): ValidationReport;
|
|
58
|
+
export declare function deriveValidationRunStatus(results: NormalizedValidationResult[], synthesisStatus: ValidationReportInput["synthesis"]["status"]): ValidationReportInput["status"];
|
|
59
|
+
export declare function renderHumanReport(content: ValidationReport["structuredContent"]): string;
|
|
@@ -37,6 +37,23 @@ export function buildValidationReport(input) {
|
|
|
37
37
|
structuredContent,
|
|
38
38
|
};
|
|
39
39
|
}
|
|
40
|
+
export function deriveValidationRunStatus(results, synthesisStatus) {
|
|
41
|
+
if (results.length === 0)
|
|
42
|
+
return "not_started";
|
|
43
|
+
const dispatched = results.filter(result => result.status !== "skipped");
|
|
44
|
+
if (dispatched.length === 0)
|
|
45
|
+
return "not_started";
|
|
46
|
+
const allDispatchedTerminal = dispatched.every(result => result.status === "completed" ||
|
|
47
|
+
result.status === "failed" ||
|
|
48
|
+
result.status === "canceled" ||
|
|
49
|
+
result.status === "orphaned");
|
|
50
|
+
const hasSkipped = results.some(result => result.status === "skipped");
|
|
51
|
+
const judgePending = synthesisStatus === "running" || synthesisStatus === "waiting_for_provider_results";
|
|
52
|
+
if (!allDispatchedTerminal || judgePending) {
|
|
53
|
+
return hasSkipped ? "partial" : "running";
|
|
54
|
+
}
|
|
55
|
+
return "completed";
|
|
56
|
+
}
|
|
40
57
|
function summarizeDisagreement(results) {
|
|
41
58
|
const completed = results.filter(result => result.status === "completed");
|
|
42
59
|
const terminalProblems = results.filter(result => ["failed", "canceled", "orphaned", "skipped"].includes(result.status));
|
|
@@ -102,7 +119,7 @@ function recommendationFor(results, hasMaterialDisagreement) {
|
|
|
102
119
|
}
|
|
103
120
|
return "Completed provider outputs show no normalized verdict disagreement; review rationales and risks before acting.";
|
|
104
121
|
}
|
|
105
|
-
function renderHumanReport(content) {
|
|
122
|
+
export function renderHumanReport(content) {
|
|
106
123
|
const lines = [
|
|
107
124
|
`Validation report ${content.validationId}`,
|
|
108
125
|
`Status: ${content.status}`,
|
package/dist/validation-tools.js
CHANGED
|
@@ -2,6 +2,8 @@ import { z } from "zod/v3";
|
|
|
2
2
|
import { CLI_TYPES } from "./session-manager.js";
|
|
3
3
|
import { getAvailableCliInfo } from "./model-registry.js";
|
|
4
4
|
import { apiProviderCatalogEntry } from "./api-request.js";
|
|
5
|
+
import { getRequestContext, principalCanAccess, resolveOwnerPrincipal } from "./request-context.js";
|
|
6
|
+
import { currentCaller, eagerMintFromJobId, eagerMintFromValidationId, resolveValidationReceipt, } from "./validation-receipt.js";
|
|
5
7
|
import { collectValidationJobResult, startJudgeSynthesis, startValidationRun, } from "./validation-orchestrator.js";
|
|
6
8
|
export function buildValidationSchemas(deps) {
|
|
7
9
|
const apiNames = (deps.apiProviders ?? []).map(p => p.name);
|
|
@@ -194,22 +196,32 @@ export function registerValidationTools(server, deps) {
|
|
|
194
196
|
.min(1)
|
|
195
197
|
.describe("Terminal normalized provider results from job_result."),
|
|
196
198
|
judgeModel: providerSchema.default("codex").describe("Provider to run the judge synthesis."),
|
|
199
|
+
validationId: z
|
|
200
|
+
.string()
|
|
201
|
+
.optional()
|
|
202
|
+
.describe("Optional run id (from the kickoff response) to link this judge job back into the durable validation run."),
|
|
197
203
|
}, {
|
|
198
204
|
title: "Synthesize validation",
|
|
199
205
|
readOnlyHint: false,
|
|
200
206
|
destructiveHint: true,
|
|
201
207
|
idempotentHint: false,
|
|
202
208
|
openWorldHint: true,
|
|
203
|
-
}, async ({ question, providerResults, judgeModel }) =>
|
|
204
|
-
|
|
205
|
-
tool: "synthesize_validation",
|
|
206
|
-
readMostly: true,
|
|
207
|
-
synthesis: startJudgeSynthesis(deps, {
|
|
209
|
+
}, async ({ question, providerResults, judgeModel, validationId }) => {
|
|
210
|
+
const synthesis = startJudgeSynthesis(deps, {
|
|
208
211
|
question,
|
|
209
212
|
providerResults,
|
|
210
213
|
judgeProvider: judgeModel,
|
|
211
|
-
|
|
212
|
-
|
|
214
|
+
validationId,
|
|
215
|
+
});
|
|
216
|
+
if (validationId)
|
|
217
|
+
eagerMintFromValidationId(deps, validationId);
|
|
218
|
+
return textResponse({
|
|
219
|
+
success: true,
|
|
220
|
+
tool: "synthesize_validation",
|
|
221
|
+
readMostly: true,
|
|
222
|
+
synthesis,
|
|
223
|
+
});
|
|
224
|
+
});
|
|
213
225
|
server.tool("list_available_models", "List models and capabilities for every available provider CLI (takes no arguments; complements per-provider list_models).", {}, {
|
|
214
226
|
title: "All provider models",
|
|
215
227
|
readOnlyHint: true,
|
|
@@ -234,7 +246,8 @@ export function registerValidationTools(server, deps) {
|
|
|
234
246
|
openWorldHint: false,
|
|
235
247
|
}, async ({ jobId }) => {
|
|
236
248
|
const job = deps.asyncJobManager.getJobSnapshot(jobId);
|
|
237
|
-
|
|
249
|
+
const caller = resolveOwnerPrincipal(getRequestContext());
|
|
250
|
+
if (!job || !principalCanAccess(deps.asyncJobManager.getJobOwner(jobId), caller)) {
|
|
238
251
|
return textResponse({ success: false, error: "Job not found", jobId });
|
|
239
252
|
}
|
|
240
253
|
return textResponse({ success: true, job });
|
|
@@ -259,9 +272,11 @@ export function registerValidationTools(server, deps) {
|
|
|
259
272
|
openWorldHint: false,
|
|
260
273
|
}, async ({ jobId, provider, maxChars }) => {
|
|
261
274
|
const result = deps.asyncJobManager.getJobResult(jobId, maxChars);
|
|
262
|
-
|
|
275
|
+
const caller = resolveOwnerPrincipal(getRequestContext());
|
|
276
|
+
if (!result || !principalCanAccess(deps.asyncJobManager.getJobOwner(jobId), caller)) {
|
|
263
277
|
return textResponse({ success: false, error: "Job not found", jobId });
|
|
264
278
|
}
|
|
279
|
+
eagerMintFromJobId(deps, jobId);
|
|
265
280
|
return textResponse({
|
|
266
281
|
success: true,
|
|
267
282
|
result,
|
|
@@ -270,4 +285,38 @@ export function registerValidationTools(server, deps) {
|
|
|
270
285
|
: null,
|
|
271
286
|
});
|
|
272
287
|
});
|
|
288
|
+
if (deps.validationRunStore) {
|
|
289
|
+
server.tool("validation_receipt", "Retrieve the immutable receipt of a terminal cross-LLM validation run by validationId. Returns minted | pending | expired_unminted | not_found (own-or-not-found).", {
|
|
290
|
+
validationId: z
|
|
291
|
+
.string()
|
|
292
|
+
.min(1)
|
|
293
|
+
.describe("The run-level validationId from a validation kickoff response (not a job/correlation id)."),
|
|
294
|
+
format: z
|
|
295
|
+
.enum(["json", "markdown"])
|
|
296
|
+
.default("json")
|
|
297
|
+
.describe("Response format. markdown returns the human-readable rendering (derived on read, never stored or hashed)."),
|
|
298
|
+
includeRawResponses: z
|
|
299
|
+
.boolean()
|
|
300
|
+
.default(false)
|
|
301
|
+
.describe("Inline full provider answer text (read-time only; pulled live per job under the same owner check; never persisted or hashed)."),
|
|
302
|
+
}, {
|
|
303
|
+
title: "Validation receipt",
|
|
304
|
+
readOnlyHint: true,
|
|
305
|
+
destructiveHint: false,
|
|
306
|
+
idempotentHint: true,
|
|
307
|
+
openWorldHint: false,
|
|
308
|
+
}, async ({ validationId, format, includeRawResponses }) => {
|
|
309
|
+
const result = resolveValidationReceipt(deps, validationId, {
|
|
310
|
+
caller: currentCaller(),
|
|
311
|
+
includeRawResponses,
|
|
312
|
+
});
|
|
313
|
+
if (format === "markdown" && result.status === "minted") {
|
|
314
|
+
return {
|
|
315
|
+
content: [{ type: "text", text: result.receipt.humanReadable }],
|
|
316
|
+
structuredContent: result,
|
|
317
|
+
};
|
|
318
|
+
}
|
|
319
|
+
return textResponse(result);
|
|
320
|
+
});
|
|
321
|
+
}
|
|
273
322
|
}
|
package/npm-shrinkwrap.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "llm-cli-gateway",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.12.0",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "llm-cli-gateway",
|
|
9
|
-
"version": "2.
|
|
9
|
+
"version": "2.12.0",
|
|
10
10
|
"license": "MIT",
|
|
11
11
|
"dependencies": {
|
|
12
12
|
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
@@ -1066,9 +1066,9 @@
|
|
|
1066
1066
|
}
|
|
1067
1067
|
},
|
|
1068
1068
|
"node_modules/smol-toml": {
|
|
1069
|
-
"version": "1.
|
|
1070
|
-
"resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.
|
|
1071
|
-
"integrity": "sha512-
|
|
1069
|
+
"version": "1.7.0",
|
|
1070
|
+
"resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.7.0.tgz",
|
|
1071
|
+
"integrity": "sha512-aqVvWoyO21L23mb+drl4RmMXbf6N7FdHjAhTRA9ZBL7apWBgfWC16KjrASI+1p9GAroljyMHj6fK67i0UiTNvQ==",
|
|
1072
1072
|
"license": "BSD-3-Clause",
|
|
1073
1073
|
"engines": {
|
|
1074
1074
|
"node": ">= 18"
|
package/package.json
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "llm-cli-gateway",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.12.0",
|
|
4
4
|
"mcpName": "io.github.verivus-oss/llm-cli-gateway",
|
|
5
|
-
"description": "MCP server providing unified access to Claude Code, Codex, Gemini, Grok,
|
|
5
|
+
"description": "MCP server providing unified access to Claude Code, Codex, Gemini, Grok, Mistral Vibe, and Devin CLIs with session management, retry logic, async job orchestration, durable job results, and cross-LLM validation.",
|
|
6
6
|
"license": "MIT",
|
|
7
7
|
"author": "VerivusAI Labs (https://github.com/verivus-oss)",
|
|
8
8
|
"repository": {
|
|
@@ -52,7 +52,13 @@
|
|
|
52
52
|
"README.md",
|
|
53
53
|
"CHANGELOG.md",
|
|
54
54
|
"socket.yml",
|
|
55
|
-
"LICENSE"
|
|
55
|
+
"LICENSE",
|
|
56
|
+
".agents/skills/async-job-orchestration/SKILL.md",
|
|
57
|
+
".agents/skills/multi-llm-review/SKILL.md",
|
|
58
|
+
".agents/skills/session-workflow/SKILL.md",
|
|
59
|
+
".agents/skills/secure-orchestration/SKILL.md",
|
|
60
|
+
".agents/skills/implement-review-fix/SKILL.md",
|
|
61
|
+
".agents/skills/public-demo-session/SKILL.md"
|
|
56
62
|
],
|
|
57
63
|
"scripts": {
|
|
58
64
|
"build": "tsc -p tsconfig.build.json",
|
|
@@ -105,7 +111,7 @@
|
|
|
105
111
|
"devDependencies": {
|
|
106
112
|
"@eslint/js": "^10.0.1",
|
|
107
113
|
"@types/better-sqlite3": "^7.6.0",
|
|
108
|
-
"@types/node": "^
|
|
114
|
+
"@types/node": "^26.0.0",
|
|
109
115
|
"@types/pg": "^8.11.10",
|
|
110
116
|
"@typescript-eslint/eslint-plugin": "^8.61.1",
|
|
111
117
|
"@typescript-eslint/parser": "^8.59.4",
|