melchizedek-agents 0.17.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +23 -3
- package/AGENT_SETUP.md +2 -1
- package/CHANGELOG.md +104 -0
- package/DOCUMENTATION.md +75 -9
- package/config/agents/syndicateSchema.yaml +13 -0
- package/db/hardening.sql +4 -3
- package/db/migrations/0012_audit_log.sql +98 -0
- package/dist/lib/a2a/app.d.ts +20 -0
- package/dist/lib/a2a/app.js +94 -4
- package/dist/lib/a2a/executor.d.ts +34 -4
- package/dist/lib/a2a/executor.js +68 -9
- package/dist/lib/compile.d.ts +1 -2
- package/dist/lib/compile.js +33 -6
- package/dist/lib/config.d.ts +7 -0
- package/dist/lib/config.js +7 -0
- package/dist/lib/loadEnv.d.ts +5 -0
- package/dist/lib/loadEnv.js +7 -0
- package/dist/lib/loadSyndicate.d.ts +4 -0
- package/dist/lib/memory/providers.d.ts +27 -0
- package/dist/lib/memory/providers.js +28 -0
- package/dist/lib/models/fallback.d.ts +41 -0
- package/dist/lib/models/fallback.js +105 -0
- package/dist/lib/observability/audit.d.ts +21 -0
- package/dist/lib/observability/audit.js +39 -0
- package/dist/lib/observability/metrics.d.ts +6 -0
- package/dist/lib/observability/tracer.d.ts +13 -0
- package/dist/lib/observability/tracer.js +16 -2
- package/dist/lib/runtime/syndicateTurn.d.ts +3 -1
- package/dist/lib/runtime/syndicateTurn.js +3 -1
- package/dist/lib/storage/postgres/index.d.ts +3 -0
- package/dist/lib/storage/postgres/index.js +2 -0
- package/dist/lib/syndicateSchema.d.ts +8 -0
- package/dist/lib/syndicateSchema.js +19 -4
- package/dist/lib/tools/mcpToolFactory.d.ts +26 -0
- package/dist/lib/tools/mcpToolFactory.js +61 -4
- package/dist/lib/tools/openapiTools.d.ts +6 -0
- package/dist/lib/tools/openapiTools.js +31 -1
- package/dist/scripts/a2a_server.js +51 -8
- package/dist/scripts/db.d.ts +9 -0
- package/dist/scripts/db.js +30 -15
- package/package.json +3 -3
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* lib/models/fallback.ts — a fallback model and a per-provider circuit
|
|
3
|
+
* breaker for an agent that declares `fallback_model:` (ADR 0044).
|
|
4
|
+
*
|
|
5
|
+
* WHY: lib/models/retry.ts makes one call survive a blip. It does nothing for
|
|
6
|
+
* an outage: every turn on a provider that is down spends its retries and
|
|
7
|
+
* fails. An agent that names a fallback model now answers from it instead,
|
|
8
|
+
* and a provider that keeps failing is skipped for a cooldown rather than
|
|
9
|
+
* tried on every turn.
|
|
10
|
+
*
|
|
11
|
+
* THE RULES:
|
|
12
|
+
* - Only provider-side failures count: what retry.ts classifies retryable
|
|
13
|
+
* (408/409/425/429/5xx, connection resets) once the adapter's own retries
|
|
14
|
+
* are spent. A 4xx is the request's fault and is thrown as it is; a
|
|
15
|
+
* canceled turn is never counted or redirected.
|
|
16
|
+
* - The fallback runs only if the primary produced nothing yet: a stream
|
|
17
|
+
* that failed midway is thrown, never replayed on another model.
|
|
18
|
+
* - The breaker is per provider, shared by every agent in the process:
|
|
19
|
+
* MODEL_BREAKER_THRESHOLD consecutive failures (default 5; 0 disables)
|
|
20
|
+
* open it for MODEL_BREAKER_COOLDOWN_MS (default 30 s). While open, a
|
|
21
|
+
* wrapped agent goes straight to its fallback; after the cooldown one
|
|
22
|
+
* call is let through, and a success closes the circuit.
|
|
23
|
+
*/
|
|
24
|
+
import { BaseLlm } from '@google/adk';
|
|
25
|
+
import type { BaseLlmConnection, LlmRequest, LlmResponse } from '@google/adk';
|
|
26
|
+
/** True while `provider`'s circuit is open (calls skip it). */
|
|
27
|
+
export declare function circuitOpen(provider: string, now?: number): boolean;
|
|
28
|
+
/** For tests: close every circuit. */
|
|
29
|
+
export declare function resetCircuits(): void;
|
|
30
|
+
/** A provider-side failure: worth counting against the provider and worth a fallback. */
|
|
31
|
+
export declare function isProviderFailure(err: unknown): boolean;
|
|
32
|
+
export declare class FallbackLlm extends BaseLlm {
|
|
33
|
+
readonly primary: BaseLlm;
|
|
34
|
+
readonly fallback: BaseLlm;
|
|
35
|
+
private readonly log;
|
|
36
|
+
private readonly primaryProvider;
|
|
37
|
+
constructor(primary: BaseLlm, fallback: BaseLlm, log?: (m: string) => void);
|
|
38
|
+
generateContentAsync(llmRequest: LlmRequest, stream?: boolean, abortSignal?: AbortSignal): AsyncGenerator<LlmResponse, void>;
|
|
39
|
+
private onFallback;
|
|
40
|
+
connect(llmRequest: LlmRequest): Promise<BaseLlmConnection>;
|
|
41
|
+
}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* lib/models/fallback.ts — a fallback model and a per-provider circuit
|
|
3
|
+
* breaker for an agent that declares `fallback_model:` (ADR 0044).
|
|
4
|
+
*
|
|
5
|
+
* WHY: lib/models/retry.ts makes one call survive a blip. It does nothing for
|
|
6
|
+
* an outage: every turn on a provider that is down spends its retries and
|
|
7
|
+
* fails. An agent that names a fallback model now answers from it instead,
|
|
8
|
+
* and a provider that keeps failing is skipped for a cooldown rather than
|
|
9
|
+
* tried on every turn.
|
|
10
|
+
*
|
|
11
|
+
* THE RULES:
|
|
12
|
+
* - Only provider-side failures count: what retry.ts classifies retryable
|
|
13
|
+
* (408/409/425/429/5xx, connection resets) once the adapter's own retries
|
|
14
|
+
* are spent. A 4xx is the request's fault and is thrown as it is; a
|
|
15
|
+
* canceled turn is never counted or redirected.
|
|
16
|
+
* - The fallback runs only if the primary produced nothing yet: a stream
|
|
17
|
+
* that failed midway is thrown, never replayed on another model.
|
|
18
|
+
* - The breaker is per provider, shared by every agent in the process:
|
|
19
|
+
* MODEL_BREAKER_THRESHOLD consecutive failures (default 5; 0 disables)
|
|
20
|
+
* open it for MODEL_BREAKER_COOLDOWN_MS (default 30 s). While open, a
|
|
21
|
+
* wrapped agent goes straight to its fallback; after the cooldown one
|
|
22
|
+
* call is let through, and a success closes the circuit.
|
|
23
|
+
*/
|
|
24
|
+
import { BaseLlm } from '@google/adk';
|
|
25
|
+
import { classifyError } from './retry.js';
|
|
26
|
+
import { providerForModel } from './providerMap.js';
|
|
27
|
+
const circuits = new Map();
|
|
28
|
+
function breakerSettings(env = process.env) {
|
|
29
|
+
const n = (name, fallback) => {
|
|
30
|
+
const v = Number(env[name]);
|
|
31
|
+
return Number.isFinite(v) && v >= 0 ? Math.floor(v) : fallback;
|
|
32
|
+
};
|
|
33
|
+
return { threshold: n('MODEL_BREAKER_THRESHOLD', 5), cooldownMs: n('MODEL_BREAKER_COOLDOWN_MS', 30_000) };
|
|
34
|
+
}
|
|
35
|
+
/** True while `provider`'s circuit is open (calls skip it). */
|
|
36
|
+
export function circuitOpen(provider, now = Date.now()) {
|
|
37
|
+
const c = circuits.get(provider);
|
|
38
|
+
return !!c && c.openUntil > now;
|
|
39
|
+
}
|
|
40
|
+
function recordFailure(provider, now = Date.now()) {
|
|
41
|
+
const { threshold, cooldownMs } = breakerSettings();
|
|
42
|
+
if (threshold === 0)
|
|
43
|
+
return;
|
|
44
|
+
const c = circuits.get(provider) ?? { failures: 0, openUntil: 0 };
|
|
45
|
+
c.failures += 1;
|
|
46
|
+
if (c.failures >= threshold)
|
|
47
|
+
c.openUntil = now + cooldownMs;
|
|
48
|
+
circuits.set(provider, c);
|
|
49
|
+
}
|
|
50
|
+
function recordSuccess(provider) {
|
|
51
|
+
circuits.delete(provider);
|
|
52
|
+
}
|
|
53
|
+
/** For tests: close every circuit. */
|
|
54
|
+
export function resetCircuits() {
|
|
55
|
+
circuits.clear();
|
|
56
|
+
}
|
|
57
|
+
/** A provider-side failure: worth counting against the provider and worth a fallback. */
|
|
58
|
+
export function isProviderFailure(err) {
|
|
59
|
+
if (err?.name === 'AbortError')
|
|
60
|
+
return false;
|
|
61
|
+
return classifyError(err).retryable;
|
|
62
|
+
}
|
|
63
|
+
export class FallbackLlm extends BaseLlm {
|
|
64
|
+
primary;
|
|
65
|
+
fallback;
|
|
66
|
+
log;
|
|
67
|
+
primaryProvider;
|
|
68
|
+
constructor(primary, fallback, log = (m) => console.warn(m)) {
|
|
69
|
+
super({ model: primary.model });
|
|
70
|
+
this.primary = primary;
|
|
71
|
+
this.fallback = fallback;
|
|
72
|
+
this.log = log;
|
|
73
|
+
this.primaryProvider = providerForModel(primary.model);
|
|
74
|
+
}
|
|
75
|
+
async *generateContentAsync(llmRequest, stream, abortSignal) {
|
|
76
|
+
if (circuitOpen(this.primaryProvider)) {
|
|
77
|
+
yield* this.onFallback(llmRequest, stream, abortSignal, `${this.primaryProvider} circuit is open`);
|
|
78
|
+
return;
|
|
79
|
+
}
|
|
80
|
+
let yielded = false;
|
|
81
|
+
try {
|
|
82
|
+
for await (const r of this.primary.generateContentAsync(llmRequest, stream, abortSignal)) {
|
|
83
|
+
yielded = true;
|
|
84
|
+
yield r;
|
|
85
|
+
}
|
|
86
|
+
recordSuccess(this.primaryProvider);
|
|
87
|
+
}
|
|
88
|
+
catch (err) {
|
|
89
|
+
if (abortSignal?.aborted || !isProviderFailure(err))
|
|
90
|
+
throw err;
|
|
91
|
+
recordFailure(this.primaryProvider);
|
|
92
|
+
if (yielded)
|
|
93
|
+
throw err;
|
|
94
|
+
yield* this.onFallback(llmRequest, stream, abortSignal, `${this.primary.model} failed (${err instanceof Error ? err.message.slice(0, 120) : String(err)})`);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
async *onFallback(llmRequest, stream, abortSignal, why) {
|
|
98
|
+
this.log(`[models] ${why}; answering from ${this.fallback.model}.`);
|
|
99
|
+
const request = { ...llmRequest, model: this.fallback.model };
|
|
100
|
+
yield* this.fallback.generateContentAsync(request, stream, abortSignal);
|
|
101
|
+
}
|
|
102
|
+
connect(llmRequest) {
|
|
103
|
+
return this.primary.connect(llmRequest);
|
|
104
|
+
}
|
|
105
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { Pool } from 'pg';
|
|
2
|
+
export type AuditEventName = 'auth.failure' | 'task.end' | 'memory.erase';
|
|
3
|
+
export interface AuditEvent {
|
|
4
|
+
event: AuditEventName;
|
|
5
|
+
/** denied | ok | the task's final status. */
|
|
6
|
+
outcome: string;
|
|
7
|
+
/** The authenticated caller's name ('shared-secret', a caller token's name, 'jwt'). */
|
|
8
|
+
caller?: string;
|
|
9
|
+
/** SHA-256 prefix of the scope key, never the key. */
|
|
10
|
+
scopeHash?: string;
|
|
11
|
+
sourceIp?: string;
|
|
12
|
+
agentId?: string;
|
|
13
|
+
taskId?: string;
|
|
14
|
+
/** Small, content-free facts: a reason code, counts, a path. */
|
|
15
|
+
detail?: Record<string, unknown>;
|
|
16
|
+
}
|
|
17
|
+
export type AuditSink = (event: AuditEvent) => void;
|
|
18
|
+
/** The scope's SHA-256 prefix, as the task log writes it. */
|
|
19
|
+
export declare function scopeHashOf(scopeKey: string | undefined): string | undefined;
|
|
20
|
+
/** Appends events to melchizedek_audit through the storage pool. */
|
|
21
|
+
export declare function postgresAuditSink(pool: Pick<Pool, 'query'>, warn?: (m: string) => void): AuditSink;
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* lib/observability/audit.ts — the audit trail: who did what, when, from
|
|
3
|
+
* where (ADR 0042).
|
|
4
|
+
*
|
|
5
|
+
* The ledger (adk_turns) records what a turn said. An auditor asks a
|
|
6
|
+
* different question: which caller, authenticated how, from which address,
|
|
7
|
+
* did what, and was it allowed. Those events are written to
|
|
8
|
+
* melchizedek_audit (db/migrations/0012), which a trigger keeps
|
|
9
|
+
* append-only. An event never carries conversation content or a scope
|
|
10
|
+
* key: a scope is a SHA-256 prefix, as in the task log.
|
|
11
|
+
*
|
|
12
|
+
* The sink is fire-and-forget: an audit write never delays or fails a
|
|
13
|
+
* request. A failed write is logged once per outage, and the event is
|
|
14
|
+
* written to stderr as one JSON line instead, so it is not lost silently.
|
|
15
|
+
*/
|
|
16
|
+
import { createHash } from 'node:crypto';
|
|
17
|
+
/** The scope's SHA-256 prefix, as the task log writes it. */
|
|
18
|
+
export function scopeHashOf(scopeKey) {
|
|
19
|
+
return scopeKey ? createHash('sha256').update(scopeKey).digest('hex').slice(0, 12) : undefined;
|
|
20
|
+
}
|
|
21
|
+
/** Appends events to melchizedek_audit through the storage pool. */
|
|
22
|
+
export function postgresAuditSink(pool, warn = (m) => console.warn(m)) {
|
|
23
|
+
let failing = false;
|
|
24
|
+
return (e) => {
|
|
25
|
+
pool
|
|
26
|
+
.query('INSERT INTO melchizedek_audit (event, outcome, caller, scope_hash, source_ip, agent_id, task_id, detail) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)', [e.event, e.outcome, e.caller ?? null, e.scopeHash ?? null, e.sourceIp ?? null, e.agentId ?? null, e.taskId ?? null, JSON.stringify(e.detail ?? {})])
|
|
27
|
+
.then(() => {
|
|
28
|
+
if (failing)
|
|
29
|
+
warn('[audit] writes to melchizedek_audit succeed again.');
|
|
30
|
+
failing = false;
|
|
31
|
+
})
|
|
32
|
+
.catch((err) => {
|
|
33
|
+
if (!failing)
|
|
34
|
+
warn(`[audit] could not write to melchizedek_audit (${err instanceof Error ? err.message : String(err)}); events go to stderr until it recovers.`);
|
|
35
|
+
failing = true;
|
|
36
|
+
console.error(JSON.stringify({ audit: { ...e, at: new Date().toISOString() } }));
|
|
37
|
+
});
|
|
38
|
+
};
|
|
39
|
+
}
|
|
@@ -10,6 +10,12 @@
|
|
|
10
10
|
*/
|
|
11
11
|
/** One finished (or refused) task, as the executor reports it. */
|
|
12
12
|
export interface TaskRecord {
|
|
13
|
+
/** The A2A task id: joins this record to the task store and the audit trail. */
|
|
14
|
+
taskId?: string;
|
|
15
|
+
/** The turn's trace id: joins this record to its spans and its ledger row. */
|
|
16
|
+
traceId?: string;
|
|
17
|
+
/** The trace id of the caller's `traceparent`, when it sent one: the turn's root span links to it. */
|
|
18
|
+
callerTraceId?: string;
|
|
13
19
|
/** The agent id the task was sent to; '' is the default syndicate. */
|
|
14
20
|
agentId: string;
|
|
15
21
|
syndicate: string;
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { SpanContext } from '@opentelemetry/api';
|
|
1
2
|
import type { ReadableSpan } from '@opentelemetry/sdk-trace-base';
|
|
2
3
|
import type { Event, LlmResponse } from '@google/adk';
|
|
3
4
|
/**
|
|
@@ -67,6 +68,14 @@ export interface TraceMetadata {
|
|
|
67
68
|
traceId: string;
|
|
68
69
|
spanId: string;
|
|
69
70
|
}) => void;
|
|
71
|
+
/**
|
|
72
|
+
* A caller's W3C `traceparent`. The turn's root span LINKS to the caller's
|
|
73
|
+
* span (and records `caller.trace_id`) instead of joining its trace: a turn's
|
|
74
|
+
* trace id keys its ledger attribution, its in-process stats and erasure,
|
|
75
|
+
* so it must stay unique to the turn and never be chosen by a caller.
|
|
76
|
+
* Ignored unless well formed (validTraceparent).
|
|
77
|
+
*/
|
|
78
|
+
traceparent?: string;
|
|
70
79
|
/**
|
|
71
80
|
* Identity of the turn. These are what make a stored turn joinable to
|
|
72
81
|
* its session row (`adk_sessions.id` = app:user:session) and to the ADK
|
|
@@ -103,6 +112,10 @@ export declare function serverToolEvents(ev: any): Array<{
|
|
|
103
112
|
name: 'ToolCall' | 'ToolResponse';
|
|
104
113
|
attributes: Record<string, string | boolean>;
|
|
105
114
|
}>;
|
|
115
|
+
/** A well-formed version-00 W3C traceparent, or undefined. Never throws. */
|
|
116
|
+
export declare function validTraceparent(value: unknown): string | undefined;
|
|
117
|
+
/** The caller's span a turn links to, from its `traceparent`; undefined when absent or malformed. */
|
|
118
|
+
export declare function callerSpanContext(traceparent: string | undefined): SpanContext | undefined;
|
|
106
119
|
export declare function traceAgentRun(stream: AsyncIterableIterator<Event>, metadata: TraceMetadata): AsyncGenerator<Event, void, void>;
|
|
107
120
|
export interface LlmCallMeta {
|
|
108
121
|
/** Provider id, e.g. 'anthropic', 'openai', 'xai', 'moonshot', 'ollama', 'gemini'. */
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { trace, context } from '@opentelemetry/api';
|
|
1
|
+
import { trace, context, defaultTextMapGetter, ROOT_CONTEXT } from '@opentelemetry/api';
|
|
2
2
|
import { createRequire } from 'node:module';
|
|
3
3
|
import { TELEMETRY_SCHEMA_VERSION, engineVersion } from './lineage.js';
|
|
4
4
|
import { FilteringSpanExporter, otlpContentMode } from './otlpFilter.js';
|
|
@@ -345,9 +345,23 @@ export function serverToolEvents(ev) {
|
|
|
345
345
|
}
|
|
346
346
|
return out;
|
|
347
347
|
}
|
|
348
|
+
const TRACEPARENT = /^00-(?!0{32})[0-9a-f]{32}-(?!0{16})[0-9a-f]{16}-[0-9a-f]{2}$/;
|
|
349
|
+
/** A well-formed version-00 W3C traceparent, or undefined. Never throws. */
|
|
350
|
+
export function validTraceparent(value) {
|
|
351
|
+
return typeof value === 'string' && TRACEPARENT.test(value.trim().toLowerCase()) ? value.trim().toLowerCase() : undefined;
|
|
352
|
+
}
|
|
353
|
+
/** The caller's span a turn links to, from its `traceparent`; undefined when absent or malformed. */
|
|
354
|
+
export function callerSpanContext(traceparent) {
|
|
355
|
+
const tp = validTraceparent(traceparent);
|
|
356
|
+
if (!tp)
|
|
357
|
+
return undefined;
|
|
358
|
+
const ctx = new core.W3CTraceContextPropagator().extract(ROOT_CONTEXT, { traceparent: tp }, defaultTextMapGetter);
|
|
359
|
+
return trace.getSpanContext(ctx);
|
|
360
|
+
}
|
|
348
361
|
export async function* traceAgentRun(stream, metadata) {
|
|
349
362
|
initializeTracing();
|
|
350
|
-
const
|
|
363
|
+
const caller = callerSpanContext(metadata.traceparent);
|
|
364
|
+
const span = tracer.startSpan(`Syndicate Execution: ${metadata.syndicateName}`, caller ? { links: [{ context: caller }], attributes: { 'caller.trace_id': caller.traceId } } : undefined);
|
|
351
365
|
turnContexts.set(span.spanContext().traceId, {
|
|
352
366
|
sessionId: metadata.sessionId,
|
|
353
367
|
userId: metadata.userId,
|
|
@@ -77,6 +77,8 @@ export interface TraceOptions {
|
|
|
77
77
|
traceId: string;
|
|
78
78
|
spanId: string;
|
|
79
79
|
}) => void;
|
|
80
|
+
/** A caller's W3C traceparent: the turn's root span links to it (never joins it). */
|
|
81
|
+
traceparent?: string;
|
|
80
82
|
}
|
|
81
83
|
export interface SyndicateTurnOptions {
|
|
82
84
|
config: SyndicateYamlConfig;
|
|
@@ -100,7 +102,7 @@ export interface SyndicateTurnOptions {
|
|
|
100
102
|
signal?: AbortSignal;
|
|
101
103
|
/** Wall-clock budget for the turn, in ms. Default: none. */
|
|
102
104
|
deadlineMs?: number;
|
|
103
|
-
/** Model-call ceiling for the whole turn. Default: the YAML's `max_steps
|
|
105
|
+
/** Model-call ceiling for the whole turn. Default: the YAML's `max_steps`, else DEFAULT_MAX_STEPS (50). */
|
|
104
106
|
maxLlmCalls?: number;
|
|
105
107
|
/** Ask adapters for token-by-token partial events (the REPL's live view). */
|
|
106
108
|
streaming?: boolean;
|
|
@@ -27,6 +27,7 @@
|
|
|
27
27
|
* request in flight, not just the loop around it.
|
|
28
28
|
*/
|
|
29
29
|
import { InMemorySessionService, ReflectAndRetryModelPlugin, ReflectAndRetryToolPlugin, Runner, StreamingMode, getFunctionCalls, getFunctionResponses } from '@google/adk';
|
|
30
|
+
import { DEFAULT_MAX_STEPS } from '../config.js';
|
|
30
31
|
import { agentGates, compileGraph, compileSubagent } from '../compile.js';
|
|
31
32
|
import { ROUTE_STEP_SUFFIX, compileWorkflow, describeInput, inputRequestFrom, isWorkflowSyndicate } from '../workflow.js';
|
|
32
33
|
export { describeInput } from '../workflow.js';
|
|
@@ -265,7 +266,7 @@ export async function runSyndicateTurn(opts) {
|
|
|
265
266
|
if (!opts.compile?.resolveModel)
|
|
266
267
|
registerAvailableProviders();
|
|
267
268
|
const control = createTurnControl({
|
|
268
|
-
maxLlmCalls: opts.maxLlmCalls ?? config.max_steps,
|
|
269
|
+
maxLlmCalls: opts.maxLlmCalls ?? config.max_steps ?? DEFAULT_MAX_STEPS,
|
|
269
270
|
deadlineMs: opts.deadlineMs,
|
|
270
271
|
signal: opts.signal,
|
|
271
272
|
});
|
|
@@ -408,6 +409,7 @@ async function runTurnInner(opts, control) {
|
|
|
408
409
|
configHash: trace.configHash,
|
|
409
410
|
attributes: trace.attributes,
|
|
410
411
|
onSpanStart: trace.onSpanStart,
|
|
412
|
+
traceparent: trace.traceparent,
|
|
411
413
|
onEnd: () => ({
|
|
412
414
|
'syndicate.relay_fallback': params.stage === 'delegate' && !!params.relayFallback?.(),
|
|
413
415
|
...(control.stopReason ? { 'syndicate.stop_reason': control.stopReason } : {}),
|
|
@@ -17,6 +17,7 @@ import type { Embedder, MemoryExtractor } from '../../memory/providers.ts';
|
|
|
17
17
|
import type { EraseCounts, EraseOptions } from '../../memory/erase.ts';
|
|
18
18
|
import { PostgresSessionService } from './sessionService.ts';
|
|
19
19
|
import { PostgresTaskStore } from './taskStore.ts';
|
|
20
|
+
import type { AuditSink } from '../../observability/audit.ts';
|
|
20
21
|
import type { RlsHardeningStatus } from '../rlsStatus.ts';
|
|
21
22
|
import type { TaskBackend } from '../../tools/taskTools.ts';
|
|
22
23
|
import type { TurnLock } from '../../a2a/turnLock.ts';
|
|
@@ -76,6 +77,8 @@ export interface PostgresStorage {
|
|
|
76
77
|
* tables (lib/storage/rlsStatus.ts). Never throws.
|
|
77
78
|
*/
|
|
78
79
|
rlsHardening: () => Promise<RlsHardeningStatus>;
|
|
80
|
+
/** Appends to melchizedek_audit (migration 0012, ADR 0042). */
|
|
81
|
+
audit: AuditSink;
|
|
79
82
|
/**
|
|
80
83
|
* One turn at a time per conversation, across every instance on this
|
|
81
84
|
* database: a session-level advisory lock on a dedicated connection, so a
|
|
@@ -20,6 +20,7 @@ import { PostgresTaskStore, reapExpiredTasks, renewTaskLeases } from './taskStor
|
|
|
20
20
|
import { postgresTaskBackend } from './taskQueue.js';
|
|
21
21
|
import { searchPathOption } from '../schema.js';
|
|
22
22
|
import { POSTGRES_RLS_QUERY, evaluatePostgresRls } from '../rlsStatus.js';
|
|
23
|
+
import { postgresAuditSink } from '../../observability/audit.js';
|
|
23
24
|
import { createHash, randomUUID } from 'node:crypto';
|
|
24
25
|
import { hostname } from 'node:os';
|
|
25
26
|
export { PostgresSessionService } from './sessionService.js';
|
|
@@ -122,6 +123,7 @@ export function postgresStorage(options) {
|
|
|
122
123
|
throw err;
|
|
123
124
|
}
|
|
124
125
|
},
|
|
126
|
+
audit: postgresAuditSink(pool),
|
|
125
127
|
async rlsHardening() {
|
|
126
128
|
try {
|
|
127
129
|
const r = await pool.query(POSTGRES_RLS_QUERY);
|
|
@@ -29,6 +29,7 @@ export declare const agentSchema: z.ZodObject<{
|
|
|
29
29
|
name: z.ZodString;
|
|
30
30
|
description: z.ZodOptional<z.ZodString>;
|
|
31
31
|
model: z.ZodString;
|
|
32
|
+
fallback_model: z.ZodOptional<z.ZodString>;
|
|
32
33
|
instruction: z.ZodString;
|
|
33
34
|
globalInstruction: z.ZodOptional<z.ZodString>;
|
|
34
35
|
tools: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
@@ -62,6 +63,7 @@ export declare const agentSchema: z.ZodObject<{
|
|
|
62
63
|
}, z.core.$loose>>;
|
|
63
64
|
outputSchema: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
64
65
|
mcp_server_url: z.ZodOptional<z.ZodString>;
|
|
66
|
+
mcp_tools: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
65
67
|
skills: z.ZodOptional<z.ZodObject<{
|
|
66
68
|
dir: z.ZodString;
|
|
67
69
|
scripts: z.ZodOptional<z.ZodEnum<{
|
|
@@ -114,6 +116,7 @@ export declare const agentSchema: z.ZodObject<{
|
|
|
114
116
|
*/
|
|
115
117
|
export declare const subagentSchema: z.ZodObject<{
|
|
116
118
|
name: z.ZodString;
|
|
119
|
+
fallback_model: z.ZodOptional<z.ZodString>;
|
|
117
120
|
globalInstruction: z.ZodOptional<z.ZodString>;
|
|
118
121
|
tools: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
119
122
|
require_approval: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
@@ -146,6 +149,7 @@ export declare const subagentSchema: z.ZodObject<{
|
|
|
146
149
|
}, z.core.$loose>>;
|
|
147
150
|
outputSchema: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
148
151
|
mcp_server_url: z.ZodOptional<z.ZodString>;
|
|
152
|
+
mcp_tools: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
149
153
|
skills: z.ZodOptional<z.ZodObject<{
|
|
150
154
|
dir: z.ZodString;
|
|
151
155
|
scripts: z.ZodOptional<z.ZodEnum<{
|
|
@@ -201,6 +205,7 @@ export declare const syndicateSchema: z.ZodObject<{
|
|
|
201
205
|
name: z.ZodString;
|
|
202
206
|
description: z.ZodOptional<z.ZodString>;
|
|
203
207
|
model: z.ZodString;
|
|
208
|
+
fallback_model: z.ZodOptional<z.ZodString>;
|
|
204
209
|
instruction: z.ZodString;
|
|
205
210
|
globalInstruction: z.ZodOptional<z.ZodString>;
|
|
206
211
|
tools: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
@@ -234,6 +239,7 @@ export declare const syndicateSchema: z.ZodObject<{
|
|
|
234
239
|
}, z.core.$loose>>;
|
|
235
240
|
outputSchema: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
236
241
|
mcp_server_url: z.ZodOptional<z.ZodString>;
|
|
242
|
+
mcp_tools: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
237
243
|
skills: z.ZodOptional<z.ZodObject<{
|
|
238
244
|
dir: z.ZodString;
|
|
239
245
|
scripts: z.ZodOptional<z.ZodEnum<{
|
|
@@ -280,6 +286,7 @@ export declare const syndicateSchema: z.ZodObject<{
|
|
|
280
286
|
}, z.core.$strict>;
|
|
281
287
|
subagents: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
282
288
|
name: z.ZodString;
|
|
289
|
+
fallback_model: z.ZodOptional<z.ZodString>;
|
|
283
290
|
globalInstruction: z.ZodOptional<z.ZodString>;
|
|
284
291
|
tools: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
285
292
|
require_approval: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
@@ -312,6 +319,7 @@ export declare const syndicateSchema: z.ZodObject<{
|
|
|
312
319
|
}, z.core.$loose>>;
|
|
313
320
|
outputSchema: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
314
321
|
mcp_server_url: z.ZodOptional<z.ZodString>;
|
|
322
|
+
mcp_tools: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
315
323
|
skills: z.ZodOptional<z.ZodObject<{
|
|
316
324
|
dir: z.ZodString;
|
|
317
325
|
scripts: z.ZodOptional<z.ZodEnum<{
|
|
@@ -125,6 +125,11 @@ const agentFields = {
|
|
|
125
125
|
.string()
|
|
126
126
|
.min(1)
|
|
127
127
|
.describe('Model id; its prefix picks the provider (gemini-*, claude-*, gpt-*/o*, grok-*, ollama/*).'),
|
|
128
|
+
fallback_model: z
|
|
129
|
+
.string()
|
|
130
|
+
.min(1)
|
|
131
|
+
.optional()
|
|
132
|
+
.describe('A model on another provider that answers when this agent\'s model fails provider-side (5xx, 429, network) before producing anything, or while that provider\'s circuit is open (ADR 0044).'),
|
|
128
133
|
instruction: z.string().min(1).describe('System prompt / persona.'),
|
|
129
134
|
globalInstruction: z
|
|
130
135
|
.string()
|
|
@@ -154,6 +159,11 @@ const agentFields = {
|
|
|
154
159
|
.string()
|
|
155
160
|
.optional()
|
|
156
161
|
.describe('MCP server (SSE) whose tools are discovered at runtime and merged with `tools`.'),
|
|
162
|
+
mcp_tools: z
|
|
163
|
+
.array(z.string().min(1))
|
|
164
|
+
.min(1)
|
|
165
|
+
.optional()
|
|
166
|
+
.describe('The MCP server\'s tools this agent may use, by name; any other tool the server lists is not exposed. Without it, every listed tool is. require_approval may name these. Needs mcp_server_url.'),
|
|
157
167
|
skills: skillsSchema.optional(),
|
|
158
168
|
openapi: z.array(openapiEntry).min(1).optional(),
|
|
159
169
|
code_execution: z
|
|
@@ -330,7 +340,7 @@ export const syndicateSchema = z
|
|
|
330
340
|
.int()
|
|
331
341
|
.positive()
|
|
332
342
|
.optional()
|
|
333
|
-
.describe('Hard cap on
|
|
343
|
+
.describe('Hard cap on model calls per turn, subagents included. Default 50 (DEFAULT_MAX_STEPS) when unset.'),
|
|
334
344
|
bundled_references: z
|
|
335
345
|
.record(z.string(), z.record(z.string(), z.unknown()))
|
|
336
346
|
.optional()
|
|
@@ -511,11 +521,13 @@ function crossFieldProblems(raw) {
|
|
|
511
521
|
return;
|
|
512
522
|
}
|
|
513
523
|
const tools = Array.isArray(agent.tools) ? agent.tools : [];
|
|
514
|
-
// An OpenAPI operation can be gated once it is named under `operations
|
|
524
|
+
// An OpenAPI operation can be gated once it is named under `operations`,
|
|
525
|
+
// and an MCP tool once it is named under `mcp_tools`.
|
|
515
526
|
const operations = (Array.isArray(agent.openapi) ? agent.openapi : []).flatMap((e) => (isObj(e) && Array.isArray(e.operations) ? e.operations : []));
|
|
527
|
+
const mcpTools = Array.isArray(agent.mcp_tools) ? agent.mcp_tools : [];
|
|
516
528
|
(Array.isArray(agent.require_approval) ? agent.require_approval : []).forEach((name, j) => {
|
|
517
|
-
if (typeof name === 'string' && !tools.includes(name) && !operations.includes(name)) {
|
|
518
|
-
out.push({ path: [...path, 'require_approval', j], message: `'${name}' is not in this agent's tools
|
|
529
|
+
if (typeof name === 'string' && !tools.includes(name) && !operations.includes(name) && !mcpTools.includes(name)) {
|
|
530
|
+
out.push({ path: [...path, 'require_approval', j], message: `'${name}' is not in this agent's tools, its openapi operations or its mcp_tools` });
|
|
519
531
|
}
|
|
520
532
|
});
|
|
521
533
|
};
|
|
@@ -567,6 +579,9 @@ function crossFieldProblems(raw) {
|
|
|
567
579
|
if (agent.mode !== undefined && !isObj(raw.workflow)) {
|
|
568
580
|
out.push({ path: [...path, 'mode'], message: 'mode: task applies to workflow nodes, whose output is the finish_task arguments' });
|
|
569
581
|
}
|
|
582
|
+
if (agent.mcp_tools !== undefined && typeof agent.mcp_server_url !== 'string') {
|
|
583
|
+
out.push({ path: [...path, 'mcp_tools'], message: 'mcp_tools chooses among an MCP server\'s tools; it needs mcp_server_url' });
|
|
584
|
+
}
|
|
570
585
|
};
|
|
571
586
|
if (isObj(raw.orchestrator))
|
|
572
587
|
executionProblems(raw.orchestrator, ['orchestrator'], true);
|
|
@@ -1,4 +1,30 @@
|
|
|
1
1
|
import { FunctionTool } from '@google/adk';
|
|
2
|
+
import type { RedirectPolicy } from '../net/redirects.ts';
|
|
2
3
|
export declare function assertSafeMcpUrl(raw: string, env?: NodeJS.ProcessEnv): Promise<URL>;
|
|
3
4
|
export declare function mcpAuthHeaders(url: URL, env?: NodeJS.ProcessEnv): Record<string, string>;
|
|
5
|
+
/**
|
|
6
|
+
* Where an MCP connection may be redirected (ADR 0036's rule, applied to MCP):
|
|
7
|
+
* a hop on the server's own origin gets the server's check (ALLOW_PRIVATE_MCP
|
|
8
|
+
* applies), a hop anywhere else the full guard with no development exception,
|
|
9
|
+
* and lib/net/redirects.ts strips every credential header from it, so a
|
|
10
|
+
* bearer token from MCP_BEARER_TOKENS never follows a redirect off its host.
|
|
11
|
+
* The SDK already refuses a POST endpoint on another origin.
|
|
12
|
+
*/
|
|
13
|
+
export declare const MCP_REDIRECTS: RedirectPolicy;
|
|
14
|
+
/** The fetch an MCP transport uses: every redirect hop checked before it is followed. */
|
|
15
|
+
export declare const mcpFetch: (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
|
|
16
|
+
/**
|
|
17
|
+
* A remote MCP server is an untrusted tool vendor: its tool descriptions go
|
|
18
|
+
* into the prompt and its results into the conversation. Both are bounded,
|
|
19
|
+
* so a server cannot flood the context (or the bill) with either.
|
|
20
|
+
*/
|
|
21
|
+
export declare const MAX_MCP_DESCRIPTION_CHARS = 1000;
|
|
22
|
+
export declare const MAX_MCP_RESULT_CHARS = 20000;
|
|
23
|
+
/**
|
|
24
|
+
* Close every open MCP connection. A connection otherwise lives as long as
|
|
25
|
+
* the process (its tools are called for the life of the compiled agent), and
|
|
26
|
+
* its stream reconnects when the server restarts; an embedding app that
|
|
27
|
+
* stops cleanly, or a test, calls this.
|
|
28
|
+
*/
|
|
29
|
+
export declare function closeMcpConnections(): Promise<void>;
|
|
4
30
|
export declare function createMcpTools(mcpServerUrl: string): Promise<FunctionTool[]>;
|
|
@@ -2,6 +2,7 @@ import { FunctionTool } from '@google/adk';
|
|
|
2
2
|
import { Client } from '@modelcontextprotocol/sdk/client/index.js';
|
|
3
3
|
import { SSEClientTransport } from '@modelcontextprotocol/sdk/client/sse.js';
|
|
4
4
|
import { checkHost } from '../net/addressGuard.js';
|
|
5
|
+
import { fetchWithRedirectPolicy } from '../net/redirects.js';
|
|
5
6
|
// Security (SSRF): mcp_server_url can arrive from a registry-stored syndicate
|
|
6
7
|
// config. Only http(s), and the host must pass lib/net/addressGuard.ts (the
|
|
7
8
|
// same guard web_extract uses): local names and non-public addresses —
|
|
@@ -53,11 +54,61 @@ export function mcpAuthHeaders(url, env = process.env) {
|
|
|
53
54
|
const token = tokens[host];
|
|
54
55
|
return typeof token === 'string' && token ? { Authorization: `Bearer ${token}` } : {};
|
|
55
56
|
}
|
|
57
|
+
/**
|
|
58
|
+
* Where an MCP connection may be redirected (ADR 0036's rule, applied to MCP):
|
|
59
|
+
* a hop on the server's own origin gets the server's check (ALLOW_PRIVATE_MCP
|
|
60
|
+
* applies), a hop anywhere else the full guard with no development exception,
|
|
61
|
+
* and lib/net/redirects.ts strips every credential header from it, so a
|
|
62
|
+
* bearer token from MCP_BEARER_TOKENS never follows a redirect off its host.
|
|
63
|
+
* The SDK already refuses a POST endpoint on another origin.
|
|
64
|
+
*/
|
|
65
|
+
export const MCP_REDIRECTS = {
|
|
66
|
+
async hopProblem(url, crossOrigin) {
|
|
67
|
+
if (!crossOrigin) {
|
|
68
|
+
try {
|
|
69
|
+
await assertSafeMcpUrl(url.href);
|
|
70
|
+
return null;
|
|
71
|
+
}
|
|
72
|
+
catch (err) {
|
|
73
|
+
return err instanceof Error ? err.message : String(err);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
if (url.protocol !== 'http:' && url.protocol !== 'https:')
|
|
77
|
+
return `${url.protocol} is not http(s)`;
|
|
78
|
+
const reason = await checkHost(url.hostname);
|
|
79
|
+
return reason ? `refusing MCP redirect to ${url.hostname}: ${reason}` : null;
|
|
80
|
+
},
|
|
81
|
+
};
|
|
82
|
+
/** The fetch an MCP transport uses: every redirect hop checked before it is followed. */
|
|
83
|
+
export const mcpFetch = (input, init) => fetchWithRedirectPolicy(input, init, MCP_REDIRECTS, (i, o) => fetch(i, o));
|
|
84
|
+
/**
|
|
85
|
+
* A remote MCP server is an untrusted tool vendor: its tool descriptions go
|
|
86
|
+
* into the prompt and its results into the conversation. Both are bounded,
|
|
87
|
+
* so a server cannot flood the context (or the bill) with either.
|
|
88
|
+
*/
|
|
89
|
+
export const MAX_MCP_DESCRIPTION_CHARS = 1_000;
|
|
90
|
+
export const MAX_MCP_RESULT_CHARS = 20_000;
|
|
91
|
+
const bounded = (text, max, what) => text.length <= max ? text : `${text.slice(0, max)}… [${what} cut at ${max} characters]`;
|
|
92
|
+
/** Every MCP connection opened and still in use, so a shutdown can close them. */
|
|
93
|
+
const openTransports = new Set();
|
|
94
|
+
/**
|
|
95
|
+
* Close every open MCP connection. A connection otherwise lives as long as
|
|
96
|
+
* the process (its tools are called for the life of the compiled agent), and
|
|
97
|
+
* its stream reconnects when the server restarts; an embedding app that
|
|
98
|
+
* stops cleanly, or a test, calls this.
|
|
99
|
+
*/
|
|
100
|
+
export async function closeMcpConnections() {
|
|
101
|
+
const all = [...openTransports];
|
|
102
|
+
openTransports.clear();
|
|
103
|
+
await Promise.all(all.map((t) => t.close().catch(() => { })));
|
|
104
|
+
}
|
|
56
105
|
export async function createMcpTools(mcpServerUrl) {
|
|
106
|
+
let transport;
|
|
57
107
|
try {
|
|
58
108
|
const url = await assertSafeMcpUrl(mcpServerUrl);
|
|
59
|
-
|
|
60
|
-
requestInit: { headers: mcpAuthHeaders(url) }
|
|
109
|
+
transport = new SSEClientTransport(url, {
|
|
110
|
+
requestInit: { headers: mcpAuthHeaders(url) },
|
|
111
|
+
fetch: mcpFetch,
|
|
61
112
|
});
|
|
62
113
|
const client = new Client({
|
|
63
114
|
name: 'melchizedek-a2a-client',
|
|
@@ -66,6 +117,7 @@ export async function createMcpTools(mcpServerUrl) {
|
|
|
66
117
|
capabilities: {}
|
|
67
118
|
});
|
|
68
119
|
await client.connect(transport);
|
|
120
|
+
openTransports.add(transport);
|
|
69
121
|
// Fetch available tools from the MCP server
|
|
70
122
|
const toolsResponse = await client.listTools();
|
|
71
123
|
// MCP servers describe tools in standard lowercase JSON Schema; the ADK
|
|
@@ -106,7 +158,7 @@ export async function createMcpTools(mcpServerUrl) {
|
|
|
106
158
|
}
|
|
107
159
|
return new FunctionTool({
|
|
108
160
|
name: tool.name,
|
|
109
|
-
description: tool.description || `MCP Tool: ${tool.name}`,
|
|
161
|
+
description: bounded(tool.description || `MCP Tool: ${tool.name}`, MAX_MCP_DESCRIPTION_CHARS, 'description'),
|
|
110
162
|
parameters: {
|
|
111
163
|
type: 'OBJECT',
|
|
112
164
|
properties,
|
|
@@ -120,7 +172,7 @@ export async function createMcpTools(mcpServerUrl) {
|
|
|
120
172
|
});
|
|
121
173
|
const content = result.content;
|
|
122
174
|
const texts = content.filter(c => c.type === 'text').map(c => c.text ?? '');
|
|
123
|
-
return texts.join('\n');
|
|
175
|
+
return bounded(texts.join('\n'), MAX_MCP_RESULT_CHARS, 'result');
|
|
124
176
|
}
|
|
125
177
|
catch (error) {
|
|
126
178
|
return `[MCP ERROR] Tool ${tool.name} failed: ${error.message}`;
|
|
@@ -131,6 +183,11 @@ export async function createMcpTools(mcpServerUrl) {
|
|
|
131
183
|
}
|
|
132
184
|
catch (error) {
|
|
133
185
|
console.warn(`[MCP] Failed to connect or load tools from ${mcpServerUrl}`, error);
|
|
186
|
+
// A failed connect leaves the SSE stream's reconnect timer running; close
|
|
187
|
+
// it, or every unreachable server keeps retrying for the process's life.
|
|
188
|
+
if (transport)
|
|
189
|
+
openTransports.delete(transport);
|
|
190
|
+
await transport?.close().catch(() => { });
|
|
134
191
|
return [];
|
|
135
192
|
}
|
|
136
193
|
}
|
|
@@ -34,6 +34,12 @@ export declare function namesTool(configured: string, tool: {
|
|
|
34
34
|
operationId?: string;
|
|
35
35
|
};
|
|
36
36
|
}): boolean;
|
|
37
|
+
/**
|
|
38
|
+
* Why an `auth` may not read this variable, or null. With
|
|
39
|
+
* OPENAPI_CREDENTIAL_ENVS set, only the names it lists are allowed; without
|
|
40
|
+
* it, anything but the framework's own variables is.
|
|
41
|
+
*/
|
|
42
|
+
export declare function credentialEnvProblem(name: string, env?: NodeJS.ProcessEnv): string | null;
|
|
37
43
|
/** Cut an API result to MAX_RESULT_CHARS, saying so. */
|
|
38
44
|
export declare function boundResult(result: unknown): unknown;
|
|
39
45
|
/**
|