melchizedek-agents 1.4.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +15 -0
- package/CHANGELOG.md +77 -0
- package/DOCUMENTATION.md +43 -2
- package/config/agents/syndicate.schema.json +80 -6
- package/db/hardening.sql +38 -12
- package/dist/lib/a2a/app.js +2 -1
- package/dist/lib/chatgpt/state.js +4 -4
- package/dist/lib/compile.js +15 -2
- package/dist/lib/index.d.ts +2 -0
- package/dist/lib/index.js +2 -0
- package/dist/lib/loadSyndicate.js +4 -0
- package/dist/lib/memory/providers.js +2 -1
- package/dist/lib/models/claudeAdapter.js +2 -1
- package/dist/lib/models/credentialSource.d.ts +28 -0
- package/dist/lib/models/credentialSource.js +49 -0
- package/dist/lib/models/endpoints.js +2 -1
- package/dist/lib/models/gateway.js +2 -1
- package/dist/lib/models/gatewayAdapter.js +2 -1
- package/dist/lib/models/geminiAdapter.js +2 -1
- package/dist/lib/models/gptAdapter.js +2 -1
- package/dist/lib/models/grokAdapter.js +2 -1
- package/dist/lib/models/kimiAdapter.js +2 -1
- package/dist/lib/models/providerMap.js +3 -2
- package/dist/lib/observability/embeddings.js +2 -1
- package/dist/lib/observability/lineage.js +11 -3
- package/dist/lib/observability/supabaseSpanExporter.d.ts +19 -1
- package/dist/lib/observability/supabaseSpanExporter.js +37 -1
- package/dist/lib/observability/tracer.d.ts +9 -0
- package/dist/lib/observability/tracer.js +45 -0
- package/dist/lib/paths.d.ts +31 -0
- package/dist/lib/paths.js +76 -0
- package/dist/lib/storage/postgres/index.d.ts +25 -0
- package/dist/lib/storage/postgres/index.js +10 -1
- package/dist/lib/syndicateSchema.d.ts +26 -4
- package/dist/lib/syndicateSchema.js +45 -1
- package/dist/lib/tools/generateImageTool.js +6 -6
- package/dist/lib/tools/inspectImageTool.d.ts +2 -0
- package/dist/lib/tools/inspectImageTool.js +17 -10
- package/dist/lib/tools/mcpToolFactory.d.ts +34 -1
- package/dist/lib/tools/mcpToolFactory.js +117 -17
- package/dist/lib/tools/taskTools.js +3 -3
- package/dist/lib/tools/xApiSearchTool.js +2 -1
- package/dist/scripts/a2a_server.js +5 -0
- package/package.json +7 -4
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* lib/models/credentialSource.ts — where the engine reads a credential by
|
|
3
|
+
* its environment variable name (ADR 0133).
|
|
4
|
+
*
|
|
5
|
+
* By default every provider key, gateway key and embedding key is read from
|
|
6
|
+
* `process.env`. An embedding application that holds keys elsewhere (an OS
|
|
7
|
+
* keychain, a secret manager) sets a source once:
|
|
8
|
+
*
|
|
9
|
+
* setCredentialSource((name) => keychain.get(name));
|
|
10
|
+
*
|
|
11
|
+
* and every read below asks the source first, by the same names the
|
|
12
|
+
* environment uses (ANTHROPIC_API_KEY, MODEL_GATEWAY_API_KEY, …), at the
|
|
13
|
+
* moment of use: nothing is cached, so a key the user removes stops working
|
|
14
|
+
* on the next call. A source that returns undefined for a name falls back to
|
|
15
|
+
* the environment. A dependency-free leaf, so providerMap.ts stays one.
|
|
16
|
+
*/
|
|
17
|
+
let source;
|
|
18
|
+
/** Sets (or, with undefined, clears) the process's credential source. */
|
|
19
|
+
export function setCredentialSource(next) {
|
|
20
|
+
source = next;
|
|
21
|
+
}
|
|
22
|
+
/** The credential named `name`: the source's, else `env`'s. A source that throws counts as holding nothing. */
|
|
23
|
+
export function credentialValue(name, env = process.env) {
|
|
24
|
+
if (source) {
|
|
25
|
+
try {
|
|
26
|
+
const held = source(name);
|
|
27
|
+
if (held !== undefined && held !== '')
|
|
28
|
+
return held;
|
|
29
|
+
}
|
|
30
|
+
catch {
|
|
31
|
+
/* a broken source must not take the engine down; the environment still answers */
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
return env[name];
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* `env` as the engine's env-reading helpers see it: a read by name asks the
|
|
38
|
+
* source first. For functions that take an `env` parameter; enumeration and
|
|
39
|
+
* spreading see only `env`'s own variables.
|
|
40
|
+
*/
|
|
41
|
+
export function credentialEnv(env = process.env) {
|
|
42
|
+
if (!source)
|
|
43
|
+
return env;
|
|
44
|
+
return new Proxy(env, {
|
|
45
|
+
get(target, key, receiver) {
|
|
46
|
+
return typeof key === 'string' ? credentialValue(key, target) : Reflect.get(target, key, receiver);
|
|
47
|
+
},
|
|
48
|
+
});
|
|
49
|
+
}
|
|
@@ -31,6 +31,7 @@ var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExte
|
|
|
31
31
|
};
|
|
32
32
|
import { PROVIDERS, providerKeyPresent } from './providerMap.js';
|
|
33
33
|
import { trimTrailingSlashes } from './urls.js';
|
|
34
|
+
import { credentialEnv } from './credentialSource.js';
|
|
34
35
|
export const PLATFORMS_FOR = {
|
|
35
36
|
gemini: ['direct', 'vertex'],
|
|
36
37
|
anthropic: ['direct', 'bedrock', 'vertex'],
|
|
@@ -81,7 +82,7 @@ export function platformFromEnv(provider, env = process.env) {
|
|
|
81
82
|
return raw;
|
|
82
83
|
}
|
|
83
84
|
/** A provider's endpoint as the environment configures it. */
|
|
84
|
-
export function endpointFromEnv(provider, env =
|
|
85
|
+
export function endpointFromEnv(provider, env = credentialEnv()) {
|
|
85
86
|
const platform = platformFromEnv(provider, env);
|
|
86
87
|
const mapEnv = MODEL_MAP_ENV[provider];
|
|
87
88
|
const models = mapEnv ? parseModelMap(env[mapEnv], mapEnv) : undefined;
|
|
@@ -41,6 +41,7 @@ import { chatGptSignInRoutesOpenAi } from '../chatgpt/state.js';
|
|
|
41
41
|
import { providerReady } from './endpoints.js';
|
|
42
42
|
import { PROVIDERS, providerForModel } from './providerMap.js';
|
|
43
43
|
import { trimTrailingSlashes } from './urls.js';
|
|
44
|
+
import { credentialValue } from './credentialSource.js';
|
|
44
45
|
export const GATEWAY_ENV = 'MODEL_GATEWAY';
|
|
45
46
|
export const GATEWAY_KEY_ENV = 'MODEL_GATEWAY_API_KEY';
|
|
46
47
|
export const GATEWAY_BASE_URL_ENV = 'MODEL_GATEWAY_BASE_URL';
|
|
@@ -78,7 +79,7 @@ export function gatewayConfig() {
|
|
|
78
79
|
return {
|
|
79
80
|
gateway,
|
|
80
81
|
baseUrl: trimTrailingSlashes(override || gateway.baseUrl),
|
|
81
|
-
keyPresent: !!
|
|
82
|
+
keyPresent: !!credentialValue(GATEWAY_KEY_ENV),
|
|
82
83
|
};
|
|
83
84
|
}
|
|
84
85
|
/** A human-readable misconfiguration, or undefined when the gateway is
|
|
@@ -31,6 +31,7 @@
|
|
|
31
31
|
import { ChatCompletionsAdapter } from './chatCompletionsAdapter.js';
|
|
32
32
|
import { GATEWAY_ENV, GATEWAY_KEY_ENV, GATEWAY_MODEL_MAP_ENV, gatewayConfig, gatewayWireModel } from './gateway.js';
|
|
33
33
|
import { providerForModel } from './providerMap.js';
|
|
34
|
+
import { credentialValue } from './credentialSource.js';
|
|
34
35
|
export class GatewayAdapter extends ChatCompletionsAdapter {
|
|
35
36
|
/** Attribution stays with the upstream provider named by the YAML id. */
|
|
36
37
|
provider;
|
|
@@ -47,7 +48,7 @@ export class GatewayAdapter extends ChatCompletionsAdapter {
|
|
|
47
48
|
return `${this.#cfg?.baseUrl ?? ''}/chat/completions`;
|
|
48
49
|
}
|
|
49
50
|
headers() {
|
|
50
|
-
return { Authorization: `Bearer ${
|
|
51
|
+
return { Authorization: `Bearer ${credentialValue(GATEWAY_KEY_ENV) ?? ''}` };
|
|
51
52
|
}
|
|
52
53
|
wireModelName(model) {
|
|
53
54
|
return this.#cfg ? gatewayWireModel(model, this.#cfg.gateway) : model;
|
|
@@ -52,6 +52,7 @@ import { currentTurnStart, providerStateOf } from './providerState.js';
|
|
|
52
52
|
import { classifyError, errorStatus, retryUntilFirstYield } from './retry.js';
|
|
53
53
|
import { isAboveHigh, reasoningConfig } from './reasoning.js';
|
|
54
54
|
import { setLlmSpanAttribute } from '../observability/tracer.js';
|
|
55
|
+
import { credentialValue } from './credentialSource.js';
|
|
55
56
|
/**
|
|
56
57
|
* The provider id this adapter reports and writes its state under, and the
|
|
57
58
|
* providerState kind for a Gemini thought signature (ADR 0046). Defined once
|
|
@@ -846,7 +847,7 @@ function nonEmpty(value) {
|
|
|
846
847
|
}
|
|
847
848
|
/** The Gemini API key from the environment, in ADK's order. */
|
|
848
849
|
function envApiKey() {
|
|
849
|
-
return nonEmpty(
|
|
850
|
+
return nonEmpty(credentialValue('GOOGLE_GENAI_API_KEY')) ?? nonEmpty(credentialValue('GOOGLE_API_KEY')) ?? nonEmpty(credentialValue('GEMINI_API_KEY'));
|
|
850
851
|
}
|
|
851
852
|
function messageOf(err) {
|
|
852
853
|
return err instanceof Error ? err.message : String(err);
|
|
@@ -48,6 +48,7 @@ import { currentTurnStart, providerStateOf } from './providerState.js';
|
|
|
48
48
|
import { effortWord } from './reasoning.js';
|
|
49
49
|
import { toContractJsonSchema } from './schemaNormalize.js';
|
|
50
50
|
import { setLlmSpanAttribute } from '../observability/tracer.js';
|
|
51
|
+
import { credentialValue } from './credentialSource.js';
|
|
51
52
|
/** The providerState kind the Responses adapters write: the reasoning output items that preceded a part (ADR 0046, ADR 0050). */
|
|
52
53
|
export const REASONING_STATE_KIND = 'reasoning_items';
|
|
53
54
|
/** The `include` value that returns reasoning items with their encrypted content. */
|
|
@@ -410,7 +411,7 @@ export class GptAdapter {
|
|
|
410
411
|
return undefined;
|
|
411
412
|
}
|
|
412
413
|
apiKeyFromEnv() {
|
|
413
|
-
return
|
|
414
|
+
return credentialValue('OPENAI_API_KEY');
|
|
414
415
|
}
|
|
415
416
|
missingKeyMessage() {
|
|
416
417
|
return 'OPENAI_API_KEY is not set in environment.';
|
|
@@ -29,6 +29,7 @@ import { GptAdapter } from './gptAdapter.js';
|
|
|
29
29
|
import { effortWord } from './reasoning.js';
|
|
30
30
|
import { collectionIdsFromEnv, collectionsMaxResultsFromEnv, xaiWebSearchParamsFromEnv, xSearchParamsFromEnv, } from '../tools/xaiSearchParams.js';
|
|
31
31
|
import { setLlmSpanAttribute } from '../observability/tracer.js';
|
|
32
|
+
import { credentialValue } from './credentialSource.js';
|
|
32
33
|
export const XAI_BASE_URL = 'https://api.x.ai/v1';
|
|
33
34
|
/**
|
|
34
35
|
* The ids that take a reasoning effort and return encrypted reasoning:
|
|
@@ -60,7 +61,7 @@ export class GrokAdapter extends GptAdapter {
|
|
|
60
61
|
return XAI_BASE_URL;
|
|
61
62
|
}
|
|
62
63
|
apiKeyFromEnv() {
|
|
63
|
-
return
|
|
64
|
+
return credentialValue('XAI_API_KEY');
|
|
64
65
|
}
|
|
65
66
|
missingKeyMessage() {
|
|
66
67
|
return 'XAI_API_KEY is not set in environment.';
|
|
@@ -120,6 +120,7 @@
|
|
|
120
120
|
import { DEFAULT_KIMI_REASONING_EFFORT } from '../config.js';
|
|
121
121
|
import { ChatCompletionsAdapter, reasonsNotAtAll } from './chatCompletionsAdapter.js';
|
|
122
122
|
import { trimTrailingSlashes } from './urls.js';
|
|
123
|
+
import { credentialValue } from './credentialSource.js';
|
|
123
124
|
export const MOONSHOT_BASE_URL = 'https://api.moonshot.ai/v1';
|
|
124
125
|
/** The flagship takes `reasoning_effort`; the K2 generation takes a `thinking` switch. */
|
|
125
126
|
export function isKimiK3(model) {
|
|
@@ -153,7 +154,7 @@ export class KimiAdapter extends ChatCompletionsAdapter {
|
|
|
153
154
|
return `${this.#baseUrl}/chat/completions`;
|
|
154
155
|
}
|
|
155
156
|
#key() {
|
|
156
|
-
return this.#apiKey ??
|
|
157
|
+
return this.#apiKey ?? credentialValue('MOONSHOT_API_KEY');
|
|
157
158
|
}
|
|
158
159
|
headers() {
|
|
159
160
|
return { Authorization: `Bearer ${this.#key() ?? ''}` };
|
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
* grok-* → xai ollama/<model> → ollama (local, keyless)
|
|
14
14
|
* kimi-* → moonshot everything else → gemini (the default)
|
|
15
15
|
*/
|
|
16
|
+
import { credentialValue } from './credentialSource.js';
|
|
16
17
|
export const PROVIDERS = {
|
|
17
18
|
gemini: { keyEnv: 'GOOGLE_GENAI_API_KEY', label: 'Google Gemini' },
|
|
18
19
|
anthropic: { keyEnv: 'ANTHROPIC_API_KEY', label: 'Anthropic Claude' },
|
|
@@ -43,9 +44,9 @@ export function providerKeyPresent(provider) {
|
|
|
43
44
|
return true;
|
|
44
45
|
// Gemini historically accepts either env name.
|
|
45
46
|
if (provider === 'gemini') {
|
|
46
|
-
return realValue(
|
|
47
|
+
return realValue(credentialValue('GOOGLE_GENAI_API_KEY')) || realValue(credentialValue('GEMINI_API_KEY'));
|
|
47
48
|
}
|
|
48
|
-
return realValue(
|
|
49
|
+
return realValue(credentialValue(keyEnv));
|
|
49
50
|
}
|
|
50
51
|
/** Set, and not a `.env.example` placeholder (`your_..._here`, `<...>`).
|
|
51
52
|
* Inlined rather than imported: this module stays a dependency-free leaf. */
|
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
*/
|
|
11
11
|
import { EMBEDDING_DIMENSIONS, EMBEDDING_MODEL } from '../config.js';
|
|
12
12
|
import { memoryProvidersFromEnv } from '../memory/providers.js';
|
|
13
|
+
import { credentialValue } from '../models/credentialSource.js';
|
|
13
14
|
const MAX_CHARS = 8_000;
|
|
14
15
|
/**
|
|
15
16
|
* Embeds each text with the SAME embedder long-term memory uses, so the two
|
|
@@ -20,7 +21,7 @@ const MAX_CHARS = 8_000;
|
|
|
20
21
|
export async function embedTexts(texts, options = {}) {
|
|
21
22
|
let embedder = options.embedder;
|
|
22
23
|
if (!embedder) {
|
|
23
|
-
const apiKey = options.apiKey ??
|
|
24
|
+
const apiKey = options.apiKey ?? credentialValue('GOOGLE_GENAI_API_KEY') ?? credentialValue('GEMINI_API_KEY');
|
|
24
25
|
const provider = (process.env.MEMORY_EMBEDDING_PROVIDER?.trim() || 'gemini').toLowerCase();
|
|
25
26
|
if (provider === 'gemini' && !apiKey)
|
|
26
27
|
throw new Error('GOOGLE_GENAI_API_KEY is required for Gemini embeddings');
|
|
@@ -26,6 +26,7 @@ import { createHash } from 'node:crypto';
|
|
|
26
26
|
import { execSync } from 'node:child_process';
|
|
27
27
|
import { existsSync, readFileSync } from 'node:fs';
|
|
28
28
|
import { join } from 'node:path';
|
|
29
|
+
import { enginePackageRoot } from '../paths.js';
|
|
29
30
|
/** Schema version stamped on every ledger row (adk_turns / adk_payloads). */
|
|
30
31
|
export const TELEMETRY_SCHEMA_VERSION = 2;
|
|
31
32
|
// ── Span lineage: the names spans are given ──────────────────────────────────
|
|
@@ -105,8 +106,10 @@ export function engineVersion() {
|
|
|
105
106
|
return cachedEngineVersion;
|
|
106
107
|
let pkgVersion = '0.0.0';
|
|
107
108
|
try {
|
|
108
|
-
|
|
109
|
-
|
|
109
|
+
// The engine's own package.json, not the embedding application's.
|
|
110
|
+
const root = enginePackageRoot();
|
|
111
|
+
const pkgPath = root ? join(root, 'package.json') : '';
|
|
112
|
+
if (pkgPath && existsSync(pkgPath)) {
|
|
110
113
|
pkgVersion = String(JSON.parse(readFileSync(pkgPath, 'utf-8')).version ?? pkgVersion);
|
|
111
114
|
}
|
|
112
115
|
}
|
|
@@ -116,7 +119,12 @@ export function engineVersion() {
|
|
|
116
119
|
let commit = process.env.SOURCE_VERSION ?? process.env.HEROKU_SLUG_COMMIT ?? process.env.GIT_COMMIT ?? '';
|
|
117
120
|
if (!commit) {
|
|
118
121
|
try {
|
|
119
|
-
|
|
122
|
+
// Only from the engine's own checkout: an installed package inside an
|
|
123
|
+
// application's repository would otherwise report the application's commit.
|
|
124
|
+
const root = enginePackageRoot();
|
|
125
|
+
if (!root || !existsSync(join(root, '.git')))
|
|
126
|
+
throw new Error('not a checkout');
|
|
127
|
+
commit = execSync('git rev-parse --short HEAD', { cwd: root, stdio: ['ignore', 'pipe', 'ignore'] })
|
|
120
128
|
.toString()
|
|
121
129
|
.trim();
|
|
122
130
|
}
|
|
@@ -174,7 +174,8 @@ export interface ExportStats {
|
|
|
174
174
|
deadLettered: number;
|
|
175
175
|
payloadsDropped: number;
|
|
176
176
|
}
|
|
177
|
-
|
|
177
|
+
/** Where the exporter writes: supabase-js, or `postgresLedgerClient` on a pg pool. */
|
|
178
|
+
export interface LedgerClient {
|
|
178
179
|
from(table: string): {
|
|
179
180
|
insert(rows: unknown[]): PromiseLike<{
|
|
180
181
|
error: {
|
|
@@ -183,6 +184,23 @@ interface SupabaseLikeClient {
|
|
|
183
184
|
}>;
|
|
184
185
|
};
|
|
185
186
|
}
|
|
187
|
+
type SupabaseLikeClient = LedgerClient;
|
|
188
|
+
/** The tables the exporter writes; nothing else is ever named in its SQL. */
|
|
189
|
+
export declare const LEDGER_TABLES: readonly ['adk_telemetry', 'adk_turns', 'adk_payloads'];
|
|
190
|
+
/** The subset of a pg Pool the ledger client needs. */
|
|
191
|
+
export interface LedgerPool {
|
|
192
|
+
query(text: string, values?: unknown[]): Promise<unknown>;
|
|
193
|
+
}
|
|
194
|
+
/**
|
|
195
|
+
* The ledger over a direct Postgres connection (ADR 0131): the same rows the
|
|
196
|
+
* Supabase sink writes, through one parameterised statement per batch. The
|
|
197
|
+
* rows go in as one JSON array and `jsonb_populate_recordset` gives each
|
|
198
|
+
* value its column's type, so JSON, arrays and timestamps need no mapping
|
|
199
|
+
* here. Only the columns the rows carry are named, so the rest keep their
|
|
200
|
+
* defaults. The table must be one of LEDGER_TABLES and every column name
|
|
201
|
+
* comes from the row builders above, never from span content.
|
|
202
|
+
*/
|
|
203
|
+
export declare function postgresLedgerClient(pool: LedgerPool): LedgerClient;
|
|
186
204
|
export interface ExporterOptions {
|
|
187
205
|
/** Injected client (tests); default: lazily created supabase-js client. */
|
|
188
206
|
client?: SupabaseLikeClient;
|
|
@@ -43,6 +43,7 @@ const { ExportResultCode } = core;
|
|
|
43
43
|
import { hasSupabaseCredentials } from '../persistence/supabaseProvider.js';
|
|
44
44
|
import { TELEMETRY_SCHEMA_VERSION, isModelCallSpan } from './lineage.js';
|
|
45
45
|
import { redactRow, telemetryRedactor } from './redact.js';
|
|
46
|
+
import { outputsDir } from '../paths.js';
|
|
46
47
|
/**
|
|
47
48
|
* TELEMETRY_PAYLOADS: off | errors | sample (default) | all
|
|
48
49
|
* TELEMETRY_PAYLOAD_SAMPLE: 0..1, default 0.10
|
|
@@ -300,7 +301,42 @@ export function telemetrySinkEnabled() {
|
|
|
300
301
|
return process.env.TELEMETRY_SUPABASE === 'true' && hasSupabaseCredentials();
|
|
301
302
|
}
|
|
302
303
|
export function deadLetterPath() {
|
|
303
|
-
return join(
|
|
304
|
+
return join(outputsDir(), 'telemetry-deadletter.ndjson');
|
|
305
|
+
}
|
|
306
|
+
/** The tables the exporter writes; nothing else is ever named in its SQL. */
|
|
307
|
+
export const LEDGER_TABLES = ['adk_telemetry', 'adk_turns', 'adk_payloads'];
|
|
308
|
+
const quoteIdent = (name) => `"${name.replace(/"/g, '""')}"`;
|
|
309
|
+
/**
|
|
310
|
+
* The ledger over a direct Postgres connection (ADR 0131): the same rows the
|
|
311
|
+
* Supabase sink writes, through one parameterised statement per batch. The
|
|
312
|
+
* rows go in as one JSON array and `jsonb_populate_recordset` gives each
|
|
313
|
+
* value its column's type, so JSON, arrays and timestamps need no mapping
|
|
314
|
+
* here. Only the columns the rows carry are named, so the rest keep their
|
|
315
|
+
* defaults. The table must be one of LEDGER_TABLES and every column name
|
|
316
|
+
* comes from the row builders above, never from span content.
|
|
317
|
+
*/
|
|
318
|
+
export function postgresLedgerClient(pool) {
|
|
319
|
+
return {
|
|
320
|
+
from(table) {
|
|
321
|
+
return {
|
|
322
|
+
async insert(rows) {
|
|
323
|
+
if (!LEDGER_TABLES.includes(table))
|
|
324
|
+
return { error: { message: `not a ledger table: ${table}` } };
|
|
325
|
+
const columns = [...new Set(rows.flatMap((r) => Object.entries(r).filter(([, v]) => v !== undefined).map(([k]) => k)))];
|
|
326
|
+
if (!columns.length)
|
|
327
|
+
return { error: null };
|
|
328
|
+
const list = columns.map(quoteIdent).join(', ');
|
|
329
|
+
try {
|
|
330
|
+
await pool.query(`INSERT INTO ${quoteIdent(table)} (${list}) SELECT ${list} FROM jsonb_populate_recordset(NULL::${quoteIdent(table)}, $1::jsonb)`, [JSON.stringify(rows)]);
|
|
331
|
+
return { error: null };
|
|
332
|
+
}
|
|
333
|
+
catch (err) {
|
|
334
|
+
return { error: { message: err instanceof Error ? err.message : String(err) } };
|
|
335
|
+
}
|
|
336
|
+
},
|
|
337
|
+
};
|
|
338
|
+
},
|
|
339
|
+
};
|
|
304
340
|
}
|
|
305
341
|
const PENDING_TTL_MS = 10 * 60 * 1000;
|
|
306
342
|
const DECISION_TTL_MS = 30 * 60 * 1000;
|
|
@@ -3,6 +3,7 @@ import type { ReadableSpan } from '@opentelemetry/sdk-trace-base';
|
|
|
3
3
|
import type { LlmResponse } from '../models/genaiMapping.ts';
|
|
4
4
|
import type { TurnEvent as Event } from '../runtime/events.ts';
|
|
5
5
|
import type { ModelRequest } from '../models/contract.ts';
|
|
6
|
+
import type { ExporterOptions, LedgerClient } from './supabaseSpanExporter.ts';
|
|
6
7
|
/**
|
|
7
8
|
* Optional OTLP export for a live trace viewer (Phoenix, Langfuse, Grafana
|
|
8
9
|
* Tempo, Jaeger — anything that speaks OTLP/HTTP). Set
|
|
@@ -15,6 +16,14 @@ import type { ModelRequest } from '../models/contract.ts';
|
|
|
15
16
|
* Supabase stays the system of record; this is a viewer.
|
|
16
17
|
*/
|
|
17
18
|
export declare function otlpEndpoint(): string | undefined;
|
|
19
|
+
/**
|
|
20
|
+
* Writes the ledger (adk_turns, adk_telemetry, adk_payloads) through
|
|
21
|
+
* `client`, batched off the turn's path, with the same rows, payload policy
|
|
22
|
+
* and redaction as the Supabase sink. One ledger is attached at a time;
|
|
23
|
+
* attaching another replaces it. Returns a detach that flushes what is
|
|
24
|
+
* buffered first.
|
|
25
|
+
*/
|
|
26
|
+
export declare function attachLedger(client: LedgerClient, options?: Omit<ExporterOptions, 'client'>): () => Promise<void>;
|
|
18
27
|
/**
|
|
19
28
|
* Flushes buffered span processors (console is synchronous; the Supabase
|
|
20
29
|
* batch sink is not). Call before process exit in short-lived scripts so
|
|
@@ -51,6 +51,49 @@ function otlpProcessor() {
|
|
|
51
51
|
let isOtelInitialized = false;
|
|
52
52
|
let tracerProvider;
|
|
53
53
|
let telemetryExporter;
|
|
54
|
+
/**
|
|
55
|
+
* The ledger an embedding application attaches at run time (ADR 0131):
|
|
56
|
+
* `attachLedger` (or `postgresStorage({ ledger: true })`) sets it, before or
|
|
57
|
+
* after tracing starts. The processor below is always registered and does
|
|
58
|
+
* nothing until then.
|
|
59
|
+
*/
|
|
60
|
+
let attachedLedger;
|
|
61
|
+
class AttachedLedgerProcessor {
|
|
62
|
+
onStart() { }
|
|
63
|
+
onEnd(span) {
|
|
64
|
+
attachedLedger?.processor.onEnd(span);
|
|
65
|
+
}
|
|
66
|
+
forceFlush() {
|
|
67
|
+
return attachedLedger ? attachedLedger.processor.forceFlush().then(() => attachedLedger?.exporter.forceFlush()) : Promise.resolve();
|
|
68
|
+
}
|
|
69
|
+
shutdown() {
|
|
70
|
+
return this.forceFlush();
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Writes the ledger (adk_turns, adk_telemetry, adk_payloads) through
|
|
75
|
+
* `client`, batched off the turn's path, with the same rows, payload policy
|
|
76
|
+
* and redaction as the Supabase sink. One ledger is attached at a time;
|
|
77
|
+
* attaching another replaces it. Returns a detach that flushes what is
|
|
78
|
+
* buffered first.
|
|
79
|
+
*/
|
|
80
|
+
export function attachLedger(client, options = {}) {
|
|
81
|
+
if (telemetrySinkEnabled()) {
|
|
82
|
+
console.warn('[TELEMETRY] ⚠ a ledger is attached while TELEMETRY_SUPABASE=true: every row is written twice. Use one sink.');
|
|
83
|
+
}
|
|
84
|
+
const exporter = new SupabaseSpanExporter({ ...options, client });
|
|
85
|
+
const entry = { exporter, processor: new sdkBase.BatchSpanProcessor(exporter) };
|
|
86
|
+
const previous = attachedLedger;
|
|
87
|
+
attachedLedger = entry;
|
|
88
|
+
void previous?.processor.shutdown();
|
|
89
|
+
initializeTracing();
|
|
90
|
+
return async () => {
|
|
91
|
+
if (attachedLedger === entry)
|
|
92
|
+
attachedLedger = undefined;
|
|
93
|
+
await entry.processor.forceFlush();
|
|
94
|
+
await entry.exporter.shutdown();
|
|
95
|
+
};
|
|
96
|
+
}
|
|
54
97
|
/**
|
|
55
98
|
* Flushes buffered span processors (console is synchronous; the Supabase
|
|
56
99
|
* batch sink is not). Call before process exit in short-lived scripts so
|
|
@@ -59,6 +102,7 @@ let telemetryExporter;
|
|
|
59
102
|
export async function flushTracing() {
|
|
60
103
|
await tracerProvider?.forceFlush?.();
|
|
61
104
|
await telemetryExporter?.forceFlush?.();
|
|
105
|
+
await attachedLedger?.exporter.forceFlush();
|
|
62
106
|
}
|
|
63
107
|
// ── In-process span listeners ────────────────────────────────────────────────
|
|
64
108
|
// Lets a script (e.g. scripts/demo_model_optionality.ts) read finished spans
|
|
@@ -231,6 +275,7 @@ export function initializeTracing() {
|
|
|
231
275
|
const spanProcessors = [
|
|
232
276
|
new SpanLineageProcessor(),
|
|
233
277
|
new SimpleSpanProcessor(new JsonConsoleExporter()),
|
|
278
|
+
new AttachedLedgerProcessor(),
|
|
234
279
|
];
|
|
235
280
|
// Optional durable sink (TELEMETRY_SUPABASE=true + Supabase credentials):
|
|
236
281
|
// llm.request and Syndicate Execution spans also land in adk_telemetry.
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* lib/paths.ts — where the engine writes on disk (ADR 0134).
|
|
3
|
+
*
|
|
4
|
+
* Each location keeps its own variable, which wins. Without one, the
|
|
5
|
+
* default is under MELCHIZEDEK_DATA_DIR when that is set, so an embedding
|
|
6
|
+
* application points every location at its own data directory with one
|
|
7
|
+
* variable; with neither, the defaults are the historical ones (the working
|
|
8
|
+
* directory's outputs/, and ~/.melchizedek for the ChatGPT sign-in).
|
|
9
|
+
*
|
|
10
|
+
* location own variable with MELCHIZEDEK_DATA_DIR default
|
|
11
|
+
* generated files, spool OUTPUTS_DIR <data>/outputs <cwd>/outputs
|
|
12
|
+
* task lists and jobs MELCHIZEDEK_TASKS_FILE <data>/outputs/tasks.json <cwd>/outputs/tasks.json
|
|
13
|
+
* ChatGPT sign-in MELCHIZEDEK_CHATGPT_SIGNIN_FILE <data>/chatgpt-signin.json ~/.melchizedek/chatgpt-signin.json
|
|
14
|
+
*
|
|
15
|
+
* Read at each call, never cached. Imports only node built-ins.
|
|
16
|
+
*/
|
|
17
|
+
export declare const DATA_DIR_ENV = "MELCHIZEDEK_DATA_DIR";
|
|
18
|
+
/** MELCHIZEDEK_DATA_DIR, resolved; undefined when unset. */
|
|
19
|
+
export declare function dataDir(env?: NodeJS.ProcessEnv): string | undefined;
|
|
20
|
+
/** Where generated files (images) and the telemetry spool go. */
|
|
21
|
+
export declare function outputsDir(env?: NodeJS.ProcessEnv): string;
|
|
22
|
+
/** The task tools' JSON file, when tasks are not in Postgres. */
|
|
23
|
+
export declare function tasksFile(env?: NodeJS.ProcessEnv): string;
|
|
24
|
+
/** The local Sign in with ChatGPT credential file. */
|
|
25
|
+
export declare function chatGptSignInFile(env?: NodeJS.ProcessEnv): string;
|
|
26
|
+
/**
|
|
27
|
+
* The engine's own package root (the directory of its package.json named
|
|
28
|
+
* melchizedek-agents), found from this module's location, whatever the
|
|
29
|
+
* working directory is; undefined if it cannot be found.
|
|
30
|
+
*/
|
|
31
|
+
export declare function enginePackageRoot(from?: string): string | undefined;
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* lib/paths.ts — where the engine writes on disk (ADR 0134).
|
|
3
|
+
*
|
|
4
|
+
* Each location keeps its own variable, which wins. Without one, the
|
|
5
|
+
* default is under MELCHIZEDEK_DATA_DIR when that is set, so an embedding
|
|
6
|
+
* application points every location at its own data directory with one
|
|
7
|
+
* variable; with neither, the defaults are the historical ones (the working
|
|
8
|
+
* directory's outputs/, and ~/.melchizedek for the ChatGPT sign-in).
|
|
9
|
+
*
|
|
10
|
+
* location own variable with MELCHIZEDEK_DATA_DIR default
|
|
11
|
+
* generated files, spool OUTPUTS_DIR <data>/outputs <cwd>/outputs
|
|
12
|
+
* task lists and jobs MELCHIZEDEK_TASKS_FILE <data>/outputs/tasks.json <cwd>/outputs/tasks.json
|
|
13
|
+
* ChatGPT sign-in MELCHIZEDEK_CHATGPT_SIGNIN_FILE <data>/chatgpt-signin.json ~/.melchizedek/chatgpt-signin.json
|
|
14
|
+
*
|
|
15
|
+
* Read at each call, never cached. Imports only node built-ins.
|
|
16
|
+
*/
|
|
17
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
18
|
+
import { homedir } from 'node:os';
|
|
19
|
+
import { dirname, join, resolve } from 'node:path';
|
|
20
|
+
import { fileURLToPath } from 'node:url';
|
|
21
|
+
export const DATA_DIR_ENV = 'MELCHIZEDEK_DATA_DIR';
|
|
22
|
+
const set = (env, name) => env[name]?.trim() || undefined;
|
|
23
|
+
/** MELCHIZEDEK_DATA_DIR, resolved; undefined when unset. */
|
|
24
|
+
export function dataDir(env = process.env) {
|
|
25
|
+
const d = set(env, DATA_DIR_ENV);
|
|
26
|
+
return d ? resolve(d) : undefined;
|
|
27
|
+
}
|
|
28
|
+
/** Where generated files (images) and the telemetry spool go. */
|
|
29
|
+
export function outputsDir(env = process.env) {
|
|
30
|
+
const own = set(env, 'OUTPUTS_DIR');
|
|
31
|
+
if (own)
|
|
32
|
+
return resolve(own);
|
|
33
|
+
const data = dataDir(env);
|
|
34
|
+
return data ? join(data, 'outputs') : join(process.cwd(), 'outputs');
|
|
35
|
+
}
|
|
36
|
+
/** The task tools' JSON file, when tasks are not in Postgres. */
|
|
37
|
+
export function tasksFile(env = process.env) {
|
|
38
|
+
const own = set(env, 'MELCHIZEDEK_TASKS_FILE');
|
|
39
|
+
if (own)
|
|
40
|
+
return resolve(own);
|
|
41
|
+
const data = dataDir(env);
|
|
42
|
+
return data ? join(data, 'outputs', 'tasks.json') : join(process.cwd(), 'outputs', 'tasks.json');
|
|
43
|
+
}
|
|
44
|
+
/** The local Sign in with ChatGPT credential file. */
|
|
45
|
+
export function chatGptSignInFile(env = process.env) {
|
|
46
|
+
const own = set(env, 'MELCHIZEDEK_CHATGPT_SIGNIN_FILE');
|
|
47
|
+
if (own)
|
|
48
|
+
return resolve(own);
|
|
49
|
+
const data = dataDir(env);
|
|
50
|
+
return data ? join(data, 'chatgpt-signin.json') : join(homedir(), '.melchizedek', 'chatgpt-signin.json');
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* The engine's own package root (the directory of its package.json named
|
|
54
|
+
* melchizedek-agents), found from this module's location, whatever the
|
|
55
|
+
* working directory is; undefined if it cannot be found.
|
|
56
|
+
*/
|
|
57
|
+
export function enginePackageRoot(from = fileURLToPath(import.meta.url)) {
|
|
58
|
+
let dir = dirname(from);
|
|
59
|
+
for (let i = 0; i < 6; i++) {
|
|
60
|
+
const pkg = join(dir, 'package.json');
|
|
61
|
+
if (existsSync(pkg)) {
|
|
62
|
+
try {
|
|
63
|
+
if (JSON.parse(readFileSync(pkg, 'utf-8')).name === 'melchizedek-agents')
|
|
64
|
+
return dir;
|
|
65
|
+
}
|
|
66
|
+
catch {
|
|
67
|
+
/* not ours; keep climbing */
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
const parent = dirname(dir);
|
|
71
|
+
if (parent === dir)
|
|
72
|
+
break;
|
|
73
|
+
dir = parent;
|
|
74
|
+
}
|
|
75
|
+
return undefined;
|
|
76
|
+
}
|
|
@@ -19,6 +19,7 @@ import { PostgresSessionService } from './sessionService.ts';
|
|
|
19
19
|
import { PostgresTaskStore } from './taskStore.ts';
|
|
20
20
|
import type { CredentialStoreOptions } from '../../tools/credentialStore.ts';
|
|
21
21
|
import type { CredentialStore } from '../../tools/auth.ts';
|
|
22
|
+
import type { PayloadPolicy } from '../../observability/supabaseSpanExporter.ts';
|
|
22
23
|
import type { AuditSink } from '../../observability/audit.ts';
|
|
23
24
|
import type { RlsHardeningStatus } from '../rlsStatus.ts';
|
|
24
25
|
import type { TaskBackend, TaskBackendOptions } from '../../tools/taskTools.ts';
|
|
@@ -28,6 +29,8 @@ export { PostgresSessionService } from './sessionService.ts';
|
|
|
28
29
|
export { PostgresTaskStore } from './taskStore.ts';
|
|
29
30
|
export { postgresMemoryStore } from './memoryStore.ts';
|
|
30
31
|
export { postgresCredentialRows, postgresCredentialStore } from './credentialStore.ts';
|
|
32
|
+
export { inProcessTurnLock } from '../../a2a/turnLock.ts';
|
|
33
|
+
export type { TurnLock } from '../../a2a/turnLock.ts';
|
|
31
34
|
export interface PostgresStorageOptions {
|
|
32
35
|
/** A Postgres URL. Ignored when `pool` is given. */
|
|
33
36
|
connectionString?: string;
|
|
@@ -54,6 +57,28 @@ export interface PostgresStorageOptions {
|
|
|
54
57
|
};
|
|
55
58
|
/** Owner of an A2A call, for the task store. Default: the SDK's user scope. */
|
|
56
59
|
taskOwner?: PostgresTaskStoreOptions['ownerResolver'];
|
|
60
|
+
/**
|
|
61
|
+
* The turn lock to use instead of the advisory lock below. The advisory
|
|
62
|
+
* lock is what makes several instances on one database take turns; it
|
|
63
|
+
* needs a database that gives each connection its own session. Embedded
|
|
64
|
+
* Postgres (PGlite behind pglite-socket) runs every connection in ONE
|
|
65
|
+
* session, where an advisory lock is re-entrant and so excludes nothing:
|
|
66
|
+
* there, one process owns the database, and `inProcessTurnLock()`
|
|
67
|
+
* (exported here) is the correct lock. When given, no turn-lock
|
|
68
|
+
* pool is opened.
|
|
69
|
+
*/
|
|
70
|
+
turnLock?: TurnLock;
|
|
71
|
+
/**
|
|
72
|
+
* Write the telemetry ledger (adk_turns, adk_telemetry, adk_payloads) to
|
|
73
|
+
* this database (ADR 0131): the rows the Supabase sink writes, with the
|
|
74
|
+
* same payload policy (TELEMETRY_PAYLOADS, or `policy`) and redaction.
|
|
75
|
+
* Needs db/telemetry.sql applied. `close()` flushes and detaches it.
|
|
76
|
+
* Default off.
|
|
77
|
+
*/
|
|
78
|
+
ledger?: boolean | {
|
|
79
|
+
policy?: PayloadPolicy;
|
|
80
|
+
deadLetterFile?: string;
|
|
81
|
+
};
|
|
57
82
|
/**
|
|
58
83
|
* Connections reserved for turn locks: each running turn holds one for its
|
|
59
84
|
* length, apart from the main pool so a long turn never starves queries.
|
|
@@ -22,12 +22,16 @@ import { postgresCredentialStore } from './credentialStore.js';
|
|
|
22
22
|
import { searchPathOption } from '../schema.js';
|
|
23
23
|
import { POSTGRES_RLS_QUERY, evaluatePostgresRls } from '../rlsStatus.js';
|
|
24
24
|
import { postgresAuditSink } from '../../observability/audit.js';
|
|
25
|
+
import { attachLedger } from '../../observability/tracer.js';
|
|
26
|
+
import { postgresLedgerClient } from '../../observability/supabaseSpanExporter.js';
|
|
25
27
|
import { createHash, randomUUID } from 'node:crypto';
|
|
26
28
|
import { hostname } from 'node:os';
|
|
27
29
|
export { PostgresSessionService } from './sessionService.js';
|
|
28
30
|
export { PostgresTaskStore } from './taskStore.js';
|
|
29
31
|
export { postgresMemoryStore } from './memoryStore.js';
|
|
30
32
|
export { postgresCredentialRows, postgresCredentialStore } from './credentialStore.js';
|
|
33
|
+
// The single-process turn lock, for an embedded database (`turnLock` below).
|
|
34
|
+
export { inProcessTurnLock } from '../../a2a/turnLock.js';
|
|
31
35
|
/**
|
|
32
36
|
* A pool emits 'error' when an idle connection dies (a database restart, a
|
|
33
37
|
* failover, an administrator's terminate); unhandled, that event crashes the
|
|
@@ -69,7 +73,7 @@ export function postgresStorage(options) {
|
|
|
69
73
|
...(options.pool ? options.pool.options : { connectionString: options.connectionString, ...options.poolConfig }),
|
|
70
74
|
max: options.lockPoolMax ?? 20,
|
|
71
75
|
}), 'turn-lock pool'));
|
|
72
|
-
const
|
|
76
|
+
const advisoryTurnLock = async (key, { waitMs, signal }) => {
|
|
73
77
|
const lockId = advisoryLockId(key);
|
|
74
78
|
const client = await locks().connect();
|
|
75
79
|
const deadline = Date.now() + Math.max(0, waitMs);
|
|
@@ -105,6 +109,10 @@ export function postgresStorage(options) {
|
|
|
105
109
|
};
|
|
106
110
|
return release;
|
|
107
111
|
};
|
|
112
|
+
const turnLock = options.turnLock ?? advisoryTurnLock;
|
|
113
|
+
const detachLedger = options.ledger
|
|
114
|
+
? attachLedger(postgresLedgerClient(pool), options.ledger === true ? {} : options.ledger)
|
|
115
|
+
: undefined;
|
|
108
116
|
const audit = postgresAuditSink(pool);
|
|
109
117
|
const credentials = options.credentials ? postgresCredentialStore(pool, { ...options.credentials, audit }) : undefined;
|
|
110
118
|
return {
|
|
@@ -160,6 +168,7 @@ export function postgresStorage(options) {
|
|
|
160
168
|
return counts;
|
|
161
169
|
},
|
|
162
170
|
async close() {
|
|
171
|
+
await detachLedger?.();
|
|
163
172
|
if (lockPool)
|
|
164
173
|
await lockPool.end();
|
|
165
174
|
if (owned)
|