melchizedek-agents 1.4.0 → 1.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/.env.example +15 -0
  2. package/CHANGELOG.md +77 -0
  3. package/DOCUMENTATION.md +43 -2
  4. package/config/agents/syndicate.schema.json +80 -6
  5. package/db/hardening.sql +38 -12
  6. package/dist/lib/a2a/app.js +2 -1
  7. package/dist/lib/chatgpt/state.js +4 -4
  8. package/dist/lib/compile.js +15 -2
  9. package/dist/lib/index.d.ts +2 -0
  10. package/dist/lib/index.js +2 -0
  11. package/dist/lib/loadSyndicate.js +4 -0
  12. package/dist/lib/memory/providers.js +2 -1
  13. package/dist/lib/models/claudeAdapter.js +2 -1
  14. package/dist/lib/models/credentialSource.d.ts +28 -0
  15. package/dist/lib/models/credentialSource.js +49 -0
  16. package/dist/lib/models/endpoints.js +2 -1
  17. package/dist/lib/models/gateway.js +2 -1
  18. package/dist/lib/models/gatewayAdapter.js +2 -1
  19. package/dist/lib/models/geminiAdapter.js +2 -1
  20. package/dist/lib/models/gptAdapter.js +2 -1
  21. package/dist/lib/models/grokAdapter.js +2 -1
  22. package/dist/lib/models/kimiAdapter.js +2 -1
  23. package/dist/lib/models/providerMap.js +3 -2
  24. package/dist/lib/observability/embeddings.js +2 -1
  25. package/dist/lib/observability/lineage.js +11 -3
  26. package/dist/lib/observability/supabaseSpanExporter.d.ts +19 -1
  27. package/dist/lib/observability/supabaseSpanExporter.js +37 -1
  28. package/dist/lib/observability/tracer.d.ts +9 -0
  29. package/dist/lib/observability/tracer.js +45 -0
  30. package/dist/lib/paths.d.ts +31 -0
  31. package/dist/lib/paths.js +76 -0
  32. package/dist/lib/storage/postgres/index.d.ts +25 -0
  33. package/dist/lib/storage/postgres/index.js +10 -1
  34. package/dist/lib/syndicateSchema.d.ts +26 -4
  35. package/dist/lib/syndicateSchema.js +45 -1
  36. package/dist/lib/tools/generateImageTool.js +6 -6
  37. package/dist/lib/tools/inspectImageTool.d.ts +2 -0
  38. package/dist/lib/tools/inspectImageTool.js +17 -10
  39. package/dist/lib/tools/mcpToolFactory.d.ts +34 -1
  40. package/dist/lib/tools/mcpToolFactory.js +117 -17
  41. package/dist/lib/tools/taskTools.js +3 -3
  42. package/dist/lib/tools/xApiSearchTool.js +2 -1
  43. package/dist/scripts/a2a_server.js +5 -0
  44. package/package.json +7 -4
@@ -0,0 +1,49 @@
1
+ /**
2
+ * lib/models/credentialSource.ts — where the engine reads a credential by
3
+ * its environment variable name (ADR 0133).
4
+ *
5
+ * By default every provider key, gateway key and embedding key is read from
6
+ * `process.env`. An embedding application that holds keys elsewhere (an OS
7
+ * keychain, a secret manager) sets a source once:
8
+ *
9
+ * setCredentialSource((name) => keychain.get(name));
10
+ *
11
+ * and every read below asks the source first, by the same names the
12
+ * environment uses (ANTHROPIC_API_KEY, MODEL_GATEWAY_API_KEY, …), at the
13
+ * moment of use: nothing is cached, so a key the user removes stops working
14
+ * on the next call. A source that returns undefined for a name falls back to
15
+ * the environment. A dependency-free leaf, so providerMap.ts stays one.
16
+ */
17
+ let source;
18
+ /** Sets (or, with undefined, clears) the process's credential source. */
19
+ export function setCredentialSource(next) {
20
+ source = next;
21
+ }
22
+ /** The credential named `name`: the source's, else `env`'s. A source that throws counts as holding nothing. */
23
+ export function credentialValue(name, env = process.env) {
24
+ if (source) {
25
+ try {
26
+ const held = source(name);
27
+ if (held !== undefined && held !== '')
28
+ return held;
29
+ }
30
+ catch {
31
+ /* a broken source must not take the engine down; the environment still answers */
32
+ }
33
+ }
34
+ return env[name];
35
+ }
36
+ /**
37
+ * `env` as the engine's env-reading helpers see it: a read by name asks the
38
+ * source first. For functions that take an `env` parameter; enumeration and
39
+ * spreading see only `env`'s own variables.
40
+ */
41
+ export function credentialEnv(env = process.env) {
42
+ if (!source)
43
+ return env;
44
+ return new Proxy(env, {
45
+ get(target, key, receiver) {
46
+ return typeof key === 'string' ? credentialValue(key, target) : Reflect.get(target, key, receiver);
47
+ },
48
+ });
49
+ }
@@ -31,6 +31,7 @@ var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExte
31
31
  };
32
32
  import { PROVIDERS, providerKeyPresent } from './providerMap.js';
33
33
  import { trimTrailingSlashes } from './urls.js';
34
+ import { credentialEnv } from './credentialSource.js';
34
35
  export const PLATFORMS_FOR = {
35
36
  gemini: ['direct', 'vertex'],
36
37
  anthropic: ['direct', 'bedrock', 'vertex'],
@@ -81,7 +82,7 @@ export function platformFromEnv(provider, env = process.env) {
81
82
  return raw;
82
83
  }
83
84
  /** A provider's endpoint as the environment configures it. */
84
- export function endpointFromEnv(provider, env = process.env) {
85
+ export function endpointFromEnv(provider, env = credentialEnv()) {
85
86
  const platform = platformFromEnv(provider, env);
86
87
  const mapEnv = MODEL_MAP_ENV[provider];
87
88
  const models = mapEnv ? parseModelMap(env[mapEnv], mapEnv) : undefined;
@@ -41,6 +41,7 @@ import { chatGptSignInRoutesOpenAi } from '../chatgpt/state.js';
41
41
  import { providerReady } from './endpoints.js';
42
42
  import { PROVIDERS, providerForModel } from './providerMap.js';
43
43
  import { trimTrailingSlashes } from './urls.js';
44
+ import { credentialValue } from './credentialSource.js';
44
45
  export const GATEWAY_ENV = 'MODEL_GATEWAY';
45
46
  export const GATEWAY_KEY_ENV = 'MODEL_GATEWAY_API_KEY';
46
47
  export const GATEWAY_BASE_URL_ENV = 'MODEL_GATEWAY_BASE_URL';
@@ -78,7 +79,7 @@ export function gatewayConfig() {
78
79
  return {
79
80
  gateway,
80
81
  baseUrl: trimTrailingSlashes(override || gateway.baseUrl),
81
- keyPresent: !!process.env[GATEWAY_KEY_ENV],
82
+ keyPresent: !!credentialValue(GATEWAY_KEY_ENV),
82
83
  };
83
84
  }
84
85
  /** A human-readable misconfiguration, or undefined when the gateway is
@@ -31,6 +31,7 @@
31
31
  import { ChatCompletionsAdapter } from './chatCompletionsAdapter.js';
32
32
  import { GATEWAY_ENV, GATEWAY_KEY_ENV, GATEWAY_MODEL_MAP_ENV, gatewayConfig, gatewayWireModel } from './gateway.js';
33
33
  import { providerForModel } from './providerMap.js';
34
+ import { credentialValue } from './credentialSource.js';
34
35
  export class GatewayAdapter extends ChatCompletionsAdapter {
35
36
  /** Attribution stays with the upstream provider named by the YAML id. */
36
37
  provider;
@@ -47,7 +48,7 @@ export class GatewayAdapter extends ChatCompletionsAdapter {
47
48
  return `${this.#cfg?.baseUrl ?? ''}/chat/completions`;
48
49
  }
49
50
  headers() {
50
- return { Authorization: `Bearer ${process.env[GATEWAY_KEY_ENV] ?? ''}` };
51
+ return { Authorization: `Bearer ${credentialValue(GATEWAY_KEY_ENV) ?? ''}` };
51
52
  }
52
53
  wireModelName(model) {
53
54
  return this.#cfg ? gatewayWireModel(model, this.#cfg.gateway) : model;
@@ -52,6 +52,7 @@ import { currentTurnStart, providerStateOf } from './providerState.js';
52
52
  import { classifyError, errorStatus, retryUntilFirstYield } from './retry.js';
53
53
  import { isAboveHigh, reasoningConfig } from './reasoning.js';
54
54
  import { setLlmSpanAttribute } from '../observability/tracer.js';
55
+ import { credentialValue } from './credentialSource.js';
55
56
  /**
56
57
  * The provider id this adapter reports and writes its state under, and the
57
58
  * providerState kind for a Gemini thought signature (ADR 0046). Defined once
@@ -846,7 +847,7 @@ function nonEmpty(value) {
846
847
  }
847
848
  /** The Gemini API key from the environment, in ADK's order. */
848
849
  function envApiKey() {
849
- return nonEmpty(process.env.GOOGLE_GENAI_API_KEY) ?? nonEmpty(process.env.GOOGLE_API_KEY) ?? nonEmpty(process.env.GEMINI_API_KEY);
850
+ return nonEmpty(credentialValue('GOOGLE_GENAI_API_KEY')) ?? nonEmpty(credentialValue('GOOGLE_API_KEY')) ?? nonEmpty(credentialValue('GEMINI_API_KEY'));
850
851
  }
851
852
  function messageOf(err) {
852
853
  return err instanceof Error ? err.message : String(err);
@@ -48,6 +48,7 @@ import { currentTurnStart, providerStateOf } from './providerState.js';
48
48
  import { effortWord } from './reasoning.js';
49
49
  import { toContractJsonSchema } from './schemaNormalize.js';
50
50
  import { setLlmSpanAttribute } from '../observability/tracer.js';
51
+ import { credentialValue } from './credentialSource.js';
51
52
  /** The providerState kind the Responses adapters write: the reasoning output items that preceded a part (ADR 0046, ADR 0050). */
52
53
  export const REASONING_STATE_KIND = 'reasoning_items';
53
54
  /** The `include` value that returns reasoning items with their encrypted content. */
@@ -410,7 +411,7 @@ export class GptAdapter {
410
411
  return undefined;
411
412
  }
412
413
  apiKeyFromEnv() {
413
- return process.env.OPENAI_API_KEY;
414
+ return credentialValue('OPENAI_API_KEY');
414
415
  }
415
416
  missingKeyMessage() {
416
417
  return 'OPENAI_API_KEY is not set in environment.';
@@ -29,6 +29,7 @@ import { GptAdapter } from './gptAdapter.js';
29
29
  import { effortWord } from './reasoning.js';
30
30
  import { collectionIdsFromEnv, collectionsMaxResultsFromEnv, xaiWebSearchParamsFromEnv, xSearchParamsFromEnv, } from '../tools/xaiSearchParams.js';
31
31
  import { setLlmSpanAttribute } from '../observability/tracer.js';
32
+ import { credentialValue } from './credentialSource.js';
32
33
  export const XAI_BASE_URL = 'https://api.x.ai/v1';
33
34
  /**
34
35
  * The ids that take a reasoning effort and return encrypted reasoning:
@@ -60,7 +61,7 @@ export class GrokAdapter extends GptAdapter {
60
61
  return XAI_BASE_URL;
61
62
  }
62
63
  apiKeyFromEnv() {
63
- return process.env.XAI_API_KEY;
64
+ return credentialValue('XAI_API_KEY');
64
65
  }
65
66
  missingKeyMessage() {
66
67
  return 'XAI_API_KEY is not set in environment.';
@@ -120,6 +120,7 @@
120
120
  import { DEFAULT_KIMI_REASONING_EFFORT } from '../config.js';
121
121
  import { ChatCompletionsAdapter, reasonsNotAtAll } from './chatCompletionsAdapter.js';
122
122
  import { trimTrailingSlashes } from './urls.js';
123
+ import { credentialValue } from './credentialSource.js';
123
124
  export const MOONSHOT_BASE_URL = 'https://api.moonshot.ai/v1';
124
125
  /** The flagship takes `reasoning_effort`; the K2 generation takes a `thinking` switch. */
125
126
  export function isKimiK3(model) {
@@ -153,7 +154,7 @@ export class KimiAdapter extends ChatCompletionsAdapter {
153
154
  return `${this.#baseUrl}/chat/completions`;
154
155
  }
155
156
  #key() {
156
- return this.#apiKey ?? process.env.MOONSHOT_API_KEY;
157
+ return this.#apiKey ?? credentialValue('MOONSHOT_API_KEY');
157
158
  }
158
159
  headers() {
159
160
  return { Authorization: `Bearer ${this.#key() ?? ''}` };
@@ -13,6 +13,7 @@
13
13
  * grok-* → xai ollama/<model> → ollama (local, keyless)
14
14
  * kimi-* → moonshot everything else → gemini (the default)
15
15
  */
16
+ import { credentialValue } from './credentialSource.js';
16
17
  export const PROVIDERS = {
17
18
  gemini: { keyEnv: 'GOOGLE_GENAI_API_KEY', label: 'Google Gemini' },
18
19
  anthropic: { keyEnv: 'ANTHROPIC_API_KEY', label: 'Anthropic Claude' },
@@ -43,9 +44,9 @@ export function providerKeyPresent(provider) {
43
44
  return true;
44
45
  // Gemini historically accepts either env name.
45
46
  if (provider === 'gemini') {
46
- return realValue(process.env.GOOGLE_GENAI_API_KEY) || realValue(process.env.GEMINI_API_KEY);
47
+ return realValue(credentialValue('GOOGLE_GENAI_API_KEY')) || realValue(credentialValue('GEMINI_API_KEY'));
47
48
  }
48
- return realValue(process.env[keyEnv]);
49
+ return realValue(credentialValue(keyEnv));
49
50
  }
50
51
  /** Set, and not a `.env.example` placeholder (`your_..._here`, `<...>`).
51
52
  * Inlined rather than imported: this module stays a dependency-free leaf. */
@@ -10,6 +10,7 @@
10
10
  */
11
11
  import { EMBEDDING_DIMENSIONS, EMBEDDING_MODEL } from '../config.js';
12
12
  import { memoryProvidersFromEnv } from '../memory/providers.js';
13
+ import { credentialValue } from '../models/credentialSource.js';
13
14
  const MAX_CHARS = 8_000;
14
15
  /**
15
16
  * Embeds each text with the SAME embedder long-term memory uses, so the two
@@ -20,7 +21,7 @@ const MAX_CHARS = 8_000;
20
21
  export async function embedTexts(texts, options = {}) {
21
22
  let embedder = options.embedder;
22
23
  if (!embedder) {
23
- const apiKey = options.apiKey ?? process.env.GOOGLE_GENAI_API_KEY ?? process.env.GEMINI_API_KEY;
24
+ const apiKey = options.apiKey ?? credentialValue('GOOGLE_GENAI_API_KEY') ?? credentialValue('GEMINI_API_KEY');
24
25
  const provider = (process.env.MEMORY_EMBEDDING_PROVIDER?.trim() || 'gemini').toLowerCase();
25
26
  if (provider === 'gemini' && !apiKey)
26
27
  throw new Error('GOOGLE_GENAI_API_KEY is required for Gemini embeddings');
@@ -26,6 +26,7 @@ import { createHash } from 'node:crypto';
26
26
  import { execSync } from 'node:child_process';
27
27
  import { existsSync, readFileSync } from 'node:fs';
28
28
  import { join } from 'node:path';
29
+ import { enginePackageRoot } from '../paths.js';
29
30
  /** Schema version stamped on every ledger row (adk_turns / adk_payloads). */
30
31
  export const TELEMETRY_SCHEMA_VERSION = 2;
31
32
  // ── Span lineage: the names spans are given ──────────────────────────────────
@@ -105,8 +106,10 @@ export function engineVersion() {
105
106
  return cachedEngineVersion;
106
107
  let pkgVersion = '0.0.0';
107
108
  try {
108
- const pkgPath = join(process.cwd(), 'package.json');
109
- if (existsSync(pkgPath)) {
109
+ // The engine's own package.json, not the embedding application's.
110
+ const root = enginePackageRoot();
111
+ const pkgPath = root ? join(root, 'package.json') : '';
112
+ if (pkgPath && existsSync(pkgPath)) {
110
113
  pkgVersion = String(JSON.parse(readFileSync(pkgPath, 'utf-8')).version ?? pkgVersion);
111
114
  }
112
115
  }
@@ -116,7 +119,12 @@ export function engineVersion() {
116
119
  let commit = process.env.SOURCE_VERSION ?? process.env.HEROKU_SLUG_COMMIT ?? process.env.GIT_COMMIT ?? '';
117
120
  if (!commit) {
118
121
  try {
119
- commit = execSync('git rev-parse --short HEAD', { stdio: ['ignore', 'pipe', 'ignore'] })
122
+ // Only from the engine's own checkout: an installed package inside an
123
+ // application's repository would otherwise report the application's commit.
124
+ const root = enginePackageRoot();
125
+ if (!root || !existsSync(join(root, '.git')))
126
+ throw new Error('not a checkout');
127
+ commit = execSync('git rev-parse --short HEAD', { cwd: root, stdio: ['ignore', 'pipe', 'ignore'] })
120
128
  .toString()
121
129
  .trim();
122
130
  }
@@ -174,7 +174,8 @@ export interface ExportStats {
174
174
  deadLettered: number;
175
175
  payloadsDropped: number;
176
176
  }
177
- interface SupabaseLikeClient {
177
+ /** Where the exporter writes: supabase-js, or `postgresLedgerClient` on a pg pool. */
178
+ export interface LedgerClient {
178
179
  from(table: string): {
179
180
  insert(rows: unknown[]): PromiseLike<{
180
181
  error: {
@@ -183,6 +184,23 @@ interface SupabaseLikeClient {
183
184
  }>;
184
185
  };
185
186
  }
187
+ type SupabaseLikeClient = LedgerClient;
188
+ /** The tables the exporter writes; nothing else is ever named in its SQL. */
189
+ export declare const LEDGER_TABLES: readonly ['adk_telemetry', 'adk_turns', 'adk_payloads'];
190
+ /** The subset of a pg Pool the ledger client needs. */
191
+ export interface LedgerPool {
192
+ query(text: string, values?: unknown[]): Promise<unknown>;
193
+ }
194
+ /**
195
+ * The ledger over a direct Postgres connection (ADR 0131): the same rows the
196
+ * Supabase sink writes, through one parameterised statement per batch. The
197
+ * rows go in as one JSON array and `jsonb_populate_recordset` gives each
198
+ * value its column's type, so JSON, arrays and timestamps need no mapping
199
+ * here. Only the columns the rows carry are named, so the rest keep their
200
+ * defaults. The table must be one of LEDGER_TABLES and every column name
201
+ * comes from the row builders above, never from span content.
202
+ */
203
+ export declare function postgresLedgerClient(pool: LedgerPool): LedgerClient;
186
204
  export interface ExporterOptions {
187
205
  /** Injected client (tests); default: lazily created supabase-js client. */
188
206
  client?: SupabaseLikeClient;
@@ -43,6 +43,7 @@ const { ExportResultCode } = core;
43
43
  import { hasSupabaseCredentials } from '../persistence/supabaseProvider.js';
44
44
  import { TELEMETRY_SCHEMA_VERSION, isModelCallSpan } from './lineage.js';
45
45
  import { redactRow, telemetryRedactor } from './redact.js';
46
+ import { outputsDir } from '../paths.js';
46
47
  /**
47
48
  * TELEMETRY_PAYLOADS: off | errors | sample (default) | all
48
49
  * TELEMETRY_PAYLOAD_SAMPLE: 0..1, default 0.10
@@ -300,7 +301,42 @@ export function telemetrySinkEnabled() {
300
301
  return process.env.TELEMETRY_SUPABASE === 'true' && hasSupabaseCredentials();
301
302
  }
302
303
  export function deadLetterPath() {
303
- return join(process.env.OUTPUTS_DIR ?? join(process.cwd(), 'outputs'), 'telemetry-deadletter.ndjson');
304
+ return join(outputsDir(), 'telemetry-deadletter.ndjson');
305
+ }
306
+ /** The tables the exporter writes; nothing else is ever named in its SQL. */
307
+ export const LEDGER_TABLES = ['adk_telemetry', 'adk_turns', 'adk_payloads'];
308
+ const quoteIdent = (name) => `"${name.replace(/"/g, '""')}"`;
309
+ /**
310
+ * The ledger over a direct Postgres connection (ADR 0131): the same rows the
311
+ * Supabase sink writes, through one parameterised statement per batch. The
312
+ * rows go in as one JSON array and `jsonb_populate_recordset` gives each
313
+ * value its column's type, so JSON, arrays and timestamps need no mapping
314
+ * here. Only the columns the rows carry are named, so the rest keep their
315
+ * defaults. The table must be one of LEDGER_TABLES and every column name
316
+ * comes from the row builders above, never from span content.
317
+ */
318
+ export function postgresLedgerClient(pool) {
319
+ return {
320
+ from(table) {
321
+ return {
322
+ async insert(rows) {
323
+ if (!LEDGER_TABLES.includes(table))
324
+ return { error: { message: `not a ledger table: ${table}` } };
325
+ const columns = [...new Set(rows.flatMap((r) => Object.entries(r).filter(([, v]) => v !== undefined).map(([k]) => k)))];
326
+ if (!columns.length)
327
+ return { error: null };
328
+ const list = columns.map(quoteIdent).join(', ');
329
+ try {
330
+ await pool.query(`INSERT INTO ${quoteIdent(table)} (${list}) SELECT ${list} FROM jsonb_populate_recordset(NULL::${quoteIdent(table)}, $1::jsonb)`, [JSON.stringify(rows)]);
331
+ return { error: null };
332
+ }
333
+ catch (err) {
334
+ return { error: { message: err instanceof Error ? err.message : String(err) } };
335
+ }
336
+ },
337
+ };
338
+ },
339
+ };
304
340
  }
305
341
  const PENDING_TTL_MS = 10 * 60 * 1000;
306
342
  const DECISION_TTL_MS = 30 * 60 * 1000;
@@ -3,6 +3,7 @@ import type { ReadableSpan } from '@opentelemetry/sdk-trace-base';
3
3
  import type { LlmResponse } from '../models/genaiMapping.ts';
4
4
  import type { TurnEvent as Event } from '../runtime/events.ts';
5
5
  import type { ModelRequest } from '../models/contract.ts';
6
+ import type { ExporterOptions, LedgerClient } from './supabaseSpanExporter.ts';
6
7
  /**
7
8
  * Optional OTLP export for a live trace viewer (Phoenix, Langfuse, Grafana
8
9
  * Tempo, Jaeger — anything that speaks OTLP/HTTP). Set
@@ -15,6 +16,14 @@ import type { ModelRequest } from '../models/contract.ts';
15
16
  * Supabase stays the system of record; this is a viewer.
16
17
  */
17
18
  export declare function otlpEndpoint(): string | undefined;
19
+ /**
20
+ * Writes the ledger (adk_turns, adk_telemetry, adk_payloads) through
21
+ * `client`, batched off the turn's path, with the same rows, payload policy
22
+ * and redaction as the Supabase sink. One ledger is attached at a time;
23
+ * attaching another replaces it. Returns a detach that flushes what is
24
+ * buffered first.
25
+ */
26
+ export declare function attachLedger(client: LedgerClient, options?: Omit<ExporterOptions, 'client'>): () => Promise<void>;
18
27
  /**
19
28
  * Flushes buffered span processors (console is synchronous; the Supabase
20
29
  * batch sink is not). Call before process exit in short-lived scripts so
@@ -51,6 +51,49 @@ function otlpProcessor() {
51
51
  let isOtelInitialized = false;
52
52
  let tracerProvider;
53
53
  let telemetryExporter;
54
+ /**
55
+ * The ledger an embedding application attaches at run time (ADR 0131):
56
+ * `attachLedger` (or `postgresStorage({ ledger: true })`) sets it, before or
57
+ * after tracing starts. The processor below is always registered and does
58
+ * nothing until then.
59
+ */
60
+ let attachedLedger;
61
+ class AttachedLedgerProcessor {
62
+ onStart() { }
63
+ onEnd(span) {
64
+ attachedLedger?.processor.onEnd(span);
65
+ }
66
+ forceFlush() {
67
+ return attachedLedger ? attachedLedger.processor.forceFlush().then(() => attachedLedger?.exporter.forceFlush()) : Promise.resolve();
68
+ }
69
+ shutdown() {
70
+ return this.forceFlush();
71
+ }
72
+ }
73
+ /**
74
+ * Writes the ledger (adk_turns, adk_telemetry, adk_payloads) through
75
+ * `client`, batched off the turn's path, with the same rows, payload policy
76
+ * and redaction as the Supabase sink. One ledger is attached at a time;
77
+ * attaching another replaces it. Returns a detach that flushes what is
78
+ * buffered first.
79
+ */
80
+ export function attachLedger(client, options = {}) {
81
+ if (telemetrySinkEnabled()) {
82
+ console.warn('[TELEMETRY] ⚠ a ledger is attached while TELEMETRY_SUPABASE=true: every row is written twice. Use one sink.');
83
+ }
84
+ const exporter = new SupabaseSpanExporter({ ...options, client });
85
+ const entry = { exporter, processor: new sdkBase.BatchSpanProcessor(exporter) };
86
+ const previous = attachedLedger;
87
+ attachedLedger = entry;
88
+ void previous?.processor.shutdown();
89
+ initializeTracing();
90
+ return async () => {
91
+ if (attachedLedger === entry)
92
+ attachedLedger = undefined;
93
+ await entry.processor.forceFlush();
94
+ await entry.exporter.shutdown();
95
+ };
96
+ }
54
97
  /**
55
98
  * Flushes buffered span processors (console is synchronous; the Supabase
56
99
  * batch sink is not). Call before process exit in short-lived scripts so
@@ -59,6 +102,7 @@ let telemetryExporter;
59
102
  export async function flushTracing() {
60
103
  await tracerProvider?.forceFlush?.();
61
104
  await telemetryExporter?.forceFlush?.();
105
+ await attachedLedger?.exporter.forceFlush();
62
106
  }
63
107
  // ── In-process span listeners ────────────────────────────────────────────────
64
108
  // Lets a script (e.g. scripts/demo_model_optionality.ts) read finished spans
@@ -231,6 +275,7 @@ export function initializeTracing() {
231
275
  const spanProcessors = [
232
276
  new SpanLineageProcessor(),
233
277
  new SimpleSpanProcessor(new JsonConsoleExporter()),
278
+ new AttachedLedgerProcessor(),
234
279
  ];
235
280
  // Optional durable sink (TELEMETRY_SUPABASE=true + Supabase credentials):
236
281
  // llm.request and Syndicate Execution spans also land in adk_telemetry.
@@ -0,0 +1,31 @@
1
+ /**
2
+ * lib/paths.ts — where the engine writes on disk (ADR 0134).
3
+ *
4
+ * Each location keeps its own variable, which wins. Without one, the
5
+ * default is under MELCHIZEDEK_DATA_DIR when that is set, so an embedding
6
+ * application points every location at its own data directory with one
7
+ * variable; with neither, the defaults are the historical ones (the working
8
+ * directory's outputs/, and ~/.melchizedek for the ChatGPT sign-in).
9
+ *
10
+ * location own variable with MELCHIZEDEK_DATA_DIR default
11
+ * generated files, spool OUTPUTS_DIR <data>/outputs <cwd>/outputs
12
+ * task lists and jobs MELCHIZEDEK_TASKS_FILE <data>/outputs/tasks.json <cwd>/outputs/tasks.json
13
+ * ChatGPT sign-in MELCHIZEDEK_CHATGPT_SIGNIN_FILE <data>/chatgpt-signin.json ~/.melchizedek/chatgpt-signin.json
14
+ *
15
+ * Read at each call, never cached. Imports only node built-ins.
16
+ */
17
+ export declare const DATA_DIR_ENV = "MELCHIZEDEK_DATA_DIR";
18
+ /** MELCHIZEDEK_DATA_DIR, resolved; undefined when unset. */
19
+ export declare function dataDir(env?: NodeJS.ProcessEnv): string | undefined;
20
+ /** Where generated files (images) and the telemetry spool go. */
21
+ export declare function outputsDir(env?: NodeJS.ProcessEnv): string;
22
+ /** The task tools' JSON file, when tasks are not in Postgres. */
23
+ export declare function tasksFile(env?: NodeJS.ProcessEnv): string;
24
+ /** The local Sign in with ChatGPT credential file. */
25
+ export declare function chatGptSignInFile(env?: NodeJS.ProcessEnv): string;
26
+ /**
27
+ * The engine's own package root (the directory of its package.json named
28
+ * melchizedek-agents), found from this module's location, whatever the
29
+ * working directory is; undefined if it cannot be found.
30
+ */
31
+ export declare function enginePackageRoot(from?: string): string | undefined;
@@ -0,0 +1,76 @@
1
+ /**
2
+ * lib/paths.ts — where the engine writes on disk (ADR 0134).
3
+ *
4
+ * Each location keeps its own variable, which wins. Without one, the
5
+ * default is under MELCHIZEDEK_DATA_DIR when that is set, so an embedding
6
+ * application points every location at its own data directory with one
7
+ * variable; with neither, the defaults are the historical ones (the working
8
+ * directory's outputs/, and ~/.melchizedek for the ChatGPT sign-in).
9
+ *
10
+ * location own variable with MELCHIZEDEK_DATA_DIR default
11
+ * generated files, spool OUTPUTS_DIR <data>/outputs <cwd>/outputs
12
+ * task lists and jobs MELCHIZEDEK_TASKS_FILE <data>/outputs/tasks.json <cwd>/outputs/tasks.json
13
+ * ChatGPT sign-in MELCHIZEDEK_CHATGPT_SIGNIN_FILE <data>/chatgpt-signin.json ~/.melchizedek/chatgpt-signin.json
14
+ *
15
+ * Read at each call, never cached. Imports only node built-ins.
16
+ */
17
+ import { existsSync, readFileSync } from 'node:fs';
18
+ import { homedir } from 'node:os';
19
+ import { dirname, join, resolve } from 'node:path';
20
+ import { fileURLToPath } from 'node:url';
21
+ export const DATA_DIR_ENV = 'MELCHIZEDEK_DATA_DIR';
22
+ const set = (env, name) => env[name]?.trim() || undefined;
23
+ /** MELCHIZEDEK_DATA_DIR, resolved; undefined when unset. */
24
+ export function dataDir(env = process.env) {
25
+ const d = set(env, DATA_DIR_ENV);
26
+ return d ? resolve(d) : undefined;
27
+ }
28
+ /** Where generated files (images) and the telemetry spool go. */
29
+ export function outputsDir(env = process.env) {
30
+ const own = set(env, 'OUTPUTS_DIR');
31
+ if (own)
32
+ return resolve(own);
33
+ const data = dataDir(env);
34
+ return data ? join(data, 'outputs') : join(process.cwd(), 'outputs');
35
+ }
36
+ /** The task tools' JSON file, when tasks are not in Postgres. */
37
+ export function tasksFile(env = process.env) {
38
+ const own = set(env, 'MELCHIZEDEK_TASKS_FILE');
39
+ if (own)
40
+ return resolve(own);
41
+ const data = dataDir(env);
42
+ return data ? join(data, 'outputs', 'tasks.json') : join(process.cwd(), 'outputs', 'tasks.json');
43
+ }
44
+ /** The local Sign in with ChatGPT credential file. */
45
+ export function chatGptSignInFile(env = process.env) {
46
+ const own = set(env, 'MELCHIZEDEK_CHATGPT_SIGNIN_FILE');
47
+ if (own)
48
+ return resolve(own);
49
+ const data = dataDir(env);
50
+ return data ? join(data, 'chatgpt-signin.json') : join(homedir(), '.melchizedek', 'chatgpt-signin.json');
51
+ }
52
+ /**
53
+ * The engine's own package root (the directory of its package.json named
54
+ * melchizedek-agents), found from this module's location, whatever the
55
+ * working directory is; undefined if it cannot be found.
56
+ */
57
+ export function enginePackageRoot(from = fileURLToPath(import.meta.url)) {
58
+ let dir = dirname(from);
59
+ for (let i = 0; i < 6; i++) {
60
+ const pkg = join(dir, 'package.json');
61
+ if (existsSync(pkg)) {
62
+ try {
63
+ if (JSON.parse(readFileSync(pkg, 'utf-8')).name === 'melchizedek-agents')
64
+ return dir;
65
+ }
66
+ catch {
67
+ /* not ours; keep climbing */
68
+ }
69
+ }
70
+ const parent = dirname(dir);
71
+ if (parent === dir)
72
+ break;
73
+ dir = parent;
74
+ }
75
+ return undefined;
76
+ }
@@ -19,6 +19,7 @@ import { PostgresSessionService } from './sessionService.ts';
19
19
  import { PostgresTaskStore } from './taskStore.ts';
20
20
  import type { CredentialStoreOptions } from '../../tools/credentialStore.ts';
21
21
  import type { CredentialStore } from '../../tools/auth.ts';
22
+ import type { PayloadPolicy } from '../../observability/supabaseSpanExporter.ts';
22
23
  import type { AuditSink } from '../../observability/audit.ts';
23
24
  import type { RlsHardeningStatus } from '../rlsStatus.ts';
24
25
  import type { TaskBackend, TaskBackendOptions } from '../../tools/taskTools.ts';
@@ -28,6 +29,8 @@ export { PostgresSessionService } from './sessionService.ts';
28
29
  export { PostgresTaskStore } from './taskStore.ts';
29
30
  export { postgresMemoryStore } from './memoryStore.ts';
30
31
  export { postgresCredentialRows, postgresCredentialStore } from './credentialStore.ts';
32
+ export { inProcessTurnLock } from '../../a2a/turnLock.ts';
33
+ export type { TurnLock } from '../../a2a/turnLock.ts';
31
34
  export interface PostgresStorageOptions {
32
35
  /** A Postgres URL. Ignored when `pool` is given. */
33
36
  connectionString?: string;
@@ -54,6 +57,28 @@ export interface PostgresStorageOptions {
54
57
  };
55
58
  /** Owner of an A2A call, for the task store. Default: the SDK's user scope. */
56
59
  taskOwner?: PostgresTaskStoreOptions['ownerResolver'];
60
+ /**
61
+ * The turn lock to use instead of the advisory lock below. The advisory
62
+ * lock is what makes several instances on one database take turns; it
63
+ * needs a database that gives each connection its own session. Embedded
64
+ * Postgres (PGlite behind pglite-socket) runs every connection in ONE
65
+ * session, where an advisory lock is re-entrant and so excludes nothing:
66
+ * there, one process owns the database, and `inProcessTurnLock()`
67
+ * (exported here) is the correct lock. When given, no turn-lock
68
+ * pool is opened.
69
+ */
70
+ turnLock?: TurnLock;
71
+ /**
72
+ * Write the telemetry ledger (adk_turns, adk_telemetry, adk_payloads) to
73
+ * this database (ADR 0131): the rows the Supabase sink writes, with the
74
+ * same payload policy (TELEMETRY_PAYLOADS, or `policy`) and redaction.
75
+ * Needs db/telemetry.sql applied. `close()` flushes and detaches it.
76
+ * Default off.
77
+ */
78
+ ledger?: boolean | {
79
+ policy?: PayloadPolicy;
80
+ deadLetterFile?: string;
81
+ };
57
82
  /**
58
83
  * Connections reserved for turn locks: each running turn holds one for its
59
84
  * length, apart from the main pool so a long turn never starves queries.
@@ -22,12 +22,16 @@ import { postgresCredentialStore } from './credentialStore.js';
22
22
  import { searchPathOption } from '../schema.js';
23
23
  import { POSTGRES_RLS_QUERY, evaluatePostgresRls } from '../rlsStatus.js';
24
24
  import { postgresAuditSink } from '../../observability/audit.js';
25
+ import { attachLedger } from '../../observability/tracer.js';
26
+ import { postgresLedgerClient } from '../../observability/supabaseSpanExporter.js';
25
27
  import { createHash, randomUUID } from 'node:crypto';
26
28
  import { hostname } from 'node:os';
27
29
  export { PostgresSessionService } from './sessionService.js';
28
30
  export { PostgresTaskStore } from './taskStore.js';
29
31
  export { postgresMemoryStore } from './memoryStore.js';
30
32
  export { postgresCredentialRows, postgresCredentialStore } from './credentialStore.js';
33
+ // The single-process turn lock, for an embedded database (`turnLock` below).
34
+ export { inProcessTurnLock } from '../../a2a/turnLock.js';
31
35
  /**
32
36
  * A pool emits 'error' when an idle connection dies (a database restart, a
33
37
  * failover, an administrator's terminate); unhandled, that event crashes the
@@ -69,7 +73,7 @@ export function postgresStorage(options) {
69
73
  ...(options.pool ? options.pool.options : { connectionString: options.connectionString, ...options.poolConfig }),
70
74
  max: options.lockPoolMax ?? 20,
71
75
  }), 'turn-lock pool'));
72
- const turnLock = async (key, { waitMs, signal }) => {
76
+ const advisoryTurnLock = async (key, { waitMs, signal }) => {
73
77
  const lockId = advisoryLockId(key);
74
78
  const client = await locks().connect();
75
79
  const deadline = Date.now() + Math.max(0, waitMs);
@@ -105,6 +109,10 @@ export function postgresStorage(options) {
105
109
  };
106
110
  return release;
107
111
  };
112
+ const turnLock = options.turnLock ?? advisoryTurnLock;
113
+ const detachLedger = options.ledger
114
+ ? attachLedger(postgresLedgerClient(pool), options.ledger === true ? {} : options.ledger)
115
+ : undefined;
108
116
  const audit = postgresAuditSink(pool);
109
117
  const credentials = options.credentials ? postgresCredentialStore(pool, { ...options.credentials, audit }) : undefined;
110
118
  return {
@@ -160,6 +168,7 @@ export function postgresStorage(options) {
160
168
  return counts;
161
169
  },
162
170
  async close() {
171
+ await detachLedger?.();
163
172
  if (lockPool)
164
173
  await lockPool.end();
165
174
  if (owned)