@powerhousedao/switchboard 6.2.3-dev.49 → 6.2.3-dev.50

package/CHANGELOG.md

@@ -1,3 +1,25 @@
+## 6.2.3-dev.50 (2026-10-09)
+
+### 🚀 Features
+
+- **switchboard:** start the Pyroscope profiler in embedded servers ([3dc361721d](https://github.com/powerhouse-inc/powerhouse/commit/3dc361721d))
+- **reactor-workflow:** stream, cache and authorize step attachments ([2c33603d58](https://github.com/powerhouse-inc/powerhouse/commit/2c33603d58))
+- **reactor-workflow:** trace runs, steps, workers and reactor calls with a host-supplied tracer and meter ([d42c5d9053](https://github.com/powerhouse-inc/powerhouse/commit/d42c5d9053))
+
+### 🩹 Fixes
+
+- **registry:** stop Verdaccio dropping a package write when Windows refuses the rename ([79636fd438](https://github.com/powerhouse-inc/powerhouse/commit/79636fd438))
+- **switchboard:** shut down and restart when the reactor's pglite session is poisoned ([ccbb0ecef1](https://github.com/powerhouse-inc/powerhouse/commit/ccbb0ecef1))
+- **switchboard:** build the pglite store on the hardened dialect ([8c359d9278](https://github.com/powerhouse-inc/powerhouse/commit/8c359d9278))
+- **reactor-workflow:** carry INDETERMINATE through test reporting ([ada7daaa61](https://github.com/powerhouse-inc/powerhouse/commit/ada7daaa61))
+
+### ❤️ Thank You
+
+- acaldas
+- Benjamin Jordan
+- Claude Opus 5.5
+- Wouter Kampmann
+
 ## 6.2.3-dev.49 (2026-10-08)
 
 This was a version bump only for @powerhousedao/switchboard to align it with other projects, there were no code changes.

package/dist/index.mjs

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 
-!function(){try{var e="undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof globalThis?globalThis:"undefined"!=typeof self?self:{},n=(new e.Error).stack;n&&(e._sentryDebugIds=e._sentryDebugIds||{},e._sentryDebugIds[n]="1fbb2406-730e-5d3e-a4cf-ac7dea3cedd5")}catch(e){}}();
-import { i as StoredDocumentsRefusedError, r as startSwitchboard, s as parseForcePgVersion } from "./server-Dvw8_NeM.mjs";
+!function(){try{var e="undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof globalThis?globalThis:"undefined"!=typeof self?self:{},n=(new e.Error).stack;n&&(e._sentryDebugIds=e._sentryDebugIds||{},e._sentryDebugIds[n]="d30347ec-a382-56a4-b3ef-bdacc9d409fd")}catch(e){}}();
+import { a as StoredDocumentsRefusedError, c as initProfilerFromEnv, i as startSwitchboard, l as parseForcePgVersion } from "./server-Zz7SqTwO.mjs";
 import { metrics } from "@opentelemetry/api";
 import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
 import { ExpressInstrumentation } from "@opentelemetry/instrumentation-express";
@@ -219,42 +219,6 @@ const config = {
 	}
 };
 //#endregion
-//#region src/profiler.ts
-async function initProfilerFromEnv(env) {
-	const { PYROSCOPE_SERVER_ADDRESS: serverAddress, PYROSCOPE_APPLICATION_NAME: appName, PYROSCOPE_USER: basicAuthUser, PYROSCOPE_PASSWORD: basicAuthPassword, PYROSCOPE_WALL_ENABLED: wallEnabled, PYROSCOPE_HEAP_ENABLED: heapEnabled } = env;
-	return initProfiler({
-		serverAddress,
-		appName,
-		basicAuthUser,
-		basicAuthPassword,
-		wall: {
-			samplingDurationMs: 1e4,
-			samplingIntervalMicros: 1e4,
-			collectCpuTime: true
-		},
-		heap: {
-			samplingIntervalBytes: 524288,
-			stackDepth: 64
-		}
-	}, {
-		wallEnabled: wallEnabled !== "false",
-		heapEnabled: heapEnabled === "true"
-	});
-}
-async function initProfiler(options, flags = {
-	wallEnabled: true,
-	heapEnabled: false
-}) {
-	console.log("Initializing Pyroscope profiler at:", options?.serverAddress);
-	console.log("  Wall profiling:", flags.wallEnabled ? "enabled" : "disabled");
-	console.log("  Heap profiling:", flags.heapEnabled ? "enabled" : "disabled");
-	const { default: Pyroscope } = await import("@pyroscope/nodejs");
-	Pyroscope.init(options);
-	if (flags.wallEnabled) Pyroscope.startWallProfiling();
-	Pyroscope.startCpuProfiling();
-	if (flags.heapEnabled) Pyroscope.startHeapProfiling();
-}
-//#endregion
 //#region src/index.mts
 const logger = childLogger(["switchboard"]);
 function ensureNodeVersion(minVersion = "24") {
@@ -293,4 +257,4 @@ try {
 export {};
 
 //# sourceMappingURL=index.mjs.map
-//# debugId=1fbb2406-730e-5d3e-a4cf-ac7dea3cedd5
+//# debugId=d30347ec-a382-56a4-b3ef-bdacc9d409fd

package/dist/index.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"index.mjs","sources":["../src/metrics.ts","../src/observability-signals.mts","../src/redact-webhook-path.ts","../src/observability.mts","../src/config.ts","../src/profiler.ts","../src/index.mts"],"sourcesContent":["import { OTLPMetricExporter } from \"@opentelemetry/exporter-metrics-otlp-http\";\nimport { resourceFromAttributes } from \"@opentelemetry/resources\";\nimport {\n  MeterProvider,\n  PeriodicExportingMetricReader,\n} from \"@opentelemetry/sdk-metrics\";\nimport { childLogger } from \"document-model\";\n\nconst logger = childLogger([\"switchboard\", \"metrics\"]);\n\nexport function createMeterProviderFromEnv(env: {\n  OTEL_EXPORTER_OTLP_ENDPOINT?: string;\n  OTEL_METRIC_EXPORT_INTERVAL?: string;\n  OTEL_SERVICE_NAME?: string;\n}): MeterProvider | undefined {\n  const endpoint = env.OTEL_EXPORTER_OTLP_ENDPOINT;\n  if (!endpoint) return undefined;\n\n  const parsed = parseInt(env.OTEL_METRIC_EXPORT_INTERVAL ?? \"\", 10);\n  const exportIntervalMillis =\n    Number.isFinite(parsed) && parsed > 0 ? parsed : 5_000;\n\n  const base = endpoint.replace(/\\/$/, \"\");\n  const exporterUrl = base.endsWith(\"/v1/metrics\")\n    ? base\n    : `${base}/v1/metrics`;\n\n  logger.info(`Initializing OpenTelemetry metrics exporter at: ${endpoint}`);\n  const meterProvider = new MeterProvider({\n    resource: resourceFromAttributes({\n      \"service.name\": env.OTEL_SERVICE_NAME ?? \"switchboard\",\n    }),\n    readers: [\n      new PeriodicExportingMetricReader({\n        exporter: new OTLPMetricExporter({\n          url: exporterUrl,\n        }),\n        exportIntervalMillis,\n        exportTimeoutMillis: Math.max(exportIntervalMillis - 250, 1),\n      }),\n    ],\n  });\n  logger.info(`Metrics export enabled (interval: ${exportIntervalMillis}ms)`);\n  return meterProvider;\n}\n","// Signal handling for the observability bootstrap, split out of\n// observability.mts so tests can exercise it without importing the module\n// whose load-time side effects start Sentry and OpenTelemetry.\n\ntype SignalProcess = Pick<NodeJS.Process, \"on\" | \"exit\"> & {\n  exitCode?: NodeJS.Process[\"exitCode\"];\n};\n\n/**\n * Flushes telemetry on SIGINT/SIGTERM, then exits only when no other module\n * has claimed a failing exit.\n *\n * Exit ownership, decided here on purpose:\n *\n * - Ordinary signal (process.exitCode unset or 0): this handler exits with\n *   the current exit code after the flush settles (the flush bounds itself,\n *   see shutdown() in observability.mts). When the reactor builder's drain\n *   handler is also attached it runs first (prependListener) and swaps\n *   process.exit for a recording shim before this handler's exit call, so\n *   the drain still finishes and performs the real exit; when the builder is\n *   not attached (startup, signalHandlers: false), this exit is what\n *   terminates the process, with code 0.\n *\n * - Fatal shutdown in progress (process.exitCode already non-zero):\n *   fatal-shutdown.mts sets exitCode = 1 and arms a 15s forced exit BEFORE\n *   raising SIGTERM, precisely so the builder's drain can flush the PGlite\n *   snapshots. Calling exit here would lower that code to 0 (supervisors\n *   would see a crash as success) and, whenever the builder's shim is not in\n *   place, cut the drain off at this handler's shorter flush budget. So this\n *   handler never exits once a non-zero code is set: the exit belongs to the\n *   builder's drain, or failing that to fatal-shutdown's 15s backstop, both\n *   of which preserve code 1.\n */\nexport function installObservabilitySignalHandlers(\n  flush: () => Promise<void>,\n  proc: SignalProcess = process,\n): void {\n  const onSignal = () => {\n    void flush().finally(() => {\n      const code = proc.exitCode;\n      if (code !== undefined && code !== 0) {\n        return;\n      }\n      proc.exit(code ?? 0);\n    });\n  };\n  proc.on(\"SIGINT\", onSignal);\n  proc.on(\"SIGTERM\", onSignal);\n}\n","const WEBHOOK_SEGMENT = \"/webhooks/\";\n\n/**\n * Strips a webhook's delivery token out of a request path.\n *\n * The token is the endpoint's whole credential and it sits in the path, so it\n * must not reach a log line or a span attribute that ships off-box. The family\n * mounts under the host's base path, so the segment can sit at any depth.\n */\nexport function redactWebhookPath(url: string | undefined): string {\n  if (!url) return \"\";\n  // Dropped with the token: `http.route` is meant to be a template, and a\n  // query string is another place a sender can put a credential.\n  const path = url.split(\"?\")[0] ?? url;\n  const at = path.indexOf(WEBHOOK_SEGMENT);\n  if (at === -1) return url;\n  // Nothing after the segment is nothing to redact.\n  if (path.length === at + WEBHOOK_SEGMENT.length) return url;\n  return `${path.slice(0, at)}${WEBHOOK_SEGMENT}[redacted]`;\n}\n","// Single observability bootstrap: Sentry + OpenTelemetry (tracing + metrics).\n//\n// MUST be imported as the very first module in apps/switchboard/src/index.mts.\n// OpenTelemetry instrumentations register require-time hooks at module load,\n// so http/express/pg/graphql must not be imported (transitively) before this\n// file runs.\n//\n// Replaces three legacy bootstrap sites:\n// - apps/switchboard/src/server.mts top-level Sentry.init\n// - apps/switchboard/src/metrics.ts standalone MeterProvider (still exported\n//   here via createMeterProviderFromEnv so its tests keep passing)\n// - packages/reactor-api/src/tracing.ts side-effect NodeSDK\nimport { metrics } from \"@opentelemetry/api\";\nimport { OTLPTraceExporter } from \"@opentelemetry/exporter-trace-otlp-http\";\nimport { ExpressInstrumentation } from \"@opentelemetry/instrumentation-express\";\nimport { GraphQLInstrumentation } from \"@opentelemetry/instrumentation-graphql\";\nimport { HttpInstrumentation } from \"@opentelemetry/instrumentation-http\";\nimport { PgInstrumentation } from \"@opentelemetry/instrumentation-pg\";\nimport { UndiciInstrumentation } from \"@opentelemetry/instrumentation-undici\";\nimport { resourceFromAttributes } from \"@opentelemetry/resources\";\nimport type { MeterProvider } from \"@opentelemetry/sdk-metrics\";\nimport { NodeSDK } from \"@opentelemetry/sdk-node\";\nimport {\n  BatchSpanProcessor,\n  type SpanProcessor,\n} from \"@opentelemetry/sdk-trace-base\";\nimport {\n  ATTR_SERVICE_NAME,\n  ATTR_SERVICE_VERSION,\n} from \"@opentelemetry/semantic-conventions\";\nimport {\n  registerEventLoopInstrumentation,\n  type EventLoopInstrumentation,\n} from \"@powerhousedao/opentelemetry-instrumentation-reactor\";\nimport * as Sentry from \"@sentry/node\";\nimport { SentryPropagator, SentrySpanProcessor } from \"@sentry/opentelemetry\";\nimport { childLogger } from \"document-model\";\nimport type { IncomingMessage } from \"node:http\";\nimport { createMeterProviderFromEnv } from \"./metrics.js\";\nimport { installObservabilitySignalHandlers } from \"./observability-signals.mjs\";\nimport { redactWebhookPath } from \"./redact-webhook-path.js\";\n\nconst logger = childLogger([\"switchboard\", \"observability\"]);\n\nconst SERVICE_NAME = process.env.OTEL_SERVICE_NAME || \"switchboard\";\nconst SERVICE_VERSION = process.env.npm_package_version || \"unknown\";\nconst TENANT_ID = process.env.TENANT_ID || \"default\";\nconst DEPLOY_ENV = process.env.NODE_ENV || \"development\";\n\nconst TEMPO_ENDPOINT = process.env.TEMPO_ENDPOINT;\nconst SENTRY_DSN = process.env.SENTRY_DSN;\n\n// Whether to forward TRANSACTIONS (spans) to Sentry. Errors are always sent\n// when SENTRY_DSN is set — this flag only gates APM/tracing into Sentry.\n// Default on (back-compat). Set SENTRY_TRACING_ENABLED=false for an\n// \"errors-only\" deployment: errors still go to Sentry, traces still go to\n// Tempo, but no transactions hit Sentry (no Kafka/ClickHouse/nodestore\n// cost). This is the recommended mode for tenant workloads at scale —\n// Sentry's value there is error grouping; traces live in Tempo.\nconst SENTRY_TRACING_TO_SENTRY =\n  Boolean(SENTRY_DSN) && process.env.SENTRY_TRACING_ENABLED !== \"false\";\n\nconst TRACING_REQUESTED =\n  process.env.ENABLE_TRACING === \"true\" ||\n  process.env.NODE_ENV === \"production\";\nconst HAS_TRACE_DESTINATION = Boolean(TEMPO_ENDPOINT) || Boolean(SENTRY_DSN);\nconst TRACING_ENABLED = TRACING_REQUESTED && HAS_TRACE_DESTINATION;\n\nif (TRACING_REQUESTED && !HAS_TRACE_DESTINATION) {\n  logger.warn(\n    \"Tracing was requested (NODE_ENV=production or ENABLE_TRACING=true) but \" +\n      \"no destination is configured — instrumentation will not run. Set \" +\n      \"TEMPO_ENDPOINT (e.g. http://tempo.monitoring.svc.cluster.local:4318/v1/traces) \" +\n      \"to export OTLP spans, and/or SENTRY_DSN to forward spans to Sentry.\",\n  );\n}\n\n// APM sampling for the Sentry-SDK-managed path (i.e. when TRACING is OFF and\n// @sentry/node runs its own bundled OTel — see skipOpenTelemetrySetup below).\n//\n// IMPORTANT: when TRACING_ENABLED, this value does NOT govern span volume.\n// Our NodeSDK (below) owns the pipeline and is constructed with no explicit\n// `sampler`, so @opentelemetry/sdk-node falls back to buildSamplerFromEnv()\n// and the REAL head-sampling knob is the standard OTEL_TRACES_SAMPLER /\n// OTEL_TRACES_SAMPLER_ARG env (set per-deploy in the k8s chart). That head\n// decision gates spans before any processor runs, so it bounds BOTH the\n// SentrySpanProcessor (→ Sentry transactions) and the Tempo OTLP export.\n// (Wiring @sentry/opentelemetry's SentrySampler here would let this rate\n// drive both backends and make DSC/sample_rand propagation spec-correct, but\n// that only matters for Sentry server-side dynamic sampling — a SaaS feature\n// our self-hosted install doesn't run — so it's intentionally deferred.)\nconst SENTRY_TRACES_SAMPLE_RATE = parseFloat(\n  process.env.SENTRY_TRACES_SAMPLE_RATE ?? \"0.1\",\n);\n\nif (SENTRY_DSN) {\n  logger.info(\"Initialized Sentry with env: @env\", process.env.SENTRY_ENV);\n  Sentry.init({\n    dsn: SENTRY_DSN,\n    environment: process.env.SENTRY_ENV,\n    // Match the version tag uploaded by release-branch.yml so source maps\n    // resolve. Populated by the CI (WORKSPACE_VERSION) or npm at runtime.\n    release:\n      process.env.SENTRY_RELEASE ||\n      (process.env.npm_package_version\n        ? `v${process.env.npm_package_version}`\n        : undefined),\n    // 0 in errors-only mode so even @sentry/node's bundled auto-OTel path\n    // (used when TRACING_ENABLED is false) produces no transactions.\n    tracesSampleRate: SENTRY_TRACING_TO_SENTRY ? SENTRY_TRACES_SAMPLE_RATE : 0,\n    // When tracing is on, our NodeSDK below owns the OTel globals and Sentry\n    // receives spans via SentrySpanProcessor. Skipping Sentry's bundled OTel\n    // setup avoids two TracerProviders fighting over setGlobalTracerProvider.\n    // When tracing is off, leave the flag unset so @sentry/node's default\n    // auto-OTel still records HTTP transactions for the APM dashboard.\n    skipOpenTelemetrySetup: TRACING_ENABLED,\n  });\n}\n\nconst meterProvider: MeterProvider | undefined = createMeterProviderFromEnv({\n  OTEL_EXPORTER_OTLP_ENDPOINT: process.env.OTEL_EXPORTER_OTLP_ENDPOINT,\n  OTEL_METRIC_EXPORT_INTERVAL: process.env.OTEL_METRIC_EXPORT_INTERVAL,\n  OTEL_SERVICE_NAME: process.env.OTEL_SERVICE_NAME,\n});\nlet eventLoopInstrumentation: EventLoopInstrumentation | undefined;\nif (meterProvider) {\n  // One-way door: must register before any code calls metrics.getMeter() —\n  // most notably ReactorInstrumentation inside the reactor module.\n  metrics.setGlobalMeterProvider(meterProvider);\n  // Host event-loop and CPU gauges (reactor.host.eventloop.*,\n  // reactor.host.cpu.utilization). The bench sweep identified the host event\n  // loop, not the executor, as the binding constraint on write throughput\n  // (bench/test/integration/BASELINE.md, Run 10) — these are the production\n  // counterpart of the metrics that finding was made from. Must come after\n  // setGlobalMeterProvider or the gauges bind to a no-op meter.\n  eventLoopInstrumentation = registerEventLoopInstrumentation();\n}\n\nlet sdk: NodeSDK | undefined;\n\nif (TRACING_ENABLED) {\n  logger.info(`Initializing OpenTelemetry tracing for ${SERVICE_NAME}`);\n  if (TEMPO_ENDPOINT) logger.info(`  Tempo endpoint: ${TEMPO_ENDPOINT}`);\n  if (SENTRY_DSN) logger.info(`  Sentry span forwarding: enabled`);\n  logger.info(`  Tenant: ${TENANT_ID}`);\n\n  const resource = resourceFromAttributes({\n    [ATTR_SERVICE_NAME]: SERVICE_NAME,\n    [ATTR_SERVICE_VERSION]: SERVICE_VERSION,\n    \"tenant.id\": TENANT_ID,\n    \"deployment.environment\": DEPLOY_ENV,\n  });\n\n  const spanProcessors: SpanProcessor[] = [];\n  if (TEMPO_ENDPOINT) {\n    spanProcessors.push(\n      new BatchSpanProcessor(new OTLPTraceExporter({ url: TEMPO_ENDPOINT })),\n    );\n  }\n  if (SENTRY_TRACING_TO_SENTRY) {\n    // Fan the same OTel spans into Sentry — same trace IDs as Tempo, so\n    // Sentry transactions cross-link to traces in Grafana. Skipped in\n    // errors-only mode (SENTRY_TRACING_ENABLED=false): spans still flow to\n    // Tempo via the BatchSpanProcessor above, just not to Sentry.\n    spanProcessors.push(new SentrySpanProcessor());\n  }\n\n  sdk = new NodeSDK({\n    resource,\n    spanProcessors,\n    textMapPropagator: SENTRY_TRACING_TO_SENTRY\n      ? new SentryPropagator()\n      : undefined,\n    instrumentations: [\n      new HttpInstrumentation({\n        ignoreIncomingRequestHook: (req) =>\n          req.url === \"/health\" || req.url === \"/ready\",\n        requireParentforIncomingSpans: false,\n        requireParentforOutgoingSpans: false,\n        requestHook: (span, request) => {\n          span.setAttribute(\n            \"http.route\",\n            redactWebhookPath((request as IncomingMessage).url),\n          );\n        },\n        responseHook: (span, response) => {\n          if (response.statusCode) {\n            span.setAttribute(\"http.status_code\", response.statusCode);\n          }\n        },\n      }),\n      new ExpressInstrumentation({\n        requestHook: (span, info) => {\n          if (info.route) span.setAttribute(\"http.route\", info.route);\n        },\n      }),\n      // HttpInstrumentation only patches node:http/https; outbound global\n      // fetch() goes through undici and is otherwise untraced — e.g. the\n      // per-request Renown credential check in AuthService.verifyCredentialExists.\n      new UndiciInstrumentation(),\n      new GraphQLInstrumentation({ mergeItems: true, allowValues: true }),\n      // requireParentSpan: only trace DB queries that run inside a request\n      // (HTTP/GraphQL) span. Parentless queries — the management\n      // switchboard's background polling loops (vetra-cloud-observability\n      // reconcile @60s + clint pull-worker @15s, each writing\n      // environment_pods / clint_runtime_endpoints) — would otherwise each\n      // become a standalone root transaction. That volume scales O(tenant\n      // count) and was ~70% of all Sentry transactions before this change.\n      // Dropping it at the instrumentation layer (no span created at all) is\n      // cheaper and cleaner than sampling it away downstream.\n      new PgInstrumentation({\n        enhancedDatabaseReporting: true,\n        requireParentSpan: true,\n      }),\n    ],\n  });\n  sdk.start();\n  if (\n    SENTRY_TRACING_TO_SENTRY &&\n    typeof Sentry.validateOpenTelemetrySetup === \"function\"\n  ) {\n    Sentry.validateOpenTelemetrySetup();\n  }\n  logger.info(\"OpenTelemetry tracing initialized\");\n}\n\nasync function shutdown() {\n  // Detach the observable callback before the provider drains, so the final\n  // export does not fire a gauge callback against a shutting-down meter.\n  eventLoopInstrumentation?.stop();\n  await Promise.race([\n    Promise.all([\n      meterProvider?.shutdown().catch(() => undefined),\n      sdk?.shutdown().catch(() => undefined),\n    ]),\n    new Promise<void>((resolve) => setTimeout(resolve, 5_000)),\n  ]);\n}\n\n// Exit-code and drain coordination with fatal-shutdown.mts and the reactor\n// builder lives in observability-signals.mts; the earlier inline handlers\n// exited with a literal 0, which turned a fatal crash (exitCode = 1) into a\n// success in the supervisor's eyes and could cut the snapshot drain short.\ninstallObservabilitySignalHandlers(shutdown);\n\nexport { meterProvider, sdk };\n","import dotenv from \"dotenv\";\ndotenv.config();\n\nimport { getConfig } from \"@powerhousedao/config/node\";\nimport { parseForcePgVersion } from \"./pglite-version.js\";\nimport type {\n  SwitchboardDriveDocumentType,\n  SwitchboardDriveInput,\n} from \"./types.js\";\nconst phConfig = getConfig();\nconst { switchboard } = phConfig;\ninterface Config {\n  database: {\n    url: string;\n  };\n  port: number;\n  mcp: boolean;\n  migratePglite: boolean;\n  forcePgVersion: 16 | 17 | null;\n  drive: SwitchboardDriveInput;\n}\n\nfunction parseDriveType(\n  raw: string | undefined,\n): SwitchboardDriveDocumentType | undefined {\n  if (!raw) return undefined;\n  if (raw === \"powerhouse/document-drive\" || raw === \"powerhouse/reactor-drive\")\n    return raw;\n  throw new Error(\n    `Invalid PH_DEFAULT_DRIVE_TYPE: ${raw}. Expected \"powerhouse/document-drive\" or \"powerhouse/reactor-drive\".`,\n  );\n}\n\nexport const config: Config = {\n  database: {\n    // url: process.env.PH_SWITCHBOARD_DATABASE_URL ?? switchboard?.database?.url ?? \"dev.db\",\n    url:\n      process.env.PH_SWITCHBOARD_DATABASE_URL ??\n      switchboard?.database?.url ??\n      \"dev.db\",\n  },\n  port:\n    process.env.PH_SWITCHBOARD_PORT &&\n    !isNaN(Number(process.env.PH_SWITCHBOARD_PORT))\n      ? Number(process.env.PH_SWITCHBOARD_PORT)\n      : (switchboard?.port ?? 4001),\n  mcp: true,\n  migratePglite: process.env.PH_MIGRATE_PGLITE === \"true\",\n  forcePgVersion: parseForcePgVersion(process.env.PH_FORCE_PG_VERSION),\n  drive: {\n    id: \"powerhouse\",\n    slug: \"powerhouse\",\n    documentType: parseDriveType(process.env.PH_DEFAULT_DRIVE_TYPE),\n    global: {\n      name: \"Powerhouse\",\n      icon: \"https://ipfs.io/ipfs/QmcaTDBYn8X2psGaXe7iQ6qd8q6oqHLgxvMX9yXf7f9uP7\",\n    },\n    local: {\n      availableOffline: true,\n      listeners: [],\n      sharingType: \"public\",\n      triggers: [],\n    },\n  },\n};\n","import type { PyroscopeConfig } from \"@pyroscope/nodejs\";\n\nexport async function initProfilerFromEnv(env: typeof process.env) {\n  const {\n    PYROSCOPE_SERVER_ADDRESS: serverAddress,\n    PYROSCOPE_APPLICATION_NAME: appName,\n    PYROSCOPE_USER: basicAuthUser,\n    PYROSCOPE_PASSWORD: basicAuthPassword,\n    PYROSCOPE_WALL_ENABLED: wallEnabled,\n    PYROSCOPE_HEAP_ENABLED: heapEnabled,\n  } = env;\n\n  const options: PyroscopeConfig = {\n    serverAddress,\n    appName,\n    basicAuthUser,\n    basicAuthPassword,\n    // Wall profiling captures wall-clock time (includes async I/O waits)\n    // This shows GraphQL resolvers even when waiting for database\n    wall: {\n      samplingDurationMs: 10000, // 10 second sampling windows\n      samplingIntervalMicros: 10000, // 10ms sampling interval (100 samples/sec)\n      collectCpuTime: true, // Also collect CPU time alongside wall time\n    },\n    // Heap profiling for memory allocation tracking\n    heap: {\n      samplingIntervalBytes: 512 * 1024, // Sample every 512KB allocated\n      stackDepth: 64, // Capture deeper stacks for better context\n    },\n  };\n  return initProfiler(options, {\n    wallEnabled: wallEnabled !== \"false\",\n    heapEnabled: heapEnabled === \"true\",\n  });\n}\n\ninterface ProfilerFlags {\n  wallEnabled?: boolean;\n  heapEnabled?: boolean;\n}\n\nexport async function initProfiler(\n  options?: PyroscopeConfig,\n  flags: ProfilerFlags = { wallEnabled: true, heapEnabled: false },\n) {\n  console.log(\"Initializing Pyroscope profiler at:\", options?.serverAddress);\n  console.log(\"  Wall profiling:\", flags.wallEnabled ? \"enabled\" : \"disabled\");\n  console.log(\"  Heap profiling:\", flags.heapEnabled ? \"enabled\" : \"disabled\");\n\n  const { default: Pyroscope } = await import(\"@pyroscope/nodejs\");\n  Pyroscope.init(options);\n\n  // Start wall profiling (captures async I/O time - shows resolvers)\n  if (flags.wallEnabled) {\n    Pyroscope.startWallProfiling();\n  }\n\n  // Start CPU profiling (captures CPU-bound work)\n  Pyroscope.startCpuProfiling();\n\n  // Optionally start heap profiling (memory allocations)\n  if (flags.heapEnabled) {\n    Pyroscope.startHeapProfiling();\n  }\n}\n","#!/usr/bin/env node\n// Observability MUST load before any module that imports http/express/pg/graphql\n// so OpenTelemetry's require-time hooks can patch them. It also owns Sentry\n// init and the SIGINT/SIGTERM flush.\nimport \"./observability.mjs\";\n\nimport * as Sentry from \"@sentry/node\";\nimport { childLogger } from \"document-model\";\nimport { config } from \"./config.js\";\nimport { initProfilerFromEnv } from \"./profiler.js\";\nimport { startSwitchboard } from \"./server.mjs\";\nimport { StoredDocumentsRefusedError } from \"./unsupported-stored-documents.mjs\";\n\nconst logger = childLogger([\"switchboard\"]);\n\nfunction ensureNodeVersion(minVersion = \"24\") {\n  const version = process.versions.node;\n  if (!version) {\n    return;\n  }\n\n  if (version < minVersion) {\n    console.error(\n      `Node version ${minVersion} or higher is required. Current version: ${version}`,\n    );\n    process.exit(1);\n  }\n}\n// Ensure minimum Node.js version\nensureNodeVersion(\"24\");\n\n// Each subgraph registers its own SIGINT/SIGTERM listeners, and the count\n// scales with dynamically-loaded document models beyond the default cap of 10.\nprocess.setMaxListeners(0);\n\nif (process.env.PYROSCOPE_SERVER_ADDRESS) {\n  try {\n    await initProfilerFromEnv(process.env);\n  } catch (e) {\n    Sentry.captureException(e);\n    logger.error(\"Error starting profiler: @error\", e);\n  }\n}\n\nconst cliMigratePglite = process.argv.slice(2).includes(\"--migrate-pglite\");\n\n// A boot failure leaves no serving process behind, so it must exit non-zero:\n// logging and continuing hides a dead switchboard from the supervisor, which\n// would otherwise never restart it.\ntry {\n  await startSwitchboard({\n    ...config,\n    migratePglite: cliMigratePglite || config.migratePglite,\n    forcePgVersion: config.forcePgVersion ?? undefined,\n    fatalErrorShutdown: true,\n  });\n} catch (e) {\n  // startSwitchboard already reported a refusal to the operator.\n  if (!StoredDocumentsRefusedError.isError(e)) {\n    Sentry.captureException(e);\n    logger.error(\"Switchboard failed to start: @error\", e);\n  }\n  await Sentry.flush(2000).catch(() => undefined);\n  process.exit(1);\n}\n"],"names":["logger","logger"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAQA,MAAMA,WAAS,YAAY,CAAC,eAAe,SAAS,CAAC;AAErD,SAAgB,2BAA2B,KAIb;CAC5B,MAAM,WAAW,IAAI;CACrB,IAAI,CAAC,UAAU,OAAO,KAAA;CAEtB,MAAM,SAAS,SAAS,IAAI,+BAA+B,IAAI,EAAE;CACjE,MAAM,uBACJ,OAAO,SAAS,MAAM,KAAK,SAAS,IAAI,SAAS;CAEnD,MAAM,OAAO,SAAS,QAAQ,OAAO,EAAE;CACvC,MAAM,cAAc,KAAK,SAAS,aAAa,IAC3C,OACA,GAAG,KAAK;CAEZ,SAAO,KAAK,mDAAmD,UAAU;CACzE,MAAM,gBAAgB,IAAI,cAAc;EACtC,UAAU,uBAAuB,EAC/B,gBAAgB,IAAI,qBAAqB,cAC3C,CAAC;EACD,SAAS,CACP,IAAI,8BAA8B;GAChC,UAAU,IAAI,mBAAmB,EAC/B,KAAK,YACP,CAAC;GACD;GACA,qBAAqB,KAAK,IAAI,uBAAuB,KAAK,CAAC;EAC7D,CAAC,CACH;CACF,CAAC;CACD,SAAO,KAAK,qCAAqC,qBAAqB,IAAI;CAC1E,OAAO;AACT;;;;;;;;;;;;;;;;;;;;;;;;;;;;ACXA,SAAgB,mCACd,OACA,OAAsB,SAChB;CACN,MAAM,iBAAiB;EACrB,MAAW,CAAC,CAAC,cAAc;GACzB,MAAM,OAAO,KAAK;GAClB,IAAI,SAAS,KAAA,KAAa,SAAS,GACjC;GAEF,KAAK,KAAK,QAAQ,CAAC;EACrB,CAAC;CACH;CACA,KAAK,GAAG,UAAU,QAAQ;CAC1B,KAAK,GAAG,WAAW,QAAQ;AAC7B;;;AChDA,MAAM,kBAAkB;;;;;;;;AASxB,SAAgB,kBAAkB,KAAiC;CACjE,IAAI,CAAC,KAAK,OAAO;CAGjB,MAAM,OAAO,IAAI,MAAM,GAAG,CAAC,CAAC,MAAM;CAClC,MAAM,KAAK,KAAK,QAAQ,eAAe;CACvC,IAAI,OAAO,IAAI,OAAO;CAEtB,IAAI,KAAK,WAAW,KAAK,IAAwB,OAAO;CACxD,OAAO,GAAG,KAAK,MAAM,GAAG,EAAE,IAAI,gBAAgB;AAChD;;;ACuBA,MAAMC,WAAS,YAAY,CAAC,eAAe,eAAe,CAAC;AAE3D,MAAM,eAAe,QAAQ,IAAI,qBAAqB;AACtD,MAAM,kBAAkB,QAAQ,IAAI,uBAAuB;AAC3D,MAAM,YAAY,QAAQ,IAAI,aAAa;AAC3C,MAAM,aAAa,QAAQ,IAAI,YAAY;AAE3C,MAAM,iBAAiB,QAAQ,IAAI;AACnC,MAAM,aAAa,QAAQ,IAAI;AAS/B,MAAM,2BACJ,QAAQ,UAAU,KAAK,QAAQ,IAAI,2BAA2B;AAEhE,MAAM,oBACJ,QAAQ,IAAI,mBAAmB,UAC/B,QAAQ,IAAI,aAAa;AAC3B,MAAM,wBAAwB,QAAQ,cAAc,KAAK,QAAQ,UAAU;AAC3E,MAAM,kBAAkB,qBAAqB;AAE7C,IAAI,qBAAqB,CAAC,uBACxB,SAAO,KACL,4RAIF;AAiBF,MAAM,4BAA4B,WAChC,QAAQ,IAAI,6BAA6B,KAC3C;AAEA,IAAI,YAAY;CACd,SAAO,KAAK,qCAAqC,QAAQ,IAAI,UAAU;CACvE,OAAO,KAAK;EACV,KAAK;EACL,aAAa,QAAQ,IAAI;EAGzB,SACE,QAAQ,IAAI,mBACX,QAAQ,IAAI,sBACT,IAAI,QAAQ,IAAI,wBAChB,KAAA;EAGN,kBAAkB,2BAA2B,4BAA4B;EAMzE,wBAAwB;CAC1B,CAAC;AACH;AAEA,MAAM,gBAA2C,2BAA2B;CAC1E,6BAA6B,QAAQ,IAAI;CACzC,6BAA6B,QAAQ,IAAI;CACzC,mBAAmB,QAAQ,IAAI;AACjC,CAAC;AACD,IAAI;AACJ,IAAI,eAAe;CAGjB,QAAQ,uBAAuB,aAAa;CAO5C,2BAA2B,iCAAiC;AAC9D;AAEA,IAAI;AAEJ,IAAI,iBAAiB;CACnB,SAAO,KAAK,0CAA0C,cAAc;CACpE,IAAI,gBAAgB,SAAO,KAAK,qBAAqB,gBAAgB;CACrE,IAAI,YAAY,SAAO,KAAK,mCAAmC;CAC/D,SAAO,KAAK,aAAa,WAAW;CAEpC,MAAM,WAAW,uBAAuB;GACrC,oBAAoB;GACpB,uBAAuB;EACxB,aAAa;EACb,0BAA0B;CAC5B,CAAC;CAED,MAAM,iBAAkC,CAAC;CACzC,IAAI,gBACF,eAAe,KACb,IAAI,mBAAmB,IAAI,kBAAkB,EAAE,KAAK,eAAe,CAAC,CAAC,CACvE;CAEF,IAAI,0BAKF,eAAe,KAAK,IAAI,oBAAoB,CAAC;CAG/C,MAAM,IAAI,QAAQ;EAChB;EACA;EACA,mBAAmB,2BACf,IAAI,iBAAiB,IACrB,KAAA;EACJ,kBAAkB;GAChB,IAAI,oBAAoB;IACtB,4BAA4B,QAC1B,IAAI,QAAQ,aAAa,IAAI,QAAQ;IACvC,+BAA+B;IAC/B,+BAA+B;IAC/B,cAAc,MAAM,YAAY;KAC9B,KAAK,aACH,cACA,kBAAmB,QAA4B,GAAG,CACpD;IACF;IACA,eAAe,MAAM,aAAa;KAChC,IAAI,SAAS,YACX,KAAK,aAAa,oBAAoB,SAAS,UAAU;IAE7D;GACF,CAAC;GACD,IAAI,uBAAuB,EACzB,cAAc,MAAM,SAAS;IAC3B,IAAI,KAAK,OAAO,KAAK,aAAa,cAAc,KAAK,KAAK;GAC5D,EACF,CAAC;GAID,IAAI,sBAAsB;GAC1B,IAAI,uBAAuB;IAAE,YAAY;IAAM,aAAa;GAAK,CAAC;GAUlE,IAAI,kBAAkB;IACpB,2BAA2B;IAC3B,mBAAmB;GACrB,CAAC;EACH;CACF,CAAC;CACD,IAAI,MAAM;CACV,IACE,4BACA,OAAO,OAAO,+BAA+B,YAE7C,OAAO,2BAA2B;CAEpC,SAAO,KAAK,mCAAmC;AACjD;AAEA,eAAe,WAAW;CAGxB,0BAA0B,KAAK;CAC/B,MAAM,QAAQ,KAAK,CACjB,QAAQ,IAAI,CACV,eAAe,SAAS,CAAC,CAAC,YAAY,KAAA,CAAS,GAC/C,KAAK,SAAS,CAAC,CAAC,YAAY,KAAA,CAAS,CACvC,CAAC,GACD,IAAI,SAAe,YAAY,WAAW,SAAS,GAAK,CAAC,CAC3D,CAAC;AACH;AAMA,mCAAmC,QAAQ;;;AClP3C,OAAO,OAAO;AASd,MAAM,EAAE,gBADS,UACc;AAY/B,SAAS,eACP,KAC0C;CAC1C,IAAI,CAAC,KAAK,OAAO,KAAA;CACjB,IAAI,QAAQ,+BAA+B,QAAQ,4BACjD,OAAO;CACT,MAAM,IAAI,MACR,kCAAkC,IAAI,sEACxC;AACF;AAEA,MAAa,SAAiB;CAC5B,UAAU,EAER,KACE,QAAQ,IAAI,+BACZ,aAAa,UAAU,OACvB,SACJ;CACA,MACE,QAAQ,IAAI,uBACZ,CAAC,MAAM,OAAO,QAAQ,IAAI,mBAAmB,CAAC,IAC1C,OAAO,QAAQ,IAAI,mBAAmB,IACrC,aAAa,QAAQ;CAC5B,KAAK;CACL,eAAe,QAAQ,IAAI,sBAAsB;CACjD,gBAAgB,oBAAoB,QAAQ,IAAI,mBAAmB;CACnE,OAAO;EACL,IAAI;EACJ,MAAM;EACN,cAAc,eAAe,QAAQ,IAAI,qBAAqB;EAC9D,QAAQ;GACN,MAAM;GACN,MAAM;EACR;EACA,OAAO;GACL,kBAAkB;GAClB,WAAW,CAAC;GACZ,aAAa;GACb,UAAU,CAAC;EACb;CACF;AACF;;;AC9DA,eAAsB,oBAAoB,KAAyB;CACjE,MAAM,EACJ,0BAA0B,eAC1B,4BAA4B,SAC5B,gBAAgB,eAChB,oBAAoB,mBACpB,wBAAwB,aACxB,wBAAwB,gBACtB;CAoBJ,OAAO,aAAa;EAjBlB;EACA;EACA;EACA;EAGA,MAAM;GACJ,oBAAoB;GACpB,wBAAwB;GACxB,gBAAgB;EAClB;EAEA,MAAM;GACJ,uBAAuB;GACvB,YAAY;EACd;CAEwB,GAAG;EAC3B,aAAa,gBAAgB;EAC7B,aAAa,gBAAgB;CAC/B,CAAC;AACH;AAOA,eAAsB,aACpB,SACA,QAAuB;CAAE,aAAa;CAAM,aAAa;AAAM,GAC/D;CACA,QAAQ,IAAI,uCAAuC,SAAS,aAAa;CACzE,QAAQ,IAAI,qBAAqB,MAAM,cAAc,YAAY,UAAU;CAC3E,QAAQ,IAAI,qBAAqB,MAAM,cAAc,YAAY,UAAU;CAE3E,MAAM,EAAE,SAAS,cAAc,MAAM,OAAO;CAC5C,UAAU,KAAK,OAAO;CAGtB,IAAI,MAAM,aACR,UAAU,mBAAmB;CAI/B,UAAU,kBAAkB;CAG5B,IAAI,MAAM,aACR,UAAU,mBAAmB;AAEjC;;;ACnDA,MAAM,SAAS,YAAY,CAAC,aAAa,CAAC;AAE1C,SAAS,kBAAkB,aAAa,MAAM;CAC5C,MAAM,UAAU,QAAQ,SAAS;CACjC,IAAI,CAAC,SACH;CAGF,IAAI,UAAU,YAAY;EACxB,QAAQ,MACN,gBAAgB,WAAW,2CAA2C,SACxE;EACA,QAAQ,KAAK,CAAC;CAChB;AACF;AAEA,kBAAkB,IAAI;AAItB,QAAQ,gBAAgB,CAAC;AAEzB,IAAI,QAAQ,IAAI,0BACd,IAAI;CACF,MAAM,oBAAoB,QAAQ,GAAG;AACvC,SAAS,GAAG;CACV,OAAO,iBAAiB,CAAC;CACzB,OAAO,MAAM,mCAAmC,CAAC;AACnD;AAGF,MAAM,mBAAmB,QAAQ,KAAK,MAAM,CAAC,CAAC,CAAC,SAAS,kBAAkB;AAK1E,IAAI;CACF,MAAM,iBAAiB;EACrB,GAAG;EACH,eAAe,oBAAoB,OAAO;EAC1C,gBAAgB,OAAO,kBAAkB,KAAA;EACzC,oBAAoB;CACtB,CAAC;AACH,SAAS,GAAG;CAEV,IAAI,CAAC,4BAA4B,QAAQ,CAAC,GAAG;EAC3C,OAAO,iBAAiB,CAAC;EACzB,OAAO,MAAM,uCAAuC,CAAC;CACvD;CACA,MAAM,OAAO,MAAM,GAAI,CAAC,CAAC,YAAY,KAAA,CAAS;CAC9C,QAAQ,KAAK,CAAC;AAChB","debug_id":"1fbb2406-730e-5d3e-a4cf-ac7dea3cedd5"}
+{"version":3,"file":"index.mjs","sources":["../src/metrics.ts","../src/observability-signals.mts","../src/redact-webhook-path.ts","../src/observability.mts","../src/config.ts","../src/index.mts"],"sourcesContent":["import { OTLPMetricExporter } from \"@opentelemetry/exporter-metrics-otlp-http\";\nimport { resourceFromAttributes } from \"@opentelemetry/resources\";\nimport {\n  MeterProvider,\n  PeriodicExportingMetricReader,\n} from \"@opentelemetry/sdk-metrics\";\nimport { childLogger } from \"document-model\";\n\nconst logger = childLogger([\"switchboard\", \"metrics\"]);\n\nexport function createMeterProviderFromEnv(env: {\n  OTEL_EXPORTER_OTLP_ENDPOINT?: string;\n  OTEL_METRIC_EXPORT_INTERVAL?: string;\n  OTEL_SERVICE_NAME?: string;\n}): MeterProvider | undefined {\n  const endpoint = env.OTEL_EXPORTER_OTLP_ENDPOINT;\n  if (!endpoint) return undefined;\n\n  const parsed = parseInt(env.OTEL_METRIC_EXPORT_INTERVAL ?? \"\", 10);\n  const exportIntervalMillis =\n    Number.isFinite(parsed) && parsed > 0 ? parsed : 5_000;\n\n  const base = endpoint.replace(/\\/$/, \"\");\n  const exporterUrl = base.endsWith(\"/v1/metrics\")\n    ? base\n    : `${base}/v1/metrics`;\n\n  logger.info(`Initializing OpenTelemetry metrics exporter at: ${endpoint}`);\n  const meterProvider = new MeterProvider({\n    resource: resourceFromAttributes({\n      \"service.name\": env.OTEL_SERVICE_NAME ?? \"switchboard\",\n    }),\n    readers: [\n      new PeriodicExportingMetricReader({\n        exporter: new OTLPMetricExporter({\n          url: exporterUrl,\n        }),\n        exportIntervalMillis,\n        exportTimeoutMillis: Math.max(exportIntervalMillis - 250, 1),\n      }),\n    ],\n  });\n  logger.info(`Metrics export enabled (interval: ${exportIntervalMillis}ms)`);\n  return meterProvider;\n}\n","// Signal handling for the observability bootstrap, split out of\n// observability.mts so tests can exercise it without importing the module\n// whose load-time side effects start Sentry and OpenTelemetry.\n\ntype SignalProcess = Pick<NodeJS.Process, \"on\" | \"exit\"> & {\n  exitCode?: NodeJS.Process[\"exitCode\"];\n};\n\n/**\n * Flushes telemetry on SIGINT/SIGTERM, then exits only when no other module\n * has claimed a failing exit.\n *\n * Exit ownership, decided here on purpose:\n *\n * - Ordinary signal (process.exitCode unset or 0): this handler exits with\n *   the current exit code after the flush settles (the flush bounds itself,\n *   see shutdown() in observability.mts). When the reactor builder's drain\n *   handler is also attached it runs first (prependListener) and swaps\n *   process.exit for a recording shim before this handler's exit call, so\n *   the drain still finishes and performs the real exit; when the builder is\n *   not attached (startup, signalHandlers: false), this exit is what\n *   terminates the process, with code 0.\n *\n * - Fatal shutdown in progress (process.exitCode already non-zero):\n *   fatal-shutdown.mts sets exitCode = 1 and arms a 15s forced exit BEFORE\n *   raising SIGTERM, precisely so the builder's drain can flush the PGlite\n *   snapshots. Calling exit here would lower that code to 0 (supervisors\n *   would see a crash as success) and, whenever the builder's shim is not in\n *   place, cut the drain off at this handler's shorter flush budget. So this\n *   handler never exits once a non-zero code is set: the exit belongs to the\n *   builder's drain, or failing that to fatal-shutdown's 15s backstop, both\n *   of which preserve code 1.\n */\nexport function installObservabilitySignalHandlers(\n  flush: () => Promise<void>,\n  proc: SignalProcess = process,\n): void {\n  const onSignal = () => {\n    void flush().finally(() => {\n      const code = proc.exitCode;\n      if (code !== undefined && code !== 0) {\n        return;\n      }\n      proc.exit(code ?? 0);\n    });\n  };\n  proc.on(\"SIGINT\", onSignal);\n  proc.on(\"SIGTERM\", onSignal);\n}\n","const WEBHOOK_SEGMENT = \"/webhooks/\";\n\n/**\n * Strips a webhook's delivery token out of a request path.\n *\n * The token is the endpoint's whole credential and it sits in the path, so it\n * must not reach a log line or a span attribute that ships off-box. The family\n * mounts under the host's base path, so the segment can sit at any depth.\n */\nexport function redactWebhookPath(url: string | undefined): string {\n  if (!url) return \"\";\n  // Dropped with the token: `http.route` is meant to be a template, and a\n  // query string is another place a sender can put a credential.\n  const path = url.split(\"?\")[0] ?? url;\n  const at = path.indexOf(WEBHOOK_SEGMENT);\n  if (at === -1) return url;\n  // Nothing after the segment is nothing to redact.\n  if (path.length === at + WEBHOOK_SEGMENT.length) return url;\n  return `${path.slice(0, at)}${WEBHOOK_SEGMENT}[redacted]`;\n}\n","// Single observability bootstrap: Sentry + OpenTelemetry (tracing + metrics).\n//\n// MUST be imported as the very first module in apps/switchboard/src/index.mts.\n// OpenTelemetry instrumentations register require-time hooks at module load,\n// so http/express/pg/graphql must not be imported (transitively) before this\n// file runs.\n//\n// Replaces three legacy bootstrap sites:\n// - apps/switchboard/src/server.mts top-level Sentry.init\n// - apps/switchboard/src/metrics.ts standalone MeterProvider (still exported\n//   here via createMeterProviderFromEnv so its tests keep passing)\n// - packages/reactor-api/src/tracing.ts side-effect NodeSDK\nimport { metrics } from \"@opentelemetry/api\";\nimport { OTLPTraceExporter } from \"@opentelemetry/exporter-trace-otlp-http\";\nimport { ExpressInstrumentation } from \"@opentelemetry/instrumentation-express\";\nimport { GraphQLInstrumentation } from \"@opentelemetry/instrumentation-graphql\";\nimport { HttpInstrumentation } from \"@opentelemetry/instrumentation-http\";\nimport { PgInstrumentation } from \"@opentelemetry/instrumentation-pg\";\nimport { UndiciInstrumentation } from \"@opentelemetry/instrumentation-undici\";\nimport { resourceFromAttributes } from \"@opentelemetry/resources\";\nimport type { MeterProvider } from \"@opentelemetry/sdk-metrics\";\nimport { NodeSDK } from \"@opentelemetry/sdk-node\";\nimport {\n  BatchSpanProcessor,\n  type SpanProcessor,\n} from \"@opentelemetry/sdk-trace-base\";\nimport {\n  ATTR_SERVICE_NAME,\n  ATTR_SERVICE_VERSION,\n} from \"@opentelemetry/semantic-conventions\";\nimport {\n  registerEventLoopInstrumentation,\n  type EventLoopInstrumentation,\n} from \"@powerhousedao/opentelemetry-instrumentation-reactor\";\nimport * as Sentry from \"@sentry/node\";\nimport { SentryPropagator, SentrySpanProcessor } from \"@sentry/opentelemetry\";\nimport { childLogger } from \"document-model\";\nimport type { IncomingMessage } from \"node:http\";\nimport { createMeterProviderFromEnv } from \"./metrics.js\";\nimport { installObservabilitySignalHandlers } from \"./observability-signals.mjs\";\nimport { redactWebhookPath } from \"./redact-webhook-path.js\";\n\nconst logger = childLogger([\"switchboard\", \"observability\"]);\n\nconst SERVICE_NAME = process.env.OTEL_SERVICE_NAME || \"switchboard\";\nconst SERVICE_VERSION = process.env.npm_package_version || \"unknown\";\nconst TENANT_ID = process.env.TENANT_ID || \"default\";\nconst DEPLOY_ENV = process.env.NODE_ENV || \"development\";\n\nconst TEMPO_ENDPOINT = process.env.TEMPO_ENDPOINT;\nconst SENTRY_DSN = process.env.SENTRY_DSN;\n\n// Whether to forward TRANSACTIONS (spans) to Sentry. Errors are always sent\n// when SENTRY_DSN is set — this flag only gates APM/tracing into Sentry.\n// Default on (back-compat). Set SENTRY_TRACING_ENABLED=false for an\n// \"errors-only\" deployment: errors still go to Sentry, traces still go to\n// Tempo, but no transactions hit Sentry (no Kafka/ClickHouse/nodestore\n// cost). This is the recommended mode for tenant workloads at scale —\n// Sentry's value there is error grouping; traces live in Tempo.\nconst SENTRY_TRACING_TO_SENTRY =\n  Boolean(SENTRY_DSN) && process.env.SENTRY_TRACING_ENABLED !== \"false\";\n\nconst TRACING_REQUESTED =\n  process.env.ENABLE_TRACING === \"true\" ||\n  process.env.NODE_ENV === \"production\";\nconst HAS_TRACE_DESTINATION = Boolean(TEMPO_ENDPOINT) || Boolean(SENTRY_DSN);\nconst TRACING_ENABLED = TRACING_REQUESTED && HAS_TRACE_DESTINATION;\n\nif (TRACING_REQUESTED && !HAS_TRACE_DESTINATION) {\n  logger.warn(\n    \"Tracing was requested (NODE_ENV=production or ENABLE_TRACING=true) but \" +\n      \"no destination is configured — instrumentation will not run. Set \" +\n      \"TEMPO_ENDPOINT (e.g. http://tempo.monitoring.svc.cluster.local:4318/v1/traces) \" +\n      \"to export OTLP spans, and/or SENTRY_DSN to forward spans to Sentry.\",\n  );\n}\n\n// APM sampling for the Sentry-SDK-managed path (i.e. when TRACING is OFF and\n// @sentry/node runs its own bundled OTel — see skipOpenTelemetrySetup below).\n//\n// IMPORTANT: when TRACING_ENABLED, this value does NOT govern span volume.\n// Our NodeSDK (below) owns the pipeline and is constructed with no explicit\n// `sampler`, so @opentelemetry/sdk-node falls back to buildSamplerFromEnv()\n// and the REAL head-sampling knob is the standard OTEL_TRACES_SAMPLER /\n// OTEL_TRACES_SAMPLER_ARG env (set per-deploy in the k8s chart). That head\n// decision gates spans before any processor runs, so it bounds BOTH the\n// SentrySpanProcessor (→ Sentry transactions) and the Tempo OTLP export.\n// (Wiring @sentry/opentelemetry's SentrySampler here would let this rate\n// drive both backends and make DSC/sample_rand propagation spec-correct, but\n// that only matters for Sentry server-side dynamic sampling — a SaaS feature\n// our self-hosted install doesn't run — so it's intentionally deferred.)\nconst SENTRY_TRACES_SAMPLE_RATE = parseFloat(\n  process.env.SENTRY_TRACES_SAMPLE_RATE ?? \"0.1\",\n);\n\nif (SENTRY_DSN) {\n  logger.info(\"Initialized Sentry with env: @env\", process.env.SENTRY_ENV);\n  Sentry.init({\n    dsn: SENTRY_DSN,\n    environment: process.env.SENTRY_ENV,\n    // Match the version tag uploaded by release-branch.yml so source maps\n    // resolve. Populated by the CI (WORKSPACE_VERSION) or npm at runtime.\n    release:\n      process.env.SENTRY_RELEASE ||\n      (process.env.npm_package_version\n        ? `v${process.env.npm_package_version}`\n        : undefined),\n    // 0 in errors-only mode so even @sentry/node's bundled auto-OTel path\n    // (used when TRACING_ENABLED is false) produces no transactions.\n    tracesSampleRate: SENTRY_TRACING_TO_SENTRY ? SENTRY_TRACES_SAMPLE_RATE : 0,\n    // When tracing is on, our NodeSDK below owns the OTel globals and Sentry\n    // receives spans via SentrySpanProcessor. Skipping Sentry's bundled OTel\n    // setup avoids two TracerProviders fighting over setGlobalTracerProvider.\n    // When tracing is off, leave the flag unset so @sentry/node's default\n    // auto-OTel still records HTTP transactions for the APM dashboard.\n    skipOpenTelemetrySetup: TRACING_ENABLED,\n  });\n}\n\nconst meterProvider: MeterProvider | undefined = createMeterProviderFromEnv({\n  OTEL_EXPORTER_OTLP_ENDPOINT: process.env.OTEL_EXPORTER_OTLP_ENDPOINT,\n  OTEL_METRIC_EXPORT_INTERVAL: process.env.OTEL_METRIC_EXPORT_INTERVAL,\n  OTEL_SERVICE_NAME: process.env.OTEL_SERVICE_NAME,\n});\nlet eventLoopInstrumentation: EventLoopInstrumentation | undefined;\nif (meterProvider) {\n  // One-way door: must register before any code calls metrics.getMeter() —\n  // most notably ReactorInstrumentation inside the reactor module.\n  metrics.setGlobalMeterProvider(meterProvider);\n  // Host event-loop and CPU gauges (reactor.host.eventloop.*,\n  // reactor.host.cpu.utilization). The bench sweep identified the host event\n  // loop, not the executor, as the binding constraint on write throughput\n  // (bench/test/integration/BASELINE.md, Run 10) — these are the production\n  // counterpart of the metrics that finding was made from. Must come after\n  // setGlobalMeterProvider or the gauges bind to a no-op meter.\n  eventLoopInstrumentation = registerEventLoopInstrumentation();\n}\n\nlet sdk: NodeSDK | undefined;\n\nif (TRACING_ENABLED) {\n  logger.info(`Initializing OpenTelemetry tracing for ${SERVICE_NAME}`);\n  if (TEMPO_ENDPOINT) logger.info(`  Tempo endpoint: ${TEMPO_ENDPOINT}`);\n  if (SENTRY_DSN) logger.info(`  Sentry span forwarding: enabled`);\n  logger.info(`  Tenant: ${TENANT_ID}`);\n\n  const resource = resourceFromAttributes({\n    [ATTR_SERVICE_NAME]: SERVICE_NAME,\n    [ATTR_SERVICE_VERSION]: SERVICE_VERSION,\n    \"tenant.id\": TENANT_ID,\n    \"deployment.environment\": DEPLOY_ENV,\n  });\n\n  const spanProcessors: SpanProcessor[] = [];\n  if (TEMPO_ENDPOINT) {\n    spanProcessors.push(\n      new BatchSpanProcessor(new OTLPTraceExporter({ url: TEMPO_ENDPOINT })),\n    );\n  }\n  if (SENTRY_TRACING_TO_SENTRY) {\n    // Fan the same OTel spans into Sentry — same trace IDs as Tempo, so\n    // Sentry transactions cross-link to traces in Grafana. Skipped in\n    // errors-only mode (SENTRY_TRACING_ENABLED=false): spans still flow to\n    // Tempo via the BatchSpanProcessor above, just not to Sentry.\n    spanProcessors.push(new SentrySpanProcessor());\n  }\n\n  sdk = new NodeSDK({\n    resource,\n    spanProcessors,\n    textMapPropagator: SENTRY_TRACING_TO_SENTRY\n      ? new SentryPropagator()\n      : undefined,\n    instrumentations: [\n      new HttpInstrumentation({\n        ignoreIncomingRequestHook: (req) =>\n          req.url === \"/health\" || req.url === \"/ready\",\n        requireParentforIncomingSpans: false,\n        requireParentforOutgoingSpans: false,\n        requestHook: (span, request) => {\n          span.setAttribute(\n            \"http.route\",\n            redactWebhookPath((request as IncomingMessage).url),\n          );\n        },\n        responseHook: (span, response) => {\n          if (response.statusCode) {\n            span.setAttribute(\"http.status_code\", response.statusCode);\n          }\n        },\n      }),\n      new ExpressInstrumentation({\n        requestHook: (span, info) => {\n          if (info.route) span.setAttribute(\"http.route\", info.route);\n        },\n      }),\n      // HttpInstrumentation only patches node:http/https; outbound global\n      // fetch() goes through undici and is otherwise untraced — e.g. the\n      // per-request Renown credential check in AuthService.verifyCredentialExists.\n      new UndiciInstrumentation(),\n      new GraphQLInstrumentation({ mergeItems: true, allowValues: true }),\n      // requireParentSpan: only trace DB queries that run inside a request\n      // (HTTP/GraphQL) span. Parentless queries — the management\n      // switchboard's background polling loops (vetra-cloud-observability\n      // reconcile @60s + clint pull-worker @15s, each writing\n      // environment_pods / clint_runtime_endpoints) — would otherwise each\n      // become a standalone root transaction. That volume scales O(tenant\n      // count) and was ~70% of all Sentry transactions before this change.\n      // Dropping it at the instrumentation layer (no span created at all) is\n      // cheaper and cleaner than sampling it away downstream.\n      new PgInstrumentation({\n        enhancedDatabaseReporting: true,\n        requireParentSpan: true,\n      }),\n    ],\n  });\n  sdk.start();\n  if (\n    SENTRY_TRACING_TO_SENTRY &&\n    typeof Sentry.validateOpenTelemetrySetup === \"function\"\n  ) {\n    Sentry.validateOpenTelemetrySetup();\n  }\n  logger.info(\"OpenTelemetry tracing initialized\");\n}\n\nasync function shutdown() {\n  // Detach the observable callback before the provider drains, so the final\n  // export does not fire a gauge callback against a shutting-down meter.\n  eventLoopInstrumentation?.stop();\n  await Promise.race([\n    Promise.all([\n      meterProvider?.shutdown().catch(() => undefined),\n      sdk?.shutdown().catch(() => undefined),\n    ]),\n    new Promise<void>((resolve) => setTimeout(resolve, 5_000)),\n  ]);\n}\n\n// Exit-code and drain coordination with fatal-shutdown.mts and the reactor\n// builder lives in observability-signals.mts; the earlier inline handlers\n// exited with a literal 0, which turned a fatal crash (exitCode = 1) into a\n// success in the supervisor's eyes and could cut the snapshot drain short.\ninstallObservabilitySignalHandlers(shutdown);\n\nexport { meterProvider, sdk };\n","import dotenv from \"dotenv\";\ndotenv.config();\n\nimport { getConfig } from \"@powerhousedao/config/node\";\nimport { parseForcePgVersion } from \"./pglite-version.js\";\nimport type {\n  SwitchboardDriveDocumentType,\n  SwitchboardDriveInput,\n} from \"./types.js\";\nconst phConfig = getConfig();\nconst { switchboard } = phConfig;\ninterface Config {\n  database: {\n    url: string;\n  };\n  port: number;\n  mcp: boolean;\n  migratePglite: boolean;\n  forcePgVersion: 16 | 17 | null;\n  drive: SwitchboardDriveInput;\n}\n\nfunction parseDriveType(\n  raw: string | undefined,\n): SwitchboardDriveDocumentType | undefined {\n  if (!raw) return undefined;\n  if (raw === \"powerhouse/document-drive\" || raw === \"powerhouse/reactor-drive\")\n    return raw;\n  throw new Error(\n    `Invalid PH_DEFAULT_DRIVE_TYPE: ${raw}. Expected \"powerhouse/document-drive\" or \"powerhouse/reactor-drive\".`,\n  );\n}\n\nexport const config: Config = {\n  database: {\n    // url: process.env.PH_SWITCHBOARD_DATABASE_URL ?? switchboard?.database?.url ?? \"dev.db\",\n    url:\n      process.env.PH_SWITCHBOARD_DATABASE_URL ??\n      switchboard?.database?.url ??\n      \"dev.db\",\n  },\n  port:\n    process.env.PH_SWITCHBOARD_PORT &&\n    !isNaN(Number(process.env.PH_SWITCHBOARD_PORT))\n      ? Number(process.env.PH_SWITCHBOARD_PORT)\n      : (switchboard?.port ?? 4001),\n  mcp: true,\n  migratePglite: process.env.PH_MIGRATE_PGLITE === \"true\",\n  forcePgVersion: parseForcePgVersion(process.env.PH_FORCE_PG_VERSION),\n  drive: {\n    id: \"powerhouse\",\n    slug: \"powerhouse\",\n    documentType: parseDriveType(process.env.PH_DEFAULT_DRIVE_TYPE),\n    global: {\n      name: \"Powerhouse\",\n      icon: \"https://ipfs.io/ipfs/QmcaTDBYn8X2psGaXe7iQ6qd8q6oqHLgxvMX9yXf7f9uP7\",\n    },\n    local: {\n      availableOffline: true,\n      listeners: [],\n      sharingType: \"public\",\n      triggers: [],\n    },\n  },\n};\n","#!/usr/bin/env node\n// Observability MUST load before any module that imports http/express/pg/graphql\n// so OpenTelemetry's require-time hooks can patch them. It also owns Sentry\n// init and the SIGINT/SIGTERM flush.\nimport \"./observability.mjs\";\n\nimport * as Sentry from \"@sentry/node\";\nimport { childLogger } from \"document-model\";\nimport { config } from \"./config.js\";\nimport { initProfilerFromEnv } from \"./profiler.js\";\nimport { startSwitchboard } from \"./server.mjs\";\nimport { StoredDocumentsRefusedError } from \"./unsupported-stored-documents.mjs\";\n\nconst logger = childLogger([\"switchboard\"]);\n\nfunction ensureNodeVersion(minVersion = \"24\") {\n  const version = process.versions.node;\n  if (!version) {\n    return;\n  }\n\n  if (version < minVersion) {\n    console.error(\n      `Node version ${minVersion} or higher is required. Current version: ${version}`,\n    );\n    process.exit(1);\n  }\n}\n// Ensure minimum Node.js version\nensureNodeVersion(\"24\");\n\n// Each subgraph registers its own SIGINT/SIGTERM listeners, and the count\n// scales with dynamically-loaded document models beyond the default cap of 10.\nprocess.setMaxListeners(0);\n\nif (process.env.PYROSCOPE_SERVER_ADDRESS) {\n  try {\n    await initProfilerFromEnv(process.env);\n  } catch (e) {\n    Sentry.captureException(e);\n    logger.error(\"Error starting profiler: @error\", e);\n  }\n}\n\nconst cliMigratePglite = process.argv.slice(2).includes(\"--migrate-pglite\");\n\n// A boot failure leaves no serving process behind, so it must exit non-zero:\n// logging and continuing hides a dead switchboard from the supervisor, which\n// would otherwise never restart it.\ntry {\n  await startSwitchboard({\n    ...config,\n    migratePglite: cliMigratePglite || config.migratePglite,\n    forcePgVersion: config.forcePgVersion ?? undefined,\n    fatalErrorShutdown: true,\n  });\n} catch (e) {\n  // startSwitchboard already reported a refusal to the operator.\n  if (!StoredDocumentsRefusedError.isError(e)) {\n    Sentry.captureException(e);\n    logger.error(\"Switchboard failed to start: @error\", e);\n  }\n  await Sentry.flush(2000).catch(() => undefined);\n  process.exit(1);\n}\n"],"names":["logger","logger"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAQA,MAAMA,WAAS,YAAY,CAAC,eAAe,SAAS,CAAC;AAErD,SAAgB,2BAA2B,KAIb;CAC5B,MAAM,WAAW,IAAI;CACrB,IAAI,CAAC,UAAU,OAAO,KAAA;CAEtB,MAAM,SAAS,SAAS,IAAI,+BAA+B,IAAI,EAAE;CACjE,MAAM,uBACJ,OAAO,SAAS,MAAM,KAAK,SAAS,IAAI,SAAS;CAEnD,MAAM,OAAO,SAAS,QAAQ,OAAO,EAAE;CACvC,MAAM,cAAc,KAAK,SAAS,aAAa,IAC3C,OACA,GAAG,KAAK;CAEZ,SAAO,KAAK,mDAAmD,UAAU;CACzE,MAAM,gBAAgB,IAAI,cAAc;EACtC,UAAU,uBAAuB,EAC/B,gBAAgB,IAAI,qBAAqB,cAC3C,CAAC;EACD,SAAS,CACP,IAAI,8BAA8B;GAChC,UAAU,IAAI,mBAAmB,EAC/B,KAAK,YACP,CAAC;GACD;GACA,qBAAqB,KAAK,IAAI,uBAAuB,KAAK,CAAC;EAC7D,CAAC,CACH;CACF,CAAC;CACD,SAAO,KAAK,qCAAqC,qBAAqB,IAAI;CAC1E,OAAO;AACT;;;;;;;;;;;;;;;;;;;;;;;;;;;;ACXA,SAAgB,mCACd,OACA,OAAsB,SAChB;CACN,MAAM,iBAAiB;EACrB,MAAW,CAAC,CAAC,cAAc;GACzB,MAAM,OAAO,KAAK;GAClB,IAAI,SAAS,KAAA,KAAa,SAAS,GACjC;GAEF,KAAK,KAAK,QAAQ,CAAC;EACrB,CAAC;CACH;CACA,KAAK,GAAG,UAAU,QAAQ;CAC1B,KAAK,GAAG,WAAW,QAAQ;AAC7B;;;AChDA,MAAM,kBAAkB;;;;;;;;AASxB,SAAgB,kBAAkB,KAAiC;CACjE,IAAI,CAAC,KAAK,OAAO;CAGjB,MAAM,OAAO,IAAI,MAAM,GAAG,CAAC,CAAC,MAAM;CAClC,MAAM,KAAK,KAAK,QAAQ,eAAe;CACvC,IAAI,OAAO,IAAI,OAAO;CAEtB,IAAI,KAAK,WAAW,KAAK,IAAwB,OAAO;CACxD,OAAO,GAAG,KAAK,MAAM,GAAG,EAAE,IAAI,gBAAgB;AAChD;;;ACuBA,MAAMC,WAAS,YAAY,CAAC,eAAe,eAAe,CAAC;AAE3D,MAAM,eAAe,QAAQ,IAAI,qBAAqB;AACtD,MAAM,kBAAkB,QAAQ,IAAI,uBAAuB;AAC3D,MAAM,YAAY,QAAQ,IAAI,aAAa;AAC3C,MAAM,aAAa,QAAQ,IAAI,YAAY;AAE3C,MAAM,iBAAiB,QAAQ,IAAI;AACnC,MAAM,aAAa,QAAQ,IAAI;AAS/B,MAAM,2BACJ,QAAQ,UAAU,KAAK,QAAQ,IAAI,2BAA2B;AAEhE,MAAM,oBACJ,QAAQ,IAAI,mBAAmB,UAC/B,QAAQ,IAAI,aAAa;AAC3B,MAAM,wBAAwB,QAAQ,cAAc,KAAK,QAAQ,UAAU;AAC3E,MAAM,kBAAkB,qBAAqB;AAE7C,IAAI,qBAAqB,CAAC,uBACxB,SAAO,KACL,4RAIF;AAiBF,MAAM,4BAA4B,WAChC,QAAQ,IAAI,6BAA6B,KAC3C;AAEA,IAAI,YAAY;CACd,SAAO,KAAK,qCAAqC,QAAQ,IAAI,UAAU;CACvE,OAAO,KAAK;EACV,KAAK;EACL,aAAa,QAAQ,IAAI;EAGzB,SACE,QAAQ,IAAI,mBACX,QAAQ,IAAI,sBACT,IAAI,QAAQ,IAAI,wBAChB,KAAA;EAGN,kBAAkB,2BAA2B,4BAA4B;EAMzE,wBAAwB;CAC1B,CAAC;AACH;AAEA,MAAM,gBAA2C,2BAA2B;CAC1E,6BAA6B,QAAQ,IAAI;CACzC,6BAA6B,QAAQ,IAAI;CACzC,mBAAmB,QAAQ,IAAI;AACjC,CAAC;AACD,IAAI;AACJ,IAAI,eAAe;CAGjB,QAAQ,uBAAuB,aAAa;CAO5C,2BAA2B,iCAAiC;AAC9D;AAEA,IAAI;AAEJ,IAAI,iBAAiB;CACnB,SAAO,KAAK,0CAA0C,cAAc;CACpE,IAAI,gBAAgB,SAAO,KAAK,qBAAqB,gBAAgB;CACrE,IAAI,YAAY,SAAO,KAAK,mCAAmC;CAC/D,SAAO,KAAK,aAAa,WAAW;CAEpC,MAAM,WAAW,uBAAuB;GACrC,oBAAoB;GACpB,uBAAuB;EACxB,aAAa;EACb,0BAA0B;CAC5B,CAAC;CAED,MAAM,iBAAkC,CAAC;CACzC,IAAI,gBACF,eAAe,KACb,IAAI,mBAAmB,IAAI,kBAAkB,EAAE,KAAK,eAAe,CAAC,CAAC,CACvE;CAEF,IAAI,0BAKF,eAAe,KAAK,IAAI,oBAAoB,CAAC;CAG/C,MAAM,IAAI,QAAQ;EAChB;EACA;EACA,mBAAmB,2BACf,IAAI,iBAAiB,IACrB,KAAA;EACJ,kBAAkB;GAChB,IAAI,oBAAoB;IACtB,4BAA4B,QAC1B,IAAI,QAAQ,aAAa,IAAI,QAAQ;IACvC,+BAA+B;IAC/B,+BAA+B;IAC/B,cAAc,MAAM,YAAY;KAC9B,KAAK,aACH,cACA,kBAAmB,QAA4B,GAAG,CACpD;IACF;IACA,eAAe,MAAM,aAAa;KAChC,IAAI,SAAS,YACX,KAAK,aAAa,oBAAoB,SAAS,UAAU;IAE7D;GACF,CAAC;GACD,IAAI,uBAAuB,EACzB,cAAc,MAAM,SAAS;IAC3B,IAAI,KAAK,OAAO,KAAK,aAAa,cAAc,KAAK,KAAK;GAC5D,EACF,CAAC;GAID,IAAI,sBAAsB;GAC1B,IAAI,uBAAuB;IAAE,YAAY;IAAM,aAAa;GAAK,CAAC;GAUlE,IAAI,kBAAkB;IACpB,2BAA2B;IAC3B,mBAAmB;GACrB,CAAC;EACH;CACF,CAAC;CACD,IAAI,MAAM;CACV,IACE,4BACA,OAAO,OAAO,+BAA+B,YAE7C,OAAO,2BAA2B;CAEpC,SAAO,KAAK,mCAAmC;AACjD;AAEA,eAAe,WAAW;CAGxB,0BAA0B,KAAK;CAC/B,MAAM,QAAQ,KAAK,CACjB,QAAQ,IAAI,CACV,eAAe,SAAS,CAAC,CAAC,YAAY,KAAA,CAAS,GAC/C,KAAK,SAAS,CAAC,CAAC,YAAY,KAAA,CAAS,CACvC,CAAC,GACD,IAAI,SAAe,YAAY,WAAW,SAAS,GAAK,CAAC,CAC3D,CAAC;AACH;AAMA,mCAAmC,QAAQ;;;AClP3C,OAAO,OAAO;AASd,MAAM,EAAE,gBADS,UACc;AAY/B,SAAS,eACP,KAC0C;CAC1C,IAAI,CAAC,KAAK,OAAO,KAAA;CACjB,IAAI,QAAQ,+BAA+B,QAAQ,4BACjD,OAAO;CACT,MAAM,IAAI,MACR,kCAAkC,IAAI,sEACxC;AACF;AAEA,MAAa,SAAiB;CAC5B,UAAU,EAER,KACE,QAAQ,IAAI,+BACZ,aAAa,UAAU,OACvB,SACJ;CACA,MACE,QAAQ,IAAI,uBACZ,CAAC,MAAM,OAAO,QAAQ,IAAI,mBAAmB,CAAC,IAC1C,OAAO,QAAQ,IAAI,mBAAmB,IACrC,aAAa,QAAQ;CAC5B,KAAK;CACL,eAAe,QAAQ,IAAI,sBAAsB;CACjD,gBAAgB,oBAAoB,QAAQ,IAAI,mBAAmB;CACnE,OAAO;EACL,IAAI;EACJ,MAAM;EACN,cAAc,eAAe,QAAQ,IAAI,qBAAqB;EAC9D,QAAQ;GACN,MAAM;GACN,MAAM;EACR;EACA,OAAO;GACL,kBAAkB;GAClB,WAAW,CAAC;GACZ,aAAa;GACb,UAAU,CAAC;EACb;CACF;AACF;;;ACnDA,MAAM,SAAS,YAAY,CAAC,aAAa,CAAC;AAE1C,SAAS,kBAAkB,aAAa,MAAM;CAC5C,MAAM,UAAU,QAAQ,SAAS;CACjC,IAAI,CAAC,SACH;CAGF,IAAI,UAAU,YAAY;EACxB,QAAQ,MACN,gBAAgB,WAAW,2CAA2C,SACxE;EACA,QAAQ,KAAK,CAAC;CAChB;AACF;AAEA,kBAAkB,IAAI;AAItB,QAAQ,gBAAgB,CAAC;AAEzB,IAAI,QAAQ,IAAI,0BACd,IAAI;CACF,MAAM,oBAAoB,QAAQ,GAAG;AACvC,SAAS,GAAG;CACV,OAAO,iBAAiB,CAAC;CACzB,OAAO,MAAM,mCAAmC,CAAC;AACnD;AAGF,MAAM,mBAAmB,QAAQ,KAAK,MAAM,CAAC,CAAC,CAAC,SAAS,kBAAkB;AAK1E,IAAI;CACF,MAAM,iBAAiB;EACrB,GAAG;EACH,eAAe,oBAAoB,OAAO;EAC1C,gBAAgB,OAAO,kBAAkB,KAAA;EACzC,oBAAoB;CACtB,CAAC;AACH,SAAS,GAAG;CAEV,IAAI,CAAC,4BAA4B,QAAQ,CAAC,GAAG;EAC3C,OAAO,iBAAiB,CAAC;EACzB,OAAO,MAAM,uCAAuC,CAAC;CACvD;CACA,MAAM,OAAO,MAAM,GAAI,CAAC,CAAC,YAAY,KAAA,CAAS;CAC9C,QAAQ,KAAK,CAAC;AAChB","debug_id":"d30347ec-a382-56a4-b3ef-bdacc9d409fd"}

package/dist/{server-Dvw8_NeM.mjs → server-Zz7SqTwO.mjs}

@@ -1,7 +1,8 @@
 
-!function(){try{var e="undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof globalThis?globalThis:"undefined"!=typeof self?self:{},n=(new e.Error).stack;n&&(e._sentryDebugIds=e._sentryDebugIds||{},e._sentryDebugIds[n]="b9c02424-1406-5d63-abbd-6c32caa53205")}catch(e){}}();
+!function(){try{var e="undefined"!=typeof window?window:"undefined"!=typeof global?global:"undefined"!=typeof globalThis?globalThis:"undefined"!=typeof self?self:{},n=(new e.Error).stack;n&&(e._sentryDebugIds=e._sentryDebugIds||{},e._sentryDebugIds[n]="12676ab9-2bd3-55f1-9a41-559509adace0")}catch(e){}}();
 import { n as addDefaultReactorDrive, r as isPostgresUrl, t as addDefaultDrive } from "./utils-uaBlzyjS.mjs";
 import { createRequire, register } from "node:module";
+import { metrics, trace } from "@opentelemetry/api";
 import { ReactorInstrumentation } from "@powerhousedao/opentelemetry-instrumentation-reactor";
 import * as Sentry from "@sentry/node";
 import { childLogger, documentModelDocumentModelModule, setLogLevel } from "document-model";
@@ -10,7 +11,7 @@ import { getConfig } from "@powerhousedao/config/node";
 import { existsSync, promises, realpathSync } from "node:fs";
 import path, { resolve } from "node:path";
 import { createDurableNodeFs, preparePgliteDataDir, removeStalePgliteFiles, resolvePgliteFsync, syncTree } from "@powerhousedao/reactor-api/pglite-node";
-import { ChannelScheme, DriveCollectionId, EventBus, REACTOR_SCHEMA, ReactorBuilder, ReactorClientBuilder, UnsupportedStoredProtocolError, createHybridProjectionCoordinatorFactory, instrumentPgPool, parseDriveUrl, supportsLiveReadModelRegistration } from "@powerhousedao/reactor";
+import { ChannelScheme, DriveCollectionId, EventBus, HardenedPGliteDialect, REACTOR_SCHEMA, ReactorBuilder, ReactorClientBuilder, UnsupportedStoredProtocolError, createHybridProjectionCoordinatorFactory, instrumentPgPool, parseDriveUrl, supportsLiveReadModelRegistration } from "@powerhousedao/reactor";
 import { AuthorizationPolicy, BaseSubgraph, ForbiddenError, HttpPackageLoader, ImportPackageLoader, PGLITE_UTC_PARSERS, PackageManagementService, PackagesSubgraph, PrivacyPermissionAdapter, callerSubject, createCanonicalDocumentIdResolver, getUniqueDocumentModels, initializeAndStartAPI, resolveRenownConfig } from "@powerhousedao/reactor-api";
 import { httpsHooksPath } from "@powerhousedao/reactor-api/https-hooks";
 import { AttachmentAlreadyExists, AttachmentNotFound, AttachmentPending, AttachmentReferenceReadModel, AttachmentSchemaCompiler, HashMismatch, InvalidAttachmentMetadata, InvalidAttachmentRef, ReservationNotFound, SizeMismatch, UploadTooLarge, createRef, createRemoteAttachmentService, parseAttachmentDownloadTarget, parseRef, validateReserveMetadata } from "@powerhousedao/reactor-attachments";
@@ -34,7 +35,6 @@ import { EnvVarProvider } from "@openfeature/env-var-provider";
 import { OpenFeature } from "@openfeature/server-sdk";
 import { GraphQLError, Kind } from "graphql";
 import { gql } from "graphql-tag";
-import { PGliteDialect } from "kysely-pglite-dialect";
 //#region src/pglite-version.ts
 const SUPPORTED_PG_MAJORS = [16, 17];
 async function readPgVersionFile(dataDir) {
@@ -83,6 +83,45 @@ async function loadPgDump(major) {
 	return (await import("@electric-sql/pglite-tools/pg_dump")).pgDump;
 }
 //#endregion
+//#region src/profiler.ts
+let started = false;
+async function initProfilerFromEnv(env) {
+	if (started || !env.PYROSCOPE_SERVER_ADDRESS) return;
+	started = true;
+	const { PYROSCOPE_SERVER_ADDRESS: serverAddress, PYROSCOPE_APPLICATION_NAME: appName, PYROSCOPE_USER: basicAuthUser, PYROSCOPE_PASSWORD: basicAuthPassword, PYROSCOPE_WALL_ENABLED: wallEnabled, PYROSCOPE_HEAP_ENABLED: heapEnabled } = env;
+	return initProfiler({
+		serverAddress,
+		appName,
+		basicAuthUser,
+		basicAuthPassword,
+		wall: {
+			samplingDurationMs: 1e4,
+			samplingIntervalMicros: 1e4,
+			collectCpuTime: true
+		},
+		heap: {
+			samplingIntervalBytes: 524288,
+			stackDepth: 64
+		}
+	}, {
+		wallEnabled: wallEnabled !== "false",
+		heapEnabled: heapEnabled === "true"
+	});
+}
+async function initProfiler(options, flags = {
+	wallEnabled: true,
+	heapEnabled: false
+}) {
+	console.log("Initializing Pyroscope profiler at:", options?.serverAddress);
+	console.log("  Wall profiling:", flags.wallEnabled ? "enabled" : "disabled");
+	console.log("  Heap profiling:", flags.heapEnabled ? "enabled" : "disabled");
+	const { default: Pyroscope } = await import("@pyroscope/nodejs");
+	Pyroscope.init(options);
+	if (flags.wallEnabled) Pyroscope.startWallProfiling();
+	Pyroscope.startCpuProfiling();
+	if (flags.heapEnabled) Pyroscope.startHeapProfiling();
+}
+//#endregion
 //#region src/attachments/auth.ts
 const ANONYMOUS_ACTOR = {
 	user: void 0,
@@ -1964,7 +2003,11 @@ const schema = gql`
   type WorkflowStepTestResult {
     "Null when the test never started, e.g. an upstream block is untested."
     runId: String
-    "SUCCEEDED | FAILED"
+    """
+    SUCCEEDED | FAILED | INDETERMINATE. INDETERMINATE is neither: a host call
+    the block made timed out, so a write it asked for may well have landed.
+    Show it distinctly - it is not a confirmed pass.
+    """
     status: String!
     output: Unknown
     error: String
@@ -2065,6 +2108,7 @@ function createWorkflowRuntimeSubgraph(runtime, oauth, access) {
 //#region src/workflow-runtime.mts
 /** The npm name the workflow package owns: its HTTP namespace and its models. */
 const WORKFLOW_PACKAGE_NAME = "@powerhousedao/workflow";
+const WORKFLOW_TELEMETRY_SCOPE = "@powerhousedao/reactor-workflow";
 /** The env var and OpenFeature flag key that turns workflows on. */
 const PH_WORKFLOWS_ENABLED = "PH_WORKFLOWS_ENABLED";
 /** Precedence, unchanged from the reactor-api resolver this replaces: the
@@ -2161,14 +2205,35 @@ function hostIdentityOf(access) {
 		key: identity.key
 	} : { key: identity.key };
 }
-/** A step reads any well-formed attachment, as it reads any document: a run
-* carries no caller to check against. */
-function canReadAttachmentRef(_documentId, ref) {
-	try {
-		return Promise.resolve(parseRef(ref).version === 1);
-	} catch {
-		return Promise.resolve(false);
-	}
+const REFERENCE_INDEX_RETRIES = 8;
+const REFERENCE_INDEX_RETRY_MS = 250;
+/** A step reads an attachment the way the download route serves one: when the
+* run's user may read a document that references it. The candidates are the
+* documents the run was handed; the reference index can trail the operation
+* that fired the trigger, so a denial is retried briefly. */
+function attachmentReadPolicy(access, retryMs = REFERENCE_INDEX_RETRY_MS) {
+	return async (request) => {
+		try {
+			if (parseRef(request.ref).version !== 1) return false;
+		} catch {
+			return false;
+		}
+		if (!access || request.documentIds.length === 0) return false;
+		const subject = request.runUser?.subject;
+		const caller = {
+			...subject?.address ? { userAddress: subject.address } : {},
+			...subject?.key ? { appKey: subject.key } : {}
+		};
+		for (let attempt = 0; attempt <= REFERENCE_INDEX_RETRIES; attempt++) {
+			for (const documentId of request.documentIds) if ((await access.canReadAttachment({
+				documentId,
+				attachmentRef: request.ref,
+				...caller
+			})).kind === "allowed") return true;
+			if (attempt < REFERENCE_INDEX_RETRIES) await new Promise((resolve) => setTimeout(resolve, retryMs));
+		}
+		return false;
+	};
 }
 async function registerWorkflowTriggersReadModel(engine, runtime, clientModule) {
 	const reactorModule = clientModule?.reactorModule;
@@ -2221,8 +2286,9 @@ async function composeWorkflowRuntime(deps) {
 		} : {},
 		webhooks: deps.webhooks,
 		attachments: deps.attachments,
-		canReadAttachmentRef,
-		logger: deps.logger
+		canReadAttachmentRef: attachmentReadPolicy(deps.attachmentAccess),
+		logger: deps.logger,
+		...deps.telemetry ? { telemetry: deps.telemetry } : {}
 	});
 	const triggers = await registerWorkflowTriggersReadModel(engine, runtime, deps.clientModule);
 	if (triggers.status === "available") deps.logger.info(`Workflow trigger read model registered (${engine.WORKFLOW_TRIGGERS_READ_MODEL}, ${engine.WORKFLOW_TRIGGERS_READ_MODEL_STAGE})`);
@@ -2247,14 +2313,27 @@ async function composeWorkflowRuntime(deps) {
 }
 //#endregion
 //#region src/pglite-dialect.ts
-var ClosablePGliteDialect = class extends PGliteDialect {
+/** A poisoned store cannot recover in process, so it takes the fatal shutdown and a supervisor restart. */
+function reactorPgliteDialectOptions(logger, proc) {
+	return {
+		onDiagnostic: (message, error) => logger.error(`[pglite-dialect] ${message}: @error`, error),
+		onPoisoned: (cause) => {
+			if (!triggerFatalShutdown("PGlite session poisoned", cause, proc)) logger.error("PGlite session poisoned: @error", cause);
+		}
+	};
+}
+var ClosablePGliteDialect = class {
+	#inner;
 	#pglite;
-	constructor(pglite) {
-		super(pglite);
+	constructor(pglite, options = {}) {
 		this.#pglite = pglite;
+		this.#inner = new HardenedPGliteDialect(pglite, options);
+	}
+	createAdapter() {
+		return this.#inner.createAdapter();
 	}
 	createDriver() {
-		const driver = super.createDriver();
+		const driver = this.#inner.createDriver();
 		const pglite = this.#pglite;
 		const innerDestroy = driver.destroy.bind(driver);
 		driver.destroy = async () => {
@@ -2268,6 +2347,12 @@ var ClosablePGliteDialect = class extends PGliteDialect {
 		};
 		return driver;
 	}
+	createQueryCompiler() {
+		return this.#inner.createQueryCompiler();
+	}
+	createIntrospector(db) {
+		return this.#inner.createIntrospector(db);
+	}
 };
 //#endregion
 //#region src/pglite-migration.ts
@@ -2708,7 +2793,7 @@ async function createReactorKysely(opts) {
 	const pglite = inMemory ? new PGlite() : new PGlite({ fs: (await loadDurableFsFactory(reactorPgliteMajor, logger))(reactorPgliteDir) });
 	logger.info(inMemory ? `Using in-memory PGlite (PG${reactorPgliteMajor}) for reactor storage [PH_PGLITE_IN_MEMORY=1]` : `Using PGlite (PG${reactorPgliteMajor}) for reactor storage at ${reactorPgliteDir}`);
 	return {
-		kysely: new Kysely({ dialect: new ClosablePGliteDialect(pglite) }),
+		kysely: new Kysely({ dialect: new ClosablePGliteDialect(pglite, reactorPgliteDialectOptions(logger)) }),
 		poolInstrumentation: void 0
 	};
 }
@@ -3024,13 +3109,18 @@ async function initServer(serverPort, options, renown, renownConfig) {
 			relationalDb: api.relationalDb,
 			secretsKeyFile: readModelPgliteDir === null ? false : void 0,
 			attachments: createAttachmentClient(api.attachments.service),
+			attachmentAccess: api.attachmentAccess,
 			webhooks: api.httpRoutes.scopeFor(WORKFLOW_PACKAGE_NAME).webhooks,
 			http: api.httpRoutes.scopeFor(WORKFLOW_PACKAGE_NAME),
 			authorizationService: api.authorizationService,
 			pieces: api.packageManager,
 			pieceRegistryUrl: registryUrl,
 			models: workerModels,
-			logger: logger.child(["workflow-runtime"])
+			logger: logger.child(["workflow-runtime"]),
+			telemetry: {
+				tracer: trace.getTracer(WORKFLOW_TELEMETRY_SCOPE),
+				meter: metrics.getMeter(WORKFLOW_TELEMETRY_SCOPE)
+			}
 		});
 		const WorkflowRuntimeSubgraph = workflows.subgraph;
 		const workflowSubgraph = new WorkflowRuntimeSubgraph({
@@ -3195,6 +3285,7 @@ async function initServer(serverPort, options, renown, renownConfig) {
 const startSwitchboard = async (options = {}) => {
 	const requestedPort = options.port ?? DEFAULT_PORT;
 	const logger = options.logger ?? defaultLogger;
+	if (process.env.PYROSCOPE_SERVER_ADDRESS) await initProfilerFromEnv(process.env).catch((error) => logger.error("Error starting profiler: @error", error));
 	const serverPort = await resolveServerPort(requestedPort, options.strictPort ?? false, logger);
 	const featureFlags = await initFeatureFlags();
 	const enableDocumentModelSubgraphs = await featureFlags.getBooleanValue(DOCUMENT_MODEL_SUBGRAPHS_ENABLED, options.enableDocumentModelSubgraphs ?? DOCUMENT_MODEL_SUBGRAPHS_ENABLED_DEFAULT);
@@ -3238,7 +3329,7 @@ const startSwitchboard = async (options = {}) => {
 };
 if (import.meta.main) await startSwitchboard({ fatalErrorShutdown: true });
 //#endregion
-export { resolveUnsupportedStoredDocuments as a, StoredDocumentsRefusedError as i, isPortAvailable as n, applySwitchboardReactorDefaults as o, startSwitchboard as r, parseForcePgVersion as s, deriveAttachmentServiceConfig as t };
+export { StoredDocumentsRefusedError as a, initProfilerFromEnv as c, startSwitchboard as i, parseForcePgVersion as l, deriveAttachmentServiceConfig as n, resolveUnsupportedStoredDocuments as o, isPortAvailable as r, applySwitchboardReactorDefaults as s, createReactorKysely as t };
 
-//# sourceMappingURL=server-Dvw8_NeM.mjs.map
-//# debugId=b9c02424-1406-5d63-abbd-6c32caa53205
+//# sourceMappingURL=server-Zz7SqTwO.mjs.map
+//# debugId=12676ab9-2bd3-55f1-9a41-559509adace0