@inflexa-ai/harness 0.35.1 → 0.35.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/execution/dbos-run-launcher.d.ts +13 -2
- package/dist/execution/dbos-run-launcher.js +28 -3
- package/dist/execution/reconcile-manifest.js +8 -8
- package/dist/lib/metrics.d.ts +20 -2
- package/dist/lib/metrics.js +39 -11
- package/dist/lib/otel.d.ts +42 -5
- package/dist/lib/otel.js +82 -16
- package/dist/runtime/dbos.d.ts +47 -4
- package/dist/runtime/dbos.js +75 -13
- package/dist/workflows/metrics.d.ts +1 -16
- package/dist/workflows/metrics.js +1 -19
- package/package.json +1 -1
|
@@ -1,7 +1,18 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* DBOS realization of the `RunLauncher` seam — the single production adapter,
|
|
3
|
-
* shared by every embedder. Host-neutral: it references only DBOS
|
|
4
|
-
* already depends on, so it ships in the harness.
|
|
3
|
+
* shared by every embedder. Host-neutral: it references only DBOS and the OTel
|
|
4
|
+
* API, which the harness already depends on, so it ships in the harness.
|
|
5
|
+
*
|
|
6
|
+
* A launch detaches the run from the caller's trace. The DBOS workflow span
|
|
7
|
+
* takes the active context as its parent, and the caller is a chat request
|
|
8
|
+
* (or another short-lived operation) whose span ends in milliseconds while the
|
|
9
|
+
* run lasts hours. So the launch opens a root span of its own, runs
|
|
10
|
+
* `startWorkflow` under it, and joins the two traces by a span link back to
|
|
11
|
+
* the originating span plus `inflexa.run_id` on both sides. With no
|
|
12
|
+
* TracerProvider registered every call here is a no-op on the API's
|
|
13
|
+
* non-recording span.
|
|
5
14
|
*/
|
|
6
15
|
import type { RunLauncher } from "./run-launcher.js";
|
|
16
|
+
/** Attribute carrying the run id on the originating span and on the run's root span. */
|
|
17
|
+
export declare const ATTR_INFLEXA_RUN_ID = "inflexa.run_id";
|
|
7
18
|
export declare function createDbosRunLauncher(): RunLauncher;
|
|
@@ -1,13 +1,38 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* DBOS realization of the `RunLauncher` seam — the single production adapter,
|
|
3
|
-
* shared by every embedder. Host-neutral: it references only DBOS
|
|
4
|
-
* already depends on, so it ships in the harness.
|
|
3
|
+
* shared by every embedder. Host-neutral: it references only DBOS and the OTel
|
|
4
|
+
* API, which the harness already depends on, so it ships in the harness.
|
|
5
|
+
*
|
|
6
|
+
* A launch detaches the run from the caller's trace. The DBOS workflow span
|
|
7
|
+
* takes the active context as its parent, and the caller is a chat request
|
|
8
|
+
* (or another short-lived operation) whose span ends in milliseconds while the
|
|
9
|
+
* run lasts hours. So the launch opens a root span of its own, runs
|
|
10
|
+
* `startWorkflow` under it, and joins the two traces by a span link back to
|
|
11
|
+
* the originating span plus `inflexa.run_id` on both sides. With no
|
|
12
|
+
* TracerProvider registered every call here is a no-op on the API's
|
|
13
|
+
* non-recording span.
|
|
5
14
|
*/
|
|
6
15
|
import { DBOS } from "@dbos-inc/dbos-sdk";
|
|
16
|
+
import { context, isSpanContextValid, ROOT_CONTEXT, trace } from "@opentelemetry/api";
|
|
17
|
+
/** Attribute carrying the run id on the originating span and on the run's root span. */
|
|
18
|
+
export const ATTR_INFLEXA_RUN_ID = "inflexa.run_id";
|
|
19
|
+
const TRACER_NAME = "inflexa.harness.run-launcher";
|
|
7
20
|
export function createDbosRunLauncher() {
|
|
8
21
|
return {
|
|
9
22
|
async launch(workflow, opts, input) {
|
|
10
|
-
|
|
23
|
+
const originating = trace.getSpan(context.active());
|
|
24
|
+
const links = [];
|
|
25
|
+
if (originating !== undefined && isSpanContextValid(originating.spanContext())) {
|
|
26
|
+
originating.setAttribute(ATTR_INFLEXA_RUN_ID, opts.workflowId);
|
|
27
|
+
links.push({ context: originating.spanContext() });
|
|
28
|
+
}
|
|
29
|
+
const root = trace.getTracer(TRACER_NAME).startSpan("launch run", { links, attributes: { [ATTR_INFLEXA_RUN_ID]: opts.workflowId } }, ROOT_CONTEXT);
|
|
30
|
+
try {
|
|
31
|
+
await context.with(trace.setSpan(ROOT_CONTEXT, root), () => DBOS.startWorkflow(workflow, { workflowID: opts.workflowId })(input));
|
|
32
|
+
}
|
|
33
|
+
finally {
|
|
34
|
+
root.end();
|
|
35
|
+
}
|
|
11
36
|
},
|
|
12
37
|
};
|
|
13
38
|
}
|
|
@@ -23,7 +23,7 @@ import { stat } from "node:fs/promises";
|
|
|
23
23
|
import path from "node:path";
|
|
24
24
|
import { createNoopLogger } from "../lib/console-logger.js";
|
|
25
25
|
import { classifyWithinRoot, computeSha256File, hasValidContentHash } from "../lib/fs-helpers.js";
|
|
26
|
-
import {
|
|
26
|
+
import { recordArtifactReconcileDropped, recordLineageInputDropped } from "../lib/metrics.js";
|
|
27
27
|
export async function reconcileManifestWithDisk(input) {
|
|
28
28
|
const { workspaceRoot, resourceId, runId, stepId, agentId, manifest, collector } = input;
|
|
29
29
|
const logger = (input.logger ?? createNoopLogger()).named("reconcile-manifest").with({ runId, stepId, agentId });
|
|
@@ -62,7 +62,7 @@ export async function reconcileManifestWithDisk(input) {
|
|
|
62
62
|
logger.debug("dropping phantom entry", { path: entry.path });
|
|
63
63
|
collector.removeRecord(entry.path);
|
|
64
64
|
droppedCount++;
|
|
65
|
-
|
|
65
|
+
recordArtifactReconcileDropped({ agentId });
|
|
66
66
|
continue;
|
|
67
67
|
}
|
|
68
68
|
throw err;
|
|
@@ -75,7 +75,7 @@ export async function reconcileManifestWithDisk(input) {
|
|
|
75
75
|
logger.debug("dropping non-file entry", { path: entry.path });
|
|
76
76
|
collector.removeRecord(entry.path);
|
|
77
77
|
droppedCount++;
|
|
78
|
-
|
|
78
|
+
recordArtifactReconcileDropped({ agentId });
|
|
79
79
|
continue;
|
|
80
80
|
}
|
|
81
81
|
// The registration payload reads output hashes from `entries` (this
|
|
@@ -140,14 +140,14 @@ async function fillInputHashesFromDisk(collector, workspaceRoot, resourceId, run
|
|
|
140
140
|
const containerPrefix = `/${resourceId}`;
|
|
141
141
|
if (ref.path !== containerPrefix && !ref.path.startsWith(containerPrefix + "/")) {
|
|
142
142
|
logger.warn("dropping out-of-tree input from lineage", { ...attestation, boundSite: "container-prefix" });
|
|
143
|
-
|
|
143
|
+
recordLineageInputDropped({ agentId, reason: "container-prefix" });
|
|
144
144
|
collector.dropInput(ref);
|
|
145
145
|
continue;
|
|
146
146
|
}
|
|
147
147
|
const hostPath = path.join(resourceRoot, ref.path.slice(containerPrefix.length + 1));
|
|
148
148
|
if (hostPath !== resourceRoot && !hostPath.startsWith(resourceRoot + path.sep)) {
|
|
149
149
|
logger.warn("dropping out-of-tree input from lineage", { ...attestation, hostPath, boundSite: "workspace-root" });
|
|
150
|
-
|
|
150
|
+
recordLineageInputDropped({ agentId, reason: "workspace-root" });
|
|
151
151
|
collector.dropInput(ref);
|
|
152
152
|
continue;
|
|
153
153
|
}
|
|
@@ -159,7 +159,7 @@ async function fillInputHashesFromDisk(collector, workspaceRoot, resourceId, run
|
|
|
159
159
|
// verdict (a dangling link) falls through to the ENOENT drop below.
|
|
160
160
|
if ((await classifyWithinRoot(resourceRoot, hostPath)) === "escaped") {
|
|
161
161
|
logger.warn("dropping out-of-tree input from lineage", { ...attestation, hostPath, boundSite: "realpath" });
|
|
162
|
-
|
|
162
|
+
recordLineageInputDropped({ agentId, reason: "symlink-escape" });
|
|
163
163
|
collector.dropInput(ref);
|
|
164
164
|
continue;
|
|
165
165
|
}
|
|
@@ -173,7 +173,7 @@ async function fillInputHashesFromDisk(collector, workspaceRoot, resourceId, run
|
|
|
173
173
|
// consumed, and the count of those is how a reader tells a noisy
|
|
174
174
|
// hook from an artifact that genuinely vanished under the step.
|
|
175
175
|
logger.warn("dropping input not present at reconcile from lineage", { ...attestation, hostPath, dropSite: "input-enoent" });
|
|
176
|
-
|
|
176
|
+
recordLineageInputDropped({ agentId, reason: "missing" });
|
|
177
177
|
collector.dropInput(ref);
|
|
178
178
|
continue;
|
|
179
179
|
}
|
|
@@ -187,7 +187,7 @@ async function fillInputHashesFromDisk(collector, workspaceRoot, resourceId, run
|
|
|
187
187
|
// non-file drop above. A genuinely missing FILE (ENOENT) still fails fast
|
|
188
188
|
// as drift; a directory is not drift.
|
|
189
189
|
logger.debug("dropping non-file input from lineage", attestation);
|
|
190
|
-
|
|
190
|
+
recordLineageInputDropped({ agentId, reason: "directory" });
|
|
191
191
|
collector.dropInput(ref);
|
|
192
192
|
continue;
|
|
193
193
|
}
|
package/dist/lib/metrics.d.ts
CHANGED
|
@@ -4,6 +4,24 @@
|
|
|
4
4
|
* Instruments:
|
|
5
5
|
* - cortex.artifact.reconcile.dropped — counter for missing manifest entries
|
|
6
6
|
* - cortex.artifact.reconcile.input_dropped — counter for lineage input drops
|
|
7
|
+
*
|
|
8
|
+
* Both carry `agent_id` (about 15 values) and never a per-step or per-run
|
|
9
|
+
* identifier: a label with unbounded values multiplies the series count.
|
|
10
|
+
*
|
|
11
|
+
* Instruments are lazy so the record site binds to whichever `MeterProvider`
|
|
12
|
+
* is globally registered at first use. The metrics API has no late-binding
|
|
13
|
+
* proxy: a meter taken at module load, before `initOtel` runs, would stay
|
|
14
|
+
* on the no-op provider for the life of the process.
|
|
7
15
|
*/
|
|
8
|
-
|
|
9
|
-
export declare
|
|
16
|
+
/** Record one manifest entry dropped at reconcile. */
|
|
17
|
+
export declare function recordArtifactReconcileDropped(args: {
|
|
18
|
+
readonly agentId: string;
|
|
19
|
+
}): void;
|
|
20
|
+
export type LineageInputDropReason = "container-prefix" | "workspace-root" | "symlink-escape" | "missing" | "directory";
|
|
21
|
+
/** Record one tracked input read dropped from lineage at reconcile. */
|
|
22
|
+
export declare function recordLineageInputDropped(args: {
|
|
23
|
+
readonly agentId: string;
|
|
24
|
+
readonly reason: LineageInputDropReason;
|
|
25
|
+
}): void;
|
|
26
|
+
/** Test hook — drop memoised instruments to rebind a fresh MeterProvider. */
|
|
27
|
+
export declare function __resetReconcileMetricsForTest(): void;
|
package/dist/lib/metrics.js
CHANGED
|
@@ -4,16 +4,44 @@
|
|
|
4
4
|
* Instruments:
|
|
5
5
|
* - cortex.artifact.reconcile.dropped — counter for missing manifest entries
|
|
6
6
|
* - cortex.artifact.reconcile.input_dropped — counter for lineage input drops
|
|
7
|
+
*
|
|
8
|
+
* Both carry `agent_id` (about 15 values) and never a per-step or per-run
|
|
9
|
+
* identifier: a label with unbounded values multiplies the series count.
|
|
10
|
+
*
|
|
11
|
+
* Instruments are lazy so the record site binds to whichever `MeterProvider`
|
|
12
|
+
* is globally registered at first use. The metrics API has no late-binding
|
|
13
|
+
* proxy: a meter taken at module load, before `initOtel` runs, would stay
|
|
14
|
+
* on the no-op provider for the life of the process.
|
|
7
15
|
*/
|
|
8
16
|
import { metrics } from "@opentelemetry/api";
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
17
|
+
let instruments;
|
|
18
|
+
function getInstruments() {
|
|
19
|
+
if (instruments === undefined) {
|
|
20
|
+
const meter = metrics.getMeter("cortex");
|
|
21
|
+
instruments = {
|
|
22
|
+
artifactReconcileDropped: meter.createCounter("cortex.artifact.reconcile.dropped", {
|
|
23
|
+
description: "Manifest entries dropped because their on-disk file was missing at " +
|
|
24
|
+
"registration time (writes-then-deletes, renames). Tagged by agent_id.",
|
|
25
|
+
}),
|
|
26
|
+
lineageInputDropped: meter.createCounter("cortex.artifact.reconcile.input_dropped", {
|
|
27
|
+
description: "Tracked input reads dropped from lineage at reconcile because they are " +
|
|
28
|
+
"not content-attestable files of the analysis (directory reads, " +
|
|
29
|
+
"out-of-tree resolutions, paths absent at reconcile). Tagged by " +
|
|
30
|
+
"agent_id, reason.",
|
|
31
|
+
}),
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
return instruments;
|
|
35
|
+
}
|
|
36
|
+
/** Record one manifest entry dropped at reconcile. */
|
|
37
|
+
export function recordArtifactReconcileDropped(args) {
|
|
38
|
+
getInstruments().artifactReconcileDropped.add(1, { agent_id: args.agentId });
|
|
39
|
+
}
|
|
40
|
+
/** Record one tracked input read dropped from lineage at reconcile. */
|
|
41
|
+
export function recordLineageInputDropped(args) {
|
|
42
|
+
getInstruments().lineageInputDropped.add(1, { agent_id: args.agentId, reason: args.reason });
|
|
43
|
+
}
|
|
44
|
+
/** Test hook — drop memoised instruments to rebind a fresh MeterProvider. */
|
|
45
|
+
export function __resetReconcileMetricsForTest() {
|
|
46
|
+
instruments = undefined;
|
|
47
|
+
}
|
package/dist/lib/otel.d.ts
CHANGED
|
@@ -1,14 +1,22 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* OpenTelemetry SDK initialization — traces + metrics.
|
|
3
3
|
*
|
|
4
|
-
* Exported as `initOtel()` and called explicitly from
|
|
5
|
-
* the bundler
|
|
4
|
+
* Exported as `initOtel()` and called explicitly from the host's boot sequence
|
|
5
|
+
* so the bundler cannot tree-shake it (side-effect-only imports get dropped).
|
|
6
6
|
*
|
|
7
7
|
* Traces: NodeTracerProvider exports to OTLP when OTEL_EXPORTER_OTLP_ENDPOINT
|
|
8
8
|
* is set.
|
|
9
9
|
*
|
|
10
|
-
* Metrics: MeterProvider exports to OTLP when OTEL_EXPORTER_OTLP_ENDPOINT
|
|
11
|
-
*
|
|
10
|
+
* Metrics: MeterProvider exports to OTLP when OTEL_EXPORTER_OTLP_ENDPOINT is
|
|
11
|
+
* set and OTEL_METRICS_EXPORTER is not `none`. The export interval is
|
|
12
|
+
* OTEL_METRIC_EXPORT_INTERVAL (ms), default 60 s. Custom Cortex
|
|
13
|
+
* metrics are defined in metrics.ts.
|
|
14
|
+
*
|
|
15
|
+
* Resource: the host's `serviceName` / `serviceVersion` merged with the SDK
|
|
16
|
+
* env detector (`OTEL_SERVICE_NAME`, `OTEL_RESOURCE_ATTRIBUTES`);
|
|
17
|
+
* the environment wins on a conflict, so a deployment can set
|
|
18
|
+
* `deployment.environment.name` or override the service name without
|
|
19
|
+
* a code change. Traces and metrics share the one resource.
|
|
12
20
|
*
|
|
13
21
|
* Note: OTEL's instrumentation-http patches node:http but does not cover
|
|
14
22
|
* Hono's request handling or Node 22's undici-based globalThis.fetch,
|
|
@@ -16,16 +24,45 @@
|
|
|
16
24
|
* not currently wired.
|
|
17
25
|
*/
|
|
18
26
|
import type { Logger } from "./logger.js";
|
|
27
|
+
/** The SDK's own default when neither the host nor the environment says otherwise. */
|
|
28
|
+
export declare const DEFAULT_METRIC_EXPORT_INTERVAL_MS = 60000;
|
|
29
|
+
export interface InitOtelOptions {
|
|
30
|
+
/** Diagnostics sink; omitted falls back to no-op. */
|
|
31
|
+
readonly logger?: Logger;
|
|
32
|
+
/** `service.name` when `OTEL_SERVICE_NAME` is unset. Default `cortex`. */
|
|
33
|
+
readonly serviceName?: string;
|
|
34
|
+
/** `service.version` — the host's package version. Omitted leaves the attribute unset. */
|
|
35
|
+
readonly serviceVersion?: string;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* `OTEL_METRICS_EXPORTER=none` turns metric export off while the OTLP endpoint
|
|
39
|
+
* stays set for traces. Any other value (including unset) keeps the OTLP
|
|
40
|
+
* exporter, the only one the harness ships.
|
|
41
|
+
*/
|
|
42
|
+
export declare function metricsExportDisabled(env?: NodeJS.ProcessEnv): boolean;
|
|
43
|
+
/**
|
|
44
|
+
* Export interval from `OTEL_METRIC_EXPORT_INTERVAL` (milliseconds). An unset,
|
|
45
|
+
* non-numeric, or non-positive value falls back to the 60 s default.
|
|
46
|
+
*/
|
|
47
|
+
export declare function metricExportIntervalMs(env?: NodeJS.ProcessEnv): number;
|
|
19
48
|
/**
|
|
20
49
|
* Initialize the OpenTelemetry SDK. Must be called before any code that
|
|
21
50
|
* calls trace.getTracer() or metrics.getMeter().
|
|
22
51
|
*
|
|
23
52
|
* Safe to call multiple times — only the first call has effect.
|
|
24
53
|
*/
|
|
25
|
-
export declare function initOtel(
|
|
54
|
+
export declare function initOtel(options?: InitOtelOptions): void;
|
|
26
55
|
/**
|
|
27
56
|
* Flush and shut down the OTel exporters. Called by the graceful-shutdown
|
|
28
57
|
* sequence so in-flight batch spans/metrics make it to the collector
|
|
29
58
|
* before the process exits. Never throws.
|
|
30
59
|
*/
|
|
31
60
|
export declare function shutdownOtel(): Promise<void>;
|
|
61
|
+
/**
|
|
62
|
+
* Test hook: shut the providers down and unregister every global the init
|
|
63
|
+
* touched, so a test can drive `initOtel` again under a different environment.
|
|
64
|
+
* Test-only.
|
|
65
|
+
*/
|
|
66
|
+
export declare function __resetOtelForTest(): Promise<void>;
|
|
67
|
+
/** Test hook: whether `initOtel` registered a metric-exporting MeterProvider. Test-only. */
|
|
68
|
+
export declare function __otelMeterProviderRegisteredForTest(): boolean;
|
package/dist/lib/otel.js
CHANGED
|
@@ -1,21 +1,29 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* OpenTelemetry SDK initialization — traces + metrics.
|
|
3
3
|
*
|
|
4
|
-
* Exported as `initOtel()` and called explicitly from
|
|
5
|
-
* the bundler
|
|
4
|
+
* Exported as `initOtel()` and called explicitly from the host's boot sequence
|
|
5
|
+
* so the bundler cannot tree-shake it (side-effect-only imports get dropped).
|
|
6
6
|
*
|
|
7
7
|
* Traces: NodeTracerProvider exports to OTLP when OTEL_EXPORTER_OTLP_ENDPOINT
|
|
8
8
|
* is set.
|
|
9
9
|
*
|
|
10
|
-
* Metrics: MeterProvider exports to OTLP when OTEL_EXPORTER_OTLP_ENDPOINT
|
|
11
|
-
*
|
|
10
|
+
* Metrics: MeterProvider exports to OTLP when OTEL_EXPORTER_OTLP_ENDPOINT is
|
|
11
|
+
* set and OTEL_METRICS_EXPORTER is not `none`. The export interval is
|
|
12
|
+
* OTEL_METRIC_EXPORT_INTERVAL (ms), default 60 s. Custom Cortex
|
|
13
|
+
* metrics are defined in metrics.ts.
|
|
14
|
+
*
|
|
15
|
+
* Resource: the host's `serviceName` / `serviceVersion` merged with the SDK
|
|
16
|
+
* env detector (`OTEL_SERVICE_NAME`, `OTEL_RESOURCE_ATTRIBUTES`);
|
|
17
|
+
* the environment wins on a conflict, so a deployment can set
|
|
18
|
+
* `deployment.environment.name` or override the service name without
|
|
19
|
+
* a code change. Traces and metrics share the one resource.
|
|
12
20
|
*
|
|
13
21
|
* Note: OTEL's instrumentation-http patches node:http but does not cover
|
|
14
22
|
* Hono's request handling or Node 22's undici-based globalThis.fetch,
|
|
15
23
|
* so inbound/outbound trace-context propagation across those paths is
|
|
16
24
|
* not currently wired.
|
|
17
25
|
*/
|
|
18
|
-
import { propagation, metrics, trace } from "@opentelemetry/api";
|
|
26
|
+
import { context, propagation, metrics, trace } from "@opentelemetry/api";
|
|
19
27
|
import { createNoopLogger } from "./console-logger.js";
|
|
20
28
|
import { W3CTraceContextPropagator } from "@opentelemetry/core";
|
|
21
29
|
import { MeterProvider, PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics";
|
|
@@ -23,25 +31,59 @@ import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-http";
|
|
|
23
31
|
import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node";
|
|
24
32
|
import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-base";
|
|
25
33
|
import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-proto";
|
|
26
|
-
import { resourceFromAttributes } from "@opentelemetry/resources";
|
|
27
|
-
import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";
|
|
34
|
+
import { detectResources, envDetector, resourceFromAttributes } from "@opentelemetry/resources";
|
|
35
|
+
import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from "@opentelemetry/semantic-conventions";
|
|
36
|
+
/** The SDK's own default when neither the host nor the environment says otherwise. */
|
|
37
|
+
export const DEFAULT_METRIC_EXPORT_INTERVAL_MS = 60_000;
|
|
28
38
|
let initialized = false;
|
|
29
39
|
let registeredTracerProvider;
|
|
30
40
|
let registeredMeterProvider;
|
|
41
|
+
/**
|
|
42
|
+
* `OTEL_METRICS_EXPORTER=none` turns metric export off while the OTLP endpoint
|
|
43
|
+
* stays set for traces. Any other value (including unset) keeps the OTLP
|
|
44
|
+
* exporter, the only one the harness ships.
|
|
45
|
+
*/
|
|
46
|
+
export function metricsExportDisabled(env = process.env) {
|
|
47
|
+
return env.OTEL_METRICS_EXPORTER === "none";
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Export interval from `OTEL_METRIC_EXPORT_INTERVAL` (milliseconds). An unset,
|
|
51
|
+
* non-numeric, or non-positive value falls back to the 60 s default.
|
|
52
|
+
*/
|
|
53
|
+
export function metricExportIntervalMs(env = process.env) {
|
|
54
|
+
const raw = env.OTEL_METRIC_EXPORT_INTERVAL;
|
|
55
|
+
if (raw === undefined || raw.trim() === "")
|
|
56
|
+
return DEFAULT_METRIC_EXPORT_INTERVAL_MS;
|
|
57
|
+
const parsed = Number(raw);
|
|
58
|
+
if (!Number.isFinite(parsed) || parsed <= 0)
|
|
59
|
+
return DEFAULT_METRIC_EXPORT_INTERVAL_MS;
|
|
60
|
+
return Math.floor(parsed);
|
|
61
|
+
}
|
|
62
|
+
function buildResource(options) {
|
|
63
|
+
const manual = {
|
|
64
|
+
[ATTR_SERVICE_NAME]: options.serviceName ?? "cortex",
|
|
65
|
+
};
|
|
66
|
+
if (options.serviceVersion)
|
|
67
|
+
manual[ATTR_SERVICE_VERSION] = options.serviceVersion;
|
|
68
|
+
// `merge` gives precedence to the argument: the environment overrides the
|
|
69
|
+
// host's values, which is the OTel SDK convention for env configuration.
|
|
70
|
+
return resourceFromAttributes(manual).merge(detectResources({ detectors: [envDetector] }));
|
|
71
|
+
}
|
|
31
72
|
/**
|
|
32
73
|
* Initialize the OpenTelemetry SDK. Must be called before any code that
|
|
33
74
|
* calls trace.getTracer() or metrics.getMeter().
|
|
34
75
|
*
|
|
35
76
|
* Safe to call multiple times — only the first call has effect.
|
|
36
77
|
*/
|
|
37
|
-
export function initOtel(
|
|
78
|
+
export function initOtel(options = {}) {
|
|
38
79
|
if (initialized)
|
|
39
80
|
return;
|
|
40
81
|
initialized = true;
|
|
41
|
-
const logger = (
|
|
82
|
+
const logger = (options.logger ?? createNoopLogger()).named("otel");
|
|
42
83
|
// ── W3C Trace Context propagation ──────────────────────────────────
|
|
43
84
|
propagation.setGlobalPropagator(new W3CTraceContextPropagator());
|
|
44
85
|
const endpoint = process.env.OTEL_EXPORTER_OTLP_ENDPOINT;
|
|
86
|
+
const resource = buildResource(options);
|
|
45
87
|
// ── Traces ─────────────────────────────────────────────────────────
|
|
46
88
|
// Always register a TracerProvider so trace.getTracer() returns real spans
|
|
47
89
|
// with unique IDs. Without this, the OtelBridge gets no-op spans (all zeros)
|
|
@@ -52,10 +94,6 @@ export function initOtel(injected) {
|
|
|
52
94
|
const base = endpoint.replace(/\/+$/, "");
|
|
53
95
|
spanProcessors.push(new BatchSpanProcessor(new OTLPTraceExporter({ url: `${base}/v1/traces` })));
|
|
54
96
|
}
|
|
55
|
-
const serviceName = process.env.OTEL_SERVICE_NAME || "cortex";
|
|
56
|
-
const resource = resourceFromAttributes({
|
|
57
|
-
[ATTR_SERVICE_NAME]: serviceName,
|
|
58
|
-
});
|
|
59
97
|
const tracerProvider = new NodeTracerProvider({
|
|
60
98
|
resource,
|
|
61
99
|
spanProcessors,
|
|
@@ -71,20 +109,29 @@ export function initOtel(injected) {
|
|
|
71
109
|
// `initTelemetry: () => {}` — its TUI owns stdout, so the banner corrupted
|
|
72
110
|
// the screen and the whole of the harness's traces + metrics were switched
|
|
73
111
|
// off to avoid it. At debug through the injected seam it costs nothing.
|
|
74
|
-
logger.debug("TracerProvider registered", {
|
|
112
|
+
logger.debug("TracerProvider registered", {
|
|
113
|
+
spanId: ctx.spanId,
|
|
114
|
+
noop: isNoop,
|
|
115
|
+
endpoint: endpoint ?? null,
|
|
116
|
+
processors: spanProcessors.length,
|
|
117
|
+
resource: resource.attributes,
|
|
118
|
+
});
|
|
75
119
|
testSpan.end();
|
|
76
120
|
}
|
|
77
121
|
// ── Metrics ────────────────────────────────────────────────────────
|
|
78
|
-
|
|
122
|
+
// With no MeterProvider registered the API's global no-op provider serves
|
|
123
|
+
// every instrument, so the record sites stay valid when export is off.
|
|
124
|
+
if (endpoint && !metricsExportDisabled()) {
|
|
79
125
|
const base = endpoint.replace(/\/+$/, "");
|
|
80
126
|
const metricExporter = new OTLPMetricExporter({
|
|
81
127
|
url: `${base}/v1/metrics`,
|
|
82
128
|
});
|
|
83
129
|
const meterProvider = new MeterProvider({
|
|
130
|
+
resource,
|
|
84
131
|
readers: [
|
|
85
132
|
new PeriodicExportingMetricReader({
|
|
86
133
|
exporter: metricExporter,
|
|
87
|
-
exportIntervalMillis:
|
|
134
|
+
exportIntervalMillis: metricExportIntervalMs(),
|
|
88
135
|
}),
|
|
89
136
|
],
|
|
90
137
|
});
|
|
@@ -105,3 +152,22 @@ export async function shutdownOtel() {
|
|
|
105
152
|
tasks.push(registeredMeterProvider.shutdown());
|
|
106
153
|
await Promise.allSettled(tasks);
|
|
107
154
|
}
|
|
155
|
+
/**
|
|
156
|
+
* Test hook: shut the providers down and unregister every global the init
|
|
157
|
+
* touched, so a test can drive `initOtel` again under a different environment.
|
|
158
|
+
* Test-only.
|
|
159
|
+
*/
|
|
160
|
+
export async function __resetOtelForTest() {
|
|
161
|
+
await shutdownOtel();
|
|
162
|
+
registeredTracerProvider = undefined;
|
|
163
|
+
registeredMeterProvider = undefined;
|
|
164
|
+
trace.disable();
|
|
165
|
+
metrics.disable();
|
|
166
|
+
context.disable();
|
|
167
|
+
propagation.disable();
|
|
168
|
+
initialized = false;
|
|
169
|
+
}
|
|
170
|
+
/** Test hook: whether `initOtel` registered a metric-exporting MeterProvider. Test-only. */
|
|
171
|
+
export function __otelMeterProviderRegisteredForTest() {
|
|
172
|
+
return registeredMeterProvider !== undefined;
|
|
173
|
+
}
|
package/dist/runtime/dbos.d.ts
CHANGED
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
*
|
|
13
13
|
* See `openspec/changes/harness-dbos-runtime`.
|
|
14
14
|
*/
|
|
15
|
+
import { type DBOSConfig, type DLogger } from "@dbos-inc/dbos-sdk";
|
|
15
16
|
import type { Pool } from "pg";
|
|
16
17
|
import type { Logger } from "../lib/logger.js";
|
|
17
18
|
/**
|
|
@@ -31,18 +32,60 @@ export interface DbosConfig {
|
|
|
31
32
|
/** Stable per-pod executor id (e.g. `process.env.HOSTNAME ?? "local-dev"`). */
|
|
32
33
|
readonly executorId: string;
|
|
33
34
|
/**
|
|
34
|
-
* SDK
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
*
|
|
35
|
+
* Least severe SDK record forwarded to the host's `Logger` (winston level
|
|
36
|
+
* names; the SDK's own default is `info`). The SDK's lines ride the
|
|
37
|
+
* harness `Logger` seam under `[dbos.sdk]`, so this threshold is applied by
|
|
38
|
+
* the bridge, not by the SDK. Interactive embedders pass `warn` so the
|
|
39
|
+
* launch banner and migration chatter stay out of their logs; the server
|
|
40
|
+
* root omits it and keeps the informative default.
|
|
38
41
|
*/
|
|
39
42
|
readonly logLevel?: string;
|
|
43
|
+
/**
|
|
44
|
+
* Emit DBOS workflow and step spans on the globally registered
|
|
45
|
+
* TracerProvider, and run every workflow and step body under an active
|
|
46
|
+
* OTel context. Default `false`.
|
|
47
|
+
*
|
|
48
|
+
* DBOS installs nothing of its own here: with no OTLP endpoint of its own it
|
|
49
|
+
* probes the global provider and, when one is registered, emits under the
|
|
50
|
+
* instrumentation scope `dbos-tracer` through that provider's processors.
|
|
51
|
+
* A host that registers no provider (the CLI passes a no-op
|
|
52
|
+
* `initTelemetry`) must leave this off, otherwise the SDK installs its own
|
|
53
|
+
* `BasicTracerProvider` and every span is created and never exported.
|
|
54
|
+
*
|
|
55
|
+
* Attribute names follow the `dbos.*` semantic-convention layout
|
|
56
|
+
* (`otelAttributeFormat: "semconv"`).
|
|
57
|
+
*
|
|
58
|
+
* Known SDK behavior a host must plan for:
|
|
59
|
+
* - a recovered workflow starts a new root trace, with no link to the trace
|
|
60
|
+
* of the original execution
|
|
61
|
+
* - every already-completed step re-emits a `cached=true` span on replay,
|
|
62
|
+
* so a restart with many in-flight steps bursts spans; size
|
|
63
|
+
* `OTEL_BSP_MAX_QUEUE_SIZE` for it
|
|
64
|
+
* - step span names embed the step, tool-use, or exec id
|
|
65
|
+
*/
|
|
66
|
+
readonly tracingEnabled?: boolean;
|
|
40
67
|
}
|
|
41
68
|
/** DBOS lifecycle facts a host's readiness probe reports on. */
|
|
42
69
|
export interface DbosState {
|
|
43
70
|
launched: boolean;
|
|
44
71
|
recoveryStarted: boolean;
|
|
45
72
|
}
|
|
73
|
+
/**
|
|
74
|
+
* Bridge the SDK's internal logger onto the harness `Logger` seam, so the
|
|
75
|
+
* SDK's own lines (`Recovering N workflows…`, migration chatter) land in the
|
|
76
|
+
* host's sink instead of on the process console. The SDK does not filter by
|
|
77
|
+
* `logLevel` before it delegates to a custom logger, so the threshold is
|
|
78
|
+
* applied here. Records arrive as strings; an `Error` arrives as its message
|
|
79
|
+
* with the stack (and cause chain) in `metadata.stack`; inside a workflow or
|
|
80
|
+
* step body `metadata.span.attributes` carries the operation context, which
|
|
81
|
+
* rides as fields.
|
|
82
|
+
*/
|
|
83
|
+
export declare function dbosSdkLogger(logger: Logger, logLevel: string | undefined): DLogger;
|
|
84
|
+
/**
|
|
85
|
+
* The SDK config `launchDbos` sets. Pure over `DbosConfig`, so a test can
|
|
86
|
+
* assert on the mapping without a launch.
|
|
87
|
+
*/
|
|
88
|
+
export declare function dbosSdkConfig(config: DbosConfig, logger: Logger): DBOSConfig;
|
|
46
89
|
/**
|
|
47
90
|
* Launch DBOS. Idempotent — a second call is a no-op so tests that drive
|
|
48
91
|
* the harness twice (or accidentally double-import) don't re-launch.
|
package/dist/runtime/dbos.js
CHANGED
|
@@ -45,6 +45,76 @@ function systemDatabaseUrl(config) {
|
|
|
45
45
|
const database = encodeURIComponent(config.dbName);
|
|
46
46
|
return `postgresql://${user}:${password}@${host}:${port}/${database}?sslmode=${dbosSslMode(config.dbSslMode)}`;
|
|
47
47
|
}
|
|
48
|
+
/** Winston severity ranks, ascending verbosity — the scale `DbosConfig.logLevel` names. */
|
|
49
|
+
const SDK_LOG_LEVEL_RANK = {
|
|
50
|
+
error: 0,
|
|
51
|
+
warn: 1,
|
|
52
|
+
info: 2,
|
|
53
|
+
http: 3,
|
|
54
|
+
verbose: 4,
|
|
55
|
+
debug: 5,
|
|
56
|
+
silly: 6,
|
|
57
|
+
};
|
|
58
|
+
const SDK_DEFAULT_LOG_LEVEL = "info";
|
|
59
|
+
/**
|
|
60
|
+
* Bridge the SDK's internal logger onto the harness `Logger` seam, so the
|
|
61
|
+
* SDK's own lines (`Recovering N workflows…`, migration chatter) land in the
|
|
62
|
+
* host's sink instead of on the process console. The SDK does not filter by
|
|
63
|
+
* `logLevel` before it delegates to a custom logger, so the threshold is
|
|
64
|
+
* applied here. Records arrive as strings; an `Error` arrives as its message
|
|
65
|
+
* with the stack (and cause chain) in `metadata.stack`; inside a workflow or
|
|
66
|
+
* step body `metadata.span.attributes` carries the operation context, which
|
|
67
|
+
* rides as fields.
|
|
68
|
+
*/
|
|
69
|
+
export function dbosSdkLogger(logger, logLevel) {
|
|
70
|
+
const sink = logger.named("sdk");
|
|
71
|
+
const threshold = SDK_LOG_LEVEL_RANK[logLevel ?? SDK_DEFAULT_LOG_LEVEL] ?? SDK_LOG_LEVEL_RANK[SDK_DEFAULT_LOG_LEVEL];
|
|
72
|
+
const passes = (level) => SDK_LOG_LEVEL_RANK[level] <= threshold;
|
|
73
|
+
const fields = (metadata) => {
|
|
74
|
+
const attributes = metadata?.span?.attributes;
|
|
75
|
+
const stack = metadata?.stack;
|
|
76
|
+
if (attributes === undefined && stack === undefined)
|
|
77
|
+
return undefined;
|
|
78
|
+
return { ...attributes, ...(stack !== undefined ? { stack } : {}) };
|
|
79
|
+
};
|
|
80
|
+
const message = (entry) => (typeof entry === "string" ? entry : String(entry));
|
|
81
|
+
return {
|
|
82
|
+
debug: (entry, metadata) => {
|
|
83
|
+
if (passes("debug"))
|
|
84
|
+
sink.debug(message(entry), fields(metadata));
|
|
85
|
+
},
|
|
86
|
+
info: (entry, metadata) => {
|
|
87
|
+
if (passes("info"))
|
|
88
|
+
sink.info(message(entry), fields(metadata));
|
|
89
|
+
},
|
|
90
|
+
warn: (entry, metadata) => {
|
|
91
|
+
if (passes("warn"))
|
|
92
|
+
sink.warn(message(entry), fields(metadata));
|
|
93
|
+
},
|
|
94
|
+
error: (entry, metadata) => {
|
|
95
|
+
if (passes("error"))
|
|
96
|
+
sink.error(message(entry), fields(metadata));
|
|
97
|
+
},
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* The SDK config `launchDbos` sets. Pure over `DbosConfig`, so a test can
|
|
102
|
+
* assert on the mapping without a launch.
|
|
103
|
+
*/
|
|
104
|
+
export function dbosSdkConfig(config, logger) {
|
|
105
|
+
return {
|
|
106
|
+
name: config.appName,
|
|
107
|
+
systemDatabaseUrl: systemDatabaseUrl(config),
|
|
108
|
+
systemDatabasePoolSize: DBOS_SYSTEM_POOL_SIZE,
|
|
109
|
+
executorID: config.executorId,
|
|
110
|
+
applicationVersion: config.applicationVersion,
|
|
111
|
+
adminPort: parseInt(config.adminPort, 10),
|
|
112
|
+
logLevel: config.logLevel,
|
|
113
|
+
logger: dbosSdkLogger(logger, config.logLevel),
|
|
114
|
+
tracingEnabled: config.tracingEnabled ?? false,
|
|
115
|
+
otelAttributeFormat: "semconv",
|
|
116
|
+
};
|
|
117
|
+
}
|
|
48
118
|
/**
|
|
49
119
|
* Launch DBOS. Idempotent — a second call is a no-op so tests that drive
|
|
50
120
|
* the harness twice (or accidentally double-import) don't re-launch.
|
|
@@ -57,25 +127,17 @@ export async function launchDbos({ config, logger: injected }) {
|
|
|
57
127
|
if (state.launched)
|
|
58
128
|
return;
|
|
59
129
|
const logger = injected.named("dbos");
|
|
60
|
-
const
|
|
61
|
-
|
|
62
|
-
DBOS.setConfig({
|
|
63
|
-
name: config.appName,
|
|
64
|
-
systemDatabaseUrl: systemDatabaseUrl(config),
|
|
65
|
-
systemDatabasePoolSize: DBOS_SYSTEM_POOL_SIZE,
|
|
66
|
-
executorID,
|
|
67
|
-
applicationVersion: config.applicationVersion,
|
|
68
|
-
adminPort,
|
|
69
|
-
logLevel: config.logLevel,
|
|
70
|
-
});
|
|
130
|
+
const sdkConfig = dbosSdkConfig(config, logger);
|
|
131
|
+
DBOS.setConfig(sdkConfig);
|
|
71
132
|
const start = performance.now();
|
|
72
133
|
await DBOS.launch();
|
|
73
134
|
state.launched = true;
|
|
74
135
|
state.recoveryStarted = true;
|
|
75
136
|
logger.info("launched", {
|
|
76
|
-
executorID,
|
|
137
|
+
executorID: sdkConfig.executorID,
|
|
77
138
|
applicationVersion: config.applicationVersion,
|
|
78
|
-
adminPort,
|
|
139
|
+
adminPort: sdkConfig.adminPort,
|
|
140
|
+
tracingEnabled: sdkConfig.tracingEnabled,
|
|
79
141
|
durationMs: Math.round(performance.now() - start),
|
|
80
142
|
});
|
|
81
143
|
}
|
|
@@ -1,15 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Workflow-side OTel metrics
|
|
2
|
+
* Workflow-side OTel metrics.
|
|
3
3
|
*
|
|
4
|
-
* Two signals so 3-hour runs and pause-resume cascades are observable:
|
|
5
4
|
* - cortex.workflow.parent.cancelled_children — counter, incremented once
|
|
6
5
|
* per child the fail-fast or pause cascade reaps. A baseline near 0
|
|
7
6
|
* means runs land cleanly; a spike means siblings keep colliding with
|
|
8
7
|
* a single failing step.
|
|
9
|
-
* - cortex.workflow.stream.bytes — counter, incremented by
|
|
10
|
-
* the byte count of every UI part written onto the DBOS stream. The
|
|
11
|
-
* leading indicator for 3-hour stream growth is monotonic — a runaway
|
|
12
|
-
* emitter shows up as an unbounded slope.
|
|
13
8
|
*
|
|
14
9
|
* Instruments are lazy so the workflow body binds to whichever
|
|
15
10
|
* `MeterProvider` is globally registered at startup.
|
|
@@ -23,15 +18,5 @@
|
|
|
23
18
|
export declare function recordCancelledChild(args: {
|
|
24
19
|
readonly cause: "fail_fast" | "budget_exceeded" | "external_cancel";
|
|
25
20
|
}): void;
|
|
26
|
-
/**
|
|
27
|
-
* Record one UI-part write. `bytes` is the serialised JSON size of the
|
|
28
|
-
* part as it lands on the stream; `kind` is the part `type` for
|
|
29
|
-
* dashboarding. The workflow body wraps its `DBOS.writeStream` call site
|
|
30
|
-
* so a single emitter records once per part.
|
|
31
|
-
*/
|
|
32
|
-
export declare function recordStreamWrite(args: {
|
|
33
|
-
readonly bytes: number;
|
|
34
|
-
readonly kind: string;
|
|
35
|
-
}): void;
|
|
36
21
|
/** Test hook — drop memoised instruments to rebind a fresh MeterProvider. */
|
|
37
22
|
export declare function __resetWorkflowMetricsForTest(): void;
|
|
@@ -1,15 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Workflow-side OTel metrics
|
|
2
|
+
* Workflow-side OTel metrics.
|
|
3
3
|
*
|
|
4
|
-
* Two signals so 3-hour runs and pause-resume cascades are observable:
|
|
5
4
|
* - cortex.workflow.parent.cancelled_children — counter, incremented once
|
|
6
5
|
* per child the fail-fast or pause cascade reaps. A baseline near 0
|
|
7
6
|
* means runs land cleanly; a spike means siblings keep colliding with
|
|
8
7
|
* a single failing step.
|
|
9
|
-
* - cortex.workflow.stream.bytes — counter, incremented by
|
|
10
|
-
* the byte count of every UI part written onto the DBOS stream. The
|
|
11
|
-
* leading indicator for 3-hour stream growth is monotonic — a runaway
|
|
12
|
-
* emitter shows up as an unbounded slope.
|
|
13
8
|
*
|
|
14
9
|
* Instruments are lazy so the workflow body binds to whichever
|
|
15
10
|
* `MeterProvider` is globally registered at startup.
|
|
@@ -23,10 +18,6 @@ function getInstruments() {
|
|
|
23
18
|
cancelledChildren: meter.createCounter("cortex.workflow.parent.cancelled_children", {
|
|
24
19
|
description: "Children cancelled by the parent's fail-fast or pause cascade",
|
|
25
20
|
}),
|
|
26
|
-
streamBytes: meter.createCounter("cortex.workflow.stream.bytes", {
|
|
27
|
-
description: "Bytes written to the parent's DBOS UI-parts stream — proxy for " + "3-hour stream growth",
|
|
28
|
-
unit: "By",
|
|
29
|
-
}),
|
|
30
21
|
};
|
|
31
22
|
}
|
|
32
23
|
return instruments;
|
|
@@ -40,15 +31,6 @@ function getInstruments() {
|
|
|
40
31
|
export function recordCancelledChild(args) {
|
|
41
32
|
getInstruments().cancelledChildren.add(1, { cause: args.cause });
|
|
42
33
|
}
|
|
43
|
-
/**
|
|
44
|
-
* Record one UI-part write. `bytes` is the serialised JSON size of the
|
|
45
|
-
* part as it lands on the stream; `kind` is the part `type` for
|
|
46
|
-
* dashboarding. The workflow body wraps its `DBOS.writeStream` call site
|
|
47
|
-
* so a single emitter records once per part.
|
|
48
|
-
*/
|
|
49
|
-
export function recordStreamWrite(args) {
|
|
50
|
-
getInstruments().streamBytes.add(args.bytes, { kind: args.kind });
|
|
51
|
-
}
|
|
52
34
|
/** Test hook — drop memoised instruments to rebind a fresh MeterProvider. */
|
|
53
35
|
export function __resetWorkflowMetricsForTest() {
|
|
54
36
|
instruments = undefined;
|
package/package.json
CHANGED