@inflexa-ai/harness 0.36.3 → 0.36.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/otel-spans.d.ts +56 -2
- package/dist/lib/otel-spans.js +93 -2
- package/dist/lib/otel.d.ts +15 -11
- package/dist/lib/otel.js +25 -18
- package/dist/loop/genai-spans.d.ts +10 -2
- package/dist/loop/genai-spans.js +44 -8
- package/dist/loop/run-agent.js +15 -13
- package/dist/providers/ai-sdk.js +12 -2
- package/dist/runtime/dbos.d.ts +3 -2
- package/dist/sandbox/await-exec.js +3 -3
- package/dist/sandbox/submit-exec.js +2 -2
- package/package.json +1 -1
package/dist/lib/otel-spans.d.ts
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* DBOS names a workflow span after the registered workflow and a step span
|
|
6
6
|
* after the step function, and it runs each body under its span, so every
|
|
7
7
|
* child (a step, a patched fetch, a pg query) sits under the workflow span.
|
|
8
|
-
*
|
|
8
|
+
* Four shapes of that output are noise in a trace store, and each is stopped
|
|
9
9
|
* in the process, before export, rather than in a collector downstream. The
|
|
10
10
|
* decisions live where the workflows and steps are written; this module holds
|
|
11
11
|
* the mechanism only:
|
|
@@ -23,9 +23,21 @@
|
|
|
23
23
|
* name itself is untouched: DBOS compares it against the recorded name on
|
|
24
24
|
* replay and throws `DBOSUnexpectedStepError` on a mismatch, so a rename
|
|
25
25
|
* would break every workflow in flight across a deploy.
|
|
26
|
+
* - A step whose span only repeats the spans under it runs inside
|
|
27
|
+
* `passThroughSpan(name, ...)`. `HarnessTracerProvider` gives that span as
|
|
28
|
+
* a pass-through: it records nothing, and each child attaches to the span
|
|
29
|
+
* that was active when the step started. `passThroughTracer` applies the
|
|
30
|
+
* same mechanism to a tracer that a library takes as an option.
|
|
31
|
+
*
|
|
32
|
+
* A pass-through is the only way to omit a span in the middle of a tree. The
|
|
33
|
+
* SDK fixes the parent of a child when the child starts, so a processor that
|
|
34
|
+
* drops the span at `onEnd` leaves its children with a parent that was never
|
|
35
|
+
* exported. A sampler is no better: `ParentBased` sampling gives the children
|
|
36
|
+
* the same decision, and they are not recorded either.
|
|
26
37
|
*/
|
|
27
|
-
import { type Attributes, type Context } from "@opentelemetry/api";
|
|
38
|
+
import { type Attributes, type Context, type SpanOptions, type Tracer, type TracerOptions } from "@opentelemetry/api";
|
|
28
39
|
import { type ReadableSpan, type Sampler, type Span, type SpanProcessor } from "@opentelemetry/sdk-trace-base";
|
|
40
|
+
import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node";
|
|
29
41
|
/**
|
|
30
42
|
* Declare a registered DBOS workflow untraced: no root span for it is
|
|
31
43
|
* recorded, and no child span under it. Call it next to the
|
|
@@ -60,6 +72,48 @@ export declare const ATTR_INFLEXA_ATTEMPT = "inflexa.attempt";
|
|
|
60
72
|
* leaves a missing or non-recording span alone.
|
|
61
73
|
*/
|
|
62
74
|
export declare function stableSpan(dbosName: string, name: string, attributes: Attributes): void;
|
|
75
|
+
/**
|
|
76
|
+
* Send a request to sandbox-server with the active trace context in its
|
|
77
|
+
* headers and with tracing suppressed, so the host's HTTP instrumentation
|
|
78
|
+
* writes no client span. Sandbox-server records no spans and echoes the exec
|
|
79
|
+
* request's `traceparent` on its callbacks to the host, so a client span would
|
|
80
|
+
* show the host calling itself; the step span around the call already covers it.
|
|
81
|
+
*/
|
|
82
|
+
export declare function untracedFetch(fetchImpl: typeof fetch, url: string, init: RequestInit & {
|
|
83
|
+
headers: Record<string, string>;
|
|
84
|
+
}): Promise<Response>;
|
|
85
|
+
/**
|
|
86
|
+
* Decides whether the span that `startSpan` is about to start passes through.
|
|
87
|
+
* `context` is the parent context of that span.
|
|
88
|
+
*/
|
|
89
|
+
export type PassThroughRule = (name: string, options: SpanOptions, context: Context) => boolean;
|
|
90
|
+
/**
|
|
91
|
+
* Wrap `tracer` so that a span that `passesThrough` selects is not written.
|
|
92
|
+
* In its place the tracer gives `trace.wrapSpanContext` of the parent span
|
|
93
|
+
* context. That span records nothing, and a span started under it becomes a
|
|
94
|
+
* child of the parent. A span with no valid parent, or a span that asks to be
|
|
95
|
+
* a root, starts as usual, because a pass-through there would make each child
|
|
96
|
+
* a root of its own trace.
|
|
97
|
+
*/
|
|
98
|
+
export declare function passThroughTracer(tracer: Tracer, passesThrough: PassThroughRule): Tracer;
|
|
99
|
+
/**
|
|
100
|
+
* Run `fn` so that a span named `name` that starts inside it is a
|
|
101
|
+
* pass-through: `HarnessTracerProvider` writes no span for it, and the
|
|
102
|
+
* children of that span attach to the span that is active now. Wrap the call
|
|
103
|
+
* that starts the span, for example a `DBOS.runStep` with the step name
|
|
104
|
+
* `name`. A span with a different name starts as usual.
|
|
105
|
+
*/
|
|
106
|
+
export declare function passThroughSpan<T>(name: string, fn: () => T): T;
|
|
107
|
+
/**
|
|
108
|
+
* The TracerProvider of the harness. Each tracer that it gives obeys
|
|
109
|
+
* `passThroughSpan`. The API keeps the first provider that is registered, and
|
|
110
|
+
* DBOS asks the API for its tracer at each span, so the rule applies to the
|
|
111
|
+
* DBOS step spans only because this provider is the registered one.
|
|
112
|
+
*/
|
|
113
|
+
export declare class HarnessTracerProvider extends NodeTracerProvider {
|
|
114
|
+
private readonly passThroughTracers;
|
|
115
|
+
getTracer(name: string, version?: string, options?: TracerOptions): Tracer;
|
|
116
|
+
}
|
|
63
117
|
/**
|
|
64
118
|
* Sits in front of the exporting processor and forwards every span except a
|
|
65
119
|
* replayed one (`cached=true`), which it drops at `onEnd`.
|
package/dist/lib/otel-spans.js
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* DBOS names a workflow span after the registered workflow and a step span
|
|
6
6
|
* after the step function, and it runs each body under its span, so every
|
|
7
7
|
* child (a step, a patched fetch, a pg query) sits under the workflow span.
|
|
8
|
-
*
|
|
8
|
+
* Four shapes of that output are noise in a trace store, and each is stopped
|
|
9
9
|
* in the process, before export, rather than in a collector downstream. The
|
|
10
10
|
* decisions live where the workflows and steps are written; this module holds
|
|
11
11
|
* the mechanism only:
|
|
@@ -23,9 +23,22 @@
|
|
|
23
23
|
* name itself is untouched: DBOS compares it against the recorded name on
|
|
24
24
|
* replay and throws `DBOSUnexpectedStepError` on a mismatch, so a rename
|
|
25
25
|
* would break every workflow in flight across a deploy.
|
|
26
|
+
* - A step whose span only repeats the spans under it runs inside
|
|
27
|
+
* `passThroughSpan(name, ...)`. `HarnessTracerProvider` gives that span as
|
|
28
|
+
* a pass-through: it records nothing, and each child attaches to the span
|
|
29
|
+
* that was active when the step started. `passThroughTracer` applies the
|
|
30
|
+
* same mechanism to a tracer that a library takes as an option.
|
|
31
|
+
*
|
|
32
|
+
* A pass-through is the only way to omit a span in the middle of a tree. The
|
|
33
|
+
* SDK fixes the parent of a child when the child starts, so a processor that
|
|
34
|
+
* drops the span at `onEnd` leaves its children with a parent that was never
|
|
35
|
+
* exported. A sampler is no better: `ParentBased` sampling gives the children
|
|
36
|
+
* the same decision, and they are not recorded either.
|
|
26
37
|
*/
|
|
27
|
-
import { trace } from "@opentelemetry/api";
|
|
38
|
+
import { context, createContextKey, propagation, trace, } from "@opentelemetry/api";
|
|
39
|
+
import { suppressTracing } from "@opentelemetry/core";
|
|
28
40
|
import { ParentBasedSampler, SamplingDecision, } from "@opentelemetry/sdk-trace-base";
|
|
41
|
+
import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node";
|
|
29
42
|
/**
|
|
30
43
|
* Names of the DBOS workflows whose traces are not recorded. A registering
|
|
31
44
|
* module adds its own name at registration, which runs at boot inside
|
|
@@ -92,6 +105,84 @@ export function stableSpan(dbosName, name, attributes) {
|
|
|
92
105
|
span.updateName(name);
|
|
93
106
|
span.setAttributes(attributes);
|
|
94
107
|
}
|
|
108
|
+
/**
|
|
109
|
+
* Send a request to sandbox-server with the active trace context in its
|
|
110
|
+
* headers and with tracing suppressed, so the host's HTTP instrumentation
|
|
111
|
+
* writes no client span. Sandbox-server records no spans and echoes the exec
|
|
112
|
+
* request's `traceparent` on its callbacks to the host, so a client span would
|
|
113
|
+
* show the host calling itself; the step span around the call already covers it.
|
|
114
|
+
*/
|
|
115
|
+
export function untracedFetch(fetchImpl, url, init) {
|
|
116
|
+
const active = context.active();
|
|
117
|
+
const headers = { ...init.headers };
|
|
118
|
+
propagation.inject(active, headers);
|
|
119
|
+
return context.with(suppressTracing(active), () => fetchImpl(url, { ...init, headers }));
|
|
120
|
+
}
|
|
121
|
+
/**
|
|
122
|
+
* Wrap `tracer` so that a span that `passesThrough` selects is not written.
|
|
123
|
+
* In its place the tracer gives `trace.wrapSpanContext` of the parent span
|
|
124
|
+
* context. That span records nothing, and a span started under it becomes a
|
|
125
|
+
* child of the parent. A span with no valid parent, or a span that asks to be
|
|
126
|
+
* a root, starts as usual, because a pass-through there would make each child
|
|
127
|
+
* a root of its own trace.
|
|
128
|
+
*/
|
|
129
|
+
export function passThroughTracer(tracer, passesThrough) {
|
|
130
|
+
return new PassThroughTracer(tracer, passesThrough);
|
|
131
|
+
}
|
|
132
|
+
class PassThroughTracer {
|
|
133
|
+
inner;
|
|
134
|
+
passesThrough;
|
|
135
|
+
constructor(inner, passesThrough) {
|
|
136
|
+
this.inner = inner;
|
|
137
|
+
this.passesThrough = passesThrough;
|
|
138
|
+
}
|
|
139
|
+
startSpan(name, options = {}, parentContext = context.active()) {
|
|
140
|
+
const parent = options.root === true ? undefined : trace.getSpanContext(parentContext);
|
|
141
|
+
if (parent !== undefined && trace.isSpanContextValid(parent) && this.passesThrough(name, options, parentContext)) {
|
|
142
|
+
return trace.wrapSpanContext(parent);
|
|
143
|
+
}
|
|
144
|
+
return this.inner.startSpan(name, options, parentContext);
|
|
145
|
+
}
|
|
146
|
+
// The inner tracer cannot take this call: its own `startActiveSpan` calls its own `startSpan`, past the rule.
|
|
147
|
+
startActiveSpan(name, ...rest) {
|
|
148
|
+
const fn = rest[rest.length - 1];
|
|
149
|
+
const options = rest.length > 1 ? rest[0] : {};
|
|
150
|
+
const parentContext = (rest.length > 2 ? rest[1] : undefined) ?? context.active();
|
|
151
|
+
const span = this.startSpan(name, options, parentContext);
|
|
152
|
+
return context.with(trace.setSpan(parentContext, span), () => fn(span));
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
/** The name of the span that the enclosing `passThroughSpan` declared. */
|
|
156
|
+
const PASS_THROUGH_SPAN_NAME = createContextKey("inflexa.pass_through_span_name");
|
|
157
|
+
/**
|
|
158
|
+
* Run `fn` so that a span named `name` that starts inside it is a
|
|
159
|
+
* pass-through: `HarnessTracerProvider` writes no span for it, and the
|
|
160
|
+
* children of that span attach to the span that is active now. Wrap the call
|
|
161
|
+
* that starts the span, for example a `DBOS.runStep` with the step name
|
|
162
|
+
* `name`. A span with a different name starts as usual.
|
|
163
|
+
*/
|
|
164
|
+
export function passThroughSpan(name, fn) {
|
|
165
|
+
return context.with(context.active().setValue(PASS_THROUGH_SPAN_NAME, name), fn);
|
|
166
|
+
}
|
|
167
|
+
const declaredPassThrough = (name, _options, parentContext) => parentContext.getValue(PASS_THROUGH_SPAN_NAME) === name;
|
|
168
|
+
/**
|
|
169
|
+
* The TracerProvider of the harness. Each tracer that it gives obeys
|
|
170
|
+
* `passThroughSpan`. The API keeps the first provider that is registered, and
|
|
171
|
+
* DBOS asks the API for its tracer at each span, so the rule applies to the
|
|
172
|
+
* DBOS step spans only because this provider is the registered one.
|
|
173
|
+
*/
|
|
174
|
+
export class HarnessTracerProvider extends NodeTracerProvider {
|
|
175
|
+
passThroughTracers = new WeakMap();
|
|
176
|
+
getTracer(name, version, options) {
|
|
177
|
+
const tracer = super.getTracer(name, version, options);
|
|
178
|
+
let wrapped = this.passThroughTracers.get(tracer);
|
|
179
|
+
if (wrapped === undefined) {
|
|
180
|
+
wrapped = passThroughTracer(tracer, declaredPassThrough);
|
|
181
|
+
this.passThroughTracers.set(tracer, wrapped);
|
|
182
|
+
}
|
|
183
|
+
return wrapped;
|
|
184
|
+
}
|
|
185
|
+
}
|
|
95
186
|
/**
|
|
96
187
|
* Sits in front of the exporting processor and forwards every span except a
|
|
97
188
|
* replayed one (`cached=true`), which it drops at `onEnd`.
|
package/dist/lib/otel.d.ts
CHANGED
|
@@ -4,13 +4,15 @@
|
|
|
4
4
|
* Exported as `initOtel()` and called explicitly from the host's boot sequence
|
|
5
5
|
* so the bundler cannot tree-shake it (side-effect-only imports get dropped).
|
|
6
6
|
*
|
|
7
|
-
* Traces:
|
|
8
|
-
* is set. The provider carries the span
|
|
9
|
-
* root span of a workflow declared with
|
|
10
|
-
* recorded (and its children with it), a
|
|
11
|
-
* is dropped before export,
|
|
12
|
-
* is exported under a stable name with its id
|
|
13
|
-
* attribute
|
|
7
|
+
* Traces: HarnessTracerProvider exports to OTLP when
|
|
8
|
+
* OTEL_EXPORTER_OTLP_ENDPOINT is set. The provider carries the span
|
|
9
|
+
* policy of otel-spans.ts: the root span of a workflow declared with
|
|
10
|
+
* `untracedWorkflow` is not recorded (and its children with it), a
|
|
11
|
+
* replayed `cached=true` span is dropped before export, a step body
|
|
12
|
+
* that calls `stableSpan` is exported under a stable name with its id
|
|
13
|
+
* in an `inflexa.*` attribute, and a span declared with
|
|
14
|
+
* `passThroughSpan` is not written, its children attached to its
|
|
15
|
+
* parent.
|
|
14
16
|
*
|
|
15
17
|
* Metrics: MeterProvider exports to OTLP when OTEL_EXPORTER_OTLP_ENDPOINT is
|
|
16
18
|
* set and OTEL_METRICS_EXPORTER is not `none`. The export interval is
|
|
@@ -21,11 +23,13 @@
|
|
|
21
23
|
* to their record sites. Every instrument binds lazily to the
|
|
22
24
|
* provider registered here.
|
|
23
25
|
*
|
|
24
|
-
* Resource: the
|
|
25
|
-
*
|
|
26
|
-
*
|
|
26
|
+
* Resource: the SDK's default resource (`telemetry.sdk.*`), under the host's
|
|
27
|
+
* `serviceName` / `serviceVersion`, under the SDK env detector
|
|
28
|
+
* (`OTEL_SERVICE_NAME`, `OTEL_RESOURCE_ATTRIBUTES`). A later layer
|
|
29
|
+
* wins on a conflict, so a deployment can set
|
|
27
30
|
* `deployment.environment.name` or override the service name without
|
|
28
|
-
* a code change
|
|
31
|
+
* a code change, and the default `unknown_service` name never
|
|
32
|
+
* reaches an export. Traces and metrics share the one resource.
|
|
29
33
|
*
|
|
30
34
|
* Note: OTEL's instrumentation-http patches node:http but does not cover
|
|
31
35
|
* Hono's request handling or Node 22's undici-based globalThis.fetch,
|
package/dist/lib/otel.js
CHANGED
|
@@ -4,13 +4,15 @@
|
|
|
4
4
|
* Exported as `initOtel()` and called explicitly from the host's boot sequence
|
|
5
5
|
* so the bundler cannot tree-shake it (side-effect-only imports get dropped).
|
|
6
6
|
*
|
|
7
|
-
* Traces:
|
|
8
|
-
* is set. The provider carries the span
|
|
9
|
-
* root span of a workflow declared with
|
|
10
|
-
* recorded (and its children with it), a
|
|
11
|
-
* is dropped before export,
|
|
12
|
-
* is exported under a stable name with its id
|
|
13
|
-
* attribute
|
|
7
|
+
* Traces: HarnessTracerProvider exports to OTLP when
|
|
8
|
+
* OTEL_EXPORTER_OTLP_ENDPOINT is set. The provider carries the span
|
|
9
|
+
* policy of otel-spans.ts: the root span of a workflow declared with
|
|
10
|
+
* `untracedWorkflow` is not recorded (and its children with it), a
|
|
11
|
+
* replayed `cached=true` span is dropped before export, a step body
|
|
12
|
+
* that calls `stableSpan` is exported under a stable name with its id
|
|
13
|
+
* in an `inflexa.*` attribute, and a span declared with
|
|
14
|
+
* `passThroughSpan` is not written, its children attached to its
|
|
15
|
+
* parent.
|
|
14
16
|
*
|
|
15
17
|
* Metrics: MeterProvider exports to OTLP when OTEL_EXPORTER_OTLP_ENDPOINT is
|
|
16
18
|
* set and OTEL_METRICS_EXPORTER is not `none`. The export interval is
|
|
@@ -21,11 +23,13 @@
|
|
|
21
23
|
* to their record sites. Every instrument binds lazily to the
|
|
22
24
|
* provider registered here.
|
|
23
25
|
*
|
|
24
|
-
* Resource: the
|
|
25
|
-
*
|
|
26
|
-
*
|
|
26
|
+
* Resource: the SDK's default resource (`telemetry.sdk.*`), under the host's
|
|
27
|
+
* `serviceName` / `serviceVersion`, under the SDK env detector
|
|
28
|
+
* (`OTEL_SERVICE_NAME`, `OTEL_RESOURCE_ATTRIBUTES`). A later layer
|
|
29
|
+
* wins on a conflict, so a deployment can set
|
|
27
30
|
* `deployment.environment.name` or override the service name without
|
|
28
|
-
* a code change
|
|
31
|
+
* a code change, and the default `unknown_service` name never
|
|
32
|
+
* reaches an export. Traces and metrics share the one resource.
|
|
29
33
|
*
|
|
30
34
|
* Note: OTEL's instrumentation-http patches node:http but does not cover
|
|
31
35
|
* Hono's request handling or Node 22's undici-based globalThis.fetch,
|
|
@@ -34,14 +38,13 @@
|
|
|
34
38
|
*/
|
|
35
39
|
import { context, propagation, metrics, trace } from "@opentelemetry/api";
|
|
36
40
|
import { createNoopLogger } from "./console-logger.js";
|
|
37
|
-
import { createHarnessSampler, DbosSpanProcessor } from "./otel-spans.js";
|
|
41
|
+
import { createHarnessSampler, DbosSpanProcessor, HarnessTracerProvider } from "./otel-spans.js";
|
|
38
42
|
import { W3CTraceContextPropagator } from "@opentelemetry/core";
|
|
39
43
|
import { MeterProvider, PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics";
|
|
40
44
|
import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-http";
|
|
41
|
-
import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node";
|
|
42
45
|
import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-base";
|
|
43
46
|
import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-proto";
|
|
44
|
-
import { detectResources, envDetector, resourceFromAttributes } from "@opentelemetry/resources";
|
|
47
|
+
import { defaultResource, detectResources, envDetector, resourceFromAttributes } from "@opentelemetry/resources";
|
|
45
48
|
import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from "@opentelemetry/semantic-conventions";
|
|
46
49
|
/** The SDK's own default when neither the host nor the environment says otherwise. */
|
|
47
50
|
export const DEFAULT_METRIC_EXPORT_INTERVAL_MS = 60_000;
|
|
@@ -75,9 +78,13 @@ function buildResource(options) {
|
|
|
75
78
|
};
|
|
76
79
|
if (options.serviceVersion)
|
|
77
80
|
manual[ATTR_SERVICE_VERSION] = options.serviceVersion;
|
|
78
|
-
// `merge` gives precedence to the argument
|
|
79
|
-
//
|
|
80
|
-
|
|
81
|
+
// `merge` gives precedence to the argument. A provider that gets an explicit
|
|
82
|
+
// resource does not add the SDK default itself, so it is the bottom layer here.
|
|
83
|
+
// The environment overrides the host's values, which is the OTel SDK
|
|
84
|
+
// convention for env configuration.
|
|
85
|
+
return defaultResource()
|
|
86
|
+
.merge(resourceFromAttributes(manual))
|
|
87
|
+
.merge(detectResources({ detectors: [envDetector] }));
|
|
81
88
|
}
|
|
82
89
|
/**
|
|
83
90
|
* Initialize the OpenTelemetry SDK. Must be called before any code that
|
|
@@ -104,7 +111,7 @@ export function initOtel(options = {}) {
|
|
|
104
111
|
const base = endpoint.replace(/\/+$/, "");
|
|
105
112
|
spanProcessors.push(new DbosSpanProcessor(new BatchSpanProcessor(new OTLPTraceExporter({ url: `${base}/v1/traces` }))));
|
|
106
113
|
}
|
|
107
|
-
const tracerProvider = new
|
|
114
|
+
const tracerProvider = new HarnessTracerProvider({
|
|
108
115
|
resource,
|
|
109
116
|
sampler: createHarnessSampler(),
|
|
110
117
|
spanProcessors,
|
|
@@ -4,6 +4,11 @@
|
|
|
4
4
|
* SDK traces the model calls (`providers/ai-sdk.ts`). It never sees the loop or
|
|
5
5
|
* the tools, because the harness runs both itself.
|
|
6
6
|
*
|
|
7
|
+
* The root of a turn opens its own trace, and a link carries it back to the run.
|
|
8
|
+
* A run lasts hours and writes tens of thousands of spans, thus one trace of a
|
|
9
|
+
* whole run passes the size limit of the trace store and no reader can follow
|
|
10
|
+
* it. `inflexa.run_id` holds the turns of one run together.
|
|
11
|
+
*
|
|
7
12
|
* An `execute_tool` span is ERROR when our code fails the call. A failure that a
|
|
8
13
|
* tool returns as data, for example a sandbox command that exits non-zero, is
|
|
9
14
|
* normal iteration of the agent, and its span stays OK.
|
|
@@ -17,8 +22,11 @@ interface AgentRun {
|
|
|
17
22
|
readonly cappedOut: boolean;
|
|
18
23
|
};
|
|
19
24
|
}
|
|
20
|
-
/**
|
|
21
|
-
|
|
25
|
+
/**
|
|
26
|
+
* Run one agent loop under an `invoke_agent` span. A turn root starts a new
|
|
27
|
+
* trace; a loop under a turn stays in the trace of that turn.
|
|
28
|
+
*/
|
|
29
|
+
export declare function traceAgentRun<T extends AgentRun>(agentName: string, session: AgentSession, turnRoot: boolean, run: () => Promise<T>): Promise<T>;
|
|
22
30
|
/** Run one tool call under an `execute_tool` span that records how the call ended. */
|
|
23
31
|
export declare function traceToolCall<T>(call: Pick<ToolCallPart, "toolName" | "toolCallId">, run: () => Promise<T>, outcomeOf: (value: T) => ToolOutcome): Promise<T>;
|
|
24
32
|
/**
|
package/dist/loop/genai-spans.js
CHANGED
|
@@ -4,15 +4,29 @@
|
|
|
4
4
|
* SDK traces the model calls (`providers/ai-sdk.ts`). It never sees the loop or
|
|
5
5
|
* the tools, because the harness runs both itself.
|
|
6
6
|
*
|
|
7
|
+
* The root of a turn opens its own trace, and a link carries it back to the run.
|
|
8
|
+
* A run lasts hours and writes tens of thousands of spans, thus one trace of a
|
|
9
|
+
* whole run passes the size limit of the trace store and no reader can follow
|
|
10
|
+
* it. `inflexa.run_id` holds the turns of one run together.
|
|
11
|
+
*
|
|
7
12
|
* An `execute_tool` span is ERROR when our code fails the call. A failure that a
|
|
8
13
|
* tool returns as data, for example a sandbox command that exits non-zero, is
|
|
9
14
|
* normal iteration of the agent, and its span stays OK.
|
|
10
15
|
*/
|
|
11
|
-
import { SpanStatusCode, trace } from "@opentelemetry/api";
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
+
import { context, SpanStatusCode, trace, TraceFlags } from "@opentelemetry/api";
|
|
17
|
+
import { isTracingSuppressed } from "@opentelemetry/core";
|
|
18
|
+
/**
|
|
19
|
+
* Run one agent loop under an `invoke_agent` span. A turn root starts a new
|
|
20
|
+
* trace; a loop under a turn stays in the trace of that turn.
|
|
21
|
+
*/
|
|
22
|
+
export function traceAgentRun(agentName, session, turnRoot, run) {
|
|
23
|
+
const attributes = {
|
|
24
|
+
"gen_ai.operation.name": "invoke_agent",
|
|
25
|
+
"gen_ai.agent.name": agentName,
|
|
26
|
+
...conversationAttributes(session),
|
|
27
|
+
...runFrameAttributes(session),
|
|
28
|
+
};
|
|
29
|
+
return withSpan(`invoke_agent ${agentName}`, turnRoot ? turnRootOptions(attributes) : { attributes }, async (span) => {
|
|
16
30
|
const result = await run();
|
|
17
31
|
span.setAttributes({ "gen_ai.response.finish_reasons": [result.finish.reason], "inflexa.agent.capped_out": result.finish.cappedOut });
|
|
18
32
|
return result;
|
|
@@ -21,7 +35,7 @@ export function traceAgentRun(agentName, session, run) {
|
|
|
21
35
|
/** Run one tool call under an `execute_tool` span that records how the call ended. */
|
|
22
36
|
export function traceToolCall(call, run, outcomeOf) {
|
|
23
37
|
const attributes = { "gen_ai.operation.name": "execute_tool", "gen_ai.tool.name": call.toolName, "gen_ai.tool.call.id": call.toolCallId };
|
|
24
|
-
return withSpan(`execute_tool ${call.toolName}`, attributes, async (span) => {
|
|
38
|
+
return withSpan(`execute_tool ${call.toolName}`, { attributes }, async (span) => {
|
|
25
39
|
const value = await run();
|
|
26
40
|
const outcome = outcomeOf(value);
|
|
27
41
|
span.setAttribute("inflexa.tool.outcome", outcome);
|
|
@@ -71,8 +85,26 @@ function errorTypeOf(err) {
|
|
|
71
85
|
return String(code);
|
|
72
86
|
return err.name === "" ? "_OTHER" : err.name;
|
|
73
87
|
}
|
|
74
|
-
|
|
75
|
-
|
|
88
|
+
/**
|
|
89
|
+
* The options of a turn root. `root: true` drops the parent before the sampler
|
|
90
|
+
* runs, thus the sampler takes its root branch and a decision from above is
|
|
91
|
+
* lost. A turn under suppressed tracing, or under a trace that is not sampled
|
|
92
|
+
* (an untraced workflow), therefore keeps its parent and stays unrecorded.
|
|
93
|
+
*/
|
|
94
|
+
function turnRootOptions(attributes) {
|
|
95
|
+
const active = context.active();
|
|
96
|
+
if (isTracingSuppressed(active))
|
|
97
|
+
return { attributes };
|
|
98
|
+
const ambient = trace.getSpanContext(active);
|
|
99
|
+
if (ambient === undefined || !trace.isSpanContextValid(ambient))
|
|
100
|
+
return { root: true, attributes };
|
|
101
|
+
if ((ambient.traceFlags & TraceFlags.SAMPLED) === 0)
|
|
102
|
+
return { attributes };
|
|
103
|
+
// The span that started the turn: the workflow span of the run, or the request span.
|
|
104
|
+
return { root: true, links: [{ context: ambient, attributes: { "inflexa.link.kind": "caller" } }], attributes };
|
|
105
|
+
}
|
|
106
|
+
function withSpan(name, options, run) {
|
|
107
|
+
return trace.getTracer("@inflexa-ai/harness").startActiveSpan(name, options, async (span) => {
|
|
76
108
|
try {
|
|
77
109
|
return await run(span);
|
|
78
110
|
}
|
|
@@ -96,3 +128,7 @@ function runFrameAttributes({ runFrame }) {
|
|
|
96
128
|
return {};
|
|
97
129
|
return { "inflexa.run_id": runFrame.runId, ...(runFrame.stepId === undefined ? {} : { "inflexa.step_id": runFrame.stepId }) };
|
|
98
130
|
}
|
|
131
|
+
/** The thread is the conversation. The analysis is not: it holds many threads. */
|
|
132
|
+
function conversationAttributes({ scope }) {
|
|
133
|
+
return scope.threadId === undefined ? {} : { "gen_ai.conversation.id": scope.threadId };
|
|
134
|
+
}
|
package/dist/loop/run-agent.js
CHANGED
|
@@ -3,7 +3,7 @@ import { jsonSchema, tool as aiTool, } from "ai";
|
|
|
3
3
|
import { createNoopUsageRecorder } from "../billing/noop-usage-recorder.js";
|
|
4
4
|
import { stripNulCharacters } from "../input-sanitization.js";
|
|
5
5
|
import { createNoopLogger } from "../lib/console-logger.js";
|
|
6
|
-
import { ATTR_INFLEXA_TOOL_USE_ID, stableSpan } from "../lib/otel-spans.js";
|
|
6
|
+
import { ATTR_INFLEXA_TOOL_USE_ID, passThroughSpan, stableSpan } from "../lib/otel-spans.js";
|
|
7
7
|
import { ResultError } from "../lib/result.js";
|
|
8
8
|
import { describeZodIssueShapes } from "../lib/zod-issue-shape.js";
|
|
9
9
|
import { hintForZodIssue, repairToolInput } from "../lib/zod-issues.js";
|
|
@@ -25,11 +25,17 @@ export const DEFAULT_STEP_NAME_FORMATTER = {
|
|
|
25
25
|
llm: (i) => `llm-${i}`,
|
|
26
26
|
tool: (name, id) => `tool-${name}-${id}`,
|
|
27
27
|
};
|
|
28
|
+
/** A run that receives no turn accumulator is the turn's root. See `RunAgentOptions.turnUsage`. */
|
|
29
|
+
const isTurnRoot = (opts) => opts.turnUsage === undefined;
|
|
28
30
|
export function runAgent(agent, initial, session, opts) {
|
|
29
|
-
return traceAgentRun(agent.id, session, () => runAgentLoop(agent, initial, session, opts));
|
|
31
|
+
return traceAgentRun(agent.id, session, isTurnRoot(opts), () => runAgentLoop(agent, initial, session, opts));
|
|
30
32
|
}
|
|
31
33
|
async function runAgentLoop(agent, initial, session, opts) {
|
|
32
34
|
const { provider, signal, emit, runStep } = opts;
|
|
35
|
+
// The step of a model call or a tool call writes no span. The `chat` span and
|
|
36
|
+
// the `execute_tool` span under it describe the call already, and they attach
|
|
37
|
+
// to the `invoke_agent` span of this run.
|
|
38
|
+
const callStep = (name, fn) => passThroughSpan(name, () => runStep(name, fn));
|
|
33
39
|
const formatStepName = opts.formatStepName ?? DEFAULT_STEP_NAME_FORMATTER;
|
|
34
40
|
const configuredFatalLoopError = opts.isFatalLoopError ?? (() => false);
|
|
35
41
|
// AbortError is control flow, not a tool failure. Always compose it with the
|
|
@@ -71,7 +77,7 @@ async function runAgentLoop(agent, initial, session, opts) {
|
|
|
71
77
|
// descendant loop folds into, and it is the only loop entitled to report a
|
|
72
78
|
// turn total. Descendants receive the root's object and mutate it in place.
|
|
73
79
|
const turnUsage = opts.turnUsage ?? {};
|
|
74
|
-
const
|
|
80
|
+
const turnRoot = isTurnRoot(opts);
|
|
75
81
|
const toolCtx = (tu) => ({
|
|
76
82
|
invocationId: tu.toolCallId,
|
|
77
83
|
session,
|
|
@@ -168,7 +174,7 @@ async function runAgentLoop(agent, initial, session, opts) {
|
|
|
168
174
|
/** The rollups this loop stamps on its finish — each absent when nothing reported. */
|
|
169
175
|
const finishUsage = () => ({
|
|
170
176
|
...(hasReportedUsage(usage) ? { usage: { ...usage } } : {}),
|
|
171
|
-
...(
|
|
177
|
+
...(turnRoot && hasReportedUsage(turnUsage) ? { turnUsage: { ...turnUsage } } : {}),
|
|
172
178
|
});
|
|
173
179
|
// The user said no. A subsequent model call would only let the agent argue
|
|
174
180
|
// with the decision, or spend a call acknowledging it; the denial tool result
|
|
@@ -252,7 +258,7 @@ async function runAgentLoop(agent, initial, session, opts) {
|
|
|
252
258
|
reasoning,
|
|
253
259
|
};
|
|
254
260
|
const llmStepName = formatStepName.llm(i);
|
|
255
|
-
const reply = await resultStep(
|
|
261
|
+
const reply = await resultStep(callStep)(llmStepName, () => provider.chat(request, session, signal));
|
|
256
262
|
accountForCall(reply, llmStepName);
|
|
257
263
|
if (reply.finishReason === "aborted") {
|
|
258
264
|
// An interrupted turn keeps whatever the model produced before the cut, but
|
|
@@ -290,7 +296,7 @@ async function runAgentLoop(agent, initial, session, opts) {
|
|
|
290
296
|
for (const [idx, tu] of earlier.entries()) {
|
|
291
297
|
await emit({ type: "tool-started", source, toolUseId: tu.toolCallId, name: tu.toolName, input: tu.input, ...detailField(earlierDetails[idx]) });
|
|
292
298
|
}
|
|
293
|
-
const { results, durations, resultDetails } = await dispatchTools(earlier, toolsById, toolCtx, isFatalLoopError,
|
|
299
|
+
const { results, durations, resultDetails } = await dispatchTools(earlier, toolsById, toolCtx, isFatalLoopError, callStep, formatStepName.tool, encoding);
|
|
294
300
|
const errored = await settleRound(earlier, results, earlierDetails, resultDetails, durations);
|
|
295
301
|
results.push(errorResult(trailing, TRUNCATED_TOOL_USE_ERROR));
|
|
296
302
|
// The trailing call was never dispatched, but it reaches the model as an error
|
|
@@ -342,7 +348,7 @@ async function runAgentLoop(agent, initial, session, opts) {
|
|
|
342
348
|
for (const [idx, tu] of toolCalls.entries()) {
|
|
343
349
|
await emit({ type: "tool-started", source, toolUseId: tu.toolCallId, name: tu.toolName, input: tu.input, ...detailField(details[idx]) });
|
|
344
350
|
}
|
|
345
|
-
const { results, durations, resultDetails } = await dispatchTools(toolCalls, toolsById, toolCtx, isFatalLoopError,
|
|
351
|
+
const { results, durations, resultDetails } = await dispatchTools(toolCalls, toolsById, toolCtx, isFatalLoopError, callStep, formatStepName.tool, encoding);
|
|
346
352
|
const errored = await settleRound(toolCalls, results, details, resultDetails, durations);
|
|
347
353
|
if (errored.length > 0)
|
|
348
354
|
log.debug("tool results returned errors", { iteration: i, tools: errored });
|
|
@@ -360,7 +366,7 @@ async function runAgentLoop(agent, initial, session, opts) {
|
|
|
360
366
|
// because it is the one call whose write is pure waste, and the
|
|
361
367
|
// cache_write_tokens counter is what makes that waste visible.
|
|
362
368
|
const wrapUpStepName = formatStepName.llm(agent.maxIterations);
|
|
363
|
-
const wrapUp = await resultStep(
|
|
369
|
+
const wrapUp = await resultStep(callStep)(wrapUpStepName, () => provider.chat({ system: agent.systemPrompt, messages: withPromptCacheBreakpoint(messages, promptCache), tools: {}, toolChoice: "none", reasoning }, session, signal));
|
|
364
370
|
accountForCall(wrapUp, wrapUpStepName);
|
|
365
371
|
if (wrapUp.finishReason === "aborted") {
|
|
366
372
|
// An abort during the tool-less wrap-up is still the user cutting the turn — the
|
|
@@ -554,11 +560,7 @@ async function dispatchTools(toolUses, toolsById, toolCtx, isFatalLoopError, run
|
|
|
554
560
|
}
|
|
555
561
|
await Promise.all(stepTools.map(({ tu, idx }) => {
|
|
556
562
|
const startedAt = performance.now();
|
|
557
|
-
|
|
558
|
-
return runStep(stepName, () => {
|
|
559
|
-
stableSpan(stepName, `tool:${tu.toolName}`, { [ATTR_INFLEXA_TOOL_USE_ID]: tu.toolCallId });
|
|
560
|
-
return dispatch(tu);
|
|
561
|
-
}).then((settled) => {
|
|
563
|
+
return runStep(toolStepName(tu.toolName, tu.toolCallId), () => dispatch(tu)).then((settled) => {
|
|
562
564
|
durations[idx] = elapsedMs(startedAt);
|
|
563
565
|
const dispatched = readDispatchedStep(settled);
|
|
564
566
|
results[idx] = dispatched.result;
|
package/dist/providers/ai-sdk.js
CHANGED
|
@@ -5,9 +5,11 @@ import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
|
|
|
5
5
|
import { OpenTelemetry } from "@ai-sdk/otel";
|
|
6
6
|
import { APICallError } from "@ai-sdk/provider";
|
|
7
7
|
import { retryWithExponentialBackoff } from "@ai-sdk/provider-utils";
|
|
8
|
+
import { trace } from "@opentelemetry/api";
|
|
8
9
|
import { ResultAsync, err, ok } from "neverthrow";
|
|
9
10
|
import { scopeWorkloadId } from "../auth/types.js";
|
|
10
11
|
import { createNoopLogger } from "../lib/console-logger.js";
|
|
12
|
+
import { passThroughTracer } from "../lib/otel-spans.js";
|
|
11
13
|
import { classifyProviderError, RequestTimeoutError, toProviderError } from "./errors.js";
|
|
12
14
|
/**
|
|
13
15
|
* The harness-owned retry envelope. The AI SDK's built-in retry exposes only a
|
|
@@ -414,6 +416,10 @@ function abortedStreamFailure(signal, reason, requestTimeoutMs) {
|
|
|
414
416
|
return new RequestTimeoutError(requestTimeoutMs);
|
|
415
417
|
return new Error(reason ?? "The provider aborted the stream.");
|
|
416
418
|
}
|
|
419
|
+
/** The `gen_ai.operation.name` of the operation span and of the step span that the AI SDK opens around each `chat` span. */
|
|
420
|
+
function isSdkWrapperSpan(operation) {
|
|
421
|
+
return operation === "invoke_agent" || operation === "agent_step";
|
|
422
|
+
}
|
|
417
423
|
export function createAiSdkProvider(deps) {
|
|
418
424
|
const capabilities = {
|
|
419
425
|
toolCalling: deps.capabilities?.toolCalling ?? true,
|
|
@@ -435,8 +441,12 @@ export function createAiSdkProvider(deps) {
|
|
|
435
441
|
// Each model call emits OpenTelemetry GenAI spans through the AI SDK, with no
|
|
436
442
|
// prompt or completion text on them. The integration rides on each call
|
|
437
443
|
// instead of the global registry, so the harness never traces the embedder's
|
|
438
|
-
// own AI SDK calls.
|
|
439
|
-
|
|
444
|
+
// own AI SDK calls. Of the SDK spans only `chat` is written. The operation
|
|
445
|
+
// span and the step span around it repeat what `chat` and the span of the
|
|
446
|
+
// caller (the loop's `invoke_agent`) say already. Thus they pass through, and
|
|
447
|
+
// `chat` attaches to the span of the caller.
|
|
448
|
+
const tracer = passThroughTracer(trace.getTracer("gen_ai"), (_name, options) => isSdkWrapperSpan(options.attributes?.["gen_ai.operation.name"]));
|
|
449
|
+
const telemetry = { integrations: new OpenTelemetry({ tracer }), recordInputs: false, recordOutputs: false };
|
|
440
450
|
routeSdkWarningsTo(logger);
|
|
441
451
|
/**
|
|
442
452
|
* Record a call that the envelope gave up on.
|
package/dist/runtime/dbos.d.ts
CHANGED
|
@@ -57,9 +57,10 @@ export interface DbosConfig {
|
|
|
57
57
|
*
|
|
58
58
|
* The harness TracerProvider shapes what the SDK emits (`lib/otel-spans.ts`):
|
|
59
59
|
* a workflow declared with `untracedWorkflow` is not recorded, a replayed
|
|
60
|
-
* `cached=true` span is dropped before export,
|
|
60
|
+
* `cached=true` span is dropped before export, a step body that calls
|
|
61
61
|
* `stableSpan` is exported under a stable name with its id in an
|
|
62
|
-
* `inflexa.*` attribute
|
|
62
|
+
* `inflexa.*` attribute, and the step of an agent-loop model call or tool
|
|
63
|
+
* call writes no span. The step name DBOS records is unchanged.
|
|
63
64
|
*
|
|
64
65
|
* Known SDK behavior a host must plan for: a recovered workflow starts a
|
|
65
66
|
* new root trace, with no link to the trace of the original execution.
|
|
@@ -55,7 +55,7 @@
|
|
|
55
55
|
* varied between replays would desynchronise the recorded function-ID sequence.
|
|
56
56
|
*/
|
|
57
57
|
import { DBOS } from "@dbos-inc/dbos-sdk";
|
|
58
|
-
import { ATTR_INFLEXA_ATTEMPT, ATTR_INFLEXA_EXEC_ID, stableSpan } from "../lib/otel-spans.js";
|
|
58
|
+
import { ATTR_INFLEXA_ATTEMPT, ATTR_INFLEXA_EXEC_ID, stableSpan, untracedFetch } from "../lib/otel-spans.js";
|
|
59
59
|
import { digestBody } from "./digest.js";
|
|
60
60
|
import { signCallback, verifyCallback } from "./hmac.js";
|
|
61
61
|
import { createEscalationPolicy, probeLiveness, syntheticFailureReason, syntheticFailureResult } from "./liveness.js";
|
|
@@ -105,7 +105,7 @@ async function pullExecResult(fetchImpl, ref, execId) {
|
|
|
105
105
|
// cached result is replayed without re-executing the body.
|
|
106
106
|
const reqTimestamp = Math.floor(Date.now() / 1000);
|
|
107
107
|
const reqSignature = signCallback({ execId, body: "", timestamp: reqTimestamp, secret: ref.callbackSecret });
|
|
108
|
-
const res = await fetchImpl
|
|
108
|
+
const res = await untracedFetch(fetchImpl, `http://${ref.host}:${ref.port}/exec/${encodeURIComponent(execId)}`, {
|
|
109
109
|
method: "GET",
|
|
110
110
|
headers: {
|
|
111
111
|
"x-sandbox-signature": reqSignature,
|
|
@@ -272,7 +272,7 @@ async function pollExecOnce(fetchImpl, ref, execId, cursor) {
|
|
|
272
272
|
// replayed without re-executing the body.
|
|
273
273
|
const reqTimestamp = Math.floor(Date.now() / 1000);
|
|
274
274
|
const reqSignature = signCallback({ execId, body: "", timestamp: reqTimestamp, secret: ref.callbackSecret });
|
|
275
|
-
const res = await fetchImpl
|
|
275
|
+
const res = await untracedFetch(fetchImpl, `http://${ref.host}:${ref.port}/exec/${encodeURIComponent(execId)}?since=${cursor}`, {
|
|
276
276
|
method: "GET",
|
|
277
277
|
headers: {
|
|
278
278
|
"x-sandbox-signature": reqSignature,
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
* untouched.
|
|
18
18
|
*/
|
|
19
19
|
import { DBOS } from "@dbos-inc/dbos-sdk";
|
|
20
|
-
import { ATTR_INFLEXA_EXEC_ID, stableSpan } from "../lib/otel-spans.js";
|
|
20
|
+
import { ATTR_INFLEXA_EXEC_ID, stableSpan, untracedFetch } from "../lib/otel-spans.js";
|
|
21
21
|
import { EXEC_STREAM_BYTE_CAP } from "../tools/workspace/result-bounds.js";
|
|
22
22
|
import { signCallback } from "./hmac.js";
|
|
23
23
|
const SUBMIT_HTTP_TIMEOUT_MS = 30_000;
|
|
@@ -33,7 +33,7 @@ async function postExec(fetchImpl, ref, body) {
|
|
|
33
33
|
const raw = JSON.stringify(body);
|
|
34
34
|
const timestamp = Math.floor(Date.now() / 1000);
|
|
35
35
|
const signature = signCallback({ execId: body.execId, body: raw, timestamp, secret: ref.callbackSecret });
|
|
36
|
-
const res = await fetchImpl
|
|
36
|
+
const res = await untracedFetch(fetchImpl, `http://${ref.host}:${ref.port}/exec`, {
|
|
37
37
|
method: "POST",
|
|
38
38
|
headers: {
|
|
39
39
|
"content-type": "application/json",
|
package/package.json
CHANGED