@hue-run/sdk 0.11.4 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI.md +3 -3
- package/EVALUATIONS.md +25 -9
- package/dist/cli/eval.js +17 -1
- package/dist/client.d.ts +3 -0
- package/dist/client.js +36 -24
- package/dist/evals/client.d.ts +25 -5
- package/dist/evals/client.js +93 -15
- package/dist/evals/environment-target.d.ts +15 -8
- package/dist/evals/environment-target.js +61 -21
- package/dist/evals/local-worker.d.ts +38 -3
- package/dist/evals/local-worker.js +158 -34
- package/dist/evals/runner.d.ts +24 -0
- package/dist/evals/runner.js +154 -12
- package/dist/evals.d.ts +3 -3
- package/dist/evals.js +2 -2
- package/dist/transport.d.ts +7 -0
- package/dist/transport.js +76 -15
- package/dist/types.d.ts +7 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
package/dist/evals/runner.js
CHANGED
|
@@ -65,6 +65,36 @@ export class TargetCancelledError extends Error {
|
|
|
65
65
|
this.name = "TargetCancelledError";
|
|
66
66
|
}
|
|
67
67
|
}
|
|
68
|
+
/** Thrown by the environment target when the target did not return before its world's deadline.
|
|
69
|
+
* The world is sealed abandoned and the attempt saved as an `error` of type `TargetTimeout`, so a
|
|
70
|
+
* hung agent ends its case instead of holding the run open. */
|
|
71
|
+
export class TargetTimeoutError extends Error {
|
|
72
|
+
environmentRunId;
|
|
73
|
+
expiresAt;
|
|
74
|
+
constructor(
|
|
75
|
+
/** The world whose deadline passed. */
|
|
76
|
+
environmentRunId,
|
|
77
|
+
/** The world's `expiresAt`. */
|
|
78
|
+
expiresAt) {
|
|
79
|
+
super(`Target did not finish before its world ${environmentRunId} expired at ${expiresAt}`);
|
|
80
|
+
this.environmentRunId = environmentRunId;
|
|
81
|
+
this.expiresAt = expiresAt;
|
|
82
|
+
this.name = "TargetTimeoutError";
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
/** The target's outcome is saved but the telemetry it requires was never acknowledged on an
|
|
86
|
+
* earlier attempt. Restore or export the trace, or complete with omitted evidence through the
|
|
87
|
+
* client; the target is never rerun. It is this case's own state, not the run's. */
|
|
88
|
+
export class TraceExportUnacknowledgedError extends Error {
|
|
89
|
+
executionId;
|
|
90
|
+
constructor(
|
|
91
|
+
/** The execution whose saved outcome waits for its trace. */
|
|
92
|
+
executionId) {
|
|
93
|
+
super("Target outcome is saved but trace export acknowledgement is unavailable. Restore/export the trace or explicitly complete with omitted evidence through the client; never rerun the target.");
|
|
94
|
+
this.executionId = executionId;
|
|
95
|
+
this.name = "TraceExportUnacknowledgedError";
|
|
96
|
+
}
|
|
97
|
+
}
|
|
68
98
|
/** Thrown when the target or world may have committed but acknowledgement is unavailable. */
|
|
69
99
|
export class TargetOutcomeUncertainError extends Error {
|
|
70
100
|
executionId;
|
|
@@ -177,25 +207,82 @@ async function allPages(page, maximum = 5000) {
|
|
|
177
207
|
} while (after !== undefined);
|
|
178
208
|
return items;
|
|
179
209
|
}
|
|
210
|
+
/** A failure the next case would meet too: Hue not answering or refusing (a transient failure
|
|
211
|
+
* past the client's retries, a lost connection, a refusal such as a finished experiment or a
|
|
212
|
+
* world that cannot be sealed), telemetry not accepted by the one transport the cases share, or
|
|
213
|
+
* a target outcome left uncertain because its world could not be sealed or read back. Starting
|
|
214
|
+
* more cases into it only piles up the same failure; the run is resumed later instead. */
|
|
215
|
+
function systemic(error) {
|
|
216
|
+
// A saved outcome still waiting for its trace (`TraceExportUnacknowledgedError`) is not
|
|
217
|
+
// systemic: the flush failed on an earlier attempt, telemetry may have recovered since, and
|
|
218
|
+
// the cases beside it can complete now; only a live export failure stops new cases.
|
|
219
|
+
return (error instanceof HueApiError ||
|
|
220
|
+
error instanceof HueExportError ||
|
|
221
|
+
error instanceof TargetOutcomeUncertainError);
|
|
222
|
+
}
|
|
223
|
+
/** Runs every case, `concurrency` at a time. One case's own failure (an attempt without a saved
|
|
224
|
+
* outcome, an output that cannot be saved, inputs the SDK refuses) does not stop the cases
|
|
225
|
+
* beside or after it: each is attempted, and the failures are thrown together at the end, so a
|
|
226
|
+
* resume has only the failed cases left to publish. A systemic failure stops new cases from
|
|
227
|
+
* starting, while the cases in flight finish; the run resumes when Hue answers again. */
|
|
180
228
|
async function pool(items, concurrency, execute) {
|
|
181
229
|
let position = 0;
|
|
182
230
|
const failures = [];
|
|
231
|
+
let stop = false;
|
|
183
232
|
await Promise.all(Array.from({ length: Math.min(concurrency, items.length) }, async () => {
|
|
184
|
-
while (position < items.length && !
|
|
233
|
+
while (position < items.length && !stop) {
|
|
185
234
|
const item = items[position++];
|
|
186
235
|
try {
|
|
187
236
|
await execute(item);
|
|
188
237
|
}
|
|
189
238
|
catch (error) {
|
|
190
239
|
failures.push(error);
|
|
240
|
+
if (systemic(error))
|
|
241
|
+
stop = true;
|
|
191
242
|
}
|
|
192
243
|
}
|
|
193
244
|
}));
|
|
194
245
|
if (failures.length === 1)
|
|
195
246
|
throw failures[0];
|
|
196
|
-
if (failures.length)
|
|
197
|
-
|
|
247
|
+
if (failures.length) {
|
|
248
|
+
const first = failures[0];
|
|
249
|
+
throw new AggregateError(failures, `${failures.length} case operations failed; resume uses saved outcomes. First: ${first instanceof Error ? first.message : String(first)}`);
|
|
250
|
+
}
|
|
198
251
|
}
|
|
252
|
+
/** Whether Hue holds the case's telemetry, read from the trace receipt: its root span and at
|
|
253
|
+
* least as many spans as the case ended; false when they are not there by the wait's end or the
|
|
254
|
+
* receipt cannot be read, and for a checkpoint an older runner saved without the span. */
|
|
255
|
+
async function traceLanded(hue, trace) {
|
|
256
|
+
// Without the count of spans the case ended, the receipt cannot tell a lost child span from a
|
|
257
|
+
// landed trace, and does not accept.
|
|
258
|
+
if (!trace || trace.spans === undefined)
|
|
259
|
+
return false;
|
|
260
|
+
const deadline = Date.now() + RECEIPT_WAIT_MS;
|
|
261
|
+
try {
|
|
262
|
+
for (;;) {
|
|
263
|
+
// The root span present and every span the case ended held: a lost child span is not
|
|
264
|
+
// hidden behind a root that landed. The receipt answers as soon as the root is there, so
|
|
265
|
+
// spans still arriving behind it are polled for until the wait ends.
|
|
266
|
+
const verification = await hue.verifyTrace(trace.traceId, {
|
|
267
|
+
expectedSpanIds: [trace.spanId],
|
|
268
|
+
timeoutMillis: Math.max(250, deadline - Date.now()),
|
|
269
|
+
});
|
|
270
|
+
if (verification.verified && (verification.receipt?.spanCount ?? 0) >= trace.spans)
|
|
271
|
+
return true;
|
|
272
|
+
if (!verification.verified || Date.now() + RECEIPT_POLL_MS >= deadline)
|
|
273
|
+
return false;
|
|
274
|
+
await new Promise((resolve) => setTimeout(resolve, RECEIPT_POLL_MS));
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
catch {
|
|
278
|
+
return false;
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
/** How long a receipt is polled for the root span after a flush reported a failure: ingestion
|
|
282
|
+
* of an accepted batch is not instant, and a span that is not there by then was not delivered. */
|
|
283
|
+
const RECEIPT_WAIT_MS = 5_000;
|
|
284
|
+
/** How often the receipt is read again while the root is there but spans are still arriving. */
|
|
285
|
+
const RECEIPT_POLL_MS = 500;
|
|
199
286
|
async function scoresFor(versions, context, options, executionId, hasEnvironment = true) {
|
|
200
287
|
const scores = [];
|
|
201
288
|
let environmentUnavailable = false;
|
|
@@ -329,8 +416,14 @@ export async function runExperiment(options) {
|
|
|
329
416
|
try {
|
|
330
417
|
const filesRoot = resolve(options.filesDirectory ?? join(options.checkpointDirectory, "files"));
|
|
331
418
|
const sanitize = (message) => message.slice(0, 4000).toWellFormed().replaceAll("\u0000", "");
|
|
419
|
+
// A target stopped at its world's deadline is a `TargetTimeout`, told apart from an error the
|
|
420
|
+
// target raised itself: the agent did not finish, rather than finishing wrongly.
|
|
332
421
|
const errorPayload = (error) => ({
|
|
333
|
-
type: error instanceof OutputTooLargeError
|
|
422
|
+
type: error instanceof OutputTooLargeError
|
|
423
|
+
? "OutputTooLarge"
|
|
424
|
+
: error instanceof TargetTimeoutError
|
|
425
|
+
? "TargetTimeout"
|
|
426
|
+
: "TargetError",
|
|
334
427
|
...(options.persistResultContent && error instanceof Error
|
|
335
428
|
? { message: sanitize(error.message) }
|
|
336
429
|
: {}),
|
|
@@ -378,6 +471,7 @@ export async function runExperiment(options) {
|
|
|
378
471
|
complete,
|
|
379
472
|
scores,
|
|
380
473
|
exportState: "pending",
|
|
474
|
+
...(saved.trace ? { trace: saved.trace } : {}),
|
|
381
475
|
};
|
|
382
476
|
await store.write(file, prepared);
|
|
383
477
|
return prepared;
|
|
@@ -533,6 +627,7 @@ export async function runExperiment(options) {
|
|
|
533
627
|
const uploading = {
|
|
534
628
|
stage: "uploading",
|
|
535
629
|
executionId: execution.id,
|
|
630
|
+
trace: { traceId: span.traceId, spanId: span.spanId },
|
|
536
631
|
state,
|
|
537
632
|
hasOutput: output !== undefined,
|
|
538
633
|
...(options.persistResultContent && output !== undefined ? { output } : {}),
|
|
@@ -553,13 +648,50 @@ export async function runExperiment(options) {
|
|
|
553
648
|
// End root before waiting for both OTLP signals. Export failure leaves the prepared checkpoint intact.
|
|
554
649
|
let exportError;
|
|
555
650
|
try {
|
|
556
|
-
|
|
557
|
-
//
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
651
|
+
// The flush itself throws for any failure since the last flush; the issues it recorded
|
|
652
|
+
// are read below, where they are attributed.
|
|
653
|
+
await options.hue.flush().catch((error) => {
|
|
654
|
+
if (!(error instanceof HueExportError))
|
|
655
|
+
throw error;
|
|
656
|
+
});
|
|
657
|
+
// The transport is shared by every case in flight, so a failure the flush reports may
|
|
658
|
+
// be another case's. Each issue names the traces of the records it concerned: one that
|
|
659
|
+
// names this trace is this case's failure, final because its evidence is incomplete;
|
|
660
|
+
// one that names only others is not this case's; one that names no trace (a processor
|
|
661
|
+
// flush that failed) may be anyone's, and this case's trace receipt decides: its root
|
|
662
|
+
// span landed, or the case stays pending for the receipt to decide again on resume.
|
|
663
|
+
const decided = checkpoint;
|
|
664
|
+
const traceId = decided.trace?.traceId;
|
|
665
|
+
// Every span the case ended has reached the exporter by now; the receipt is held to
|
|
666
|
+
// that count, so a lost child span is not hidden behind a root span that landed. The
|
|
667
|
+
// count is saved before anything is decided, so a process that stops here leaves a
|
|
668
|
+
// pending checkpoint the receipt can be held to; a transport that does not count (an
|
|
669
|
+
// older or borrowed one) leaves no count, and the receipt then cannot accept.
|
|
670
|
+
if (decided.trace && traceId !== undefined) {
|
|
671
|
+
const spans = options.hue.transport.spansEnded?.(traceId);
|
|
672
|
+
if (spans !== undefined) {
|
|
673
|
+
decided.trace.spans = spans;
|
|
674
|
+
await store.write(file, decided);
|
|
675
|
+
}
|
|
676
|
+
}
|
|
677
|
+
if (options.hue.transport.getFailureSequence() !== failureSequenceBefore) {
|
|
678
|
+
const retained = options.hue.transport.getIssues();
|
|
679
|
+
const issues = retained.filter((issue) => issue.sequence > failureSequenceBefore && issue.kind !== "warning");
|
|
680
|
+
const named = issues.filter((issue) => issue.traceIds && (traceId === undefined || issue.traceIds.includes(traceId)));
|
|
681
|
+
// An issue naming no trace, and a failure whose issue the bounded history has already
|
|
682
|
+
// rolled past (the oldest retained issue is younger than the first this case could
|
|
683
|
+
// have caused), may be anyone's.
|
|
684
|
+
const unattributed = issues.filter((issue) => !issue.traceIds);
|
|
685
|
+
const evicted = issues.length === 0 || (retained[0]?.sequence ?? 0) > failureSequenceBefore + 1;
|
|
686
|
+
if (named.length) {
|
|
687
|
+
checkpoint.exportState = "failed";
|
|
688
|
+
await store.write(file, checkpoint);
|
|
689
|
+
throw new HueExportError([...named, ...unattributed], options.hue.transport.getReport());
|
|
690
|
+
}
|
|
691
|
+
if ((unattributed.length || evicted) &&
|
|
692
|
+
!(await traceLanded(options.hue, decided.trace)))
|
|
693
|
+
throw new HueExportError(unattributed, options.hue.transport.getReport());
|
|
694
|
+
}
|
|
563
695
|
}
|
|
564
696
|
catch (error) {
|
|
565
697
|
exportError = error;
|
|
@@ -606,8 +738,18 @@ export async function runExperiment(options) {
|
|
|
606
738
|
}
|
|
607
739
|
const prepared = checkpoint;
|
|
608
740
|
await options.client.getExecution(prepared.executionId);
|
|
741
|
+
// A saved outcome no flush decided on is asked about through its trace receipt: the root
|
|
742
|
+
// span there says the case's export landed before the process stopped, or that a later
|
|
743
|
+
// flush of the same transport delivered it. An export failure that named this trace is
|
|
744
|
+
// final: the evidence is incomplete whatever the receipt holds.
|
|
745
|
+
if (prepared.exportState === "pending" &&
|
|
746
|
+
prepared.complete.traceEvidence !== "omit" &&
|
|
747
|
+
(await traceLanded(options.hue, prepared.trace))) {
|
|
748
|
+
prepared.exportState = "accepted";
|
|
749
|
+
await store.write(file, prepared);
|
|
750
|
+
}
|
|
609
751
|
if (prepared.exportState !== "accepted" && prepared.complete.traceEvidence !== "omit")
|
|
610
|
-
throw new
|
|
752
|
+
throw new TraceExportUnacknowledgedError(prepared.executionId);
|
|
611
753
|
const save = () => store.write(file, prepared);
|
|
612
754
|
const notAccepted = prepared.exportState === "not_accepted"
|
|
613
755
|
? {
|
package/dist/evals.d.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
export { ArtifactSizeError, createEvaluationClient, EvaluationClient, HueApiError, } from "./evals/client.js";
|
|
1
|
+
export { ArtifactSizeError, createEvaluationClient, EvaluationClient, HueApiError, isTransientApiError, } from "./evals/client.js";
|
|
2
2
|
export type { EvaluationClientOptions } from "./evals/client.js";
|
|
3
|
-
export { runExperiment, rescore, UncertainExecutionError, OutcomeSerializationError, TargetCancelledError, TargetOutcomeUncertainError, } from "./evals/runner.js";
|
|
3
|
+
export { runExperiment, rescore, UncertainExecutionError, OutcomeSerializationError, TargetCancelledError, TargetOutcomeUncertainError, TargetTimeoutError, TraceExportUnacknowledgedError, } from "./evals/runner.js";
|
|
4
4
|
export type { RunExperimentOptions, RunExperimentTargetContext, RescoreOptions, RunnerReport, TelemetryIssueCount, TelemetryNotAccepted, } from "./evals/runner.js";
|
|
5
5
|
export { TargetResult, withFiles } from "./evals/types.js";
|
|
6
6
|
export { CaseFileError, outputContentTypes, outputFileLimits, OutputFileError, safeFilename, targetFileRoles, } from "./evals/files.js";
|
|
@@ -13,7 +13,7 @@ export { builtins, defineLocalScorer, scoreLocally } from "./evals/scorers.js";
|
|
|
13
13
|
export { sourceDigest } from "./evals/json.js";
|
|
14
14
|
export type * from "./evals/types.js";
|
|
15
15
|
export { localAgentCapabilities, registeredCapabilities, runLocalAgent, } from "./evals/local-worker.js";
|
|
16
|
-
export type { LocalAgentDirectContext, LocalAgentTargetContext, RunLocalAgentOptions, } from "./evals/local-worker.js";
|
|
16
|
+
export type { LocalAgentDirectContext, LocalAgentTargetContext, LocalRunFailure, RunLocalAgentOptions, } from "./evals/local-worker.js";
|
|
17
17
|
export { getScenario, listScenarios, matchByName, parseScenarioSelector, resolveEvalSetPins, resolveScenarioPins, } from "./evals/scenarios.js";
|
|
18
18
|
export type { NamedCandidate, NameMatch, ScenarioClient, ScenarioPins, ScenarioSelector, } from "./evals/scenarios.js";
|
|
19
19
|
export { collectExperimentVerdicts, compareVerdicts, metricPassed, summarizeVerdicts, waitForResults, } from "./evals/verdicts.js";
|
package/dist/evals.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
export { ArtifactSizeError, createEvaluationClient, EvaluationClient, HueApiError, } from "./evals/client.js";
|
|
2
|
-
export { runExperiment, rescore, UncertainExecutionError, OutcomeSerializationError, TargetCancelledError, TargetOutcomeUncertainError, } from "./evals/runner.js";
|
|
1
|
+
export { ArtifactSizeError, createEvaluationClient, EvaluationClient, HueApiError, isTransientApiError, } from "./evals/client.js";
|
|
2
|
+
export { runExperiment, rescore, UncertainExecutionError, OutcomeSerializationError, TargetCancelledError, TargetOutcomeUncertainError, TargetTimeoutError, TraceExportUnacknowledgedError, } from "./evals/runner.js";
|
|
3
3
|
export { TargetResult, withFiles } from "./evals/types.js";
|
|
4
4
|
export { CaseFileError, outputContentTypes, outputFileLimits, OutputFileError, safeFilename, targetFileRoles, } from "./evals/files.js";
|
|
5
5
|
export { runSimulation } from "./evals/simulation.js";
|
package/dist/transport.d.ts
CHANGED
|
@@ -75,6 +75,13 @@ export declare class HueTransport {
|
|
|
75
75
|
getIssues(): ExportIssue[];
|
|
76
76
|
/** Monotonic failure marker, retained even when the bounded issue history rolls over. */
|
|
77
77
|
getFailureSequence(): number;
|
|
78
|
+
/** Spans of a trace this transport has handed to its exporter, by trace id, for the newest
|
|
79
|
+
* traces only: a runner compares the count with the trace receipt's span count to know that
|
|
80
|
+
* every span of a case landed, not only the root. Undefined for a trace it does not remember,
|
|
81
|
+
* which is not the same as none: the caller must not take an unknown count for complete. */
|
|
82
|
+
spansEnded(traceId: string): number | undefined;
|
|
83
|
+
private endedSpans;
|
|
84
|
+
private countEnded;
|
|
78
85
|
/**
|
|
79
86
|
* Waits for the processors' and exporters' in-flight work; drain the providers first.
|
|
80
87
|
*
|
package/dist/transport.js
CHANGED
|
@@ -138,12 +138,13 @@ export class HueTransport {
|
|
|
138
138
|
if (!queued)
|
|
139
139
|
return;
|
|
140
140
|
admitted = queued;
|
|
141
|
+
this.countEnded(admitted);
|
|
141
142
|
spans.onEnd(admitted);
|
|
142
143
|
}
|
|
143
144
|
catch {
|
|
144
145
|
if (admitted)
|
|
145
146
|
this.finish("traces", [admitted]);
|
|
146
|
-
this.issue("traces", "invalid", 1, "Telemetry processor could not accept a record");
|
|
147
|
+
this.issue("traces", "invalid", 1, "Telemetry processor could not accept a record", undefined, traceIdsOf([span]));
|
|
147
148
|
}
|
|
148
149
|
},
|
|
149
150
|
forceFlush: () => spans.forceFlush(),
|
|
@@ -165,7 +166,7 @@ export class HueTransport {
|
|
|
165
166
|
catch {
|
|
166
167
|
if (admitted)
|
|
167
168
|
this.finish("logs", [admitted]);
|
|
168
|
-
this.issue("logs", "invalid", 1, "Telemetry processor could not accept a record");
|
|
169
|
+
this.issue("logs", "invalid", 1, "Telemetry processor could not accept a record", undefined, traceIdsOf([log]));
|
|
169
170
|
}
|
|
170
171
|
},
|
|
171
172
|
forceFlush: () => logs.forceFlush(),
|
|
@@ -181,10 +182,13 @@ export class HueTransport {
|
|
|
181
182
|
const pending = signal === "traces" ? this.spans : this.logs;
|
|
182
183
|
if (advisory && (this.closed || pending.size >= 2048 / 4))
|
|
183
184
|
return undefined;
|
|
185
|
+
// A record dropped here names its trace, so a runner with several traces in flight knows
|
|
186
|
+
// whose telemetry is missing; a log record names the trace of the span it was emitted in.
|
|
187
|
+
const traceIds = traceIdsOf([record]);
|
|
184
188
|
if (this.closed || pending.size >= 2048) {
|
|
185
189
|
this.issue(signal, "dropped", 1, this.closed
|
|
186
190
|
? "Telemetry emitted after transport shutdown"
|
|
187
|
-
: "Telemetry queue reached 2048 records");
|
|
191
|
+
: "Telemetry queue reached 2048 records", undefined, traceIds);
|
|
188
192
|
return undefined;
|
|
189
193
|
}
|
|
190
194
|
try {
|
|
@@ -205,7 +209,7 @@ export class HueTransport {
|
|
|
205
209
|
catch {
|
|
206
210
|
if (advisory)
|
|
207
211
|
return undefined;
|
|
208
|
-
this.issue(signal, "dropped", 1, "Telemetry snapshot exceeded its byte or complexity budget or contained unsupported data");
|
|
212
|
+
this.issue(signal, "dropped", 1, "Telemetry snapshot exceeded its byte or complexity budget or contained unsupported data", undefined, traceIds);
|
|
209
213
|
return undefined;
|
|
210
214
|
}
|
|
211
215
|
}
|
|
@@ -299,7 +303,7 @@ export class HueTransport {
|
|
|
299
303
|
this.accepted[signal] += count;
|
|
300
304
|
}
|
|
301
305
|
/** @internal Records a sanitized issue, updates counters and rate-limits the diagnostic callback. */
|
|
302
|
-
issue(signal, kind, count, message, status) {
|
|
306
|
+
issue(signal, kind, count, message, status, traceIds) {
|
|
303
307
|
if (kind === "dropped")
|
|
304
308
|
this.dropped[signal] += count;
|
|
305
309
|
if (kind === "rejected")
|
|
@@ -313,6 +317,7 @@ export class HueTransport {
|
|
|
313
317
|
count,
|
|
314
318
|
message,
|
|
315
319
|
...(status !== undefined ? { status } : {}),
|
|
320
|
+
...(traceIds !== undefined ? { traceIds } : {}),
|
|
316
321
|
};
|
|
317
322
|
if (kind !== "warning")
|
|
318
323
|
this.failureSequence = issue.sequence;
|
|
@@ -339,11 +344,13 @@ export class HueTransport {
|
|
|
339
344
|
}
|
|
340
345
|
}
|
|
341
346
|
/** @internal Counts a helper capture or instrumentation failure that preserved application execution. */
|
|
342
|
-
|
|
347
|
+
/** Records a capture or instrumentation failure, named to the trace it happened in when the
|
|
348
|
+
* caller knows it, so a runner with several traces in flight can tell whose it was. */
|
|
349
|
+
instrumentationFailure(signal = "traces", message = "Telemetry capture or instrumentation failed; application execution was preserved", count = 1, traceId) {
|
|
343
350
|
if (!Number.isSafeInteger(count) || count < 1)
|
|
344
351
|
return;
|
|
345
352
|
this.instrumentationFailures += count;
|
|
346
|
-
this.issue(signal, "invalid", 0, message);
|
|
353
|
+
this.issue(signal, "invalid", 0, message, undefined, traceId ? [traceId] : undefined);
|
|
347
354
|
}
|
|
348
355
|
/** Cumulative counters and current queue gauges. */
|
|
349
356
|
getReport() {
|
|
@@ -370,6 +377,26 @@ export class HueTransport {
|
|
|
370
377
|
getFailureSequence() {
|
|
371
378
|
return this.failureSequence;
|
|
372
379
|
}
|
|
380
|
+
/** Spans of a trace this transport has handed to its exporter, by trace id, for the newest
|
|
381
|
+
* traces only: a runner compares the count with the trace receipt's span count to know that
|
|
382
|
+
* every span of a case landed, not only the root. Undefined for a trace it does not remember,
|
|
383
|
+
* which is not the same as none: the caller must not take an unknown count for complete. */
|
|
384
|
+
spansEnded(traceId) {
|
|
385
|
+
return this.endedSpans.get(traceId);
|
|
386
|
+
}
|
|
387
|
+
endedSpans = new Map();
|
|
388
|
+
countEnded(span) {
|
|
389
|
+
let traceId;
|
|
390
|
+
try {
|
|
391
|
+
traceId = span.spanContext().traceId;
|
|
392
|
+
}
|
|
393
|
+
catch {
|
|
394
|
+
return;
|
|
395
|
+
}
|
|
396
|
+
this.endedSpans.set(traceId, (this.endedSpans.get(traceId) ?? 0) + 1);
|
|
397
|
+
if (this.endedSpans.size > MAX_REMEMBERED_TRACES)
|
|
398
|
+
this.endedSpans.delete(this.endedSpans.keys().next().value);
|
|
399
|
+
}
|
|
373
400
|
/**
|
|
374
401
|
* Waits for the processors' and exporters' in-flight work; drain the providers first.
|
|
375
402
|
*
|
|
@@ -427,6 +454,32 @@ export class HueTransport {
|
|
|
427
454
|
return this.shutdownPromise;
|
|
428
455
|
}
|
|
429
456
|
}
|
|
457
|
+
/** The distinct trace ids of a batch, or undefined past 64 of them, when an issue would name too
|
|
458
|
+
* many traces to tell one apart; a record whose context cannot be read is left out. */
|
|
459
|
+
const MAX_ISSUE_TRACE_IDS = 64;
|
|
460
|
+
/** How many traces' ended-span counts one transport remembers; the oldest is forgotten first. */
|
|
461
|
+
const MAX_REMEMBERED_TRACES = 10_000;
|
|
462
|
+
function traceIdsOf(records) {
|
|
463
|
+
const ids = new Set();
|
|
464
|
+
for (const record of records) {
|
|
465
|
+
try {
|
|
466
|
+
// A span reports its context; a log record carries the span's identifiers.
|
|
467
|
+
const traceId = "spanContext" in record && typeof record.spanContext === "function"
|
|
468
|
+
? record.spanContext().traceId
|
|
469
|
+
: record.spanContext?.traceId;
|
|
470
|
+
if (traceId)
|
|
471
|
+
ids.add(traceId);
|
|
472
|
+
}
|
|
473
|
+
catch {
|
|
474
|
+
// A record without a readable context names no trace.
|
|
475
|
+
}
|
|
476
|
+
if (ids.size > MAX_ISSUE_TRACE_IDS)
|
|
477
|
+
return undefined;
|
|
478
|
+
}
|
|
479
|
+
// No readable trace names nobody, which is not the same as naming no one's: the issue may
|
|
480
|
+
// concern any trace in flight.
|
|
481
|
+
return ids.size ? [...ids] : undefined;
|
|
482
|
+
}
|
|
430
483
|
class ReportingExporter {
|
|
431
484
|
transport;
|
|
432
485
|
signal;
|
|
@@ -485,12 +538,12 @@ class ReportingExporter {
|
|
|
485
538
|
const cache = new WeakMap();
|
|
486
539
|
let failed = false;
|
|
487
540
|
let redactedBytes = 0;
|
|
488
|
-
const invalid = (placeholder, message) => {
|
|
541
|
+
const invalid = (placeholder, message, record) => {
|
|
489
542
|
if (placeholder)
|
|
490
543
|
this.transport.issue(this.signal, "warning", 1, `${message} (in-progress span placeholder)`);
|
|
491
544
|
else {
|
|
492
545
|
failed = true;
|
|
493
|
-
this.transport.issue(this.signal, "invalid", 1, message);
|
|
546
|
+
this.transport.issue(this.signal, "invalid", 1, message, undefined, record ? traceIdsOf([record]) : undefined);
|
|
494
547
|
}
|
|
495
548
|
};
|
|
496
549
|
const send = async (batch) => {
|
|
@@ -539,7 +592,7 @@ class ReportingExporter {
|
|
|
539
592
|
placeholders.add(redacted);
|
|
540
593
|
}
|
|
541
594
|
catch {
|
|
542
|
-
invalid(markers !== undefined, "Telemetry record could not be redacted or exceeds supported content limits");
|
|
595
|
+
invalid(markers !== undefined, "Telemetry record could not be redacted or exceeds supported content limits", record);
|
|
543
596
|
}
|
|
544
597
|
}
|
|
545
598
|
// Each record is encoded once to measure it; a request is encoded once more when it is sent.
|
|
@@ -555,7 +608,7 @@ class ReportingExporter {
|
|
|
555
608
|
recordBytes = this.serializer.serializeRequest([record])?.byteLength ?? Infinity;
|
|
556
609
|
}
|
|
557
610
|
catch {
|
|
558
|
-
invalid(placeholders.has(record), "Telemetry record could not be serialized");
|
|
611
|
+
invalid(placeholders.has(record), "Telemetry record could not be serialized", record);
|
|
559
612
|
continue;
|
|
560
613
|
}
|
|
561
614
|
const framedBytes = recordBytes + RECORD_FRAMING_BYTES;
|
|
@@ -565,7 +618,7 @@ class ReportingExporter {
|
|
|
565
618
|
batchBytes = 0;
|
|
566
619
|
}
|
|
567
620
|
if (recordBytes > limit) {
|
|
568
|
-
invalid(placeholders.has(record), "Telemetry record exceeds the 1 MiB request limit");
|
|
621
|
+
invalid(placeholders.has(record), "Telemetry record exceeds the 1 MiB request limit", record);
|
|
569
622
|
continue;
|
|
570
623
|
}
|
|
571
624
|
batch.push(record);
|
|
@@ -580,9 +633,12 @@ class ReportingExporter {
|
|
|
580
633
|
async send(records, placeholders = 0) {
|
|
581
634
|
const options = this.transport.options;
|
|
582
635
|
const real = records.length - placeholders;
|
|
636
|
+
// The traces this batch carries, spans or the log records emitted in their spans, so a loss
|
|
637
|
+
// or rejection names whose telemetry it was.
|
|
638
|
+
const traceIds = traceIdsOf(records);
|
|
583
639
|
// A loss involving only placeholders is a warning. Mixed losses count only real records.
|
|
584
640
|
const lose = (message, status) => {
|
|
585
|
-
this.transport.issue(this.signal, real ? "failed" : "warning", real || placeholders, real ? message : `${message} (in-progress span placeholders only)`, status);
|
|
641
|
+
this.transport.issue(this.signal, real ? "failed" : "warning", real || placeholders, real ? message : `${message} (in-progress span placeholders only)`, status, traceIds);
|
|
586
642
|
return !real;
|
|
587
643
|
};
|
|
588
644
|
let rejected = 0;
|
|
@@ -611,14 +667,19 @@ class ReportingExporter {
|
|
|
611
667
|
if (downgrade)
|
|
612
668
|
this.transport.rejectPlaceholders(placeholderRejections);
|
|
613
669
|
const remaining = count - placeholderRejections;
|
|
614
|
-
// Rejections are not matched to records. Attribute them to real records first.
|
|
670
|
+
// Rejections are not matched to records. Attribute them to real records first. A
|
|
671
|
+
// rejection of spans in a batch of one trace is that trace's; in a batch of several it
|
|
672
|
+
// names none, since the innocent traces' spans may have been accepted, and each case's
|
|
673
|
+
// receipt decides. A rejection of log records names every trace in the batch: the
|
|
674
|
+
// receipt counts spans, not logs, so it could not tell whose logs were lost.
|
|
615
675
|
rejected = Math.min(remaining, real);
|
|
676
|
+
const rejectedTraceIds = this.signal === "logs" || (traceIds && traceIds.length === 1) ? traceIds : undefined;
|
|
616
677
|
if (remaining || (partial?.errorMessage && !downgrade))
|
|
617
678
|
this.transport.issue(this.signal, rejected ? "rejected" : "warning", rejected || remaining, rejected
|
|
618
679
|
? "Hue rejected telemetry records; inspect the project ingestion settings and supported limits"
|
|
619
680
|
: remaining
|
|
620
681
|
? "Hue rejected in-progress span placeholders"
|
|
621
|
-
: "Hue returned an ingestion warning");
|
|
682
|
+
: "Hue returned an ingestion warning", undefined, rejectedTraceIds);
|
|
622
683
|
// Do not pass backend error text or raw response bytes into the global OTel diagnostic logger.
|
|
623
684
|
return {};
|
|
624
685
|
}
|
package/dist/types.d.ts
CHANGED
|
@@ -87,6 +87,13 @@ export interface ExportIssue {
|
|
|
87
87
|
status?: number;
|
|
88
88
|
/** Fixed, sanitized description; never server text or content. */
|
|
89
89
|
message: string;
|
|
90
|
+
/** The traces the records concerned belonged to, when the issue arose with records in hand
|
|
91
|
+
* (a refused or rejected batch, a dropped record, a record the processor could not accept, a
|
|
92
|
+
* capture failure inside a span): up to 64 distinct ids, so a caller running several traces
|
|
93
|
+
* at once can tell whose telemetry the issue was. Absent for an issue without records (a
|
|
94
|
+
* processor flush that failed) and for a batch of more traces than that; such an issue may
|
|
95
|
+
* concern any trace in flight. */
|
|
96
|
+
traceIds?: string[];
|
|
90
97
|
}
|
|
91
98
|
/** Cumulative delivery counters and current queue gauges for one transport. */
|
|
92
99
|
export interface ExportReport {
|
package/dist/version.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
/** Package version shared by the instrumentation scope and the export User-Agent. */
|
|
2
|
-
export declare const sdkVersion = "0.
|
|
2
|
+
export declare const sdkVersion = "0.12.0";
|
package/dist/version.js
CHANGED