@hue-run/sdk 0.11.3 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI.md +33 -18
- package/EVALUATIONS.md +25 -9
- package/README.md +6 -5
- package/dist/cli/eval.js +23 -5
- package/dist/cli/login.js +2 -1
- package/dist/cli/mcp.d.ts +5 -4
- package/dist/cli/mcp.js +245 -31
- package/dist/client.d.ts +3 -0
- package/dist/client.js +36 -24
- package/dist/environment/world.d.ts +4 -2
- package/dist/environment/world.js +13 -4
- package/dist/evals/client.d.ts +25 -5
- package/dist/evals/client.js +93 -15
- package/dist/evals/environment-target.d.ts +15 -8
- package/dist/evals/environment-target.js +61 -21
- package/dist/evals/local-worker.d.ts +38 -3
- package/dist/evals/local-worker.js +158 -34
- package/dist/evals/runner.d.ts +24 -0
- package/dist/evals/runner.js +154 -12
- package/dist/evals.d.ts +3 -3
- package/dist/evals.js +2 -2
- package/dist/transport.d.ts +7 -0
- package/dist/transport.js +76 -15
- package/dist/types.d.ts +7 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
package/dist/transport.js
CHANGED
|
@@ -138,12 +138,13 @@ export class HueTransport {
|
|
|
138
138
|
if (!queued)
|
|
139
139
|
return;
|
|
140
140
|
admitted = queued;
|
|
141
|
+
this.countEnded(admitted);
|
|
141
142
|
spans.onEnd(admitted);
|
|
142
143
|
}
|
|
143
144
|
catch {
|
|
144
145
|
if (admitted)
|
|
145
146
|
this.finish("traces", [admitted]);
|
|
146
|
-
this.issue("traces", "invalid", 1, "Telemetry processor could not accept a record");
|
|
147
|
+
this.issue("traces", "invalid", 1, "Telemetry processor could not accept a record", undefined, traceIdsOf([span]));
|
|
147
148
|
}
|
|
148
149
|
},
|
|
149
150
|
forceFlush: () => spans.forceFlush(),
|
|
@@ -165,7 +166,7 @@ export class HueTransport {
|
|
|
165
166
|
catch {
|
|
166
167
|
if (admitted)
|
|
167
168
|
this.finish("logs", [admitted]);
|
|
168
|
-
this.issue("logs", "invalid", 1, "Telemetry processor could not accept a record");
|
|
169
|
+
this.issue("logs", "invalid", 1, "Telemetry processor could not accept a record", undefined, traceIdsOf([log]));
|
|
169
170
|
}
|
|
170
171
|
},
|
|
171
172
|
forceFlush: () => logs.forceFlush(),
|
|
@@ -181,10 +182,13 @@ export class HueTransport {
|
|
|
181
182
|
const pending = signal === "traces" ? this.spans : this.logs;
|
|
182
183
|
if (advisory && (this.closed || pending.size >= 2048 / 4))
|
|
183
184
|
return undefined;
|
|
185
|
+
// A record dropped here names its trace, so a runner with several traces in flight knows
|
|
186
|
+
// whose telemetry is missing; a log record names the trace of the span it was emitted in.
|
|
187
|
+
const traceIds = traceIdsOf([record]);
|
|
184
188
|
if (this.closed || pending.size >= 2048) {
|
|
185
189
|
this.issue(signal, "dropped", 1, this.closed
|
|
186
190
|
? "Telemetry emitted after transport shutdown"
|
|
187
|
-
: "Telemetry queue reached 2048 records");
|
|
191
|
+
: "Telemetry queue reached 2048 records", undefined, traceIds);
|
|
188
192
|
return undefined;
|
|
189
193
|
}
|
|
190
194
|
try {
|
|
@@ -205,7 +209,7 @@ export class HueTransport {
|
|
|
205
209
|
catch {
|
|
206
210
|
if (advisory)
|
|
207
211
|
return undefined;
|
|
208
|
-
this.issue(signal, "dropped", 1, "Telemetry snapshot exceeded its byte or complexity budget or contained unsupported data");
|
|
212
|
+
this.issue(signal, "dropped", 1, "Telemetry snapshot exceeded its byte or complexity budget or contained unsupported data", undefined, traceIds);
|
|
209
213
|
return undefined;
|
|
210
214
|
}
|
|
211
215
|
}
|
|
@@ -299,7 +303,7 @@ export class HueTransport {
|
|
|
299
303
|
this.accepted[signal] += count;
|
|
300
304
|
}
|
|
301
305
|
/** @internal Records a sanitized issue, updates counters and rate-limits the diagnostic callback. */
|
|
302
|
-
issue(signal, kind, count, message, status) {
|
|
306
|
+
issue(signal, kind, count, message, status, traceIds) {
|
|
303
307
|
if (kind === "dropped")
|
|
304
308
|
this.dropped[signal] += count;
|
|
305
309
|
if (kind === "rejected")
|
|
@@ -313,6 +317,7 @@ export class HueTransport {
|
|
|
313
317
|
count,
|
|
314
318
|
message,
|
|
315
319
|
...(status !== undefined ? { status } : {}),
|
|
320
|
+
...(traceIds !== undefined ? { traceIds } : {}),
|
|
316
321
|
};
|
|
317
322
|
if (kind !== "warning")
|
|
318
323
|
this.failureSequence = issue.sequence;
|
|
@@ -339,11 +344,13 @@ export class HueTransport {
|
|
|
339
344
|
}
|
|
340
345
|
}
|
|
341
346
|
/** @internal Counts a helper capture or instrumentation failure that preserved application execution. */
|
|
342
|
-
|
|
347
|
+
/** Records a capture or instrumentation failure, named to the trace it happened in when the
|
|
348
|
+
* caller knows it, so a runner with several traces in flight can tell whose it was. */
|
|
349
|
+
instrumentationFailure(signal = "traces", message = "Telemetry capture or instrumentation failed; application execution was preserved", count = 1, traceId) {
|
|
343
350
|
if (!Number.isSafeInteger(count) || count < 1)
|
|
344
351
|
return;
|
|
345
352
|
this.instrumentationFailures += count;
|
|
346
|
-
this.issue(signal, "invalid", 0, message);
|
|
353
|
+
this.issue(signal, "invalid", 0, message, undefined, traceId ? [traceId] : undefined);
|
|
347
354
|
}
|
|
348
355
|
/** Cumulative counters and current queue gauges. */
|
|
349
356
|
getReport() {
|
|
@@ -370,6 +377,26 @@ export class HueTransport {
|
|
|
370
377
|
getFailureSequence() {
|
|
371
378
|
return this.failureSequence;
|
|
372
379
|
}
|
|
380
|
+
/** Spans of a trace this transport has handed to its exporter, by trace id, for the newest
|
|
381
|
+
* traces only: a runner compares the count with the trace receipt's span count to know that
|
|
382
|
+
* every span of a case landed, not only the root. Undefined for a trace it does not remember,
|
|
383
|
+
* which is not the same as none: the caller must not take an unknown count for complete. */
|
|
384
|
+
spansEnded(traceId) {
|
|
385
|
+
return this.endedSpans.get(traceId);
|
|
386
|
+
}
|
|
387
|
+
endedSpans = new Map();
|
|
388
|
+
countEnded(span) {
|
|
389
|
+
let traceId;
|
|
390
|
+
try {
|
|
391
|
+
traceId = span.spanContext().traceId;
|
|
392
|
+
}
|
|
393
|
+
catch {
|
|
394
|
+
return;
|
|
395
|
+
}
|
|
396
|
+
this.endedSpans.set(traceId, (this.endedSpans.get(traceId) ?? 0) + 1);
|
|
397
|
+
if (this.endedSpans.size > MAX_REMEMBERED_TRACES)
|
|
398
|
+
this.endedSpans.delete(this.endedSpans.keys().next().value);
|
|
399
|
+
}
|
|
373
400
|
/**
|
|
374
401
|
* Waits for the processors' and exporters' in-flight work; drain the providers first.
|
|
375
402
|
*
|
|
@@ -427,6 +454,32 @@ export class HueTransport {
|
|
|
427
454
|
return this.shutdownPromise;
|
|
428
455
|
}
|
|
429
456
|
}
|
|
457
|
+
/** The distinct trace ids of a batch, or undefined past 64 of them, when an issue would name too
|
|
458
|
+
* many traces to tell one apart; a record whose context cannot be read is left out. */
|
|
459
|
+
const MAX_ISSUE_TRACE_IDS = 64;
|
|
460
|
+
/** How many traces' ended-span counts one transport remembers; the oldest is forgotten first. */
|
|
461
|
+
const MAX_REMEMBERED_TRACES = 10_000;
|
|
462
|
+
function traceIdsOf(records) {
|
|
463
|
+
const ids = new Set();
|
|
464
|
+
for (const record of records) {
|
|
465
|
+
try {
|
|
466
|
+
// A span reports its context; a log record carries the span's identifiers.
|
|
467
|
+
const traceId = "spanContext" in record && typeof record.spanContext === "function"
|
|
468
|
+
? record.spanContext().traceId
|
|
469
|
+
: record.spanContext?.traceId;
|
|
470
|
+
if (traceId)
|
|
471
|
+
ids.add(traceId);
|
|
472
|
+
}
|
|
473
|
+
catch {
|
|
474
|
+
// A record without a readable context names no trace.
|
|
475
|
+
}
|
|
476
|
+
if (ids.size > MAX_ISSUE_TRACE_IDS)
|
|
477
|
+
return undefined;
|
|
478
|
+
}
|
|
479
|
+
// No readable trace names nobody, which is not the same as naming no one's: the issue may
|
|
480
|
+
// concern any trace in flight.
|
|
481
|
+
return ids.size ? [...ids] : undefined;
|
|
482
|
+
}
|
|
430
483
|
class ReportingExporter {
|
|
431
484
|
transport;
|
|
432
485
|
signal;
|
|
@@ -485,12 +538,12 @@ class ReportingExporter {
|
|
|
485
538
|
const cache = new WeakMap();
|
|
486
539
|
let failed = false;
|
|
487
540
|
let redactedBytes = 0;
|
|
488
|
-
const invalid = (placeholder, message) => {
|
|
541
|
+
const invalid = (placeholder, message, record) => {
|
|
489
542
|
if (placeholder)
|
|
490
543
|
this.transport.issue(this.signal, "warning", 1, `${message} (in-progress span placeholder)`);
|
|
491
544
|
else {
|
|
492
545
|
failed = true;
|
|
493
|
-
this.transport.issue(this.signal, "invalid", 1, message);
|
|
546
|
+
this.transport.issue(this.signal, "invalid", 1, message, undefined, record ? traceIdsOf([record]) : undefined);
|
|
494
547
|
}
|
|
495
548
|
};
|
|
496
549
|
const send = async (batch) => {
|
|
@@ -539,7 +592,7 @@ class ReportingExporter {
|
|
|
539
592
|
placeholders.add(redacted);
|
|
540
593
|
}
|
|
541
594
|
catch {
|
|
542
|
-
invalid(markers !== undefined, "Telemetry record could not be redacted or exceeds supported content limits");
|
|
595
|
+
invalid(markers !== undefined, "Telemetry record could not be redacted or exceeds supported content limits", record);
|
|
543
596
|
}
|
|
544
597
|
}
|
|
545
598
|
// Each record is encoded once to measure it; a request is encoded once more when it is sent.
|
|
@@ -555,7 +608,7 @@ class ReportingExporter {
|
|
|
555
608
|
recordBytes = this.serializer.serializeRequest([record])?.byteLength ?? Infinity;
|
|
556
609
|
}
|
|
557
610
|
catch {
|
|
558
|
-
invalid(placeholders.has(record), "Telemetry record could not be serialized");
|
|
611
|
+
invalid(placeholders.has(record), "Telemetry record could not be serialized", record);
|
|
559
612
|
continue;
|
|
560
613
|
}
|
|
561
614
|
const framedBytes = recordBytes + RECORD_FRAMING_BYTES;
|
|
@@ -565,7 +618,7 @@ class ReportingExporter {
|
|
|
565
618
|
batchBytes = 0;
|
|
566
619
|
}
|
|
567
620
|
if (recordBytes > limit) {
|
|
568
|
-
invalid(placeholders.has(record), "Telemetry record exceeds the 1 MiB request limit");
|
|
621
|
+
invalid(placeholders.has(record), "Telemetry record exceeds the 1 MiB request limit", record);
|
|
569
622
|
continue;
|
|
570
623
|
}
|
|
571
624
|
batch.push(record);
|
|
@@ -580,9 +633,12 @@ class ReportingExporter {
|
|
|
580
633
|
async send(records, placeholders = 0) {
|
|
581
634
|
const options = this.transport.options;
|
|
582
635
|
const real = records.length - placeholders;
|
|
636
|
+
// The traces this batch carries, spans or the log records emitted in their spans, so a loss
|
|
637
|
+
// or rejection names whose telemetry it was.
|
|
638
|
+
const traceIds = traceIdsOf(records);
|
|
583
639
|
// A loss involving only placeholders is a warning. Mixed losses count only real records.
|
|
584
640
|
const lose = (message, status) => {
|
|
585
|
-
this.transport.issue(this.signal, real ? "failed" : "warning", real || placeholders, real ? message : `${message} (in-progress span placeholders only)`, status);
|
|
641
|
+
this.transport.issue(this.signal, real ? "failed" : "warning", real || placeholders, real ? message : `${message} (in-progress span placeholders only)`, status, traceIds);
|
|
586
642
|
return !real;
|
|
587
643
|
};
|
|
588
644
|
let rejected = 0;
|
|
@@ -611,14 +667,19 @@ class ReportingExporter {
|
|
|
611
667
|
if (downgrade)
|
|
612
668
|
this.transport.rejectPlaceholders(placeholderRejections);
|
|
613
669
|
const remaining = count - placeholderRejections;
|
|
614
|
-
// Rejections are not matched to records. Attribute them to real records first.
|
|
670
|
+
// Rejections are not matched to records. Attribute them to real records first. A
|
|
671
|
+
// rejection of spans in a batch of one trace is that trace's; in a batch of several it
|
|
672
|
+
// names none, since the innocent traces' spans may have been accepted, and each case's
|
|
673
|
+
// receipt decides. A rejection of log records names every trace in the batch: the
|
|
674
|
+
// receipt counts spans, not logs, so it could not tell whose logs were lost.
|
|
615
675
|
rejected = Math.min(remaining, real);
|
|
676
|
+
const rejectedTraceIds = this.signal === "logs" || (traceIds && traceIds.length === 1) ? traceIds : undefined;
|
|
616
677
|
if (remaining || (partial?.errorMessage && !downgrade))
|
|
617
678
|
this.transport.issue(this.signal, rejected ? "rejected" : "warning", rejected || remaining, rejected
|
|
618
679
|
? "Hue rejected telemetry records; inspect the project ingestion settings and supported limits"
|
|
619
680
|
: remaining
|
|
620
681
|
? "Hue rejected in-progress span placeholders"
|
|
621
|
-
: "Hue returned an ingestion warning");
|
|
682
|
+
: "Hue returned an ingestion warning", undefined, rejectedTraceIds);
|
|
622
683
|
// Do not pass backend error text or raw response bytes into the global OTel diagnostic logger.
|
|
623
684
|
return {};
|
|
624
685
|
}
|
package/dist/types.d.ts
CHANGED
|
@@ -87,6 +87,13 @@ export interface ExportIssue {
|
|
|
87
87
|
status?: number;
|
|
88
88
|
/** Fixed, sanitized description; never server text or content. */
|
|
89
89
|
message: string;
|
|
90
|
+
/** The traces the records concerned belonged to, when the issue arose with records in hand
|
|
91
|
+
* (a refused or rejected batch, a dropped record, a record the processor could not accept, a
|
|
92
|
+
* capture failure inside a span): up to 64 distinct ids, so a caller running several traces
|
|
93
|
+
* at once can tell whose telemetry the issue was. Absent for an issue without records (a
|
|
94
|
+
* processor flush that failed) and for a batch of more traces than that; such an issue may
|
|
95
|
+
* concern any trace in flight. */
|
|
96
|
+
traceIds?: string[];
|
|
90
97
|
}
|
|
91
98
|
/** Cumulative delivery counters and current queue gauges for one transport. */
|
|
92
99
|
export interface ExportReport {
|
package/dist/version.d.ts
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
/** Package version shared by the instrumentation scope and the export User-Agent. */
|
|
2
|
-
export declare const sdkVersion = "0.
|
|
2
|
+
export declare const sdkVersion = "0.12.0";
|
package/dist/version.js
CHANGED