@hue-run/sdk 0.21.0 → 0.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/CLI.md +255 -61
  2. package/README.md +61 -20
  3. package/WORLDS.md +6 -3
  4. package/dist/ai-sdk.d.ts +23 -1
  5. package/dist/ai-sdk.js +60 -2
  6. package/dist/cli/import-formats/adapter.d.ts +80 -0
  7. package/dist/cli/import-formats/adapter.js +1 -0
  8. package/dist/cli/import-formats/agent-events.d.ts +5 -0
  9. package/dist/cli/import-formats/agent-events.js +16 -0
  10. package/dist/cli/import-formats/atif.d.ts +3 -0
  11. package/dist/cli/import-formats/atif.js +15 -0
  12. package/dist/cli/import-formats/braintrust.d.ts +2 -0
  13. package/dist/cli/import-formats/braintrust.js +222 -0
  14. package/dist/cli/import-formats/containers.d.ts +32 -0
  15. package/dist/cli/import-formats/containers.js +399 -0
  16. package/dist/cli/import-formats/datadog.d.ts +2 -0
  17. package/dist/cli/import-formats/datadog.js +264 -0
  18. package/dist/cli/import-formats/helicone.d.ts +2 -0
  19. package/dist/cli/import-formats/helicone.js +161 -0
  20. package/dist/cli/import-formats/identity.d.ts +112 -0
  21. package/dist/cli/import-formats/identity.js +594 -0
  22. package/dist/cli/import-formats/jaeger.d.ts +3 -0
  23. package/dist/cli/import-formats/jaeger.js +266 -0
  24. package/dist/cli/import-formats/langfuse.d.ts +7 -0
  25. package/dist/cli/import-formats/langfuse.js +510 -0
  26. package/dist/cli/import-formats/langsmith.d.ts +2 -0
  27. package/dist/cli/import-formats/langsmith.js +244 -0
  28. package/dist/cli/import-formats/messages.d.ts +28 -0
  29. package/dist/cli/import-formats/messages.js +301 -0
  30. package/dist/cli/import-formats/mlflow.d.ts +2 -0
  31. package/dist/cli/import-formats/mlflow.js +260 -0
  32. package/dist/cli/import-formats/openai-agents.d.ts +2 -0
  33. package/dist/cli/import-formats/openai-agents.js +221 -0
  34. package/dist/cli/import-formats/opik.d.ts +2 -0
  35. package/dist/cli/import-formats/opik.js +193 -0
  36. package/dist/cli/import-formats/otel-family.d.ts +5 -0
  37. package/dist/cli/import-formats/otel-family.js +17 -0
  38. package/dist/cli/import-formats/otel-spans.d.ts +94 -0
  39. package/dist/cli/import-formats/otel-spans.js +262 -0
  40. package/dist/cli/import-formats/otlp-protobuf.d.ts +35 -0
  41. package/dist/cli/import-formats/otlp-protobuf.js +486 -0
  42. package/dist/cli/import-formats/otlp.d.ts +4 -0
  43. package/dist/cli/import-formats/otlp.js +15 -0
  44. package/dist/cli/import-formats/phoenix.d.ts +3 -0
  45. package/dist/cli/import-formats/phoenix.js +256 -0
  46. package/dist/cli/import-formats/raindrop.d.ts +21 -0
  47. package/dist/cli/import-formats/raindrop.js +255 -0
  48. package/dist/cli/import-formats/reader.d.ts +56 -0
  49. package/dist/cli/import-formats/reader.js +364 -0
  50. package/dist/cli/import-formats/registry.d.ts +36 -0
  51. package/dist/cli/import-formats/registry.js +71 -0
  52. package/dist/cli/import-formats/row-platform-fields.d.ts +32 -0
  53. package/dist/cli/import-formats/row-platform-fields.js +76 -0
  54. package/dist/cli/import-formats/rows.d.ts +165 -0
  55. package/dist/cli/import-formats/rows.js +638 -0
  56. package/dist/cli/import-formats/transcripts.d.ts +25 -0
  57. package/dist/cli/import-formats/transcripts.js +498 -0
  58. package/dist/cli/import-formats/weave.d.ts +2 -0
  59. package/dist/cli/import-formats/weave.js +159 -0
  60. package/dist/cli/import-formats/zipkin.d.ts +2 -0
  61. package/dist/cli/import-formats/zipkin.js +219 -0
  62. package/dist/cli/import.d.ts +1 -1
  63. package/dist/cli/import.js +4 -2
  64. package/dist/cli/mcp.d.ts +5 -1
  65. package/dist/cli/mcp.js +35 -23
  66. package/dist/cli/traces-import-convert.d.ts +65 -33
  67. package/dist/cli/traces-import-convert.js +211 -214
  68. package/dist/cli/traces-import-prepare.d.ts +26 -22
  69. package/dist/cli/traces-import-prepare.js +51 -122
  70. package/dist/cli/traces-import.d.ts +4 -4
  71. package/dist/cli/traces-import.js +63 -11
  72. package/dist/version.d.ts +1 -1
  73. package/dist/version.js +1 -1
  74. package/package.json +3 -3
@@ -4,14 +4,15 @@ import { z } from "zod/v4";
4
4
  import { blobEntry, blobPrefix, BLOBS_KEY, MAX_BLOB_ENTRIES, textContentType } from "../blobs.js";
5
5
  import { TRUNCATED_KEY } from "../privacy.js";
6
6
  import { hueSpanId, hueTraceId } from "./langfuse.js";
7
- import { isZipSignature } from "./traces-import-zip.js";
8
7
  /**
9
- * The conversions behind `hue traces import prepare`: a source file is recognized by its content
10
- * (an OTLP/JSON trace export, a complete Langfuse observation export or a text ATIF export),
11
- * each trace becomes OTLP/JSON entries, one span each, that keep the recorded ids, hierarchy,
12
- * timestamps, content, errors and usage, and the entries are packed into deterministic gzip
13
- * batches. Nothing here depends on where a file sat or what it was called: a renamed or repacked
14
- * bundle converts to byte-identical batches, so the manifest's digests name the same import.
8
+ * The conversions behind `hue traces import prepare` that predate the format registry
9
+ * (`import-formats/registry.ts`, which recognizes each value and hands it to its format): OTLP/JSON
10
+ * spans kept as recorded, complete Langfuse observation exports and text ATIF exports, each trace
11
+ * as OTLP/JSON entries, one span each, that keep the recorded ids, hierarchy, timestamps,
12
+ * content, errors and usage; the shared attribute, large-value and JSON helpers; and the packing
13
+ * of entries into deterministic gzip batches and the manifest. Nothing here depends on where a
14
+ * file sat or what it was called: a renamed or repacked bundle converts to byte-identical
15
+ * batches, so the manifest's digests name the same import.
15
16
  */
16
17
  export const traceImportFormats = ["otlp_json", "langfuse_observations", "atif"];
17
18
  export const traceImportOrigins = ["production", "evaluation", "simulation", "unknown"];
@@ -45,32 +46,52 @@ const BASE64_SPAN_ID = /^[A-Za-z0-9+/]{11}=$/;
45
46
  const MAX_IDENTIFIER = 16_384;
46
47
  const MAX_SESSION = 2_048;
47
48
  const MAX_NAMESPACE = 128;
48
- /** The root attributes Hue reads a trace's user, session and explicit entity from. */
49
- const USER_KEYS = [
50
- "user.id",
51
- "enduser.id",
52
- "hue.user.id",
53
- "ai.telemetry.metadata.userId",
54
- "langfuse.user.id",
55
- ];
56
- const SESSION_KEYS = [
57
- "gen_ai.conversation.id",
58
- "session.id",
59
- "hue.session.id",
60
- "ai.telemetry.metadata.sessionId",
61
- "langfuse.session.id",
62
- ];
63
- const ENTITY_NAMESPACE = "hue.world.entity.namespace";
64
- const ENTITY_ID = "hue.world.entity.id";
65
49
  /** Attributes that mark a trace as an evaluation's or a simulated world's. */
66
50
  const EVALUATION_KEYS = ["hue.execution.id", "hue.eval.execution_id"];
67
51
  const SIMULATION_KEYS = ["hue.world.id", "hue.environment.run_id"];
68
52
  const SIMULATION_SCOPE = "hue.sim-gateway";
69
53
  export class ConversionError extends Error {
70
54
  }
55
+ /**
56
+ * A failure that refuses only what was being read: a ConversionError as itself, and a RangeError
57
+ * (a value too large or nested too deep for a reader to walk) as one. Anything else is a defect
58
+ * and is undefined, for the caller to rethrow.
59
+ */
60
+ export function conversionFailure(error) {
61
+ if (error instanceof ConversionError)
62
+ return error;
63
+ if (error instanceof RangeError)
64
+ return new ConversionError("the record is too large or nested too deep to read");
65
+ return undefined;
66
+ }
67
+ /** How deep a value read from a source may nest: far beyond any export's records, and well
68
+ * inside what the readers and serializers that walk a value can follow. */
69
+ export const MAX_SOURCE_NESTING = 512;
70
+ /** Whether a parsed JSON value nests deeper than `max` levels, walked without recursion and
71
+ * holding only the path to the value being looked at. */
72
+ export function nestedTooDeep(value, max = MAX_SOURCE_NESTING) {
73
+ if (value === null || typeof value !== "object")
74
+ return false;
75
+ const members = (item) => (Array.isArray(item) ? item : Object.values(item));
76
+ const path = [{ items: members(value), next: 0 }];
77
+ while (path.length) {
78
+ const level = path[path.length - 1];
79
+ if (level.next >= level.items.length) {
80
+ path.pop();
81
+ continue;
82
+ }
83
+ const child = level.items[level.next++];
84
+ if (child === null || typeof child !== "object")
85
+ continue;
86
+ if (path.length >= max)
87
+ return true;
88
+ path.push({ items: members(child), next: 0 });
89
+ }
90
+ return false;
91
+ }
71
92
  export const sha256Hex = (value) => createHash("sha256").update(value).digest("hex");
72
- const digestHex = (text, length) => sha256Hex(text).slice(0, length);
73
- function isObject(value) {
93
+ export const digestHex = (text, length) => sha256Hex(text).slice(0, length);
94
+ export function isObject(value) {
74
95
  return value !== null && typeof value === "object" && !Array.isArray(value);
75
96
  }
76
97
  function object(value) {
@@ -122,6 +143,19 @@ export function parseJsonText(text) {
122
143
  throw error;
123
144
  }
124
145
  }
146
+ /** JSON text a record holds (metadata values, tool arguments, a column a warehouse stores as text)
147
+ * parsed as {@link parseJsonText} parses it; text nested deeper than {@link MAX_SOURCE_NESTING}
148
+ * throws, so the caller keeps it as the text it is. */
149
+ export function parseNestedJsonText(text) {
150
+ const parsed = parseJsonText(text);
151
+ if (nestedTooDeep(parsed.value))
152
+ throw new ConversionError("JSON text nested too deep");
153
+ return parsed;
154
+ }
155
+ /** Langfuse's experiment and evaluator environments (`sdk-experiment`, `langfuse-…`): traffic
156
+ * Langfuse itself marks as evaluation. */
157
+ export const langfuseEvaluationEnvironment = (environment) => environment === "sdk-experiment" ||
158
+ (typeof environment === "string" && environment.startsWith("langfuse-"));
125
159
  /** Refuses a source value that JavaScript or OTLP cannot retain exactly; never omits it. */
126
160
  export function validateSource(value, depth = 0) {
127
161
  if (depth > 64)
@@ -203,95 +237,6 @@ export function validateJsonText(raw) {
203
237
  }
204
238
  }
205
239
  }
206
- const isOtlpEnvelope = (value) => isObject(value) && (Array.isArray(value.resourceSpans) || Array.isArray(value.resource_spans));
207
- const isObservation = (value) => isObject(value) &&
208
- typeof value.id === "string" &&
209
- typeof value.traceId === "string" &&
210
- typeof value.type === "string" &&
211
- typeof value.startTime === "string";
212
- /** What a file holds, read from its content alone; never from its name. */
213
- export function detectFormat(bytes) {
214
- if (!bytes.byteLength)
215
- return { format: "unsupported", reason: "the file is empty" };
216
- if (bytes[0] === 0x1f && bytes[1] === 0x8b)
217
- return { format: "unsupported", reason: "a gzip file; decompress it first" };
218
- // An archive is read where the walk finds it on disk; one inside another is not opened.
219
- if (isZipSignature(bytes))
220
- return { format: "unsupported", reason: "a ZIP archive inside the input; extract it first" };
221
- let text;
222
- try {
223
- text = new TextDecoder("utf-8", { fatal: true, ignoreBOM: false }).decode(bytes);
224
- }
225
- catch {
226
- return { format: "unsupported", reason: "not UTF-8 text" };
227
- }
228
- let parsed;
229
- try {
230
- parsed = parseJsonText(text);
231
- }
232
- catch {
233
- parsed = undefined;
234
- }
235
- if (!parsed) {
236
- // OTLP/JSON exports are also written one request a line.
237
- const lines = text.split(/\r?\n/).filter((line) => line.trim());
238
- if (lines.length < 2)
239
- return { format: "unsupported", reason: "not JSON" };
240
- const envelopes = [];
241
- for (const line of lines) {
242
- let value;
243
- try {
244
- value = parseJsonText(line).value;
245
- }
246
- catch {
247
- return { format: "unsupported", reason: "not JSON" };
248
- }
249
- if (!isOtlpEnvelope(value))
250
- return { format: "unsupported", reason: "JSON Lines whose records are not OTLP exports" };
251
- envelopes.push(value);
252
- }
253
- return { format: "otlp_json", envelopes };
254
- }
255
- const value = parsed.value;
256
- if (Array.isArray(value)) {
257
- if (value.length && value.every(isOtlpEnvelope))
258
- return { format: "otlp_json", envelopes: value };
259
- return { format: "unsupported", reason: "a JSON array that is not a list of OTLP exports" };
260
- }
261
- if (!isObject(value))
262
- return { format: "unsupported", reason: "not a JSON object" };
263
- if (isOtlpEnvelope(value))
264
- return { format: "otlp_json", envelopes: [value] };
265
- if (Array.isArray(value.resourceLogs) || Array.isArray(value.resource_logs))
266
- return { format: "unsupported", reason: "an OTLP logs export; only traces are imported" };
267
- if (typeof value.schema_version === "string" && value.schema_version.startsWith("ATIF-")) {
268
- if (!Array.isArray(value.steps))
269
- return { format: "unsupported", reason: "an ATIF document without steps" };
270
- return { format: "atif", text, document: value, unsafeIntegers: parsed.unsafeIntegers };
271
- }
272
- const key = Array.isArray(value.data)
273
- ? "data"
274
- : Array.isArray(value.observations)
275
- ? "observations"
276
- : undefined;
277
- if (key) {
278
- if (value.data !== undefined && value.observations !== undefined)
279
- return { format: "unsupported", reason: "two observation envelopes in one file" };
280
- const records = value[key];
281
- if (!records.length)
282
- return { format: "unsupported", reason: "a Langfuse export without observations" };
283
- if (!records.every(isObservation))
284
- return { format: "unsupported", reason: "a `data` list that is not Langfuse observations" };
285
- const envelope = Object.fromEntries(Object.entries(value).filter(([name]) => name !== key));
286
- return {
287
- format: "langfuse_observations",
288
- observations: records,
289
- envelope,
290
- unsafeIntegers: parsed.unsafeIntegers,
291
- };
292
- }
293
- return { format: "unsupported", reason: "a JSON document of no recognized shape" };
294
- }
295
240
  // --- Attributes and large values ----------------------------------------------------------------
296
241
  export function anyValue(value) {
297
242
  if (value === null)
@@ -428,39 +373,16 @@ function attributeMap(attributes) {
428
373
  map.set(attribute.key, attribute.value);
429
374
  return map;
430
375
  }
431
- /** A trace's identity as its root records it, in the attributes Hue reads. */
432
- function identityOf(root, resource) {
433
- const identity = {};
434
- for (const key of USER_KEYS) {
435
- const user = identifier(stringOf(root.get(key)), MAX_IDENTIFIER);
436
- if (user) {
437
- identity.user = user;
438
- break;
439
- }
440
- }
441
- for (const source of [root, resource]) {
442
- const namespace = identifier(stringOf(source.get(ENTITY_NAMESPACE)), MAX_NAMESPACE);
443
- const id = identifier(stringOf(source.get(ENTITY_ID)), MAX_IDENTIFIER);
444
- if (namespace && id) {
445
- identity.entity = { namespace, id };
446
- break;
447
- }
448
- }
449
- for (const key of SESSION_KEYS) {
450
- const session = identifier(stringOf(root.get(key)), MAX_SESSION);
451
- if (session) {
452
- identity.session = session;
453
- break;
454
- }
455
- }
456
- return identity.user || identity.entity || identity.session ? identity : undefined;
457
- }
458
- /** Gathers the spans of OTLP/JSON exports by trace, across files: an export writes a trace's
459
- * spans in the requests that carried them, which need not be one file. */
376
+ /** The collector key of the formats that keep OTLP spans as recorded (OTLP/JSON, binary OTLP,
377
+ * Jaeger API v3): one {@link OtlpTraceCollector} gathers their spans together. */
378
+ export const OTLP_SPANS = "otlp_spans";
379
+ /** Gathers the spans of OTLP/JSON exports by trace, across files and the formats that carry
380
+ * them: an export writes a trace's spans in the requests that carried them, which need not be one
381
+ * file. Each trace is named by the format that read its first span. */
460
382
  export class OtlpTraceCollector {
461
383
  traces = new Map();
462
- /** Adds every span of the envelopes; the spans refused, with their reasons. */
463
- add(envelopes, sourceDigest) {
384
+ /** Adds every span of the envelopes `format` read; the spans refused, with their reasons. */
385
+ add(envelopes, sourceDigest, format) {
464
386
  const refused = [];
465
387
  for (const envelope of envelopes) {
466
388
  // An export may mix the proto field names with lowerCamelCase at any depth; every
@@ -471,9 +393,10 @@ export class OtlpTraceCollector {
471
393
  if (!isObject(groupValue))
472
394
  continue;
473
395
  const { scopeSpans, ...resourceGroup } = groupValue;
474
- const resource = isObject(resourceGroup.resource)
475
- ? attributeMap(readAttributes(resourceGroup.resource.attributes) ?? [])
476
- : new Map();
396
+ const resourceAttributes = isObject(resourceGroup.resource)
397
+ ? readAttributes(resourceGroup.resource.attributes)
398
+ : [];
399
+ const resource = attributeMap(resourceAttributes ?? []);
477
400
  for (const scopeValue of Array.isArray(scopeSpans) ? scopeSpans : []) {
478
401
  if (!isObject(scopeValue))
479
402
  continue;
@@ -481,12 +404,18 @@ export class OtlpTraceCollector {
481
404
  const scopeName = isObject(scopeGroup.scope) ? scopeGroup.scope.name : undefined;
482
405
  for (const spanValue of Array.isArray(spans) ? spans : []) {
483
406
  try {
484
- this.addSpan(spanValue, resourceGroup, scopeGroup, resource, scopeName, sourceDigest);
407
+ if (resourceAttributes === undefined)
408
+ throw new ConversionError("the span's resource has malformed attributes");
409
+ this.addSpan(spanValue, resourceGroup, scopeGroup, resource, scopeName, sourceDigest, format);
485
410
  }
486
411
  catch (error) {
487
- if (!(error instanceof ConversionError))
412
+ const failure = conversionFailure(error);
413
+ if (!failure)
488
414
  throw error;
489
- refused.push({ reason: error.message });
415
+ // A malformed span refuses its trace, never leaving a partial one; a span that names
416
+ // no trace is refused alone.
417
+ const traceId = isObject(spanValue) ? spanValue.traceId : undefined;
418
+ refused.push(this.refuse(traceId, failure.message));
490
419
  }
491
420
  }
492
421
  }
@@ -494,17 +423,23 @@ export class OtlpTraceCollector {
494
423
  }
495
424
  return refused;
496
425
  }
497
- addSpan(value, resourceGroup, scopeGroup, resource, scopeName, sourceDigest) {
426
+ /** Refuses the trace a span names (its id as recorded), whatever else of it is read; a span
427
+ * that names no trace is refused alone. */
428
+ refuse(traceIdValue, reason) {
429
+ const id = otlpTraceId(traceIdValue);
430
+ if (id)
431
+ this.draft(id.traceId, id.remapped).conflict ??= reason;
432
+ return { ...(id ? { traceId: id.traceId } : {}), reason };
433
+ }
434
+ addSpan(value, resourceGroup, scopeGroup, resource, scopeName, sourceDigest, format) {
498
435
  if (!isObject(value))
499
436
  throw new ConversionError("a span that is not a JSON object");
500
437
  const span = { ...value };
501
438
  const originalTraceId = span.traceId;
502
- if (typeof originalTraceId !== "string" || !originalTraceId)
439
+ const id = otlpTraceId(originalTraceId);
440
+ if (typeof originalTraceId !== "string" || !id)
503
441
  throw new ConversionError("a span without a trace id");
504
- let traceId = normalizeId(originalTraceId, 32);
505
- const remappedTraceId = traceId === undefined;
506
- if (!traceId)
507
- traceId = digestHex(`otlp.trace:${originalTraceId}`, 32);
442
+ const { traceId, remapped: remappedTraceId } = id;
508
443
  const originalSpanId = span.spanId;
509
444
  if (typeof originalSpanId !== "string" || !originalSpanId)
510
445
  throw new ConversionError("a span without a span id");
@@ -534,21 +469,8 @@ export class OtlpTraceCollector {
534
469
  const attributes = readAttributes(span.attributes);
535
470
  if (attributes === undefined)
536
471
  throw new ConversionError(`span ${spanId} has malformed attributes`);
537
- let trace = this.traces.get(traceId);
538
- if (!trace) {
539
- trace = {
540
- traceId,
541
- remappedTraceId,
542
- spans: new Map(),
543
- blobs: new BlobSink(),
544
- sourceDigests: new Set(),
545
- remappedSpans: 0,
546
- evaluation: false,
547
- simulation: false,
548
- production: false,
549
- };
550
- this.traces.set(traceId, trace);
551
- }
472
+ const trace = this.draft(traceId, remappedTraceId);
473
+ trace.format ??= format;
552
474
  const kept = [...attributes];
553
475
  if (remappedTraceId)
554
476
  kept.push({ key: "hue.import.original_trace_id", value: { stringValue: originalTraceId } });
@@ -559,11 +481,11 @@ export class OtlpTraceCollector {
559
481
  ...resourceGroup,
560
482
  scopeSpans: [{ ...scopeGroup, spans: [span] }],
561
483
  };
562
- const canonical = JSON.stringify(entry);
484
+ const digest = sha256Hex(JSON.stringify(entry));
563
485
  const held = trace.spans.get(spanId);
564
486
  if (held) {
565
487
  // The same span sent twice is one span; another payload under its id is a conflict.
566
- if (held.canonical !== canonical)
488
+ if (held.digest !== digest)
567
489
  trace.conflict ??= `span ${spanId} appears with different content`;
568
490
  trace.sourceDigests.add(sourceDigest);
569
491
  return;
@@ -588,29 +510,56 @@ export class OtlpTraceCollector {
588
510
  ...(parentSpanId ? { parentSpanId } : {}),
589
511
  start: BigInt(start),
590
512
  entry,
591
- canonical,
592
- attributes: map,
593
- resource,
513
+ digest,
594
514
  });
595
515
  }
516
+ draft(traceId, remappedTraceId) {
517
+ let trace = this.traces.get(traceId);
518
+ if (!trace) {
519
+ trace = {
520
+ traceId,
521
+ remappedTraceId,
522
+ spans: new Map(),
523
+ blobs: new BlobSink(),
524
+ sourceDigests: new Set(),
525
+ remappedSpans: 0,
526
+ evaluation: false,
527
+ simulation: false,
528
+ production: false,
529
+ };
530
+ this.traces.set(traceId, trace);
531
+ }
532
+ return trace;
533
+ }
596
534
  /** Every trace gathered so far, spans in time order, and the traces refused whole. */
597
535
  finish() {
598
536
  const traces = [];
599
537
  const refused = [];
600
- for (const draft of this.traces.values()) {
538
+ for (const [key, draft] of this.traces) {
539
+ // Each draft is let go once its trace is converted.
540
+ this.traces.delete(key);
601
541
  if (draft.conflict) {
602
542
  refused.push({ traceId: draft.traceId, reason: draft.conflict });
603
543
  continue;
604
544
  }
545
+ // A parent outside the trace is a root, as OTLP allows; a parent chain that closes on
546
+ // itself leaves the trace without one and is refused.
547
+ const parents = new Map([...draft.spans.values()].map((span) => [
548
+ span.spanId,
549
+ span.parentSpanId !== undefined && draft.spans.has(span.parentSpanId)
550
+ ? span.parentSpanId
551
+ : undefined,
552
+ ]));
553
+ if (ancestryIssue(parents) === "cyclic_ancestry") {
554
+ refused.push({ traceId: draft.traceId, reason: "The source hierarchy contains a cycle" });
555
+ continue;
556
+ }
605
557
  const spans = [...draft.spans.values()].sort((a, b) => a.start < b.start ? -1 : a.start > b.start ? 1 : a.spanId < b.spanId ? -1 : 1);
606
- const root = spans.find((span) => !span.parentSpanId || !draft.spans.has(span.parentSpanId));
607
- const identity = root ? identityOf(root.attributes, root.resource) : undefined;
608
558
  traces.push({
609
559
  traceId: draft.traceId,
610
- format: "otlp_json",
560
+ format: draft.format ?? "otlp_json",
611
561
  entries: spans.map((span) => span.entry),
612
562
  blobs: draft.blobs.sorted(),
613
- ...(identity ? { identity } : {}),
614
563
  origin: draft.evaluation
615
564
  ? "evaluation"
616
565
  : draft.simulation
@@ -626,6 +575,48 @@ export class OtlpTraceCollector {
626
575
  return { traces, refused };
627
576
  }
628
577
  }
578
+ /** A span's OpenTelemetry trace id: the recorded one, or one derived from an id that is not. */
579
+ function otlpTraceId(value) {
580
+ if (typeof value !== "string" || !value)
581
+ return undefined;
582
+ const traceId = normalizeId(value, 32);
583
+ return traceId
584
+ ? { traceId, remapped: false }
585
+ : { traceId: digestHex(`otlp.trace:${value}`, 32), remapped: true };
586
+ }
587
+ /** Whether a recorded OTLP `AnyValue` has the shape OTLP/JSON gives it: an object whose
588
+ * `arrayValue` and `kvlistValue` hold lists of values and of keyed values. Walked without
589
+ * recursion. */
590
+ function validOtlpValue(value) {
591
+ const stack = [value];
592
+ while (stack.length) {
593
+ const item = stack.pop();
594
+ if (!isObject(item))
595
+ return false;
596
+ for (const [field, keyed] of [
597
+ ["arrayValue", false],
598
+ ["kvlistValue", true],
599
+ ]) {
600
+ const list = item[field];
601
+ if (list === undefined || list === null)
602
+ continue;
603
+ if (!isObject(list))
604
+ return false;
605
+ const values = list.values ?? [];
606
+ if (!Array.isArray(values))
607
+ return false;
608
+ for (const entry of values) {
609
+ if (!keyed)
610
+ stack.push(entry);
611
+ else if (!isObject(entry) || typeof entry.key !== "string")
612
+ return false;
613
+ else if (entry.value !== undefined && entry.value !== null)
614
+ stack.push(entry.value);
615
+ }
616
+ }
617
+ }
618
+ return true;
619
+ }
629
620
  /** An OTLP attribute list as recorded, or undefined when it is malformed. */
630
621
  function readAttributes(value) {
631
622
  if (value === undefined || value === null)
@@ -634,7 +625,7 @@ function readAttributes(value) {
634
625
  return undefined;
635
626
  const attributes = [];
636
627
  for (const item of value) {
637
- if (!isObject(item) || typeof item.key !== "string" || !isObject(item.value))
628
+ if (!isObject(item) || typeof item.key !== "string" || !validOtlpValue(item.value))
638
629
  return undefined;
639
630
  attributes.push(item);
640
631
  }
@@ -642,7 +633,7 @@ function readAttributes(value) {
642
633
  }
643
634
  /** A recorded span's string values over the inline limit stored apart from it, listed beside
644
635
  * any values its exporter already listed. */
645
- function offloadOtlpValues(attributes, blobs) {
636
+ export function offloadOtlpValues(attributes, blobs) {
646
637
  const over = attributes.filter((attribute) => typeof attribute.value.stringValue === "string" &&
647
638
  Buffer.byteLength(attribute.value.stringValue, "utf8") > MAX_INLINE_VALUE_BYTES);
648
639
  if (!over.length)
@@ -708,7 +699,8 @@ export function unixNano(value) {
708
699
  throw new ConversionError("Invalid source timestamp");
709
700
  return (BigInt(seconds) * 1000000n + BigInt((match[2] ?? "").padEnd(9, "0"))).toString();
710
701
  }
711
- const canonicalJson = (value) => {
702
+ /** JSON text with object keys sorted: equal for equal values whatever their key order. */
703
+ export const canonicalJson = (value) => {
712
704
  if (Array.isArray(value))
713
705
  return `[${value.map(canonicalJson).join(",")}]`;
714
706
  if (value && typeof value === "object")
@@ -748,32 +740,44 @@ export class LangfuseTraceCollector {
748
740
  for (const observation of observations) {
749
741
  const traceId = text(observation.traceId, true);
750
742
  const id = text(observation.id, true);
751
- let draft = this.traces.get(traceId);
752
- if (!draft) {
753
- draft = {
754
- originalTraceId: traceId,
755
- observations: new Map(),
756
- envelopes: new Map(),
757
- sourceDigests: new Set(),
758
- };
759
- this.traces.set(traceId, draft);
760
- }
743
+ const draft = this.draft(traceId);
761
744
  draft.sourceDigests.add(sourceDigest);
762
745
  draft.envelopes.set(sourceDigest, envelopeText);
763
- const canonical = canonicalJson(observation);
746
+ const digest = sha256Hex(canonicalJson(observation));
764
747
  const held = draft.observations.get(id);
765
748
  if (held) {
766
- if (held.canonical !== canonical)
749
+ if (held.digest !== digest)
767
750
  draft.conflict ??= `observation ${id} appears with different content`;
768
751
  continue;
769
752
  }
770
- draft.observations.set(id, { record: observation, canonical });
753
+ draft.observations.set(id, { record: observation, digest });
771
754
  }
772
755
  }
756
+ /** Refuses a trace by its Langfuse id, whatever else of it is read. */
757
+ refuse(traceId, reason, sourceDigest) {
758
+ const draft = this.draft(traceId);
759
+ draft.sourceDigests.add(sourceDigest);
760
+ draft.conflict ??= reason;
761
+ }
762
+ draft(traceId) {
763
+ let draft = this.traces.get(traceId);
764
+ if (!draft) {
765
+ draft = {
766
+ originalTraceId: traceId,
767
+ observations: new Map(),
768
+ envelopes: new Map(),
769
+ sourceDigests: new Set(),
770
+ };
771
+ this.traces.set(traceId, draft);
772
+ }
773
+ return draft;
774
+ }
773
775
  finish() {
774
776
  const traces = [];
775
777
  const refused = [];
776
- for (const draft of this.traces.values()) {
778
+ for (const [key, draft] of this.traces) {
779
+ // Each draft is let go once its trace is converted.
780
+ this.traces.delete(key);
777
781
  const traceId = hueTraceId(draft.originalTraceId);
778
782
  if (draft.conflict) {
779
783
  refused.push({ traceId, reason: draft.conflict });
@@ -783,9 +787,10 @@ export class LangfuseTraceCollector {
783
787
  traces.push(convertLangfuseTrace(draft, traceId));
784
788
  }
785
789
  catch (error) {
786
- if (!(error instanceof ConversionError))
790
+ const failure = conversionFailure(error);
791
+ if (!failure)
787
792
  throw error;
788
- refused.push({ traceId, reason: error.message });
793
+ refused.push({ traceId, reason: failure.message });
789
794
  }
790
795
  }
791
796
  return { traces, refused };
@@ -848,10 +853,10 @@ function convertLangfuseTrace(draft, traceId) {
848
853
  imported.add(spanId);
849
854
  spanIds.set(originalId, spanId);
850
855
  }
851
- let identity;
852
856
  // The origin markers the OTLP reader applies, read from the converted attributes with the same
853
857
  // precedence: an evaluation's or a simulated world's records are never production history,
854
- // whatever environment the export was taken from.
858
+ // whatever environment the export was taken from. Langfuse's own experiment and evaluator
859
+ // environments are evaluation traffic too, as the `langfuse` format reads them.
855
860
  let production = false;
856
861
  let evaluation = false;
857
862
  let simulation = false;
@@ -940,6 +945,8 @@ function convertLangfuseTrace(draft, traceId) {
940
945
  attributes.add("hue.import.status_source", "langfuse.level");
941
946
  if (observation.environment === "production")
942
947
  production = true;
948
+ if (langfuseEvaluationEnvironment(observation.environment))
949
+ evaluation = true;
943
950
  const message = text(observation.statusMessage);
944
951
  const span = {
945
952
  traceId,
@@ -953,11 +960,6 @@ function convertLangfuseTrace(draft, traceId) {
953
960
  attributes: attributes.finish(),
954
961
  status: { code: observation.level === "ERROR" ? 2 : 0, ...(message ? { message } : {}) },
955
962
  };
956
- // The trace's owner as the OTLP reader takes it from a root: the user (`langfuse.user.id`
957
- // among the keys), an explicit entity in the root's metadata or resource attributes, and the
958
- // session.
959
- if (!parent)
960
- identity = identityOf(attributeMap(span.attributes), attributeMap(resource));
961
963
  return {
962
964
  resource: { attributes: resource },
963
965
  scopeSpans: [
@@ -978,7 +980,6 @@ function convertLangfuseTrace(draft, traceId) {
978
980
  format: "langfuse_observations",
979
981
  entries,
980
982
  blobs: blobs.sorted(),
981
- ...(identity ? { identity } : {}),
982
983
  origin: evaluation
983
984
  ? "evaluation"
984
985
  : simulation
@@ -1581,15 +1582,11 @@ export function convertAtifExport(source, rawBytes) {
1581
1582
  attributes: [{ key: "user.id", value: { stringValue: userId } }],
1582
1583
  status: { code: 0 },
1583
1584
  });
1584
- const session = identifier(document.session_id, MAX_SESSION);
1585
1585
  return {
1586
1586
  traceId,
1587
1587
  format: "atif",
1588
1588
  entries,
1589
1589
  blobs: blobs.sorted(),
1590
- ...(userId || session
1591
- ? { identity: { ...(userId ? { user: userId } : {}), ...(session ? { session } : {}) } }
1592
- : {}),
1593
1590
  origin: "unknown",
1594
1591
  sourceDigests: new Set(),
1595
1592
  remappedTraceId: false,