@hue-run/sdk 0.20.0 → 0.20.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI.md +102 -10
- package/ENVIRONMENTS.md +1 -1
- package/README.md +30 -2
- package/WORLDS.md +14 -3
- package/dist/cli/traces-import-convert.d.ts +262 -0
- package/dist/cli/traces-import-convert.js +1779 -0
- package/dist/cli/traces-import-prepare.d.ts +64 -0
- package/dist/cli/traces-import-prepare.js +443 -0
- package/dist/cli/traces-import-upload.d.ts +129 -0
- package/dist/cli/traces-import-upload.js +838 -0
- package/dist/cli/traces-import-zip.d.ts +52 -0
- package/dist/cli/traces-import-zip.js +278 -0
- package/dist/cli/traces-import.d.ts +18 -0
- package/dist/cli/traces-import.js +231 -0
- package/dist/setup/cli.js +7 -3
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/world/client.d.ts +13 -2
- package/dist/world/client.js +24 -5
- package/dist/world/model-types.d.ts +8 -2
- package/dist/world/models.d.ts +3 -1
- package/dist/world/readiness.d.ts +18 -0
- package/dist/world/readiness.js +69 -0
- package/dist/world/world.d.ts +3 -2
- package/dist/world/world.js +4 -2
- package/package.json +1 -1
|
@@ -0,0 +1,1779 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { gunzipSync, gzipSync } from "node:zlib";
|
|
3
|
+
import { z } from "zod/v4";
|
|
4
|
+
import { blobEntry, blobPrefix, BLOBS_KEY, MAX_BLOB_ENTRIES, textContentType } from "../blobs.js";
|
|
5
|
+
import { TRUNCATED_KEY } from "../privacy.js";
|
|
6
|
+
import { hueSpanId, hueTraceId } from "./langfuse.js";
|
|
7
|
+
import { isZipSignature } from "./traces-import-zip.js";
|
|
8
|
+
/**
|
|
9
|
+
* The conversions behind `hue traces import prepare`: a source file is recognized by its content
|
|
10
|
+
* (an OTLP/JSON trace export, a complete Langfuse observation export or a text ATIF export),
|
|
11
|
+
* each trace becomes OTLP/JSON entries, one span each, that keep the recorded ids, hierarchy,
|
|
12
|
+
* timestamps, content, errors and usage, and the entries are packed into deterministic gzip
|
|
13
|
+
* batches. Nothing here depends on where a file sat or what it was called: a renamed or repacked
|
|
14
|
+
* bundle converts to byte-identical batches, so the manifest's digests name the same import.
|
|
15
|
+
*/
|
|
16
|
+
export const traceImportFormats = ["otlp_json", "langfuse_observations", "atif"];
|
|
17
|
+
export const traceImportOrigins = ["production", "evaluation", "simulation", "unknown"];
|
|
18
|
+
/** The bounds the import routes hold a manifest to. */
|
|
19
|
+
export const traceImportLimits = {
|
|
20
|
+
traces: 10_000,
|
|
21
|
+
batchesPerTrace: 64,
|
|
22
|
+
blobsPerTrace: 1_000,
|
|
23
|
+
spansPerTrace: 100_000,
|
|
24
|
+
manifestBytes: 8 * 1024 * 1024,
|
|
25
|
+
batchBytes: 4 * 1024 * 1024,
|
|
26
|
+
blobBytes: 1_000_000_000,
|
|
27
|
+
};
|
|
28
|
+
/** One batch, decoded, and on the wire: well inside the receiver's request bounds. */
|
|
29
|
+
export const MAX_BATCH_DECODED_BYTES = 3 * 1024 * 1024;
|
|
30
|
+
export const MAX_BATCH_WIRE_BYTES = 900 * 1024;
|
|
31
|
+
/** The largest value a record keeps inline; a longer text is stored apart from its span. */
|
|
32
|
+
export const MAX_INLINE_VALUE_BYTES = 1024 * 1024;
|
|
33
|
+
export const SOURCE_RECORD = "hue.import.source_record";
|
|
34
|
+
export const SOURCE_ENVELOPE = "hue.import.source_envelope";
|
|
35
|
+
const BATCH_PREFIX = '{"resourceSpans":[';
|
|
36
|
+
const BATCH_SUFFIX = "]}";
|
|
37
|
+
const BATCH_FRAME_BYTES = BATCH_PREFIX.length + BATCH_SUFFIX.length;
|
|
38
|
+
const HEX32 = /^[0-9a-f]{32}$/;
|
|
39
|
+
const HEX16 = /^[0-9a-f]{16}$/;
|
|
40
|
+
const BASE64_TRACE_ID = /^[A-Za-z0-9+/]{22}==$/;
|
|
41
|
+
const BASE64_SPAN_ID = /^[A-Za-z0-9+/]{11}=$/;
|
|
42
|
+
const MAX_IDENTIFIER = 16_384;
|
|
43
|
+
const MAX_SESSION = 2_048;
|
|
44
|
+
const MAX_NAMESPACE = 128;
|
|
45
|
+
/** The root attributes Hue reads a trace's user, session and explicit entity from. */
|
|
46
|
+
const USER_KEYS = [
|
|
47
|
+
"user.id",
|
|
48
|
+
"enduser.id",
|
|
49
|
+
"hue.user.id",
|
|
50
|
+
"ai.telemetry.metadata.userId",
|
|
51
|
+
"langfuse.user.id",
|
|
52
|
+
];
|
|
53
|
+
const SESSION_KEYS = [
|
|
54
|
+
"gen_ai.conversation.id",
|
|
55
|
+
"session.id",
|
|
56
|
+
"hue.session.id",
|
|
57
|
+
"ai.telemetry.metadata.sessionId",
|
|
58
|
+
"langfuse.session.id",
|
|
59
|
+
];
|
|
60
|
+
const ENTITY_NAMESPACE = "hue.world.entity.namespace";
|
|
61
|
+
const ENTITY_ID = "hue.world.entity.id";
|
|
62
|
+
/** Attributes that mark a trace as an evaluation's or a simulated world's. */
|
|
63
|
+
const EVALUATION_KEYS = ["hue.execution.id", "hue.eval.execution_id"];
|
|
64
|
+
const SIMULATION_KEYS = ["hue.world.id", "hue.environment.run_id"];
|
|
65
|
+
const SIMULATION_SCOPE = "hue.sim-gateway";
|
|
66
|
+
export class ConversionError extends Error {
|
|
67
|
+
}
|
|
68
|
+
export const sha256Hex = (value) => createHash("sha256").update(value).digest("hex");
|
|
69
|
+
const digestHex = (text, length) => sha256Hex(text).slice(0, length);
|
|
70
|
+
function isObject(value) {
|
|
71
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
72
|
+
}
|
|
73
|
+
function object(value) {
|
|
74
|
+
if (!isObject(value))
|
|
75
|
+
throw new ConversionError("Expected a source JSON object");
|
|
76
|
+
return value;
|
|
77
|
+
}
|
|
78
|
+
/** An identifier Hue keeps as a history owner or session: nonempty, bounded, well formed. */
|
|
79
|
+
function identifier(value, max) {
|
|
80
|
+
return typeof value === "string" &&
|
|
81
|
+
value.length >= 1 &&
|
|
82
|
+
value.length <= max &&
|
|
83
|
+
!value.includes("\0") &&
|
|
84
|
+
value.isWellFormed()
|
|
85
|
+
? value
|
|
86
|
+
: undefined;
|
|
87
|
+
}
|
|
88
|
+
// --- JSON reading ------------------------------------------------------------------------------
|
|
89
|
+
const LONG_INTEGER = /\d{16}/;
|
|
90
|
+
/**
|
|
91
|
+
* JSON text parsed with every integer too large for a JavaScript number kept as its decimal
|
|
92
|
+
* text, so an OTLP timestamp written as a number keeps its nanoseconds. `unsafeIntegers` says
|
|
93
|
+
* whether any was found; the Langfuse and ATIF readers refuse such a file, as Hue's do.
|
|
94
|
+
*/
|
|
95
|
+
export function parseJsonText(text) {
|
|
96
|
+
if (!LONG_INTEGER.test(text))
|
|
97
|
+
return { value: JSON.parse(text), unsafeIntegers: false };
|
|
98
|
+
let unsafeIntegers = false;
|
|
99
|
+
const reviver = (_key, value, context) => {
|
|
100
|
+
if (typeof value === "number" &&
|
|
101
|
+
!Number.isSafeInteger(value) &&
|
|
102
|
+
typeof context?.source === "string" &&
|
|
103
|
+
/^-?\d+$/.test(context.source)) {
|
|
104
|
+
unsafeIntegers = true;
|
|
105
|
+
return context.source;
|
|
106
|
+
}
|
|
107
|
+
return value;
|
|
108
|
+
};
|
|
109
|
+
try {
|
|
110
|
+
return {
|
|
111
|
+
value: JSON.parse(text, reviver),
|
|
112
|
+
unsafeIntegers,
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
catch (error) {
|
|
116
|
+
// The reviver walks recursively; a body nested too deep for it is read without it.
|
|
117
|
+
if (error instanceof RangeError)
|
|
118
|
+
return { value: JSON.parse(text), unsafeIntegers: false };
|
|
119
|
+
throw error;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
/** Refuses a source value that JavaScript or OTLP cannot retain exactly; never omits it. */
|
|
123
|
+
export function validateSource(value, depth = 0) {
|
|
124
|
+
if (depth > 64)
|
|
125
|
+
throw new ConversionError("Source JSON exceeds the nesting bound");
|
|
126
|
+
if (typeof value === "number") {
|
|
127
|
+
if (!Number.isFinite(value) || (Number.isInteger(value) && !Number.isSafeInteger(value)))
|
|
128
|
+
throw new ConversionError("Source JSON contains an unsafe number");
|
|
129
|
+
}
|
|
130
|
+
else if (typeof value === "string") {
|
|
131
|
+
if (!value.isWellFormed() || value.includes("\u0000"))
|
|
132
|
+
throw new ConversionError("Source JSON contains invalid text");
|
|
133
|
+
}
|
|
134
|
+
else if (value && typeof value === "object") {
|
|
135
|
+
for (const [key, child] of Object.entries(value)) {
|
|
136
|
+
validateSource(key, depth + 1);
|
|
137
|
+
validateSource(child, depth + 1);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
/** Compares decimal values without rounding, allowing harmless exponent and zero formatting. */
|
|
142
|
+
function decimalIdentity(value) {
|
|
143
|
+
const match = /^(-?)(\d+)(?:\.(\d+))?(?:[eE]([+-]?\d+))?$/.exec(value);
|
|
144
|
+
if (!match)
|
|
145
|
+
throw new ConversionError("JSON has unsafe numbers");
|
|
146
|
+
const digits = match[2] + (match[3] ?? "");
|
|
147
|
+
let start = 0;
|
|
148
|
+
let end = digits.length;
|
|
149
|
+
while (start < end && digits[start] === "0")
|
|
150
|
+
start++;
|
|
151
|
+
if (start === end)
|
|
152
|
+
return `${match[1]}0`;
|
|
153
|
+
while (digits[end - 1] === "0")
|
|
154
|
+
end--;
|
|
155
|
+
const exponent = Number(match[4] ?? 0) - (match[3]?.length ?? 0) + digits.length - end;
|
|
156
|
+
if (!Number.isSafeInteger(exponent))
|
|
157
|
+
throw new ConversionError("JSON has unsafe numbers");
|
|
158
|
+
return `${match[1]}${digits.slice(start, end)}e${exponent}`;
|
|
159
|
+
}
|
|
160
|
+
/** JSON.parse otherwise loses duplicate members and can round decimal argument values. */
|
|
161
|
+
export function validateJsonText(raw) {
|
|
162
|
+
const stack = [];
|
|
163
|
+
for (let i = 0; i < raw.length; i++) {
|
|
164
|
+
const char = raw[i];
|
|
165
|
+
if (char === "{")
|
|
166
|
+
stack.push({ keys: new Set(), expectingKey: true });
|
|
167
|
+
else if (char === "[")
|
|
168
|
+
stack.push(null);
|
|
169
|
+
else if (char === "}" || char === "]")
|
|
170
|
+
stack.pop();
|
|
171
|
+
else if (char === ",") {
|
|
172
|
+
const current = stack.at(-1);
|
|
173
|
+
if (current)
|
|
174
|
+
current.expectingKey = true;
|
|
175
|
+
}
|
|
176
|
+
else if (char === '"') {
|
|
177
|
+
const start = i;
|
|
178
|
+
for (i++; i < raw.length; i++) {
|
|
179
|
+
if (raw[i] === "\\")
|
|
180
|
+
i++;
|
|
181
|
+
else if (raw[i] === '"')
|
|
182
|
+
break;
|
|
183
|
+
}
|
|
184
|
+
const current = stack.at(-1);
|
|
185
|
+
if (current?.expectingKey) {
|
|
186
|
+
const key = JSON.parse(raw.slice(start, i + 1));
|
|
187
|
+
if (current.keys.has(key))
|
|
188
|
+
throw new ConversionError("JSON has duplicate object keys");
|
|
189
|
+
current.keys.add(key);
|
|
190
|
+
current.expectingKey = false;
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
else if (char === "-" || (char >= "0" && char <= "9")) {
|
|
194
|
+
const start = i;
|
|
195
|
+
while (i + 1 < raw.length && /[\d.eE+-]/.test(raw[i + 1]))
|
|
196
|
+
i++;
|
|
197
|
+
const original = raw.slice(start, i + 1);
|
|
198
|
+
if (decimalIdentity(original) !== decimalIdentity(JSON.stringify(Number(original))))
|
|
199
|
+
throw new ConversionError("JSON has unsafe numbers");
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
const isOtlpEnvelope = (value) => isObject(value) && (Array.isArray(value.resourceSpans) || Array.isArray(value.resource_spans));
|
|
204
|
+
const isObservation = (value) => isObject(value) &&
|
|
205
|
+
typeof value.id === "string" &&
|
|
206
|
+
typeof value.traceId === "string" &&
|
|
207
|
+
typeof value.type === "string" &&
|
|
208
|
+
typeof value.startTime === "string";
|
|
209
|
+
/** What a file holds, read from its content alone; never from its name. */
|
|
210
|
+
export function detectFormat(bytes) {
|
|
211
|
+
if (!bytes.byteLength)
|
|
212
|
+
return { format: "unsupported", reason: "the file is empty" };
|
|
213
|
+
if (bytes[0] === 0x1f && bytes[1] === 0x8b)
|
|
214
|
+
return { format: "unsupported", reason: "a gzip file; decompress it first" };
|
|
215
|
+
// An archive is read where the walk finds it on disk; one inside another is not opened.
|
|
216
|
+
if (isZipSignature(bytes))
|
|
217
|
+
return { format: "unsupported", reason: "a ZIP archive inside the input; extract it first" };
|
|
218
|
+
let text;
|
|
219
|
+
try {
|
|
220
|
+
text = new TextDecoder("utf-8", { fatal: true, ignoreBOM: false }).decode(bytes);
|
|
221
|
+
}
|
|
222
|
+
catch {
|
|
223
|
+
return { format: "unsupported", reason: "not UTF-8 text" };
|
|
224
|
+
}
|
|
225
|
+
let parsed;
|
|
226
|
+
try {
|
|
227
|
+
parsed = parseJsonText(text);
|
|
228
|
+
}
|
|
229
|
+
catch {
|
|
230
|
+
parsed = undefined;
|
|
231
|
+
}
|
|
232
|
+
if (!parsed) {
|
|
233
|
+
// OTLP/JSON exports are also written one request a line.
|
|
234
|
+
const lines = text.split(/\r?\n/).filter((line) => line.trim());
|
|
235
|
+
if (lines.length < 2)
|
|
236
|
+
return { format: "unsupported", reason: "not JSON" };
|
|
237
|
+
const envelopes = [];
|
|
238
|
+
for (const line of lines) {
|
|
239
|
+
let value;
|
|
240
|
+
try {
|
|
241
|
+
value = parseJsonText(line).value;
|
|
242
|
+
}
|
|
243
|
+
catch {
|
|
244
|
+
return { format: "unsupported", reason: "not JSON" };
|
|
245
|
+
}
|
|
246
|
+
if (!isOtlpEnvelope(value))
|
|
247
|
+
return { format: "unsupported", reason: "JSON Lines whose records are not OTLP exports" };
|
|
248
|
+
envelopes.push(value);
|
|
249
|
+
}
|
|
250
|
+
return { format: "otlp_json", envelopes };
|
|
251
|
+
}
|
|
252
|
+
const value = parsed.value;
|
|
253
|
+
if (Array.isArray(value)) {
|
|
254
|
+
if (value.length && value.every(isOtlpEnvelope))
|
|
255
|
+
return { format: "otlp_json", envelopes: value };
|
|
256
|
+
return { format: "unsupported", reason: "a JSON array that is not a list of OTLP exports" };
|
|
257
|
+
}
|
|
258
|
+
if (!isObject(value))
|
|
259
|
+
return { format: "unsupported", reason: "not a JSON object" };
|
|
260
|
+
if (isOtlpEnvelope(value))
|
|
261
|
+
return { format: "otlp_json", envelopes: [value] };
|
|
262
|
+
if (Array.isArray(value.resourceLogs) || Array.isArray(value.resource_logs))
|
|
263
|
+
return { format: "unsupported", reason: "an OTLP logs export; only traces are imported" };
|
|
264
|
+
if (typeof value.schema_version === "string" && value.schema_version.startsWith("ATIF-")) {
|
|
265
|
+
if (!Array.isArray(value.steps))
|
|
266
|
+
return { format: "unsupported", reason: "an ATIF document without steps" };
|
|
267
|
+
return { format: "atif", text, document: value, unsafeIntegers: parsed.unsafeIntegers };
|
|
268
|
+
}
|
|
269
|
+
const key = Array.isArray(value.data)
|
|
270
|
+
? "data"
|
|
271
|
+
: Array.isArray(value.observations)
|
|
272
|
+
? "observations"
|
|
273
|
+
: undefined;
|
|
274
|
+
if (key) {
|
|
275
|
+
if (value.data !== undefined && value.observations !== undefined)
|
|
276
|
+
return { format: "unsupported", reason: "two observation envelopes in one file" };
|
|
277
|
+
const records = value[key];
|
|
278
|
+
if (!records.length)
|
|
279
|
+
return { format: "unsupported", reason: "a Langfuse export without observations" };
|
|
280
|
+
if (!records.every(isObservation))
|
|
281
|
+
return { format: "unsupported", reason: "a `data` list that is not Langfuse observations" };
|
|
282
|
+
const envelope = Object.fromEntries(Object.entries(value).filter(([name]) => name !== key));
|
|
283
|
+
return {
|
|
284
|
+
format: "langfuse_observations",
|
|
285
|
+
observations: records,
|
|
286
|
+
envelope,
|
|
287
|
+
unsafeIntegers: parsed.unsafeIntegers,
|
|
288
|
+
};
|
|
289
|
+
}
|
|
290
|
+
return { format: "unsupported", reason: "a JSON document of no recognized shape" };
|
|
291
|
+
}
|
|
292
|
+
// --- Attributes and large values ----------------------------------------------------------------
|
|
293
|
+
export function anyValue(value) {
|
|
294
|
+
if (value === null)
|
|
295
|
+
return {};
|
|
296
|
+
if (typeof value === "string")
|
|
297
|
+
return { stringValue: value };
|
|
298
|
+
if (typeof value === "boolean")
|
|
299
|
+
return { boolValue: value };
|
|
300
|
+
if (typeof value === "number")
|
|
301
|
+
return Number.isInteger(value) ? { intValue: String(value) } : { doubleValue: value };
|
|
302
|
+
if (Array.isArray(value))
|
|
303
|
+
return { arrayValue: { values: value.map(anyValue) } };
|
|
304
|
+
return {
|
|
305
|
+
kvlistValue: {
|
|
306
|
+
values: Object.entries(value).map(([key, item]) => ({ key, value: anyValue(item) })),
|
|
307
|
+
},
|
|
308
|
+
};
|
|
309
|
+
}
|
|
310
|
+
/** The large values one trace stores apart from its spans, each once by digest. */
|
|
311
|
+
export class BlobSink {
|
|
312
|
+
blobs = new Map();
|
|
313
|
+
/** Stores a text whole and returns what its record keeps: the first 16 KiB and the `hue.blobs`
|
|
314
|
+
* entry; undefined when the text is over the limit of one stored value. */
|
|
315
|
+
place(key, text) {
|
|
316
|
+
const bytes = Buffer.from(text, "utf8");
|
|
317
|
+
if (bytes.byteLength > traceImportLimits.blobBytes)
|
|
318
|
+
return undefined;
|
|
319
|
+
const sha256 = sha256Hex(bytes);
|
|
320
|
+
const contentType = textContentType(text);
|
|
321
|
+
if (!this.blobs.has(sha256))
|
|
322
|
+
this.blobs.set(sha256, { sha256, byteSize: bytes.byteLength, contentType, bytes });
|
|
323
|
+
return {
|
|
324
|
+
prefix: blobPrefix(text),
|
|
325
|
+
entry: blobEntry(key, sha256, bytes.byteLength, contentType),
|
|
326
|
+
};
|
|
327
|
+
}
|
|
328
|
+
/** The values in digest order: the manifest lists a trace's values the same whatever order its
|
|
329
|
+
* sources were read in, so renaming or repacking them keeps the manifest digest. */
|
|
330
|
+
sorted() {
|
|
331
|
+
return [...this.blobs.values()].sort((a, b) => a.sha256 < b.sha256 ? -1 : a.sha256 > b.sha256 ? 1 : 0);
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
/** A converted span's attributes: a value over the inline limit is stored apart from the span
|
|
335
|
+
* through the trace's {@link BlobSink} and listed in `hue.blobs`, as the SDK's exporter lists
|
|
336
|
+
* the values it uploads, with the key in `hue.truncated`. */
|
|
337
|
+
export class AttributeList {
|
|
338
|
+
blobs;
|
|
339
|
+
values = [];
|
|
340
|
+
listed = [];
|
|
341
|
+
truncated = [];
|
|
342
|
+
constructor(blobs) {
|
|
343
|
+
this.blobs = blobs;
|
|
344
|
+
}
|
|
345
|
+
add(key, value) {
|
|
346
|
+
const text = typeof value === "string" ? value : undefined;
|
|
347
|
+
const bytes = text !== undefined
|
|
348
|
+
? Buffer.byteLength(text, "utf8")
|
|
349
|
+
: Buffer.byteLength(JSON.stringify(value));
|
|
350
|
+
if (bytes <= MAX_INLINE_VALUE_BYTES || this.listed.length >= MAX_BLOB_ENTRIES) {
|
|
351
|
+
this.values.push({ key, value: anyValue(value) });
|
|
352
|
+
return;
|
|
353
|
+
}
|
|
354
|
+
const placed = this.blobs.place(key, text ?? JSON.stringify(value));
|
|
355
|
+
if (!placed)
|
|
356
|
+
throw new ConversionError("A source value exceeds the stored value limit");
|
|
357
|
+
this.values.push({ key, value: { stringValue: placed.prefix } });
|
|
358
|
+
this.listed.push(placed.entry);
|
|
359
|
+
this.truncated.push(key);
|
|
360
|
+
}
|
|
361
|
+
json(key, value) {
|
|
362
|
+
this.add(key, JSON.stringify(value));
|
|
363
|
+
}
|
|
364
|
+
finish() {
|
|
365
|
+
if (this.listed.length) {
|
|
366
|
+
this.values.push({ key: BLOBS_KEY, value: anyValue(this.listed) });
|
|
367
|
+
this.values.push({ key: TRUNCATED_KEY, value: anyValue(this.truncated) });
|
|
368
|
+
}
|
|
369
|
+
return this.values;
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
// --- OTLP/JSON --------------------------------------------------------------------------------
|
|
373
|
+
/** ProtoJSON parsers accept the proto field names beside lowerCamelCase: a body whose envelope
|
|
374
|
+
* uses them is read with every field renamed in place; attribute keys are values, never renamed. */
|
|
375
|
+
function renameProtoFields(decoded) {
|
|
376
|
+
const stack = [decoded];
|
|
377
|
+
while (stack.length) {
|
|
378
|
+
const item = stack.pop();
|
|
379
|
+
if (item === null || typeof item !== "object")
|
|
380
|
+
continue;
|
|
381
|
+
if (Array.isArray(item)) {
|
|
382
|
+
for (const entry of item)
|
|
383
|
+
stack.push(entry);
|
|
384
|
+
continue;
|
|
385
|
+
}
|
|
386
|
+
const record = item;
|
|
387
|
+
for (const key of Object.keys(record)) {
|
|
388
|
+
const value = record[key];
|
|
389
|
+
stack.push(value);
|
|
390
|
+
if (!key.includes("_"))
|
|
391
|
+
continue;
|
|
392
|
+
const renamed = key.replace(/_([a-z0-9])/g, (_, letter) => letter.toUpperCase());
|
|
393
|
+
if (renamed === key)
|
|
394
|
+
continue;
|
|
395
|
+
if (!Object.hasOwn(record, renamed))
|
|
396
|
+
record[renamed] = value;
|
|
397
|
+
delete record[key];
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
/** An OTLP id as sent (hex, or the base64 of a protobuf-JSON encoder) as lowercase hex, or
|
|
402
|
+
* undefined for anything that is not a valid nonzero id of the length. */
|
|
403
|
+
function normalizeId(value, length) {
|
|
404
|
+
if (typeof value !== "string")
|
|
405
|
+
return undefined;
|
|
406
|
+
const base64 = length === 32 ? BASE64_TRACE_ID : BASE64_SPAN_ID;
|
|
407
|
+
const candidate = base64.test(value)
|
|
408
|
+
? Buffer.from(value, "base64").toString("hex")
|
|
409
|
+
: value.toLowerCase();
|
|
410
|
+
return (length === 32 ? HEX32 : HEX16).test(candidate) && !/^0+$/.test(candidate)
|
|
411
|
+
? candidate
|
|
412
|
+
: undefined;
|
|
413
|
+
}
|
|
414
|
+
function decimalOf(value) {
|
|
415
|
+
if (typeof value === "number")
|
|
416
|
+
return Number.isSafeInteger(value) && value >= 0 ? String(value) : undefined;
|
|
417
|
+
if (typeof value === "string" && /^\d{1,20}$/.test(value))
|
|
418
|
+
return String(BigInt(value));
|
|
419
|
+
return undefined;
|
|
420
|
+
}
|
|
421
|
+
const stringOf = (value) => value && typeof value.stringValue === "string" ? value.stringValue : undefined;
|
|
422
|
+
function attributeMap(attributes) {
|
|
423
|
+
const map = new Map();
|
|
424
|
+
for (const attribute of attributes)
|
|
425
|
+
map.set(attribute.key, attribute.value);
|
|
426
|
+
return map;
|
|
427
|
+
}
|
|
428
|
+
/** A trace's identity as its root records it, in the attributes Hue reads. */
|
|
429
|
+
function identityOf(root, resource) {
|
|
430
|
+
const identity = {};
|
|
431
|
+
for (const key of USER_KEYS) {
|
|
432
|
+
const user = identifier(stringOf(root.get(key)), MAX_IDENTIFIER);
|
|
433
|
+
if (user) {
|
|
434
|
+
identity.user = user;
|
|
435
|
+
break;
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
for (const source of [root, resource]) {
|
|
439
|
+
const namespace = identifier(stringOf(source.get(ENTITY_NAMESPACE)), MAX_NAMESPACE);
|
|
440
|
+
const id = identifier(stringOf(source.get(ENTITY_ID)), MAX_IDENTIFIER);
|
|
441
|
+
if (namespace && id) {
|
|
442
|
+
identity.entity = { namespace, id };
|
|
443
|
+
break;
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
for (const key of SESSION_KEYS) {
|
|
447
|
+
const session = identifier(stringOf(root.get(key)), MAX_SESSION);
|
|
448
|
+
if (session) {
|
|
449
|
+
identity.session = session;
|
|
450
|
+
break;
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
return identity.user || identity.entity || identity.session ? identity : undefined;
|
|
454
|
+
}
|
|
455
|
+
/** Gathers the spans of OTLP/JSON exports by trace, across files: an export writes a trace's
|
|
456
|
+
* spans in the requests that carried them, which need not be one file. */
|
|
457
|
+
export class OtlpTraceCollector {
|
|
458
|
+
traces = new Map();
|
|
459
|
+
/** Adds every span of the envelopes; the spans refused, with their reasons. */
|
|
460
|
+
add(envelopes, sourceDigest) {
|
|
461
|
+
const refused = [];
|
|
462
|
+
for (const envelope of envelopes) {
|
|
463
|
+
// An export may mix the proto field names with lowerCamelCase at any depth; every
|
|
464
|
+
// envelope is normalized whole, so a `scope_spans` under a `resourceSpans` root is read.
|
|
465
|
+
renameProtoFields(envelope);
|
|
466
|
+
const groups = Array.isArray(envelope.resourceSpans) ? envelope.resourceSpans : [];
|
|
467
|
+
for (const groupValue of groups) {
|
|
468
|
+
if (!isObject(groupValue))
|
|
469
|
+
continue;
|
|
470
|
+
const { scopeSpans, ...resourceGroup } = groupValue;
|
|
471
|
+
const resource = isObject(resourceGroup.resource)
|
|
472
|
+
? attributeMap(readAttributes(resourceGroup.resource.attributes) ?? [])
|
|
473
|
+
: new Map();
|
|
474
|
+
for (const scopeValue of Array.isArray(scopeSpans) ? scopeSpans : []) {
|
|
475
|
+
if (!isObject(scopeValue))
|
|
476
|
+
continue;
|
|
477
|
+
const { spans, ...scopeGroup } = scopeValue;
|
|
478
|
+
const scopeName = isObject(scopeGroup.scope) ? scopeGroup.scope.name : undefined;
|
|
479
|
+
for (const spanValue of Array.isArray(spans) ? spans : []) {
|
|
480
|
+
try {
|
|
481
|
+
this.addSpan(spanValue, resourceGroup, scopeGroup, resource, scopeName, sourceDigest);
|
|
482
|
+
}
|
|
483
|
+
catch (error) {
|
|
484
|
+
if (!(error instanceof ConversionError))
|
|
485
|
+
throw error;
|
|
486
|
+
refused.push({ reason: error.message });
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
return refused;
|
|
493
|
+
}
|
|
494
|
+
addSpan(value, resourceGroup, scopeGroup, resource, scopeName, sourceDigest) {
|
|
495
|
+
if (!isObject(value))
|
|
496
|
+
throw new ConversionError("a span that is not a JSON object");
|
|
497
|
+
const span = { ...value };
|
|
498
|
+
const originalTraceId = span.traceId;
|
|
499
|
+
if (typeof originalTraceId !== "string" || !originalTraceId)
|
|
500
|
+
throw new ConversionError("a span without a trace id");
|
|
501
|
+
let traceId = normalizeId(originalTraceId, 32);
|
|
502
|
+
const remappedTraceId = traceId === undefined;
|
|
503
|
+
if (!traceId)
|
|
504
|
+
traceId = digestHex(`otlp.trace:${originalTraceId}`, 32);
|
|
505
|
+
const originalSpanId = span.spanId;
|
|
506
|
+
if (typeof originalSpanId !== "string" || !originalSpanId)
|
|
507
|
+
throw new ConversionError("a span without a span id");
|
|
508
|
+
const mapSpanId = (original) => normalizeId(original, 16) ?? digestHex(`otlp.span:${traceId}:${original}`, 16);
|
|
509
|
+
const spanId = mapSpanId(originalSpanId);
|
|
510
|
+
const remappedSpanId = normalizeId(originalSpanId, 16) === undefined;
|
|
511
|
+
span.traceId = traceId;
|
|
512
|
+
span.spanId = spanId;
|
|
513
|
+
const parent = span.parentSpanId;
|
|
514
|
+
let parentSpanId;
|
|
515
|
+
if (typeof parent === "string" && parent && !/^[0=A]*$/.test(parent) && !/^0+$/.test(parent))
|
|
516
|
+
parentSpanId = mapSpanId(parent);
|
|
517
|
+
if (parentSpanId)
|
|
518
|
+
span.parentSpanId = parentSpanId;
|
|
519
|
+
else
|
|
520
|
+
delete span.parentSpanId;
|
|
521
|
+
const start = decimalOf(span.startTimeUnixNano);
|
|
522
|
+
const end = decimalOf(span.endTimeUnixNano);
|
|
523
|
+
if (start === undefined || start === "0")
|
|
524
|
+
throw new ConversionError(`span ${spanId} has no valid start time`);
|
|
525
|
+
if (end === undefined || BigInt(end) < BigInt(start))
|
|
526
|
+
throw new ConversionError(`span ${spanId} has no valid end time`);
|
|
527
|
+
span.startTimeUnixNano = start;
|
|
528
|
+
span.endTimeUnixNano = end;
|
|
529
|
+
if (span.name !== undefined && typeof span.name !== "string")
|
|
530
|
+
throw new ConversionError(`span ${spanId} has a name that is not text`);
|
|
531
|
+
const attributes = readAttributes(span.attributes);
|
|
532
|
+
if (attributes === undefined)
|
|
533
|
+
throw new ConversionError(`span ${spanId} has malformed attributes`);
|
|
534
|
+
let trace = this.traces.get(traceId);
|
|
535
|
+
if (!trace) {
|
|
536
|
+
trace = {
|
|
537
|
+
traceId,
|
|
538
|
+
remappedTraceId,
|
|
539
|
+
spans: new Map(),
|
|
540
|
+
blobs: new BlobSink(),
|
|
541
|
+
sourceDigests: new Set(),
|
|
542
|
+
remappedSpans: 0,
|
|
543
|
+
evaluation: false,
|
|
544
|
+
simulation: false,
|
|
545
|
+
production: false,
|
|
546
|
+
};
|
|
547
|
+
this.traces.set(traceId, trace);
|
|
548
|
+
}
|
|
549
|
+
const kept = [...attributes];
|
|
550
|
+
if (remappedTraceId)
|
|
551
|
+
kept.push({ key: "hue.import.original_trace_id", value: { stringValue: originalTraceId } });
|
|
552
|
+
if (remappedSpanId)
|
|
553
|
+
kept.push({ key: "hue.import.original_span_id", value: { stringValue: originalSpanId } });
|
|
554
|
+
span.attributes = offloadOtlpValues(kept, trace.blobs);
|
|
555
|
+
const entry = {
|
|
556
|
+
...resourceGroup,
|
|
557
|
+
scopeSpans: [{ ...scopeGroup, spans: [span] }],
|
|
558
|
+
};
|
|
559
|
+
const canonical = JSON.stringify(entry);
|
|
560
|
+
const held = trace.spans.get(spanId);
|
|
561
|
+
if (held) {
|
|
562
|
+
// The same span sent twice is one span; another payload under its id is a conflict.
|
|
563
|
+
if (held.canonical !== canonical)
|
|
564
|
+
trace.conflict ??= `span ${spanId} appears with different content`;
|
|
565
|
+
trace.sourceDigests.add(sourceDigest);
|
|
566
|
+
return;
|
|
567
|
+
}
|
|
568
|
+
const map = attributeMap(kept);
|
|
569
|
+
for (const key of EVALUATION_KEYS)
|
|
570
|
+
if (map.has(key) || resource.has(key))
|
|
571
|
+
trace.evaluation = true;
|
|
572
|
+
for (const key of SIMULATION_KEYS)
|
|
573
|
+
if (map.has(key) || resource.has(key))
|
|
574
|
+
trace.simulation = true;
|
|
575
|
+
if (scopeName === SIMULATION_SCOPE)
|
|
576
|
+
trace.simulation = true;
|
|
577
|
+
if (stringOf(resource.get("deployment.environment.name")) === "production" ||
|
|
578
|
+
stringOf(resource.get("deployment.environment")) === "production")
|
|
579
|
+
trace.production = true;
|
|
580
|
+
if (remappedSpanId)
|
|
581
|
+
trace.remappedSpans++;
|
|
582
|
+
trace.sourceDigests.add(sourceDigest);
|
|
583
|
+
trace.spans.set(spanId, {
|
|
584
|
+
spanId,
|
|
585
|
+
...(parentSpanId ? { parentSpanId } : {}),
|
|
586
|
+
start: BigInt(start),
|
|
587
|
+
entry,
|
|
588
|
+
canonical,
|
|
589
|
+
attributes: map,
|
|
590
|
+
resource,
|
|
591
|
+
});
|
|
592
|
+
}
|
|
593
|
+
/** Every trace gathered so far, spans in time order, and the traces refused whole. */
|
|
594
|
+
finish() {
|
|
595
|
+
const traces = [];
|
|
596
|
+
const refused = [];
|
|
597
|
+
for (const draft of this.traces.values()) {
|
|
598
|
+
if (draft.conflict) {
|
|
599
|
+
refused.push({ traceId: draft.traceId, reason: draft.conflict });
|
|
600
|
+
continue;
|
|
601
|
+
}
|
|
602
|
+
const spans = [...draft.spans.values()].sort((a, b) => a.start < b.start ? -1 : a.start > b.start ? 1 : a.spanId < b.spanId ? -1 : 1);
|
|
603
|
+
const root = spans.find((span) => !span.parentSpanId || !draft.spans.has(span.parentSpanId));
|
|
604
|
+
const identity = root ? identityOf(root.attributes, root.resource) : undefined;
|
|
605
|
+
traces.push({
|
|
606
|
+
traceId: draft.traceId,
|
|
607
|
+
format: "otlp_json",
|
|
608
|
+
entries: spans.map((span) => span.entry),
|
|
609
|
+
blobs: draft.blobs.sorted(),
|
|
610
|
+
...(identity ? { identity } : {}),
|
|
611
|
+
origin: draft.evaluation
|
|
612
|
+
? "evaluation"
|
|
613
|
+
: draft.simulation
|
|
614
|
+
? "simulation"
|
|
615
|
+
: draft.production
|
|
616
|
+
? "production"
|
|
617
|
+
: "unknown",
|
|
618
|
+
sourceDigests: draft.sourceDigests,
|
|
619
|
+
remappedTraceId: draft.remappedTraceId,
|
|
620
|
+
remappedSpans: draft.remappedSpans,
|
|
621
|
+
});
|
|
622
|
+
}
|
|
623
|
+
return { traces, refused };
|
|
624
|
+
}
|
|
625
|
+
}
|
|
626
|
+
/** An OTLP attribute list as recorded, or undefined when it is malformed. */
|
|
627
|
+
function readAttributes(value) {
|
|
628
|
+
if (value === undefined || value === null)
|
|
629
|
+
return [];
|
|
630
|
+
if (!Array.isArray(value))
|
|
631
|
+
return undefined;
|
|
632
|
+
const attributes = [];
|
|
633
|
+
for (const item of value) {
|
|
634
|
+
if (!isObject(item) || typeof item.key !== "string" || !isObject(item.value))
|
|
635
|
+
return undefined;
|
|
636
|
+
attributes.push(item);
|
|
637
|
+
}
|
|
638
|
+
return attributes;
|
|
639
|
+
}
|
|
640
|
+
/** A recorded span's string values over the inline limit stored apart from it, listed beside
|
|
641
|
+
* any values its exporter already listed. */
|
|
642
|
+
function offloadOtlpValues(attributes, blobs) {
|
|
643
|
+
const over = attributes.filter((attribute) => typeof attribute.value.stringValue === "string" &&
|
|
644
|
+
Buffer.byteLength(attribute.value.stringValue, "utf8") > MAX_INLINE_VALUE_BYTES);
|
|
645
|
+
if (!over.length)
|
|
646
|
+
return attributes;
|
|
647
|
+
const listedValues = (key) => {
|
|
648
|
+
const found = attributes.find((attribute) => attribute.key === key);
|
|
649
|
+
if (!found)
|
|
650
|
+
return [];
|
|
651
|
+
const values = found.value.arrayValue?.values;
|
|
652
|
+
if (!values || values.some((value) => typeof value.stringValue !== "string"))
|
|
653
|
+
return undefined;
|
|
654
|
+
return values.map((value) => value.stringValue);
|
|
655
|
+
};
|
|
656
|
+
const listed = listedValues(BLOBS_KEY);
|
|
657
|
+
const truncated = listedValues(TRUNCATED_KEY);
|
|
658
|
+
if (listed === undefined || truncated === undefined)
|
|
659
|
+
return attributes;
|
|
660
|
+
const result = attributes.filter((attribute) => attribute.key !== BLOBS_KEY && attribute.key !== TRUNCATED_KEY);
|
|
661
|
+
for (const attribute of over) {
|
|
662
|
+
if (listed.length >= MAX_BLOB_ENTRIES)
|
|
663
|
+
break;
|
|
664
|
+
const placed = blobs.place(attribute.key, attribute.value.stringValue);
|
|
665
|
+
if (!placed)
|
|
666
|
+
continue;
|
|
667
|
+
const at = result.indexOf(attribute);
|
|
668
|
+
result[at] = { key: attribute.key, value: { stringValue: placed.prefix } };
|
|
669
|
+
listed.push(placed.entry);
|
|
670
|
+
if (!truncated.includes(attribute.key))
|
|
671
|
+
truncated.push(attribute.key);
|
|
672
|
+
}
|
|
673
|
+
if (listed.length) {
|
|
674
|
+
result.push({ key: BLOBS_KEY, value: anyValue(listed) });
|
|
675
|
+
result.push({ key: TRUNCATED_KEY, value: anyValue(truncated) });
|
|
676
|
+
}
|
|
677
|
+
return result;
|
|
678
|
+
}
|
|
679
|
+
// --- Langfuse observations --------------------------------------------------------------------
|
|
680
|
+
function text(value, required = false) {
|
|
681
|
+
if (value === undefined || value === null) {
|
|
682
|
+
if (required)
|
|
683
|
+
throw new ConversionError("A required source text field is absent");
|
|
684
|
+
return "";
|
|
685
|
+
}
|
|
686
|
+
if (typeof value !== "string")
|
|
687
|
+
throw new ConversionError("Expected a source text field");
|
|
688
|
+
if (required && !value)
|
|
689
|
+
throw new ConversionError("A required source text field is empty");
|
|
690
|
+
return value;
|
|
691
|
+
}
|
|
692
|
+
function integer(value) {
|
|
693
|
+
if (value === undefined || value === null)
|
|
694
|
+
return 0;
|
|
695
|
+
if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0)
|
|
696
|
+
throw new ConversionError("Expected a nonnegative source token count");
|
|
697
|
+
return value;
|
|
698
|
+
}
|
|
699
|
+
export function unixNano(value) {
|
|
700
|
+
const match = /^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(?:\.(\d{1,9}))?(Z|[+-]\d{2}:\d{2})$/.exec(text(value, true));
|
|
701
|
+
if (!match)
|
|
702
|
+
throw new ConversionError("Invalid source timestamp");
|
|
703
|
+
const seconds = Date.parse(`${match[1]}${match[3]}`);
|
|
704
|
+
if (!Number.isSafeInteger(seconds) || seconds < 0)
|
|
705
|
+
throw new ConversionError("Invalid source timestamp");
|
|
706
|
+
return (BigInt(seconds) * 1000000n + BigInt((match[2] ?? "").padEnd(9, "0"))).toString();
|
|
707
|
+
}
|
|
708
|
+
const canonicalJson = (value) => {
|
|
709
|
+
if (Array.isArray(value))
|
|
710
|
+
return `[${value.map(canonicalJson).join(",")}]`;
|
|
711
|
+
if (value && typeof value === "object")
|
|
712
|
+
return `{${Object.keys(value)
|
|
713
|
+
.sort()
|
|
714
|
+
.map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`)
|
|
715
|
+
.join(",")}}`;
|
|
716
|
+
return JSON.stringify(value);
|
|
717
|
+
};
|
|
718
|
+
/** Validates parent links without rewalking ancestry already settled by another child. */
|
|
719
|
+
function ancestryIssue(parents) {
|
|
720
|
+
const settled = new Set();
|
|
721
|
+
for (const id of parents.keys()) {
|
|
722
|
+
const path = [];
|
|
723
|
+
const active = new Set();
|
|
724
|
+
let current = id;
|
|
725
|
+
while (current !== undefined && !settled.has(current)) {
|
|
726
|
+
if (active.has(current))
|
|
727
|
+
return "cyclic_ancestry";
|
|
728
|
+
if (!parents.has(current))
|
|
729
|
+
return "missing_parent";
|
|
730
|
+
active.add(current);
|
|
731
|
+
path.push(current);
|
|
732
|
+
current = parents.get(current) || undefined;
|
|
733
|
+
}
|
|
734
|
+
for (const ancestor of path)
|
|
735
|
+
settled.add(ancestor);
|
|
736
|
+
}
|
|
737
|
+
return undefined;
|
|
738
|
+
}
|
|
739
|
+
/** Gathers Langfuse observations by trace across files and converts each trace as Hue's reader
|
|
740
|
+
* converts an observation export: one span per observation, the record kept whole. */
|
|
741
|
+
export class LangfuseTraceCollector {
|
|
742
|
+
traces = new Map();
|
|
743
|
+
add(observations, envelope, sourceDigest) {
|
|
744
|
+
const envelopeText = JSON.stringify(envelope);
|
|
745
|
+
for (const observation of observations) {
|
|
746
|
+
const traceId = text(observation.traceId, true);
|
|
747
|
+
const id = text(observation.id, true);
|
|
748
|
+
let draft = this.traces.get(traceId);
|
|
749
|
+
if (!draft) {
|
|
750
|
+
draft = {
|
|
751
|
+
originalTraceId: traceId,
|
|
752
|
+
observations: new Map(),
|
|
753
|
+
envelopes: new Map(),
|
|
754
|
+
sourceDigests: new Set(),
|
|
755
|
+
};
|
|
756
|
+
this.traces.set(traceId, draft);
|
|
757
|
+
}
|
|
758
|
+
draft.sourceDigests.add(sourceDigest);
|
|
759
|
+
draft.envelopes.set(sourceDigest, envelopeText);
|
|
760
|
+
const canonical = canonicalJson(observation);
|
|
761
|
+
const held = draft.observations.get(id);
|
|
762
|
+
if (held) {
|
|
763
|
+
if (held.canonical !== canonical)
|
|
764
|
+
draft.conflict ??= `observation ${id} appears with different content`;
|
|
765
|
+
continue;
|
|
766
|
+
}
|
|
767
|
+
draft.observations.set(id, { record: observation, canonical });
|
|
768
|
+
}
|
|
769
|
+
}
|
|
770
|
+
finish() {
|
|
771
|
+
const traces = [];
|
|
772
|
+
const refused = [];
|
|
773
|
+
for (const draft of this.traces.values()) {
|
|
774
|
+
const traceId = hueTraceId(draft.originalTraceId);
|
|
775
|
+
if (draft.conflict) {
|
|
776
|
+
refused.push({ traceId, reason: draft.conflict });
|
|
777
|
+
continue;
|
|
778
|
+
}
|
|
779
|
+
try {
|
|
780
|
+
traces.push(convertLangfuseTrace(draft, traceId));
|
|
781
|
+
}
|
|
782
|
+
catch (error) {
|
|
783
|
+
if (!(error instanceof ConversionError))
|
|
784
|
+
throw error;
|
|
785
|
+
refused.push({ traceId, reason: error.message });
|
|
786
|
+
}
|
|
787
|
+
}
|
|
788
|
+
return { traces, refused };
|
|
789
|
+
}
|
|
790
|
+
}
|
|
791
|
+
function convertLangfuseTrace(draft, traceId) {
|
|
792
|
+
const originalTraceId = draft.originalTraceId;
|
|
793
|
+
for (const { record } of draft.observations.values())
|
|
794
|
+
validateSource(record);
|
|
795
|
+
// Observations in time order, then by id: the same trace from any packing orders the same.
|
|
796
|
+
const observations = [...draft.observations.values()]
|
|
797
|
+
.map(({ record }) => ({ record, start: BigInt(unixNano(record.startTime)) }))
|
|
798
|
+
.sort((a, b) => a.start < b.start
|
|
799
|
+
? -1
|
|
800
|
+
: a.start > b.start
|
|
801
|
+
? 1
|
|
802
|
+
: text(a.record.id, true) < text(b.record.id, true)
|
|
803
|
+
? -1
|
|
804
|
+
: 1)
|
|
805
|
+
.map(({ record }) => record);
|
|
806
|
+
const ids = new Set();
|
|
807
|
+
let roots = 0;
|
|
808
|
+
for (const observation of observations) {
|
|
809
|
+
ids.add(text(observation.id, true));
|
|
810
|
+
if (!text(observation.parentObservationId))
|
|
811
|
+
roots++;
|
|
812
|
+
}
|
|
813
|
+
if (roots !== 1)
|
|
814
|
+
throw new ConversionError("Expected one root observation per source trace");
|
|
815
|
+
const parents = new Map();
|
|
816
|
+
for (const observation of observations) {
|
|
817
|
+
const parent = text(observation.parentObservationId);
|
|
818
|
+
if (parent && !ids.has(parent))
|
|
819
|
+
throw new ConversionError("A source parent is missing");
|
|
820
|
+
parents.set(text(observation.id, true), parent);
|
|
821
|
+
}
|
|
822
|
+
const ancestry = ancestryIssue(parents);
|
|
823
|
+
if (ancestry === "missing_parent")
|
|
824
|
+
throw new ConversionError("A source parent is missing");
|
|
825
|
+
if (ancestry === "cyclic_ancestry")
|
|
826
|
+
throw new ConversionError("The source hierarchy contains a cycle");
|
|
827
|
+
// The export's envelope is kept on the root when one file held the whole trace.
|
|
828
|
+
const envelope = draft.envelopes.size === 1 ? [...draft.envelopes.values()][0] : undefined;
|
|
829
|
+
const blobs = new BlobSink();
|
|
830
|
+
const spanIds = new Map();
|
|
831
|
+
const imported = new Set();
|
|
832
|
+
let remappedSpans = 0;
|
|
833
|
+
for (const [index, observation] of observations.entries()) {
|
|
834
|
+
const originalId = text(observation.id, true);
|
|
835
|
+
let spanId = hueSpanId(originalTraceId, originalId);
|
|
836
|
+
if (spanId !== originalId)
|
|
837
|
+
remappedSpans++;
|
|
838
|
+
if (imported.has(spanId)) {
|
|
839
|
+
let salt = 0;
|
|
840
|
+
do
|
|
841
|
+
spanId = digestHex(`langfuse.observation.duplicate:${traceId}:${index}:${salt++}`, 16);
|
|
842
|
+
while (ids.has(spanId) || imported.has(spanId) || /^0+$/.test(spanId));
|
|
843
|
+
remappedSpans++;
|
|
844
|
+
}
|
|
845
|
+
imported.add(spanId);
|
|
846
|
+
spanIds.set(originalId, spanId);
|
|
847
|
+
}
|
|
848
|
+
let identity;
|
|
849
|
+
// The origin markers the OTLP reader applies, read from the converted attributes with the same
|
|
850
|
+
// precedence: an evaluation's or a simulated world's records are never production history,
|
|
851
|
+
// whatever environment the export was taken from.
|
|
852
|
+
let production = false;
|
|
853
|
+
let evaluation = false;
|
|
854
|
+
let simulation = false;
|
|
855
|
+
const markOrigin = (key, value, resourceAttribute) => {
|
|
856
|
+
if (EVALUATION_KEYS.includes(key))
|
|
857
|
+
evaluation = true;
|
|
858
|
+
if (SIMULATION_KEYS.includes(key))
|
|
859
|
+
simulation = true;
|
|
860
|
+
if (resourceAttribute &&
|
|
861
|
+
(key === "deployment.environment.name" || key === "deployment.environment") &&
|
|
862
|
+
value === "production")
|
|
863
|
+
production = true;
|
|
864
|
+
};
|
|
865
|
+
const entries = observations.map((observation, index) => {
|
|
866
|
+
const originalId = text(observation.id, true);
|
|
867
|
+
const spanId = spanIds.get(originalId);
|
|
868
|
+
const startTimeUnixNano = unixNano(observation.startTime);
|
|
869
|
+
// An observation without an end (a Langfuse event) lasts no time, as `hue import langfuse`
|
|
870
|
+
// sends it.
|
|
871
|
+
const ended = observation.endTime !== undefined && observation.endTime !== null;
|
|
872
|
+
const endTimeUnixNano = ended ? unixNano(observation.endTime) : startTimeUnixNano;
|
|
873
|
+
if (BigInt(endTimeUnixNano) < BigInt(startTimeUnixNano))
|
|
874
|
+
throw new ConversionError("A source observation has negative duration");
|
|
875
|
+
const type = text(observation.type, true).toLowerCase();
|
|
876
|
+
const metadata = object(observation.metadata ?? {});
|
|
877
|
+
const attributes = new AttributeList(blobs);
|
|
878
|
+
const resource = [];
|
|
879
|
+
for (const [key, value] of Object.entries(metadata)) {
|
|
880
|
+
if (key.startsWith("hue.import."))
|
|
881
|
+
throw new ConversionError("Source metadata uses reserved import keys");
|
|
882
|
+
if (key.startsWith("resourceAttributes.")) {
|
|
883
|
+
const resourceKey = key.slice("resourceAttributes.".length);
|
|
884
|
+
resource.push({ key: resourceKey, value: anyValue(value) });
|
|
885
|
+
markOrigin(resourceKey, value, true);
|
|
886
|
+
}
|
|
887
|
+
else if (key !== "scope.name" && key !== "scope.version") {
|
|
888
|
+
attributes.add(key, value);
|
|
889
|
+
markOrigin(key, value, false);
|
|
890
|
+
}
|
|
891
|
+
}
|
|
892
|
+
if (text(metadata["scope.name"]) === SIMULATION_SCOPE)
|
|
893
|
+
simulation = true;
|
|
894
|
+
attributes.add("hue.import.source_index", index);
|
|
895
|
+
attributes.add("hue.import.original_span_id", originalId);
|
|
896
|
+
if (traceId !== originalTraceId)
|
|
897
|
+
attributes.add("hue.import.original_trace_id", originalTraceId);
|
|
898
|
+
attributes.add(SOURCE_RECORD, JSON.stringify(observation));
|
|
899
|
+
const parent = text(observation.parentObservationId);
|
|
900
|
+
if (!parent && envelope !== undefined)
|
|
901
|
+
attributes.add(SOURCE_ENVELOPE, envelope);
|
|
902
|
+
attributes.add("langfuse.observation.type", type);
|
|
903
|
+
for (const field of ["input", "output"]) {
|
|
904
|
+
const value = observation[field];
|
|
905
|
+
if (value === undefined || value === null || value === "")
|
|
906
|
+
continue;
|
|
907
|
+
attributes.add(`langfuse.observation.${field}`, value);
|
|
908
|
+
}
|
|
909
|
+
for (const [field, key] of [
|
|
910
|
+
["model", "langfuse.observation.model.name"],
|
|
911
|
+
["sessionId", "langfuse.session.id"],
|
|
912
|
+
["userId", "langfuse.user.id"],
|
|
913
|
+
["traceName", "langfuse.trace.name"],
|
|
914
|
+
["release", "langfuse.release"],
|
|
915
|
+
["environment", "langfuse.environment"],
|
|
916
|
+
["level", "langfuse.observation.level"],
|
|
917
|
+
]) {
|
|
918
|
+
const value = observation[field];
|
|
919
|
+
if (value !== undefined && value !== null && value !== "")
|
|
920
|
+
attributes.add(key, value);
|
|
921
|
+
}
|
|
922
|
+
if (observation.usageDetails !== undefined)
|
|
923
|
+
attributes.add("langfuse.observation.usage_details", JSON.stringify(observation.usageDetails));
|
|
924
|
+
if (type === "generation") {
|
|
925
|
+
if (observation.inputUsage !== undefined && observation.inputUsage !== null)
|
|
926
|
+
attributes.add("gen_ai.usage.input_tokens", integer(observation.inputUsage));
|
|
927
|
+
if (observation.outputUsage !== undefined && observation.outputUsage !== null)
|
|
928
|
+
attributes.add("gen_ai.usage.output_tokens", integer(observation.outputUsage));
|
|
929
|
+
const usage = object(observation.usageDetails ?? {});
|
|
930
|
+
for (const [field, key] of [
|
|
931
|
+
["input_cached_tokens", "gen_ai.usage.cache_read.input_tokens"],
|
|
932
|
+
["input_cache_write_tokens", "gen_ai.usage.cache_write.input_tokens"],
|
|
933
|
+
])
|
|
934
|
+
if (usage[field] !== undefined && usage[field] !== null)
|
|
935
|
+
attributes.add(key, integer(usage[field]));
|
|
936
|
+
}
|
|
937
|
+
attributes.add("hue.import.status_source", "langfuse.level");
|
|
938
|
+
if (observation.environment === "production")
|
|
939
|
+
production = true;
|
|
940
|
+
const message = text(observation.statusMessage);
|
|
941
|
+
const span = {
|
|
942
|
+
traceId,
|
|
943
|
+
spanId,
|
|
944
|
+
...(parent ? { parentSpanId: spanIds.get(parent) } : {}),
|
|
945
|
+
name: text(observation.name, true),
|
|
946
|
+
// The export does not retain the original OTLP span kind.
|
|
947
|
+
kind: 0,
|
|
948
|
+
startTimeUnixNano,
|
|
949
|
+
endTimeUnixNano,
|
|
950
|
+
attributes: attributes.finish(),
|
|
951
|
+
status: { code: observation.level === "ERROR" ? 2 : 0, ...(message ? { message } : {}) },
|
|
952
|
+
};
|
|
953
|
+
// The trace's owner as the OTLP reader takes it from a root: the user (`langfuse.user.id`
|
|
954
|
+
// among the keys), an explicit entity in the root's metadata or resource attributes, and the
|
|
955
|
+
// session.
|
|
956
|
+
if (!parent)
|
|
957
|
+
identity = identityOf(attributeMap(span.attributes), attributeMap(resource));
|
|
958
|
+
return {
|
|
959
|
+
resource: { attributes: resource },
|
|
960
|
+
scopeSpans: [
|
|
961
|
+
{
|
|
962
|
+
scope: {
|
|
963
|
+
...(text(metadata["scope.name"]) ? { name: text(metadata["scope.name"]) } : {}),
|
|
964
|
+
...(text(metadata["scope.version"])
|
|
965
|
+
? { version: text(metadata["scope.version"]) }
|
|
966
|
+
: {}),
|
|
967
|
+
},
|
|
968
|
+
spans: [span],
|
|
969
|
+
},
|
|
970
|
+
],
|
|
971
|
+
};
|
|
972
|
+
});
|
|
973
|
+
return {
|
|
974
|
+
traceId,
|
|
975
|
+
format: "langfuse_observations",
|
|
976
|
+
entries,
|
|
977
|
+
blobs: blobs.sorted(),
|
|
978
|
+
...(identity ? { identity } : {}),
|
|
979
|
+
origin: evaluation
|
|
980
|
+
? "evaluation"
|
|
981
|
+
: simulation
|
|
982
|
+
? "simulation"
|
|
983
|
+
: production
|
|
984
|
+
? "production"
|
|
985
|
+
: "unknown",
|
|
986
|
+
sourceDigests: draft.sourceDigests,
|
|
987
|
+
remappedTraceId: traceId !== originalTraceId,
|
|
988
|
+
remappedSpans,
|
|
989
|
+
};
|
|
990
|
+
}
|
|
991
|
+
// --- ATIF -------------------------------------------------------------------------------------
|
|
992
|
+
/** The ATIF reader's bounds: a text export of at most 4 MiB and 2,000 spans. */
|
|
993
|
+
export const MAX_ATIF_FILE_BYTES = 4 * 1024 * 1024;
|
|
994
|
+
export const MAX_ATIF_SPANS = 2000;
|
|
995
|
+
const ATIF_IMPORT_VERSION = "2";
|
|
996
|
+
/** The format and scope names Hue reads ATIF-imported spans by (their recorded step order). */
|
|
997
|
+
const ATIF_FORMAT = "auctor.atif";
|
|
998
|
+
const ATIF_SCOPE = "hue.auctor.atif-import";
|
|
999
|
+
const optionalText = z.string().nullable().optional();
|
|
1000
|
+
const identity = z.string().min(1).max(512);
|
|
1001
|
+
const count = z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER).nullable().optional();
|
|
1002
|
+
const extra = z.record(z.string(), z.json()).nullable().optional();
|
|
1003
|
+
const metrics = z.strictObject({
|
|
1004
|
+
prompt_tokens: count,
|
|
1005
|
+
completion_tokens: count,
|
|
1006
|
+
cached_tokens: count,
|
|
1007
|
+
cost_usd: z.number().nonnegative().nullable().optional(),
|
|
1008
|
+
prompt_token_ids: z.array(z.number().int().nonnegative()).nullable().optional(),
|
|
1009
|
+
completion_token_ids: z.array(z.number().int().nonnegative()).nullable().optional(),
|
|
1010
|
+
logprobs: z.array(z.number()).nullable().optional(),
|
|
1011
|
+
extra,
|
|
1012
|
+
});
|
|
1013
|
+
const call = z.strictObject({
|
|
1014
|
+
tool_call_id: identity,
|
|
1015
|
+
function_name: identity,
|
|
1016
|
+
arguments: z.record(z.string(), z.json()),
|
|
1017
|
+
extra,
|
|
1018
|
+
});
|
|
1019
|
+
/** A delegated subagent's trajectory (ATIF v1.7+). Only the embedded form resolves:
|
|
1020
|
+
* `trajectory_id` names an entry of the root's `subagent_trajectories`. */
|
|
1021
|
+
const subagentRef = z.strictObject({
|
|
1022
|
+
trajectory_id: identity.nullable().optional(),
|
|
1023
|
+
trajectory_path: optionalText,
|
|
1024
|
+
session_id: identity.nullable().optional(),
|
|
1025
|
+
extra,
|
|
1026
|
+
});
|
|
1027
|
+
const result = z.strictObject({
|
|
1028
|
+
source_call_id: identity,
|
|
1029
|
+
content: z.string(),
|
|
1030
|
+
subagent_trajectory_ref: z.array(subagentRef).min(1).nullable().optional(),
|
|
1031
|
+
extra,
|
|
1032
|
+
});
|
|
1033
|
+
const step = z.strictObject({
|
|
1034
|
+
step_id: z.number().int().positive(),
|
|
1035
|
+
timestamp: z.string(),
|
|
1036
|
+
source: z.enum(["system", "user", "agent"]),
|
|
1037
|
+
message: z.string(),
|
|
1038
|
+
model_name: optionalText,
|
|
1039
|
+
reasoning_effort: z.union([z.string(), z.number()]).nullable().optional(),
|
|
1040
|
+
reasoning_content: optionalText,
|
|
1041
|
+
tool_calls: z.array(call).nullable().optional(),
|
|
1042
|
+
observation: z
|
|
1043
|
+
.strictObject({ results: z.array(result) })
|
|
1044
|
+
.nullable()
|
|
1045
|
+
.optional(),
|
|
1046
|
+
metrics: metrics.nullable().optional(),
|
|
1047
|
+
llm_call_count: count,
|
|
1048
|
+
is_copied_context: z.boolean().nullable().optional(),
|
|
1049
|
+
extra,
|
|
1050
|
+
});
|
|
1051
|
+
// v1.8 adds only audio content parts, which the text-only `message` and `content` fields refuse.
|
|
1052
|
+
const trajectoryFields = {
|
|
1053
|
+
schema_version: z.enum(["ATIF-v1.7", "ATIF-v1.8"]),
|
|
1054
|
+
trajectory_id: identity.nullable().optional(),
|
|
1055
|
+
agent: z.strictObject({
|
|
1056
|
+
name: identity,
|
|
1057
|
+
version: identity,
|
|
1058
|
+
model_name: optionalText,
|
|
1059
|
+
tool_definitions: z.array(z.json()).nullable().optional(),
|
|
1060
|
+
extra,
|
|
1061
|
+
}),
|
|
1062
|
+
steps: z.array(step).min(1).max(MAX_ATIF_SPANS),
|
|
1063
|
+
notes: optionalText,
|
|
1064
|
+
final_metrics: z
|
|
1065
|
+
.strictObject({
|
|
1066
|
+
total_prompt_tokens: count,
|
|
1067
|
+
total_completion_tokens: count,
|
|
1068
|
+
total_cached_tokens: count,
|
|
1069
|
+
total_cost_usd: z.number().nonnegative().nullable().optional(),
|
|
1070
|
+
total_steps: count,
|
|
1071
|
+
extra,
|
|
1072
|
+
})
|
|
1073
|
+
.nullable()
|
|
1074
|
+
.optional(),
|
|
1075
|
+
extra,
|
|
1076
|
+
};
|
|
1077
|
+
const subagentTrajectory = z.strictObject({
|
|
1078
|
+
...trajectoryFields,
|
|
1079
|
+
session_id: identity.nullable().optional(),
|
|
1080
|
+
trajectory_id: identity,
|
|
1081
|
+
subagent_trajectories: z.array(z.json()).nullable().optional(),
|
|
1082
|
+
});
|
|
1083
|
+
const trajectory = z.strictObject({
|
|
1084
|
+
...trajectoryFields,
|
|
1085
|
+
session_id: identity,
|
|
1086
|
+
subagent_trajectories: z.array(subagentTrajectory).max(MAX_ATIF_SPANS).nullable().optional(),
|
|
1087
|
+
});
|
|
1088
|
+
/** Retains nanoseconds and requires an explicit timezone, valid calendar date and OTLP uint64. */
|
|
1089
|
+
export function atifUnixNano(value) {
|
|
1090
|
+
const match = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.(\d{1,9}))?(Z|[+-](\d{2}):(\d{2}))$/.exec(value);
|
|
1091
|
+
if (!match)
|
|
1092
|
+
throw new ConversionError("ATIF timestamps require an ISO date and timezone");
|
|
1093
|
+
const [year, month, day, hour, minute, second] = match.slice(1, 7).map(Number);
|
|
1094
|
+
if (month < 1 ||
|
|
1095
|
+
month > 12 ||
|
|
1096
|
+
day < 1 ||
|
|
1097
|
+
day > new Date(Date.UTC(year, month, 0)).getUTCDate() ||
|
|
1098
|
+
hour > 23 ||
|
|
1099
|
+
minute > 59 ||
|
|
1100
|
+
second > 59 ||
|
|
1101
|
+
Number(match[9] ?? 0) > 23 ||
|
|
1102
|
+
Number(match[10] ?? 0) > 59)
|
|
1103
|
+
throw new ConversionError("ATIF contains an invalid calendar timestamp");
|
|
1104
|
+
const milliseconds = Date.parse(`${value.slice(0, 19)}${match[8]}`);
|
|
1105
|
+
if (!Number.isSafeInteger(milliseconds) || milliseconds < 0)
|
|
1106
|
+
throw new ConversionError("ATIF timestamp is outside the supported range");
|
|
1107
|
+
const nanos = BigInt(milliseconds) * 1000000n + BigInt((match[7] ?? "").padEnd(9, "0"));
|
|
1108
|
+
if (nanos <= 0n || nanos > 18446744073709551615n)
|
|
1109
|
+
throw new ConversionError("ATIF timestamp is outside the supported range");
|
|
1110
|
+
return nanos.toString();
|
|
1111
|
+
}
|
|
1112
|
+
function addUsage(attributes, values) {
|
|
1113
|
+
for (const [field, key] of [
|
|
1114
|
+
["prompt_tokens", "gen_ai.usage.input_tokens"],
|
|
1115
|
+
["completion_tokens", "gen_ai.usage.output_tokens"],
|
|
1116
|
+
["cached_tokens", "gen_ai.usage.cache_read.input_tokens"],
|
|
1117
|
+
]) {
|
|
1118
|
+
const value = values?.[field];
|
|
1119
|
+
if (value !== undefined && value !== null)
|
|
1120
|
+
attributes.add(key, value);
|
|
1121
|
+
}
|
|
1122
|
+
if (values?.cost_usd !== undefined && values.cost_usd !== null)
|
|
1123
|
+
attributes.json("hue.import.source_cost_usd", values.cost_usd);
|
|
1124
|
+
}
|
|
1125
|
+
function assertStep(loaded, index, allCalls) {
|
|
1126
|
+
if (loaded.step_id !== index + 1)
|
|
1127
|
+
throw new ConversionError("ATIF step IDs must be consecutive and ordered from 1");
|
|
1128
|
+
if (loaded.is_copied_context === true)
|
|
1129
|
+
throw new ConversionError("Copied context requires a continuation-aware importer");
|
|
1130
|
+
if (loaded.source !== "agent") {
|
|
1131
|
+
if (loaded.model_name != null ||
|
|
1132
|
+
loaded.reasoning_effort != null ||
|
|
1133
|
+
loaded.reasoning_content != null ||
|
|
1134
|
+
loaded.tool_calls != null ||
|
|
1135
|
+
loaded.metrics != null ||
|
|
1136
|
+
(loaded.llm_call_count != null && loaded.llm_call_count !== 0) ||
|
|
1137
|
+
loaded.observation?.results.length)
|
|
1138
|
+
throw new ConversionError("Non-agent ATIF steps may only contain recorded input messages");
|
|
1139
|
+
return;
|
|
1140
|
+
}
|
|
1141
|
+
if (loaded.llm_call_count !== 1)
|
|
1142
|
+
throw new ConversionError("ATIF agent steps must represent one model call");
|
|
1143
|
+
const calls = loaded.tool_calls ?? [];
|
|
1144
|
+
const results = loaded.observation?.results ?? [];
|
|
1145
|
+
const stepCalls = new Set();
|
|
1146
|
+
for (const requested of calls) {
|
|
1147
|
+
if (allCalls.has(requested.tool_call_id))
|
|
1148
|
+
throw new ConversionError("ATIF contains a duplicate tool call ID");
|
|
1149
|
+
allCalls.add(requested.tool_call_id);
|
|
1150
|
+
stepCalls.add(requested.tool_call_id);
|
|
1151
|
+
}
|
|
1152
|
+
const resultCalls = new Set();
|
|
1153
|
+
for (const observed of results) {
|
|
1154
|
+
if (!stepCalls.has(observed.source_call_id))
|
|
1155
|
+
throw new ConversionError("ATIF observation does not match a tool in its step");
|
|
1156
|
+
if (resultCalls.has(observed.source_call_id))
|
|
1157
|
+
throw new ConversionError("ATIF has duplicate observations for one tool call");
|
|
1158
|
+
resultCalls.add(observed.source_call_id);
|
|
1159
|
+
}
|
|
1160
|
+
if (stepCalls.size !== resultCalls.size)
|
|
1161
|
+
throw new ConversionError("ATIF is missing a tool call observation");
|
|
1162
|
+
const usage = loaded.metrics;
|
|
1163
|
+
if (usage?.cached_tokens != null &&
|
|
1164
|
+
usage.prompt_tokens != null &&
|
|
1165
|
+
usage.cached_tokens > usage.prompt_tokens)
|
|
1166
|
+
throw new ConversionError("ATIF cached tokens exceed total input tokens");
|
|
1167
|
+
}
|
|
1168
|
+
const laterNanos = (a, b) => (BigInt(b) > BigInt(a) ? b : a);
|
|
1169
|
+
const earlierNanos = (a, b) => (BigInt(b) < BigInt(a) ? b : a);
|
|
1170
|
+
/** One trajectory's steps, instants and recorded usage, checked as a complete standalone export:
|
|
1171
|
+
* the root and each embedded subagent alike. Tool call IDs are unique across the whole export. */
|
|
1172
|
+
function checkTrajectory(source, allCalls) {
|
|
1173
|
+
const modelSteps = source.steps.filter((loaded) => loaded.source === "agent");
|
|
1174
|
+
if (!modelSteps.length)
|
|
1175
|
+
throw new ConversionError("ATIF contains no agent steps");
|
|
1176
|
+
const firstAgentIndex = source.steps.findIndex((loaded) => loaded.source === "agent");
|
|
1177
|
+
if (source.steps.some((loaded, index) => index > firstAgentIndex && loaded.source !== "agent"))
|
|
1178
|
+
throw new ConversionError("Unsupported ATIF multi-turn export; input messages must precede all agent steps");
|
|
1179
|
+
const times = source.steps.map((loaded, index) => {
|
|
1180
|
+
assertStep(loaded, index, allCalls);
|
|
1181
|
+
return atifUnixNano(loaded.timestamp);
|
|
1182
|
+
});
|
|
1183
|
+
// The input messages before the first agent step are the task as given, not timed actions;
|
|
1184
|
+
// every agent step follows all of them and the agent step before it.
|
|
1185
|
+
let latest = times[0];
|
|
1186
|
+
for (let index = 1; index < times.length; index++) {
|
|
1187
|
+
if (index < firstAgentIndex) {
|
|
1188
|
+
latest = laterNanos(latest, times[index]);
|
|
1189
|
+
continue;
|
|
1190
|
+
}
|
|
1191
|
+
if (BigInt(times[index]) < BigInt(latest))
|
|
1192
|
+
throw new ConversionError("ATIF step timestamps must follow recorded step order");
|
|
1193
|
+
latest = times[index];
|
|
1194
|
+
}
|
|
1195
|
+
const start = times.slice(0, firstAgentIndex + 1).reduce(earlierNanos);
|
|
1196
|
+
if (source.final_metrics?.total_steps != null &&
|
|
1197
|
+
source.final_metrics.total_steps !== source.steps.length)
|
|
1198
|
+
throw new ConversionError("ATIF final step count does not match the complete export");
|
|
1199
|
+
const usage = {
|
|
1200
|
+
recordedInputTokens: 0,
|
|
1201
|
+
recordedOutputTokens: 0,
|
|
1202
|
+
recordedCachedTokens: 0,
|
|
1203
|
+
stepsWithInputTokens: 0,
|
|
1204
|
+
stepsWithOutputTokens: 0,
|
|
1205
|
+
stepsWithCachedTokens: 0,
|
|
1206
|
+
};
|
|
1207
|
+
const finalInput = source.final_metrics?.total_prompt_tokens;
|
|
1208
|
+
const finalCached = source.final_metrics?.total_cached_tokens;
|
|
1209
|
+
if (finalInput != null && finalCached != null && finalCached > finalInput)
|
|
1210
|
+
throw new ConversionError("ATIF final cached tokens exceed total input tokens");
|
|
1211
|
+
for (const [field, total, tally, presence] of [
|
|
1212
|
+
["prompt_tokens", "total_prompt_tokens", "recordedInputTokens", "stepsWithInputTokens"],
|
|
1213
|
+
[
|
|
1214
|
+
"completion_tokens",
|
|
1215
|
+
"total_completion_tokens",
|
|
1216
|
+
"recordedOutputTokens",
|
|
1217
|
+
"stepsWithOutputTokens",
|
|
1218
|
+
],
|
|
1219
|
+
["cached_tokens", "total_cached_tokens", "recordedCachedTokens", "stepsWithCachedTokens"],
|
|
1220
|
+
]) {
|
|
1221
|
+
for (const loaded of modelSteps) {
|
|
1222
|
+
const value = loaded.metrics?.[field];
|
|
1223
|
+
if (value != null) {
|
|
1224
|
+
usage[tally] += value;
|
|
1225
|
+
usage[presence]++;
|
|
1226
|
+
}
|
|
1227
|
+
}
|
|
1228
|
+
if (!Number.isSafeInteger(usage[tally]))
|
|
1229
|
+
throw new ConversionError("ATIF aggregate usage exceeds the exact integer bound");
|
|
1230
|
+
const recordedTotal = source.final_metrics?.[total];
|
|
1231
|
+
if (recordedTotal != null &&
|
|
1232
|
+
(recordedTotal < usage[tally] ||
|
|
1233
|
+
(usage[presence] === modelSteps.length && recordedTotal !== usage[tally])))
|
|
1234
|
+
throw new ConversionError("ATIF final usage does not match its recorded model steps");
|
|
1235
|
+
}
|
|
1236
|
+
// Cached tokens establish a lower bound for a call whose total input is unavailable; they
|
|
1237
|
+
// never become invented per-call input usage or get added twice to known input.
|
|
1238
|
+
const minimumInput = modelSteps.reduce((sum, loaded) => sum + (loaded.metrics?.prompt_tokens ?? loaded.metrics?.cached_tokens ?? 0), 0);
|
|
1239
|
+
if (!Number.isSafeInteger(minimumInput))
|
|
1240
|
+
throw new ConversionError("ATIF aggregate usage exceeds the exact integer bound");
|
|
1241
|
+
if (finalInput != null && finalInput < minimumInput)
|
|
1242
|
+
throw new ConversionError("ATIF final input tokens contradict recorded input/cache counts");
|
|
1243
|
+
const knownUncachedInput = modelSteps.reduce((sum, loaded) => {
|
|
1244
|
+
const input = loaded.metrics?.prompt_tokens;
|
|
1245
|
+
const cached = loaded.metrics?.cached_tokens;
|
|
1246
|
+
return sum + (input != null && cached != null ? input - cached : 0);
|
|
1247
|
+
}, 0);
|
|
1248
|
+
const totalInput = finalInput ??
|
|
1249
|
+
(usage.stepsWithInputTokens === modelSteps.length ? usage.recordedInputTokens : undefined);
|
|
1250
|
+
if (finalCached != null && totalInput != null && finalCached > totalInput - knownUncachedInput)
|
|
1251
|
+
throw new ConversionError("ATIF final cached tokens contradict recorded uncached input counts");
|
|
1252
|
+
return { times, start, modelSteps, usage };
|
|
1253
|
+
}
|
|
1254
|
+
/** A trajectory's own recorded totals, as its agent span reports them. */
|
|
1255
|
+
function finalUsage(source) {
|
|
1256
|
+
return source.final_metrics == null
|
|
1257
|
+
? undefined
|
|
1258
|
+
: {
|
|
1259
|
+
prompt_tokens: source.final_metrics.total_prompt_tokens,
|
|
1260
|
+
completion_tokens: source.final_metrics.total_completion_tokens,
|
|
1261
|
+
cached_tokens: source.final_metrics.total_cached_tokens,
|
|
1262
|
+
cost_usd: source.final_metrics.total_cost_usd,
|
|
1263
|
+
};
|
|
1264
|
+
}
|
|
1265
|
+
/** An export's user id, where it is an identifier: one token, never an address or a name. */
|
|
1266
|
+
function atifUserId(value) {
|
|
1267
|
+
return typeof value === "string" && /^[^\s@]{1,200}$/.test(value) ? value : undefined;
|
|
1268
|
+
}
|
|
1269
|
+
/**
|
|
1270
|
+
* A text ATIF v1.7 or v1.8 export as one trace: an agent span for the run, a model span per
|
|
1271
|
+
* agent step with the conversation so far, a tool span per call under it, and an embedded
|
|
1272
|
+
* subagent trajectory as its own agent span under the tool span of the call that spawned it.
|
|
1273
|
+
* Identity is run scoped (`session_id` and `trajectory_id`), never a path. The complete original
|
|
1274
|
+
* document stays on the root; each model and tool span keeps its own record. ATIF supplies step
|
|
1275
|
+
* instants, not call durations: model and tool spans are zero-duration point spans.
|
|
1276
|
+
*/
|
|
1277
|
+
export function convertAtifExport(source, rawBytes) {
|
|
1278
|
+
if (rawBytes > MAX_ATIF_FILE_BYTES)
|
|
1279
|
+
throw new ConversionError("ATIF file exceeds the import size bound");
|
|
1280
|
+
if (source.unsafeIntegers)
|
|
1281
|
+
throw new ConversionError("ATIF JSON has unsafe numbers, invalid text or excessive nesting");
|
|
1282
|
+
validateJsonText(source.text);
|
|
1283
|
+
try {
|
|
1284
|
+
validateSource(source.document);
|
|
1285
|
+
}
|
|
1286
|
+
catch {
|
|
1287
|
+
throw new ConversionError("ATIF JSON has unsafe numbers, invalid text or excessive nesting");
|
|
1288
|
+
}
|
|
1289
|
+
const validation = trajectory.safeParse(source.document);
|
|
1290
|
+
if (!validation.success)
|
|
1291
|
+
throw new ConversionError("Unsupported or invalid ATIF v1.7/v1.8 shape; only standalone text exports are supported");
|
|
1292
|
+
// Retain the original object after validation: a schema's cloned records may omit special
|
|
1293
|
+
// JSON keys such as __proto__, which are safe own properties on JSON.parse's objects.
|
|
1294
|
+
const document = source.document;
|
|
1295
|
+
const allCalls = new Set();
|
|
1296
|
+
const main = checkTrajectory(document, allCalls);
|
|
1297
|
+
const embedded = new Map();
|
|
1298
|
+
for (const sub of document.subagent_trajectories ?? []) {
|
|
1299
|
+
if (embedded.has(sub.trajectory_id))
|
|
1300
|
+
throw new ConversionError("ATIF embeds two subagent trajectories with one trajectory_id");
|
|
1301
|
+
if (sub.subagent_trajectories?.length ||
|
|
1302
|
+
sub.steps.some((loaded) => (loaded.observation?.results ?? []).some((returned) => returned.subagent_trajectory_ref != null)))
|
|
1303
|
+
throw new ConversionError("ATIF subagents that spawn their own subagents are not supported");
|
|
1304
|
+
embedded.set(sub.trajectory_id, { trajectory: sub, checked: checkTrajectory(sub, allCalls) });
|
|
1305
|
+
}
|
|
1306
|
+
const spawned = new Map();
|
|
1307
|
+
for (const [index, loaded] of document.steps.entries())
|
|
1308
|
+
for (const returned of loaded.observation?.results ?? [])
|
|
1309
|
+
for (const ref of returned.subagent_trajectory_ref ?? []) {
|
|
1310
|
+
if (ref.trajectory_id == null)
|
|
1311
|
+
throw new ConversionError("ATIF references a subagent trajectory file; only embedded subagents are supported");
|
|
1312
|
+
const found = embedded.get(ref.trajectory_id);
|
|
1313
|
+
if (!found)
|
|
1314
|
+
throw new ConversionError("ATIF references a subagent trajectory the export does not embed");
|
|
1315
|
+
if (found.ref !== undefined)
|
|
1316
|
+
throw new ConversionError("ATIF references one embedded subagent trajectory twice");
|
|
1317
|
+
if (BigInt(found.checked.start) < BigInt(main.times[index]))
|
|
1318
|
+
throw new ConversionError("ATIF subagent starts before the step that spawned it");
|
|
1319
|
+
found.ref = ref;
|
|
1320
|
+
spawned.set(returned.source_call_id, [
|
|
1321
|
+
...(spawned.get(returned.source_call_id) ?? []),
|
|
1322
|
+
found.trajectory,
|
|
1323
|
+
]);
|
|
1324
|
+
}
|
|
1325
|
+
for (const found of embedded.values())
|
|
1326
|
+
if (found.ref === undefined)
|
|
1327
|
+
throw new ConversionError("ATIF embeds a subagent trajectory no observation references");
|
|
1328
|
+
const checked = [main, ...[...embedded.values()].map((found) => found.checked)];
|
|
1329
|
+
if (checked.reduce((sum, part) => sum + 1 + part.modelSteps.length, 0) + allCalls.size >
|
|
1330
|
+
MAX_ATIF_SPANS)
|
|
1331
|
+
throw new ConversionError("ATIF trace exceeds the import span bound");
|
|
1332
|
+
// Tied instants sort by `hue.import.source_index`: a step's place in execution order, where a
|
|
1333
|
+
// subagent's steps follow the step that spawned it.
|
|
1334
|
+
const order = new Map([[document, []]]);
|
|
1335
|
+
let ordinal = 0;
|
|
1336
|
+
for (const loaded of document.steps) {
|
|
1337
|
+
order.get(document).push(ordinal++);
|
|
1338
|
+
for (const requested of loaded.tool_calls ?? [])
|
|
1339
|
+
for (const sub of spawned.get(requested.tool_call_id) ?? [])
|
|
1340
|
+
order.set(sub, sub.steps.map(() => ordinal++));
|
|
1341
|
+
}
|
|
1342
|
+
const identityParts = [
|
|
1343
|
+
ATIF_FORMAT === "auctor.atif" ? "hue.auctor.atif" : ATIF_FORMAT,
|
|
1344
|
+
ATIF_IMPORT_VERSION,
|
|
1345
|
+
document.session_id,
|
|
1346
|
+
document.trajectory_id ?? null,
|
|
1347
|
+
];
|
|
1348
|
+
const traceId = sha256Hex(JSON.stringify(identityParts)).slice(0, 32);
|
|
1349
|
+
const spanId = (parts) => sha256Hex(JSON.stringify([traceId, ...parts])).slice(0, 16);
|
|
1350
|
+
const rootId = spanId(["root"]);
|
|
1351
|
+
const blobs = new BlobSink();
|
|
1352
|
+
const entries = [];
|
|
1353
|
+
// Roles are stated only where the export records delegation.
|
|
1354
|
+
const role = embedded.size ? "orchestrator" : undefined;
|
|
1355
|
+
const commonOf = (attributes, t, agentRole) => {
|
|
1356
|
+
attributes.add("gen_ai.agent.name", t.agent.name);
|
|
1357
|
+
attributes.add("gen_ai.conversation.id", t.session_id ?? document.session_id);
|
|
1358
|
+
attributes.add("hue.import.format", ATIF_FORMAT);
|
|
1359
|
+
attributes.add("hue.import.source_schema", t.schema_version);
|
|
1360
|
+
if (agentRole)
|
|
1361
|
+
attributes.add("hue.agent.role", agentRole);
|
|
1362
|
+
if (agentRole === "subagent" && t.trajectory_id != null)
|
|
1363
|
+
attributes.add("hue.import.trajectory_id", t.trajectory_id);
|
|
1364
|
+
};
|
|
1365
|
+
const addSpan = (span) => {
|
|
1366
|
+
entries.push({
|
|
1367
|
+
resource: {
|
|
1368
|
+
attributes: [
|
|
1369
|
+
{ key: "service.name", value: { stringValue: document.agent.name } },
|
|
1370
|
+
{ key: "service.version", value: { stringValue: document.agent.version } },
|
|
1371
|
+
],
|
|
1372
|
+
},
|
|
1373
|
+
scopeSpans: [
|
|
1374
|
+
{
|
|
1375
|
+
scope: { name: ATIF_SCOPE, version: ATIF_IMPORT_VERSION },
|
|
1376
|
+
spans: [{ traceId, kind: 1, ...span }],
|
|
1377
|
+
},
|
|
1378
|
+
],
|
|
1379
|
+
});
|
|
1380
|
+
};
|
|
1381
|
+
const inputOf = (t) => t.steps
|
|
1382
|
+
.filter((loaded) => loaded.source !== "agent")
|
|
1383
|
+
.map((loaded) => ({ role: loaded.source, content: loaded.message }));
|
|
1384
|
+
const replyOf = (t) => [
|
|
1385
|
+
{
|
|
1386
|
+
role: "assistant",
|
|
1387
|
+
content: t.steps.filter((loaded) => loaded.source === "agent").at(-1).message,
|
|
1388
|
+
},
|
|
1389
|
+
];
|
|
1390
|
+
const lastInstant = checked.map((part) => part.times.at(-1)).reduce(laterNanos);
|
|
1391
|
+
// The root keeps the export as imported. Where it embeds subagents, each embedded trajectory is
|
|
1392
|
+
// kept whole on its own agent span instead, so the root keeps the rest of the export.
|
|
1393
|
+
const rootAttributes = new AttributeList(blobs);
|
|
1394
|
+
commonOf(rootAttributes, document, role);
|
|
1395
|
+
rootAttributes.add("gen_ai.operation.name", "invoke_agent");
|
|
1396
|
+
rootAttributes.json("gen_ai.input.messages", inputOf(document));
|
|
1397
|
+
rootAttributes.json("gen_ai.output.messages", replyOf(document));
|
|
1398
|
+
rootAttributes.add(SOURCE_RECORD, embedded.size
|
|
1399
|
+
? JSON.stringify(Object.fromEntries(Object.entries(document).filter(([key]) => key !== "subagent_trajectories")))
|
|
1400
|
+
: source.text);
|
|
1401
|
+
if (embedded.size)
|
|
1402
|
+
rootAttributes.add("hue.import.source_record_scope", "export_without_embedded_subagents");
|
|
1403
|
+
rootAttributes.add("hue.import.timing", "first_to_last_recorded_step; call_durations_unavailable");
|
|
1404
|
+
addUsage(rootAttributes, finalUsage(document));
|
|
1405
|
+
addSpan({
|
|
1406
|
+
spanId: rootId,
|
|
1407
|
+
name: "invoke_agent auctor",
|
|
1408
|
+
startTimeUnixNano: main.start,
|
|
1409
|
+
endTimeUnixNano: lastInstant,
|
|
1410
|
+
attributes: rootAttributes.finish(),
|
|
1411
|
+
status: { code: 0 },
|
|
1412
|
+
});
|
|
1413
|
+
const addSteps = (t, { times }, agentId, idParts, agentRole) => {
|
|
1414
|
+
const sourceIndex = order.get(t);
|
|
1415
|
+
const history = [];
|
|
1416
|
+
for (const [index, loaded] of t.steps.entries()) {
|
|
1417
|
+
if (loaded.source !== "agent") {
|
|
1418
|
+
history.push({ role: loaded.source, content: loaded.message });
|
|
1419
|
+
continue;
|
|
1420
|
+
}
|
|
1421
|
+
const modelId = spanId([...idParts, "step", loaded.step_id]);
|
|
1422
|
+
const calls = loaded.tool_calls ?? [];
|
|
1423
|
+
const observed = loaded.observation?.results ?? [];
|
|
1424
|
+
const output = {
|
|
1425
|
+
role: "assistant",
|
|
1426
|
+
content: loaded.message,
|
|
1427
|
+
...(calls.length
|
|
1428
|
+
? {
|
|
1429
|
+
tool_calls: calls.map((requested) => ({
|
|
1430
|
+
id: requested.tool_call_id,
|
|
1431
|
+
type: "function",
|
|
1432
|
+
function: {
|
|
1433
|
+
name: requested.function_name,
|
|
1434
|
+
arguments: JSON.stringify(requested.arguments),
|
|
1435
|
+
},
|
|
1436
|
+
})),
|
|
1437
|
+
}
|
|
1438
|
+
: {}),
|
|
1439
|
+
};
|
|
1440
|
+
const modelAttributes = new AttributeList(blobs);
|
|
1441
|
+
commonOf(modelAttributes, t, agentRole);
|
|
1442
|
+
modelAttributes.add("gen_ai.operation.name", "chat");
|
|
1443
|
+
modelAttributes.json("gen_ai.input.messages", history);
|
|
1444
|
+
modelAttributes.add("hue.import.input_source", "preceding_recorded_steps; runtime_prompt_unavailable");
|
|
1445
|
+
modelAttributes.json("gen_ai.output.messages", [
|
|
1446
|
+
{
|
|
1447
|
+
...output,
|
|
1448
|
+
...(loaded.reasoning_content != null
|
|
1449
|
+
? { reasoning_content: loaded.reasoning_content }
|
|
1450
|
+
: {}),
|
|
1451
|
+
},
|
|
1452
|
+
]);
|
|
1453
|
+
modelAttributes.add("hue.import.source_index", sourceIndex[index]);
|
|
1454
|
+
modelAttributes.add("hue.import.step_id", loaded.step_id);
|
|
1455
|
+
modelAttributes.add("hue.import.timing", "recorded_step_instant; call_duration_unavailable");
|
|
1456
|
+
modelAttributes.json(SOURCE_RECORD, loaded);
|
|
1457
|
+
const model = loaded.model_name ?? t.agent.model_name;
|
|
1458
|
+
if (model != null)
|
|
1459
|
+
modelAttributes.add("gen_ai.request.model", model);
|
|
1460
|
+
if (loaded.reasoning_content != null)
|
|
1461
|
+
modelAttributes.add("hue.import.reasoning_content", loaded.reasoning_content);
|
|
1462
|
+
if (loaded.reasoning_effort != null)
|
|
1463
|
+
modelAttributes.json("hue.import.reasoning_effort", loaded.reasoning_effort);
|
|
1464
|
+
if (loaded.llm_call_count != null)
|
|
1465
|
+
modelAttributes.add("hue.import.llm_call_count", loaded.llm_call_count);
|
|
1466
|
+
if (t.agent.tool_definitions != null)
|
|
1467
|
+
modelAttributes.json("gen_ai.tool.definitions", t.agent.tool_definitions);
|
|
1468
|
+
addUsage(modelAttributes, loaded.metrics);
|
|
1469
|
+
addSpan({
|
|
1470
|
+
spanId: modelId,
|
|
1471
|
+
parentSpanId: agentId,
|
|
1472
|
+
name: "chat",
|
|
1473
|
+
startTimeUnixNano: times[index],
|
|
1474
|
+
endTimeUnixNano: times[index],
|
|
1475
|
+
attributes: modelAttributes.finish(),
|
|
1476
|
+
status: { code: 0 },
|
|
1477
|
+
});
|
|
1478
|
+
history.push(output);
|
|
1479
|
+
for (const [position, requested] of calls.entries()) {
|
|
1480
|
+
const observedIndex = observed.findIndex((entry) => entry.source_call_id === requested.tool_call_id);
|
|
1481
|
+
const returned = observed[observedIndex];
|
|
1482
|
+
// A spawning call stays open until its subagent's last recorded step.
|
|
1483
|
+
const children = spawned.get(requested.tool_call_id) ?? [];
|
|
1484
|
+
const toolId = spanId([...idParts, "step", loaded.step_id, "tool", requested.tool_call_id]);
|
|
1485
|
+
const attributes = new AttributeList(blobs);
|
|
1486
|
+
commonOf(attributes, t, agentRole);
|
|
1487
|
+
attributes.add("gen_ai.operation.name", "execute_tool");
|
|
1488
|
+
attributes.add("gen_ai.tool.name", requested.function_name);
|
|
1489
|
+
attributes.add("gen_ai.tool.call.id", requested.tool_call_id);
|
|
1490
|
+
attributes.add("gen_ai.tool.type", "function");
|
|
1491
|
+
attributes.json("gen_ai.tool.call.arguments", requested.arguments);
|
|
1492
|
+
attributes.json("gen_ai.tool.call.result", returned.content);
|
|
1493
|
+
attributes.add("hue.tool.call.position", position);
|
|
1494
|
+
attributes.add("hue.import.source_index", sourceIndex[index]);
|
|
1495
|
+
attributes.add("hue.import.step_id", loaded.step_id);
|
|
1496
|
+
attributes.add("hue.import.observation_index", observedIndex);
|
|
1497
|
+
attributes.add("hue.import.timing", children.length
|
|
1498
|
+
? "recorded_step_instant_to_last_subagent_step"
|
|
1499
|
+
: "recorded_step_instant; tool_duration_unavailable");
|
|
1500
|
+
attributes.json(SOURCE_RECORD, { call: requested, observation: returned });
|
|
1501
|
+
if (children.length)
|
|
1502
|
+
attributes.json("hue.import.subagent_trajectory_ids", children.map((sub) => sub.trajectory_id));
|
|
1503
|
+
addSpan({
|
|
1504
|
+
spanId: toolId,
|
|
1505
|
+
parentSpanId: modelId,
|
|
1506
|
+
name: requested.function_name,
|
|
1507
|
+
startTimeUnixNano: times[index],
|
|
1508
|
+
endTimeUnixNano: children
|
|
1509
|
+
.map((sub) => embedded.get(sub.trajectory_id).checked.times.at(-1))
|
|
1510
|
+
.reduce(laterNanos, times[index]),
|
|
1511
|
+
attributes: attributes.finish(),
|
|
1512
|
+
// ATIF records a tool's result text, never its outcome: the span keeps the content and
|
|
1513
|
+
// states no error. Any one exporter's failure wording stays in the result it wrote.
|
|
1514
|
+
status: { code: 0 },
|
|
1515
|
+
});
|
|
1516
|
+
for (const sub of children)
|
|
1517
|
+
addSubagent(sub, toolId);
|
|
1518
|
+
}
|
|
1519
|
+
// The recorded observation order is kept in the next model's conversation, even when
|
|
1520
|
+
// parallel calls returned in a different order from their requests.
|
|
1521
|
+
for (const returned of observed)
|
|
1522
|
+
history.push({
|
|
1523
|
+
role: "tool",
|
|
1524
|
+
tool_call_id: returned.source_call_id,
|
|
1525
|
+
content: returned.content,
|
|
1526
|
+
});
|
|
1527
|
+
}
|
|
1528
|
+
};
|
|
1529
|
+
// A subagent is its own agent span under the spawning tool span: it keeps the whole embedded
|
|
1530
|
+
// trajectory and the reference that resolved it; each step also keeps its own record.
|
|
1531
|
+
const addSubagent = (sub, parentSpanId) => {
|
|
1532
|
+
const found = embedded.get(sub.trajectory_id);
|
|
1533
|
+
const agentId = spanId(["subagent", sub.trajectory_id]);
|
|
1534
|
+
const attributes = new AttributeList(blobs);
|
|
1535
|
+
commonOf(attributes, sub, "subagent");
|
|
1536
|
+
attributes.add("gen_ai.operation.name", "invoke_agent");
|
|
1537
|
+
attributes.json("gen_ai.input.messages", inputOf(sub));
|
|
1538
|
+
attributes.json("gen_ai.output.messages", replyOf(sub));
|
|
1539
|
+
attributes.json(SOURCE_RECORD, sub);
|
|
1540
|
+
attributes.json("hue.import.subagent_ref", found.ref);
|
|
1541
|
+
attributes.add("hue.import.timing", "first_to_last_recorded_step; call_durations_unavailable");
|
|
1542
|
+
addUsage(attributes, finalUsage(sub));
|
|
1543
|
+
addSpan({
|
|
1544
|
+
spanId: agentId,
|
|
1545
|
+
parentSpanId,
|
|
1546
|
+
name: `invoke_agent ${sub.agent.name}`,
|
|
1547
|
+
startTimeUnixNano: found.checked.start,
|
|
1548
|
+
endTimeUnixNano: found.checked.times.at(-1),
|
|
1549
|
+
attributes: attributes.finish(),
|
|
1550
|
+
status: { code: 0 },
|
|
1551
|
+
});
|
|
1552
|
+
addSteps(sub, found.checked, agentId, ["subagent", sub.trajectory_id], "subagent");
|
|
1553
|
+
};
|
|
1554
|
+
addSteps(document, main, rootId, [], role);
|
|
1555
|
+
// The export's user (`extra.user_id`) as the standard `user.id` on a span of its own, as Hue
|
|
1556
|
+
// reads an import's user: a trace imported before it gains it as one new span.
|
|
1557
|
+
const userId = atifUserId(document.extra?.user_id);
|
|
1558
|
+
if (userId)
|
|
1559
|
+
addSpan({
|
|
1560
|
+
spanId: spanId(["user"]),
|
|
1561
|
+
parentSpanId: rootId,
|
|
1562
|
+
name: "hue.import.user",
|
|
1563
|
+
startTimeUnixNano: main.start,
|
|
1564
|
+
endTimeUnixNano: main.start,
|
|
1565
|
+
attributes: [{ key: "user.id", value: { stringValue: userId } }],
|
|
1566
|
+
status: { code: 0 },
|
|
1567
|
+
});
|
|
1568
|
+
const session = identifier(document.session_id, MAX_SESSION);
|
|
1569
|
+
return {
|
|
1570
|
+
traceId,
|
|
1571
|
+
format: "atif",
|
|
1572
|
+
entries,
|
|
1573
|
+
blobs: blobs.sorted(),
|
|
1574
|
+
...(userId || session
|
|
1575
|
+
? { identity: { ...(userId ? { user: userId } : {}), ...(session ? { session } : {}) } }
|
|
1576
|
+
: {}),
|
|
1577
|
+
origin: "unknown",
|
|
1578
|
+
sourceDigests: new Set(),
|
|
1579
|
+
remappedTraceId: false,
|
|
1580
|
+
remappedSpans: 0,
|
|
1581
|
+
};
|
|
1582
|
+
}
|
|
1583
|
+
/** `Math.floor(numerator / divisor)` for safe nonnegative integers, without a division: the
|
|
1584
|
+
* published package's import-boundary check reads a `/` after a name as a regular expression, so
|
|
1585
|
+
* the quotient is estimated from the reciprocal and corrected by whole steps, which makes it
|
|
1586
|
+
* exact on every engine. */
|
|
1587
|
+
export function integerQuotient(numerator, divisor) {
|
|
1588
|
+
let quotient = Math.floor(numerator * divisor ** -1);
|
|
1589
|
+
while (quotient * divisor > numerator)
|
|
1590
|
+
quotient--;
|
|
1591
|
+
while ((quotient + 1) * divisor <= numerator)
|
|
1592
|
+
quotient++;
|
|
1593
|
+
return quotient;
|
|
1594
|
+
}
|
|
1595
|
+
const spanIdOf = (entry) => {
|
|
1596
|
+
const span = entry.scopeSpans[0]?.spans[0];
|
|
1597
|
+
return typeof span?.spanId === "string" ? span.spanId : "";
|
|
1598
|
+
};
|
|
1599
|
+
/**
|
|
1600
|
+
* The trace's entries packed into gzip OTLP/JSON batches in order, each at most 3 MiB decoded
|
|
1601
|
+
* and 900 KiB on the wire. Packing is linear and the output depends on the entries alone, so the
|
|
1602
|
+
* same trace packs to the same batches whatever bundle it came from.
|
|
1603
|
+
*/
|
|
1604
|
+
export function batchEntries(entries) {
|
|
1605
|
+
const batches = [];
|
|
1606
|
+
const parts = [];
|
|
1607
|
+
const encode = (from, to) => {
|
|
1608
|
+
const decoded = Buffer.from(`${BATCH_PREFIX}${parts
|
|
1609
|
+
.slice(from, to)
|
|
1610
|
+
.map((part) => part.json)
|
|
1611
|
+
.join(",")}${BATCH_SUFFIX}`);
|
|
1612
|
+
return { decoded, body: gzipSync(decoded, { level: 6 }) };
|
|
1613
|
+
};
|
|
1614
|
+
const commit = (from, to, encoded) => {
|
|
1615
|
+
if (encoded.decoded.byteLength > MAX_BATCH_DECODED_BYTES ||
|
|
1616
|
+
encoded.body.byteLength > MAX_BATCH_WIRE_BYTES)
|
|
1617
|
+
throw new ConversionError("A batch exceeds the import batch bounds");
|
|
1618
|
+
// The batch must read back whole: what the receiver decodes is what was packed.
|
|
1619
|
+
const readBack = JSON.parse(gunzipSync(encoded.body).toString("utf8"));
|
|
1620
|
+
if (!Array.isArray(readBack.resourceSpans) || readBack.resourceSpans.length !== to - from)
|
|
1621
|
+
throw new ConversionError("A batch does not read back whole");
|
|
1622
|
+
batches.push({
|
|
1623
|
+
sha256: sha256Hex(encoded.body),
|
|
1624
|
+
body: encoded.body,
|
|
1625
|
+
decodedBytes: encoded.decoded.byteLength,
|
|
1626
|
+
spanIds: entries.slice(from, to).map(spanIdOf),
|
|
1627
|
+
});
|
|
1628
|
+
};
|
|
1629
|
+
/** Seals entries from `from`, returning where the next batch starts. */
|
|
1630
|
+
const seal = (from, to) => {
|
|
1631
|
+
let end = to;
|
|
1632
|
+
let encoded = encode(from, end);
|
|
1633
|
+
while (encoded.body.byteLength > MAX_BATCH_WIRE_BYTES) {
|
|
1634
|
+
if (end - from === 1)
|
|
1635
|
+
throw new ConversionError("One span exceeds the import batch bounds");
|
|
1636
|
+
// Shrink toward the wire cap by the observed compression ratio rather than re-encoding
|
|
1637
|
+
// per entry.
|
|
1638
|
+
const budget = integerQuotient(encoded.decoded.byteLength * MAX_BATCH_WIRE_BYTES * 9, encoded.body.byteLength * 10);
|
|
1639
|
+
let next = from + 1;
|
|
1640
|
+
let bytes = BATCH_FRAME_BYTES + parts[from].bytes;
|
|
1641
|
+
while (next < end - 1 && bytes + 1 + parts[next].bytes <= budget)
|
|
1642
|
+
bytes += 1 + parts[next++].bytes;
|
|
1643
|
+
end = next;
|
|
1644
|
+
encoded = encode(from, end);
|
|
1645
|
+
}
|
|
1646
|
+
commit(from, end, encoded);
|
|
1647
|
+
return end;
|
|
1648
|
+
};
|
|
1649
|
+
let start = 0;
|
|
1650
|
+
let pendingBytes = BATCH_FRAME_BYTES;
|
|
1651
|
+
let index = 0;
|
|
1652
|
+
while (index < entries.length) {
|
|
1653
|
+
let part = parts[index];
|
|
1654
|
+
if (!part) {
|
|
1655
|
+
const json = JSON.stringify(entries[index]);
|
|
1656
|
+
part = parts[index] = { json, bytes: Buffer.byteLength(json) };
|
|
1657
|
+
}
|
|
1658
|
+
if (BATCH_FRAME_BYTES + part.bytes > MAX_BATCH_DECODED_BYTES)
|
|
1659
|
+
throw new ConversionError("One span exceeds the import batch bounds");
|
|
1660
|
+
const grown = pendingBytes + (index > start ? 1 : 0) + part.bytes;
|
|
1661
|
+
if (grown <= MAX_BATCH_DECODED_BYTES) {
|
|
1662
|
+
pendingBytes = grown;
|
|
1663
|
+
index++;
|
|
1664
|
+
continue;
|
|
1665
|
+
}
|
|
1666
|
+
start = seal(start, index);
|
|
1667
|
+
index = start;
|
|
1668
|
+
pendingBytes = BATCH_FRAME_BYTES;
|
|
1669
|
+
}
|
|
1670
|
+
while (start < parts.length)
|
|
1671
|
+
start = seal(start, parts.length);
|
|
1672
|
+
return batches;
|
|
1673
|
+
}
|
|
1674
|
+
/** The totals a manifest adds up to, as Hue computes them. */
|
|
1675
|
+
export function manifestTotals(manifest) {
|
|
1676
|
+
let batches = 0, spans = 0, blobs = 0, bytes = 0;
|
|
1677
|
+
for (const trace of manifest.traces) {
|
|
1678
|
+
batches += trace.batches.length;
|
|
1679
|
+
spans += trace.span_count;
|
|
1680
|
+
blobs += trace.blobs.length;
|
|
1681
|
+
for (const batch of trace.batches)
|
|
1682
|
+
bytes += batch.bytes;
|
|
1683
|
+
}
|
|
1684
|
+
return { traces: manifest.traces.length, batches, spans, blobs, bytes };
|
|
1685
|
+
}
|
|
1686
|
+
export function manifestTraceHasIdentity(trace) {
|
|
1687
|
+
return Boolean(trace.identity?.user || trace.identity?.entity);
|
|
1688
|
+
}
|
|
1689
|
+
const HEX64 = /^[0-9a-f]{64}$/;
|
|
1690
|
+
const isCount = (value, max) => typeof value === "number" && Number.isSafeInteger(value) && value >= 1 && value <= max;
|
|
1691
|
+
/** A manifest's problem, or undefined: the shape and cross-checks Hue's registration applies
|
|
1692
|
+
* (every trace id once, every batch digest once, each trace's batches adding up to its span
|
|
1693
|
+
* count, each trace's values distinct) so a bundle is refused here, before any upload. */
|
|
1694
|
+
export function validateManifest(value) {
|
|
1695
|
+
if (!isObject(value))
|
|
1696
|
+
return "the manifest is not a JSON object";
|
|
1697
|
+
if (value.version !== 1)
|
|
1698
|
+
return "the manifest version is not 1";
|
|
1699
|
+
if (!isObject(value.source) ||
|
|
1700
|
+
!Array.isArray(value.source.formats) ||
|
|
1701
|
+
!value.source.formats.length ||
|
|
1702
|
+
!value.source.formats.every((format) => typeof format === "string" && traceImportFormats.includes(format)) ||
|
|
1703
|
+
typeof value.source.helper !== "string" ||
|
|
1704
|
+
!value.source.helper)
|
|
1705
|
+
return "the manifest source is malformed";
|
|
1706
|
+
if (!Array.isArray(value.traces) || !value.traces.length)
|
|
1707
|
+
return "the manifest declares no traces";
|
|
1708
|
+
if (value.traces.length > traceImportLimits.traces)
|
|
1709
|
+
return `the manifest declares more than ${traceImportLimits.traces} traces`;
|
|
1710
|
+
const traceIds = new Set();
|
|
1711
|
+
const digests = new Set();
|
|
1712
|
+
for (const trace of value.traces) {
|
|
1713
|
+
if (!isObject(trace))
|
|
1714
|
+
return "a declared trace is not an object";
|
|
1715
|
+
const id = trace.trace_id;
|
|
1716
|
+
if (typeof id !== "string" || !HEX32.test(id) || /^0+$/.test(id))
|
|
1717
|
+
return "a declared trace id is not an OpenTelemetry trace id";
|
|
1718
|
+
if (traceIds.has(id))
|
|
1719
|
+
return `the manifest declares trace ${id} more than once`;
|
|
1720
|
+
traceIds.add(id);
|
|
1721
|
+
if (!isCount(trace.span_count, traceImportLimits.spansPerTrace))
|
|
1722
|
+
return `trace ${id} declares an invalid span count`;
|
|
1723
|
+
if (!Array.isArray(trace.batches) ||
|
|
1724
|
+
!trace.batches.length ||
|
|
1725
|
+
trace.batches.length > traceImportLimits.batchesPerTrace)
|
|
1726
|
+
return `trace ${id} declares an invalid batch list`;
|
|
1727
|
+
let spans = 0;
|
|
1728
|
+
for (const batch of trace.batches) {
|
|
1729
|
+
if (!isObject(batch) ||
|
|
1730
|
+
typeof batch.sha256 !== "string" ||
|
|
1731
|
+
!HEX64.test(batch.sha256) ||
|
|
1732
|
+
!isCount(batch.bytes, traceImportLimits.batchBytes) ||
|
|
1733
|
+
!isCount(batch.span_count, traceImportLimits.spansPerTrace))
|
|
1734
|
+
return `trace ${id} declares a malformed batch`;
|
|
1735
|
+
if (digests.has(batch.sha256))
|
|
1736
|
+
return `the manifest declares batch ${batch.sha256} more than once`;
|
|
1737
|
+
digests.add(batch.sha256);
|
|
1738
|
+
spans += batch.span_count;
|
|
1739
|
+
}
|
|
1740
|
+
if (spans !== trace.span_count)
|
|
1741
|
+
return `trace ${id} declares ${String(trace.span_count)} spans but its batches hold ${spans}`;
|
|
1742
|
+
const blobs = trace.blobs ?? [];
|
|
1743
|
+
if (!Array.isArray(blobs) || blobs.length > traceImportLimits.blobsPerTrace)
|
|
1744
|
+
return `trace ${id} declares an invalid value list`;
|
|
1745
|
+
const seen = new Set();
|
|
1746
|
+
for (const blob of blobs) {
|
|
1747
|
+
if (!isObject(blob) ||
|
|
1748
|
+
typeof blob.sha256 !== "string" ||
|
|
1749
|
+
!HEX64.test(blob.sha256) ||
|
|
1750
|
+
!isCount(blob.byte_size, traceImportLimits.blobBytes) ||
|
|
1751
|
+
typeof blob.content_type !== "string" ||
|
|
1752
|
+
blob.content_type.length < 3 ||
|
|
1753
|
+
blob.content_type.length > 255)
|
|
1754
|
+
return `trace ${id} declares a malformed value`;
|
|
1755
|
+
if (seen.has(blob.sha256))
|
|
1756
|
+
return `trace ${id} lists value ${blob.sha256} more than once`;
|
|
1757
|
+
seen.add(blob.sha256);
|
|
1758
|
+
}
|
|
1759
|
+
if (trace.identity !== undefined) {
|
|
1760
|
+
if (!isObject(trace.identity))
|
|
1761
|
+
return `trace ${id} declares a malformed identity`;
|
|
1762
|
+
const { user, entity, session } = trace.identity;
|
|
1763
|
+
if ((user !== undefined && !identifier(user, MAX_IDENTIFIER)) ||
|
|
1764
|
+
(session !== undefined && !identifier(session, MAX_SESSION)) ||
|
|
1765
|
+
(entity !== undefined &&
|
|
1766
|
+
(!isObject(entity) ||
|
|
1767
|
+
!identifier(entity.namespace, MAX_NAMESPACE) ||
|
|
1768
|
+
!identifier(entity.id, MAX_IDENTIFIER))))
|
|
1769
|
+
return `trace ${id} declares a malformed identity`;
|
|
1770
|
+
}
|
|
1771
|
+
if (trace.origin !== undefined &&
|
|
1772
|
+
!traceImportOrigins.includes(trace.origin))
|
|
1773
|
+
return `trace ${id} declares an unknown origin`;
|
|
1774
|
+
if (trace.source_digest !== undefined &&
|
|
1775
|
+
(typeof trace.source_digest !== "string" || !HEX64.test(trace.source_digest)))
|
|
1776
|
+
return `trace ${id} declares a malformed source digest`;
|
|
1777
|
+
}
|
|
1778
|
+
return undefined;
|
|
1779
|
+
}
|