@hue-run/sdk 0.21.1 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI.md +246 -52
- package/README.md +54 -13
- package/WORLDS.md +6 -3
- package/dist/ai-sdk.d.ts +23 -1
- package/dist/ai-sdk.js +60 -2
- package/dist/cli/import-formats/adapter.d.ts +80 -0
- package/dist/cli/import-formats/adapter.js +1 -0
- package/dist/cli/import-formats/agent-events.d.ts +5 -0
- package/dist/cli/import-formats/agent-events.js +16 -0
- package/dist/cli/import-formats/atif.d.ts +3 -0
- package/dist/cli/import-formats/atif.js +15 -0
- package/dist/cli/import-formats/braintrust.d.ts +2 -0
- package/dist/cli/import-formats/braintrust.js +222 -0
- package/dist/cli/import-formats/containers.d.ts +32 -0
- package/dist/cli/import-formats/containers.js +399 -0
- package/dist/cli/import-formats/datadog.d.ts +2 -0
- package/dist/cli/import-formats/datadog.js +264 -0
- package/dist/cli/import-formats/helicone.d.ts +2 -0
- package/dist/cli/import-formats/helicone.js +161 -0
- package/dist/cli/import-formats/identity.d.ts +112 -0
- package/dist/cli/import-formats/identity.js +594 -0
- package/dist/cli/import-formats/jaeger.d.ts +3 -0
- package/dist/cli/import-formats/jaeger.js +266 -0
- package/dist/cli/import-formats/langfuse.d.ts +7 -0
- package/dist/cli/import-formats/langfuse.js +510 -0
- package/dist/cli/import-formats/langsmith.d.ts +2 -0
- package/dist/cli/import-formats/langsmith.js +244 -0
- package/dist/cli/import-formats/messages.d.ts +28 -0
- package/dist/cli/import-formats/messages.js +301 -0
- package/dist/cli/import-formats/mlflow.d.ts +2 -0
- package/dist/cli/import-formats/mlflow.js +260 -0
- package/dist/cli/import-formats/openai-agents.d.ts +2 -0
- package/dist/cli/import-formats/openai-agents.js +221 -0
- package/dist/cli/import-formats/opik.d.ts +2 -0
- package/dist/cli/import-formats/opik.js +193 -0
- package/dist/cli/import-formats/otel-family.d.ts +5 -0
- package/dist/cli/import-formats/otel-family.js +17 -0
- package/dist/cli/import-formats/otel-spans.d.ts +94 -0
- package/dist/cli/import-formats/otel-spans.js +262 -0
- package/dist/cli/import-formats/otlp-protobuf.d.ts +35 -0
- package/dist/cli/import-formats/otlp-protobuf.js +486 -0
- package/dist/cli/import-formats/otlp.d.ts +4 -0
- package/dist/cli/import-formats/otlp.js +15 -0
- package/dist/cli/import-formats/phoenix.d.ts +3 -0
- package/dist/cli/import-formats/phoenix.js +256 -0
- package/dist/cli/import-formats/raindrop.d.ts +21 -0
- package/dist/cli/import-formats/raindrop.js +255 -0
- package/dist/cli/import-formats/reader.d.ts +56 -0
- package/dist/cli/import-formats/reader.js +364 -0
- package/dist/cli/import-formats/registry.d.ts +36 -0
- package/dist/cli/import-formats/registry.js +71 -0
- package/dist/cli/import-formats/row-platform-fields.d.ts +32 -0
- package/dist/cli/import-formats/row-platform-fields.js +76 -0
- package/dist/cli/import-formats/rows.d.ts +165 -0
- package/dist/cli/import-formats/rows.js +638 -0
- package/dist/cli/import-formats/transcripts.d.ts +25 -0
- package/dist/cli/import-formats/transcripts.js +498 -0
- package/dist/cli/import-formats/weave.d.ts +2 -0
- package/dist/cli/import-formats/weave.js +159 -0
- package/dist/cli/import-formats/zipkin.d.ts +2 -0
- package/dist/cli/import-formats/zipkin.js +219 -0
- package/dist/cli/import.d.ts +1 -1
- package/dist/cli/import.js +4 -2
- package/dist/cli/traces-import-convert.d.ts +65 -33
- package/dist/cli/traces-import-convert.js +211 -214
- package/dist/cli/traces-import-prepare.d.ts +26 -22
- package/dist/cli/traces-import-prepare.js +51 -122
- package/dist/cli/traces-import.d.ts +4 -4
- package/dist/cli/traces-import.js +63 -11
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,399 @@
|
|
|
1
|
+
import * as zlib from "node:zlib";
|
|
2
|
+
import { parseJsonText } from "../traces-import-convert.js";
|
|
3
|
+
import { isZipSignature } from "../traces-import-zip.js";
|
|
4
|
+
/**
|
|
5
|
+
* What every format reads through: a file's compression (gzip, zstd) undone, its text decoded,
|
|
6
|
+
* and its JSON read as one document or, failing that, as JSON Lines whose every line is read on
|
|
7
|
+
* its own, a line that is not JSON named with its number instead of refusing the file.
|
|
8
|
+
*
|
|
9
|
+
* Damage is contained the same way: a gzip or zstd file that breaks off, or carries bytes after
|
|
10
|
+
* its last member or frame, is read up to the break and the rest named; a line that is not UTF-8
|
|
11
|
+
* is named and the other lines read; Python's bare `NaN` and `Infinity` are read as null and
|
|
12
|
+
* named.
|
|
13
|
+
*/
|
|
14
|
+
const GZIP = [0x1f, 0x8b];
|
|
15
|
+
const ZSTD = [0x28, 0xb5, 0x2f, 0xfd];
|
|
16
|
+
const startsWith = (bytes, magic, at = 0) => bytes.byteLength >= at + magic.length && magic.every((byte, index) => bytes[at + index] === byte);
|
|
17
|
+
/** A zstd skippable frame (`0x184D2A50` to `0x184D2A5F`, little-endian): `pzstd` opens with one. */
|
|
18
|
+
const skippable = (bytes, at = 0) => bytes.byteLength >= at + 4 &&
|
|
19
|
+
(bytes[at] & 0xf0) === 0x50 &&
|
|
20
|
+
bytes[at + 1] === 0x2a &&
|
|
21
|
+
bytes[at + 2] === 0x4d &&
|
|
22
|
+
bytes[at + 3] === 0x18;
|
|
23
|
+
const isZstd = (bytes, at = 0) => startsWith(bytes, ZSTD, at) || skippable(bytes, at);
|
|
24
|
+
const tooLarge = (error) => error instanceof RangeError ||
|
|
25
|
+
error?.code === "ERR_BUFFER_TOO_LARGE";
|
|
26
|
+
/** The file's content with its compression undone, or why it cannot be read. `maxBytes` bounds
|
|
27
|
+
* the decompressed size. */
|
|
28
|
+
export function decodeContainer(bytes, maxBytes) {
|
|
29
|
+
if (!bytes.byteLength)
|
|
30
|
+
return { unsupported: "the file is empty" };
|
|
31
|
+
let decoded = { bytes };
|
|
32
|
+
if (startsWith(bytes, GZIP))
|
|
33
|
+
decoded = gunzip(bytes, maxBytes);
|
|
34
|
+
else if (isZstd(bytes))
|
|
35
|
+
decoded = unzstd(bytes, maxBytes);
|
|
36
|
+
if ("unsupported" in decoded)
|
|
37
|
+
return decoded;
|
|
38
|
+
if (decoded.bytes !== bytes) {
|
|
39
|
+
if (startsWith(decoded.bytes, GZIP) || isZstd(decoded.bytes))
|
|
40
|
+
return { unsupported: "a file compressed twice; decompress it first" };
|
|
41
|
+
if (!decoded.bytes.byteLength)
|
|
42
|
+
return { unsupported: "the file is empty" };
|
|
43
|
+
}
|
|
44
|
+
// An archive is read where the walk finds it on disk; one inside another is not opened.
|
|
45
|
+
if (isZipSignature(decoded.bytes))
|
|
46
|
+
return { unsupported: "a ZIP archive inside the input; extract it first" };
|
|
47
|
+
return decoded;
|
|
48
|
+
}
|
|
49
|
+
const oversized = (format, maxBytes) => ({
|
|
50
|
+
unsupported: `a ${format} file that decompresses to more than ${maxBytes} bytes`,
|
|
51
|
+
});
|
|
52
|
+
/** gzip, every member of it. A file that breaks off, or holds bytes after its last member, is
|
|
53
|
+
* read member by member up to the break, the rest named; a member that fails its checksum leaves
|
|
54
|
+
* the file unread. */
|
|
55
|
+
function gunzip(bytes, maxBytes) {
|
|
56
|
+
try {
|
|
57
|
+
return { bytes: zlib.gunzipSync(bytes, { maxOutputLength: maxBytes }) };
|
|
58
|
+
}
|
|
59
|
+
catch (error) {
|
|
60
|
+
if (tooLarge(error))
|
|
61
|
+
return oversized("gzip", maxBytes);
|
|
62
|
+
}
|
|
63
|
+
const parts = [];
|
|
64
|
+
let size = 0;
|
|
65
|
+
let pos = 0;
|
|
66
|
+
let note;
|
|
67
|
+
while (pos < bytes.byteLength) {
|
|
68
|
+
const start = startsWith(bytes, GZIP, pos) ? memberData(bytes, pos) : undefined;
|
|
69
|
+
if (start === undefined) {
|
|
70
|
+
// Zero padding after the last member is what a tape or block device writes.
|
|
71
|
+
const rest = bytes.subarray(pos);
|
|
72
|
+
if (!rest.every((byte) => byte === 0))
|
|
73
|
+
note = `${rest.byteLength} bytes after the last gzip member are not gzip and were not read`;
|
|
74
|
+
break;
|
|
75
|
+
}
|
|
76
|
+
let inflated;
|
|
77
|
+
try {
|
|
78
|
+
inflated = zlib.inflateRawSync(bytes.subarray(start), {
|
|
79
|
+
info: true,
|
|
80
|
+
finishFlush: zlib.constants.Z_SYNC_FLUSH,
|
|
81
|
+
maxOutputLength: maxBytes - size,
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
catch (error) {
|
|
85
|
+
if (tooLarge(error))
|
|
86
|
+
return oversized("gzip", maxBytes);
|
|
87
|
+
note = "the gzip file is damaged; what follows the damage was not read";
|
|
88
|
+
break;
|
|
89
|
+
}
|
|
90
|
+
parts.push(inflated.buffer);
|
|
91
|
+
size += inflated.buffer.byteLength;
|
|
92
|
+
// A complete member ends with its 8-byte trailer (CRC-32 and size).
|
|
93
|
+
const end = start + inflated.engine.bytesWritten + 8;
|
|
94
|
+
if (end > bytes.byteLength) {
|
|
95
|
+
note = "the gzip file breaks off; what follows the last complete line was not read";
|
|
96
|
+
break;
|
|
97
|
+
}
|
|
98
|
+
// A member whose data does not match its trailer is damaged, and none of the file is trusted.
|
|
99
|
+
const trailer = new DataView(bytes.buffer, bytes.byteOffset + end - 8, 8);
|
|
100
|
+
if (trailer.getUint32(0, true) !== zlib.crc32(inflated.buffer) ||
|
|
101
|
+
trailer.getUint32(4, true) !== inflated.buffer.byteLength >>> 0)
|
|
102
|
+
return { unsupported: "a damaged gzip file" };
|
|
103
|
+
pos = end;
|
|
104
|
+
}
|
|
105
|
+
// A file that reads whole, every member matching its trailer, yet failed to decompress is
|
|
106
|
+
// damaged somewhere this cannot see: none of it is trusted.
|
|
107
|
+
if (!note || !size)
|
|
108
|
+
return {
|
|
109
|
+
unsupported: note?.includes("breaks off") ? "a truncated gzip file" : "a damaged gzip file",
|
|
110
|
+
};
|
|
111
|
+
return { bytes: Buffer.concat(parts), note };
|
|
112
|
+
}
|
|
113
|
+
/** Where a gzip member's compressed data starts, past its header; undefined when the header is
|
|
114
|
+
* damaged or cut off. */
|
|
115
|
+
function memberData(bytes, pos) {
|
|
116
|
+
if (bytes[pos + 2] !== 8)
|
|
117
|
+
return undefined;
|
|
118
|
+
const flags = bytes[pos + 3];
|
|
119
|
+
let at = pos + 10;
|
|
120
|
+
if (flags & 0x04) {
|
|
121
|
+
if (at + 2 > bytes.byteLength)
|
|
122
|
+
return undefined;
|
|
123
|
+
at += 2 + (bytes[at] | (bytes[at + 1] << 8));
|
|
124
|
+
}
|
|
125
|
+
for (const flag of [0x08, 0x10]) {
|
|
126
|
+
if (!(flags & flag))
|
|
127
|
+
continue;
|
|
128
|
+
while (at < bytes.byteLength && bytes[at] !== 0)
|
|
129
|
+
at++;
|
|
130
|
+
at++;
|
|
131
|
+
}
|
|
132
|
+
if (flags & 0x02)
|
|
133
|
+
at += 2;
|
|
134
|
+
return at < bytes.byteLength ? at : undefined;
|
|
135
|
+
}
|
|
136
|
+
/** The end of the zstd frame (or skippable frame) at `pos`, or why there is none: `truncated`
|
|
137
|
+
* when the frame runs past the content, `other` when no frame starts there. */
|
|
138
|
+
function frameEnd(bytes, pos) {
|
|
139
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
140
|
+
if (skippable(bytes, pos)) {
|
|
141
|
+
if (pos + 8 > bytes.byteLength)
|
|
142
|
+
return "truncated";
|
|
143
|
+
const end = pos + 8 + view.getUint32(pos + 4, true);
|
|
144
|
+
return end > bytes.byteLength ? "truncated" : end;
|
|
145
|
+
}
|
|
146
|
+
if (!startsWith(bytes, ZSTD, pos))
|
|
147
|
+
return "other";
|
|
148
|
+
if (pos + 5 > bytes.byteLength)
|
|
149
|
+
return "truncated";
|
|
150
|
+
const descriptor = bytes[pos + 4];
|
|
151
|
+
const contentSize = descriptor >> 6;
|
|
152
|
+
const singleSegment = (descriptor >> 5) & 1;
|
|
153
|
+
const checksum = (descriptor >> 2) & 1;
|
|
154
|
+
const dictionary = descriptor & 3;
|
|
155
|
+
let at = pos +
|
|
156
|
+
5 +
|
|
157
|
+
(singleSegment ? 0 : 1) +
|
|
158
|
+
[0, 1, 2, 4][dictionary] +
|
|
159
|
+
[singleSegment ? 1 : 0, 2, 4, 8][contentSize];
|
|
160
|
+
for (;;) {
|
|
161
|
+
if (at + 3 > bytes.byteLength)
|
|
162
|
+
return "truncated";
|
|
163
|
+
const header = bytes[at] | (bytes[at + 1] << 8) | (bytes[at + 2] << 16);
|
|
164
|
+
const type = (header >> 1) & 3;
|
|
165
|
+
if (type === 3)
|
|
166
|
+
return "other";
|
|
167
|
+
at += 3 + (type === 1 ? 1 : header >>> 3);
|
|
168
|
+
if (header & 1)
|
|
169
|
+
break;
|
|
170
|
+
}
|
|
171
|
+
at += checksum ? 4 : 0;
|
|
172
|
+
return at > bytes.byteLength ? "truncated" : at;
|
|
173
|
+
}
|
|
174
|
+
/** zstd, every frame of it: Node.js decompresses only a file's first frame on its own, so each
|
|
175
|
+
* frame is found and decompressed in turn. Skippable frames are passed over; a frame that breaks
|
|
176
|
+
* off, and bytes after the last frame, are named. */
|
|
177
|
+
function unzstd(bytes, maxBytes) {
|
|
178
|
+
// zstd is in node:zlib from Node.js 22.15; earlier releases of the supported range lack it.
|
|
179
|
+
if (typeof zlib.zstdDecompressSync !== "function")
|
|
180
|
+
return {
|
|
181
|
+
unsupported: "a zstd file, which this Node.js cannot decompress (Node.js 22.15 or later can); decompress it first",
|
|
182
|
+
};
|
|
183
|
+
const parts = [];
|
|
184
|
+
let size = 0;
|
|
185
|
+
let pos = 0;
|
|
186
|
+
let frames = 0;
|
|
187
|
+
let stop;
|
|
188
|
+
while (pos < bytes.byteLength) {
|
|
189
|
+
const end = frameEnd(bytes, pos);
|
|
190
|
+
if (end === "other" || end === "truncated") {
|
|
191
|
+
stop = end === "other" ? "trailing" : "truncated";
|
|
192
|
+
break;
|
|
193
|
+
}
|
|
194
|
+
if (!skippable(bytes, pos)) {
|
|
195
|
+
let output;
|
|
196
|
+
try {
|
|
197
|
+
output = zlib.zstdDecompressSync(bytes.subarray(pos, end), {
|
|
198
|
+
maxOutputLength: maxBytes - size,
|
|
199
|
+
});
|
|
200
|
+
}
|
|
201
|
+
catch (error) {
|
|
202
|
+
if (tooLarge(error))
|
|
203
|
+
return oversized("zstd", maxBytes);
|
|
204
|
+
stop = "damaged";
|
|
205
|
+
break;
|
|
206
|
+
}
|
|
207
|
+
parts.push(output);
|
|
208
|
+
size += output.byteLength;
|
|
209
|
+
frames++;
|
|
210
|
+
}
|
|
211
|
+
pos = end;
|
|
212
|
+
}
|
|
213
|
+
if (!frames)
|
|
214
|
+
return {
|
|
215
|
+
unsupported: stop === "truncated"
|
|
216
|
+
? "a truncated zstd file"
|
|
217
|
+
: stop
|
|
218
|
+
? "a damaged zstd file"
|
|
219
|
+
: "a zstd file without data",
|
|
220
|
+
};
|
|
221
|
+
const note = stop === "trailing"
|
|
222
|
+
? `${bytes.byteLength - pos} bytes after the last zstd frame are not zstd and were not read`
|
|
223
|
+
: stop === "truncated"
|
|
224
|
+
? "the zstd file breaks off; what follows the last complete frame was not read"
|
|
225
|
+
: stop === "damaged"
|
|
226
|
+
? "the zstd file is damaged; what follows the damage was not read"
|
|
227
|
+
: undefined;
|
|
228
|
+
return { bytes: Buffer.concat(parts), ...(note ? { note } : {}) };
|
|
229
|
+
}
|
|
230
|
+
/** Whether text that failed to parse breaks off inside a string or an open object or list. */
|
|
231
|
+
function truncated(text) {
|
|
232
|
+
if (!/^\s*[{[]/.test(text))
|
|
233
|
+
return false;
|
|
234
|
+
let depth = 0;
|
|
235
|
+
let inString = false;
|
|
236
|
+
for (let index = 0; index < text.length; index++) {
|
|
237
|
+
const char = text[index];
|
|
238
|
+
if (inString) {
|
|
239
|
+
if (char === "\\")
|
|
240
|
+
index++;
|
|
241
|
+
else if (char === '"')
|
|
242
|
+
inString = false;
|
|
243
|
+
}
|
|
244
|
+
else if (char === '"')
|
|
245
|
+
inString = true;
|
|
246
|
+
else if (char === "{" || char === "[")
|
|
247
|
+
depth++;
|
|
248
|
+
else if (char === "}" || char === "]")
|
|
249
|
+
depth--;
|
|
250
|
+
}
|
|
251
|
+
return inString || depth > 0;
|
|
252
|
+
}
|
|
253
|
+
/** Text with every bare `NaN`, `Infinity` and `-Infinity` outside a string (which Python's
|
|
254
|
+
* `json.dumps` writes by default) as `null`; undefined when it holds none. */
|
|
255
|
+
function withoutNonFinite(text) {
|
|
256
|
+
if (!/NaN|Infinity/.test(text))
|
|
257
|
+
return undefined;
|
|
258
|
+
let out = "";
|
|
259
|
+
let from = 0;
|
|
260
|
+
let replaced = false;
|
|
261
|
+
let inString = false;
|
|
262
|
+
for (let index = 0; index < text.length; index++) {
|
|
263
|
+
const char = text[index];
|
|
264
|
+
if (inString) {
|
|
265
|
+
if (char === "\\")
|
|
266
|
+
index++;
|
|
267
|
+
else if (char === '"')
|
|
268
|
+
inString = false;
|
|
269
|
+
continue;
|
|
270
|
+
}
|
|
271
|
+
if (char === '"') {
|
|
272
|
+
inString = true;
|
|
273
|
+
continue;
|
|
274
|
+
}
|
|
275
|
+
if (char !== "N" && char !== "I" && char !== "-")
|
|
276
|
+
continue;
|
|
277
|
+
const token = /^(?:-?Infinity|NaN)/.exec(text.slice(index, index + 9))?.[0];
|
|
278
|
+
if (!token ||
|
|
279
|
+
/[\w$]/.test(text[index - 1] ?? "") ||
|
|
280
|
+
/[\w$]/.test(text[index + token.length] ?? ""))
|
|
281
|
+
continue;
|
|
282
|
+
out += `${text.slice(from, index)}null`;
|
|
283
|
+
from = index + token.length;
|
|
284
|
+
index = from - 1;
|
|
285
|
+
replaced = true;
|
|
286
|
+
}
|
|
287
|
+
return replaced ? out + text.slice(from) : undefined;
|
|
288
|
+
}
|
|
289
|
+
const NON_FINITE = "NaN or Infinity, which JSON cannot hold, read as null";
|
|
290
|
+
/** JSON text parsed, with Python's bare `NaN` and `Infinity` read as null when nothing else
|
|
291
|
+
* stands in the way; throws when it is not JSON even so. */
|
|
292
|
+
function parseRepairing(text) {
|
|
293
|
+
try {
|
|
294
|
+
return parseJsonText(text);
|
|
295
|
+
}
|
|
296
|
+
catch (error) {
|
|
297
|
+
const repaired = withoutNonFinite(text);
|
|
298
|
+
if (repaired === undefined)
|
|
299
|
+
throw error;
|
|
300
|
+
return { ...parseJsonText(repaired), note: NON_FINITE };
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
/** Text decoded from bytes: UTF-8, or UTF-16 after its byte order mark. A UTF-8 file with lines
|
|
304
|
+
* that are not UTF-8 keeps its other lines, the bad ones blanked and named. */
|
|
305
|
+
function decodeText(bytes) {
|
|
306
|
+
const utf16 = bytes[0] === 0xff && bytes[1] === 0xfe
|
|
307
|
+
? "le"
|
|
308
|
+
: bytes[0] === 0xfe && bytes[1] === 0xff
|
|
309
|
+
? "be"
|
|
310
|
+
: undefined;
|
|
311
|
+
if (utf16) {
|
|
312
|
+
// UTF-16 as Windows PowerShell's `>` writes it; big-endian is swapped to little first.
|
|
313
|
+
const body = Buffer.from(bytes.subarray(2));
|
|
314
|
+
if (body.byteLength % 2)
|
|
315
|
+
return { unsupported: "not UTF-16 text" };
|
|
316
|
+
if (utf16 === "be")
|
|
317
|
+
body.swap16();
|
|
318
|
+
try {
|
|
319
|
+
return { text: new TextDecoder("utf-16le", { fatal: true }).decode(body), badLines: [] };
|
|
320
|
+
}
|
|
321
|
+
catch {
|
|
322
|
+
return { unsupported: "not UTF-16 text" };
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: false });
|
|
326
|
+
try {
|
|
327
|
+
return { text: decoder.decode(bytes), badLines: [] };
|
|
328
|
+
}
|
|
329
|
+
catch {
|
|
330
|
+
// Some line is not UTF-8: each line is decoded on its own.
|
|
331
|
+
}
|
|
332
|
+
const lines = [];
|
|
333
|
+
const badLines = [];
|
|
334
|
+
let start = 0;
|
|
335
|
+
while (start <= bytes.byteLength) {
|
|
336
|
+
let end = bytes.indexOf(0x0a, start);
|
|
337
|
+
if (end < 0)
|
|
338
|
+
end = bytes.byteLength;
|
|
339
|
+
try {
|
|
340
|
+
lines.push(decoder.decode(bytes.subarray(start, end)));
|
|
341
|
+
}
|
|
342
|
+
catch {
|
|
343
|
+
lines.push("");
|
|
344
|
+
badLines.push(lines.length);
|
|
345
|
+
}
|
|
346
|
+
start = end + 1;
|
|
347
|
+
}
|
|
348
|
+
return { text: lines.join("\n"), badLines };
|
|
349
|
+
}
|
|
350
|
+
/** The JSON values of decoded content, the lines that are not JSON, or why none can be read. */
|
|
351
|
+
export function parseValues(bytes) {
|
|
352
|
+
const decoded = decodeText(bytes);
|
|
353
|
+
if ("unsupported" in decoded)
|
|
354
|
+
return decoded;
|
|
355
|
+
const { text, badLines } = decoded;
|
|
356
|
+
const problems = badLines.map((line) => ({
|
|
357
|
+
line,
|
|
358
|
+
reason: "not UTF-8 text",
|
|
359
|
+
}));
|
|
360
|
+
if (!badLines.length)
|
|
361
|
+
try {
|
|
362
|
+
const parsed = parseRepairing(text);
|
|
363
|
+
return { values: [{ ...parsed, text }], problems: [] };
|
|
364
|
+
}
|
|
365
|
+
catch {
|
|
366
|
+
// Not one document: JSON Lines, read line by line.
|
|
367
|
+
}
|
|
368
|
+
const lines = text.split(/\r?\n/);
|
|
369
|
+
const filled = lines.filter((line) => line.trim()).length + badLines.length;
|
|
370
|
+
if (filled < 2)
|
|
371
|
+
return {
|
|
372
|
+
unsupported: badLines.length
|
|
373
|
+
? `not UTF-8 text (line ${badLines[0]})`
|
|
374
|
+
: truncated(text)
|
|
375
|
+
? "truncated JSON"
|
|
376
|
+
: "not JSON",
|
|
377
|
+
};
|
|
378
|
+
const values = [];
|
|
379
|
+
for (const [index, line] of lines.entries()) {
|
|
380
|
+
if (!line.trim())
|
|
381
|
+
continue;
|
|
382
|
+
try {
|
|
383
|
+
values.push({ ...parseRepairing(line), text: line, line: index + 1 });
|
|
384
|
+
}
|
|
385
|
+
catch {
|
|
386
|
+
problems.push({ line: index + 1, reason: truncated(line) ? "truncated JSON" : "not JSON" });
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
problems.sort((a, b) => a.line - b.line);
|
|
390
|
+
if (values.length)
|
|
391
|
+
return { values, problems };
|
|
392
|
+
return {
|
|
393
|
+
unsupported: badLines.length
|
|
394
|
+
? `not UTF-8 text (line ${badLines[0]})`
|
|
395
|
+
: truncated(text)
|
|
396
|
+
? "truncated JSON"
|
|
397
|
+
: "not JSON",
|
|
398
|
+
};
|
|
399
|
+
}
|
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
import { ConversionError, isObject, parseNestedJsonText, } from "../traces-import-convert.js";
|
|
2
|
+
import { identityText } from "./identity.js";
|
|
3
|
+
import { unsigned } from "./otel-spans.js";
|
|
4
|
+
import { readRecords } from "./rows.js";
|
|
5
|
+
const object = (value) => isObject(value) ? value : undefined;
|
|
6
|
+
const text = (value) => typeof value === "string" && value ? value : undefined;
|
|
7
|
+
const isSpan = (value) => {
|
|
8
|
+
const attributes = object(object(value)?.attributes);
|
|
9
|
+
return (object(value)?.type === "span" &&
|
|
10
|
+
attributes !== undefined &&
|
|
11
|
+
typeof attributes.span_id === "string" &&
|
|
12
|
+
typeof attributes.trace_id === "string");
|
|
13
|
+
};
|
|
14
|
+
function recordsOf(value) {
|
|
15
|
+
if (isSpan(value))
|
|
16
|
+
return [value];
|
|
17
|
+
const page = object(value);
|
|
18
|
+
return page && Array.isArray(page.data) && page.data.some(isSpan) ? page.data : undefined;
|
|
19
|
+
}
|
|
20
|
+
const KINDS = {
|
|
21
|
+
llm: "llm",
|
|
22
|
+
tool: "tool",
|
|
23
|
+
agent: "agent",
|
|
24
|
+
embedding: "embedding",
|
|
25
|
+
retrieval: "retrieval",
|
|
26
|
+
};
|
|
27
|
+
/** A root span's `parent_id`. */
|
|
28
|
+
const NO_PARENT = new Set(["undefined", "", "0"]);
|
|
29
|
+
/** `start_ns` as Unix nanoseconds, the unit it was read in (nanoseconds per unit) and the rule
|
|
30
|
+
* that read it when it was not nanoseconds. */
|
|
31
|
+
function startOf(value) {
|
|
32
|
+
if (value === undefined || value === null)
|
|
33
|
+
return undefined;
|
|
34
|
+
const count = unsigned(value);
|
|
35
|
+
if (!count)
|
|
36
|
+
throw new ConversionError("start_ns is not a positive integer");
|
|
37
|
+
if (count < 100000000000000n)
|
|
38
|
+
return { start: count * 1000000n, scale: 1000000n, unit: "ms" };
|
|
39
|
+
if (count < 100000000000000000n)
|
|
40
|
+
return { start: count * 1000n, scale: 1000n, unit: "us" };
|
|
41
|
+
return { start: count, scale: 1n };
|
|
42
|
+
}
|
|
43
|
+
/** `duration` in the unit `start_ns` was read in, as nanoseconds. */
|
|
44
|
+
function durationOf(value, scale) {
|
|
45
|
+
if (value === undefined || value === null)
|
|
46
|
+
return undefined;
|
|
47
|
+
if (typeof value === "number" && Number.isFinite(value) && value >= 0)
|
|
48
|
+
return BigInt(Math.round(value * Number(scale)));
|
|
49
|
+
const count = unsigned(value);
|
|
50
|
+
if (count === undefined)
|
|
51
|
+
throw new ConversionError("duration is not a count of nanoseconds");
|
|
52
|
+
return count * scale;
|
|
53
|
+
}
|
|
54
|
+
/** Tags (`key:value`) by key; a key tagged twice keeps its first value. */
|
|
55
|
+
function tagMap(tags) {
|
|
56
|
+
const map = new Map();
|
|
57
|
+
if (tags === undefined || tags === null)
|
|
58
|
+
return map;
|
|
59
|
+
if (!Array.isArray(tags))
|
|
60
|
+
throw new ConversionError("tags is not a list of text");
|
|
61
|
+
for (const tag of tags) {
|
|
62
|
+
if (typeof tag !== "string")
|
|
63
|
+
throw new ConversionError("tags is not a list of text");
|
|
64
|
+
const at = tag.indexOf(":");
|
|
65
|
+
const key = at < 0 ? tag : tag.slice(0, at);
|
|
66
|
+
if (!map.has(key))
|
|
67
|
+
map.set(key, at < 0 ? "" : tag.slice(at + 1));
|
|
68
|
+
}
|
|
69
|
+
return map;
|
|
70
|
+
}
|
|
71
|
+
/** Datadog's messages as GenAI messages: content as a text part, tool calls and tool results as
|
|
72
|
+
* `tool_call` and `tool_call_response` parts. */
|
|
73
|
+
function messagesOf(raw, role) {
|
|
74
|
+
if (!Array.isArray(raw) || !raw.length)
|
|
75
|
+
return undefined;
|
|
76
|
+
return raw.map((item) => {
|
|
77
|
+
const message = object(item);
|
|
78
|
+
if (!message)
|
|
79
|
+
throw new ConversionError("a message that is not an object");
|
|
80
|
+
const parts = [];
|
|
81
|
+
if (typeof message.content === "string" && message.content)
|
|
82
|
+
parts.push({ type: "text", content: message.content });
|
|
83
|
+
for (const [field, kind] of [
|
|
84
|
+
["tool_calls", "tool_call"],
|
|
85
|
+
["tool_results", "tool_call_response"],
|
|
86
|
+
]) {
|
|
87
|
+
const calls = message[field] ?? [];
|
|
88
|
+
if (!Array.isArray(calls))
|
|
89
|
+
throw new ConversionError(`${field} is not a list`);
|
|
90
|
+
for (const value of calls) {
|
|
91
|
+
const call = object(value);
|
|
92
|
+
if (!call)
|
|
93
|
+
throw new ConversionError(`${field} holds a value that is not an object`);
|
|
94
|
+
parts.push({
|
|
95
|
+
type: kind,
|
|
96
|
+
...(text(call.tool_id) ? { id: text(call.tool_id) } : {}),
|
|
97
|
+
...(text(call.name) ? { name: text(call.name) } : {}),
|
|
98
|
+
...(kind === "tool_call"
|
|
99
|
+
? { arguments: call.arguments ?? null }
|
|
100
|
+
: { response: call.result ?? null }),
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
return { role: text(message.role) ?? role, parts };
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
/** A span's input or output: its messages when it has any, else its value, else the object. */
|
|
108
|
+
function contentOf(raw, role) {
|
|
109
|
+
const io = object(raw);
|
|
110
|
+
if (!io)
|
|
111
|
+
return raw ?? undefined;
|
|
112
|
+
const messages = messagesOf(io.messages, role);
|
|
113
|
+
if (messages)
|
|
114
|
+
return messages;
|
|
115
|
+
if (typeof io.value === "string")
|
|
116
|
+
return io.value;
|
|
117
|
+
return Object.keys(io).length ? io : undefined;
|
|
118
|
+
}
|
|
119
|
+
/** A tool's arguments or result: the JSON a value holds, else the text. */
|
|
120
|
+
function toolValue(raw) {
|
|
121
|
+
const io = object(raw);
|
|
122
|
+
const value = io ? io.value : raw;
|
|
123
|
+
if (typeof value !== "string")
|
|
124
|
+
return value ?? undefined;
|
|
125
|
+
try {
|
|
126
|
+
return parseNestedJsonText(value).value;
|
|
127
|
+
}
|
|
128
|
+
catch {
|
|
129
|
+
return value;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
const count = (value) => typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : undefined;
|
|
133
|
+
function readSpan(resource, context) {
|
|
134
|
+
const span = object(resource.attributes);
|
|
135
|
+
const traceKey = span?.trace_id;
|
|
136
|
+
const spanKey = span?.span_id;
|
|
137
|
+
if (!span ||
|
|
138
|
+
resource.type !== "span" ||
|
|
139
|
+
typeof traceKey !== "string" ||
|
|
140
|
+
typeof spanKey !== "string")
|
|
141
|
+
throw new ConversionError("a record that is not a Datadog span");
|
|
142
|
+
const parent = text(span.parent_id);
|
|
143
|
+
const metadata = object(span.metadata);
|
|
144
|
+
if (span.metadata != null && !metadata)
|
|
145
|
+
throw new ConversionError("metadata is not an object");
|
|
146
|
+
const metrics = object(span.metrics);
|
|
147
|
+
if (span.metrics != null && !metrics)
|
|
148
|
+
throw new ConversionError("metrics is not an object");
|
|
149
|
+
const tags = tagMap(span.tags);
|
|
150
|
+
const spanKind = text(span.span_kind);
|
|
151
|
+
const kind = spanKind && Object.hasOwn(KINDS, spanKind) ? KINDS[spanKind] : undefined;
|
|
152
|
+
const started = startOf(span.start_ns);
|
|
153
|
+
const duration = durationOf(span.duration, started?.scale ?? 1n);
|
|
154
|
+
const timing = started?.unit
|
|
155
|
+
? [
|
|
156
|
+
`start_ns_read_as_${started.unit}`,
|
|
157
|
+
...(duration !== undefined ? [`duration_read_as_${started.unit}`] : []),
|
|
158
|
+
].join("; ")
|
|
159
|
+
: undefined;
|
|
160
|
+
const tool = kind === "tool";
|
|
161
|
+
const row = {
|
|
162
|
+
traceKey,
|
|
163
|
+
spanKey,
|
|
164
|
+
...(parent && !NO_PARENT.has(parent) ? { parentKey: parent } : {}),
|
|
165
|
+
name: text(span.name) ?? spanKind ?? "span",
|
|
166
|
+
...(kind ? { kind } : {}),
|
|
167
|
+
...(started ? { start: started.start } : {}),
|
|
168
|
+
...(started && duration !== undefined ? { end: started.start + duration } : {}),
|
|
169
|
+
...(timing ? { timing } : {}),
|
|
170
|
+
record: resource,
|
|
171
|
+
...(metadata ? { metadata } : {}),
|
|
172
|
+
attributes: {
|
|
173
|
+
...(spanKind ? { "datadog.span_kind": spanKind } : {}),
|
|
174
|
+
...(text(span.ml_app) ? { "datadog.ml_app": text(span.ml_app) } : {}),
|
|
175
|
+
...(Array.isArray(span.tags) ? { "datadog.tags": span.tags } : {}),
|
|
176
|
+
...(metadata && Object.keys(metadata).length
|
|
177
|
+
? { "datadog.metadata": JSON.stringify(metadata) }
|
|
178
|
+
: {}),
|
|
179
|
+
...(metrics ? { "datadog.metrics": JSON.stringify(metrics) } : {}),
|
|
180
|
+
...(object(span.evaluation) ? { "datadog.evaluation": JSON.stringify(span.evaluation) } : {}),
|
|
181
|
+
...(text(span.intent) ? { "datadog.intent": text(span.intent) } : {}),
|
|
182
|
+
...(Array.isArray(span.tool_definitions)
|
|
183
|
+
? { "gen_ai.tool.definitions": JSON.stringify(span.tool_definitions) }
|
|
184
|
+
: {}),
|
|
185
|
+
},
|
|
186
|
+
};
|
|
187
|
+
const service = tags.get("service");
|
|
188
|
+
if (service)
|
|
189
|
+
row.resource = { "service.name": service };
|
|
190
|
+
const environment = tags.get("env");
|
|
191
|
+
if (environment)
|
|
192
|
+
row.environment = environment;
|
|
193
|
+
if (spanKind === "experiment" || tags.get("experiment_id"))
|
|
194
|
+
row.origin = "evaluation";
|
|
195
|
+
if (tool) {
|
|
196
|
+
const args = toolValue(span.input);
|
|
197
|
+
const result = toolValue(span.output);
|
|
198
|
+
row.tool = {
|
|
199
|
+
name: text(span.name) ?? "tool",
|
|
200
|
+
...(args !== undefined ? { arguments: args } : {}),
|
|
201
|
+
...(result !== undefined ? { result } : {}),
|
|
202
|
+
};
|
|
203
|
+
}
|
|
204
|
+
else {
|
|
205
|
+
const input = contentOf(span.input, "user");
|
|
206
|
+
const output = contentOf(span.output, "assistant");
|
|
207
|
+
if (input !== undefined)
|
|
208
|
+
row.input = input;
|
|
209
|
+
if (output !== undefined)
|
|
210
|
+
row.output = output;
|
|
211
|
+
// Who asked, never from a tool span: a tool's record may name the account it acted on.
|
|
212
|
+
const user = tags.get("user_id") ??
|
|
213
|
+
tags.get("user.id") ??
|
|
214
|
+
tags.get("usr.id") ??
|
|
215
|
+
identityText(metadata?.user_id) ??
|
|
216
|
+
identityText(metadata?.userId);
|
|
217
|
+
const session = identityText(span.session_id) ??
|
|
218
|
+
tags.get("session_id") ??
|
|
219
|
+
identityText(metadata?.session_id) ??
|
|
220
|
+
identityText(metadata?.sessionId) ??
|
|
221
|
+
identityText(metadata?.thread_id) ??
|
|
222
|
+
identityText(metadata?.conversation_id);
|
|
223
|
+
if (user)
|
|
224
|
+
row.user = user;
|
|
225
|
+
if (session)
|
|
226
|
+
row.session = session;
|
|
227
|
+
}
|
|
228
|
+
if (kind === "llm") {
|
|
229
|
+
if (text(span.model_name))
|
|
230
|
+
row.model = text(span.model_name);
|
|
231
|
+
if (text(span.model_provider))
|
|
232
|
+
row.provider = text(span.model_provider);
|
|
233
|
+
row.usage = {
|
|
234
|
+
input: count(metrics?.input_tokens),
|
|
235
|
+
output: count(metrics?.output_tokens),
|
|
236
|
+
cacheRead: count(metrics?.cache_read_input_tokens),
|
|
237
|
+
cacheWrite: count(metrics?.cache_write_input_tokens),
|
|
238
|
+
reasoning: count(metrics?.reasoning_output_tokens),
|
|
239
|
+
};
|
|
240
|
+
}
|
|
241
|
+
if (span.status === "error") {
|
|
242
|
+
const error = object(span.error);
|
|
243
|
+
const message = text(error?.message) ?? text(span["error.message"]);
|
|
244
|
+
const type = text(error?.type) ?? text(span["error.type"]) ?? tags.get("error_type");
|
|
245
|
+
row.error = { ...(message ? { message } : {}), ...(type ? { type } : {}) };
|
|
246
|
+
}
|
|
247
|
+
context.row(row);
|
|
248
|
+
}
|
|
249
|
+
export const datadogLlmObs = {
|
|
250
|
+
id: "datadog_llmobs",
|
|
251
|
+
manifestFormat: "otlp_json",
|
|
252
|
+
detect: (value) => recordsOf(value) !== undefined,
|
|
253
|
+
convert(value, context) {
|
|
254
|
+
readRecords(context, recordsOf(value) ?? [], (record) => {
|
|
255
|
+
const traceId = object(object(record)?.attributes)?.trace_id;
|
|
256
|
+
return typeof traceId === "string" && traceId ? traceId : undefined;
|
|
257
|
+
}, (record) => {
|
|
258
|
+
const resource = object(record);
|
|
259
|
+
if (!resource)
|
|
260
|
+
throw new ConversionError("a record that is not an object");
|
|
261
|
+
readSpan(resource, context);
|
|
262
|
+
});
|
|
263
|
+
},
|
|
264
|
+
};
|