@hue-run/sdk 0.21.1 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI.md +246 -52
- package/README.md +69 -14
- package/WORLDS.md +6 -3
- package/dist/ai-sdk.d.ts +23 -1
- package/dist/ai-sdk.js +60 -2
- package/dist/cli/import-formats/adapter.d.ts +80 -0
- package/dist/cli/import-formats/adapter.js +1 -0
- package/dist/cli/import-formats/agent-events.d.ts +5 -0
- package/dist/cli/import-formats/agent-events.js +16 -0
- package/dist/cli/import-formats/atif.d.ts +3 -0
- package/dist/cli/import-formats/atif.js +15 -0
- package/dist/cli/import-formats/braintrust.d.ts +2 -0
- package/dist/cli/import-formats/braintrust.js +222 -0
- package/dist/cli/import-formats/containers.d.ts +32 -0
- package/dist/cli/import-formats/containers.js +399 -0
- package/dist/cli/import-formats/datadog.d.ts +2 -0
- package/dist/cli/import-formats/datadog.js +264 -0
- package/dist/cli/import-formats/helicone.d.ts +2 -0
- package/dist/cli/import-formats/helicone.js +161 -0
- package/dist/cli/import-formats/identity.d.ts +112 -0
- package/dist/cli/import-formats/identity.js +594 -0
- package/dist/cli/import-formats/jaeger.d.ts +3 -0
- package/dist/cli/import-formats/jaeger.js +266 -0
- package/dist/cli/import-formats/langfuse.d.ts +7 -0
- package/dist/cli/import-formats/langfuse.js +510 -0
- package/dist/cli/import-formats/langsmith.d.ts +2 -0
- package/dist/cli/import-formats/langsmith.js +244 -0
- package/dist/cli/import-formats/messages.d.ts +28 -0
- package/dist/cli/import-formats/messages.js +301 -0
- package/dist/cli/import-formats/mlflow.d.ts +2 -0
- package/dist/cli/import-formats/mlflow.js +260 -0
- package/dist/cli/import-formats/openai-agents.d.ts +2 -0
- package/dist/cli/import-formats/openai-agents.js +221 -0
- package/dist/cli/import-formats/opik.d.ts +2 -0
- package/dist/cli/import-formats/opik.js +193 -0
- package/dist/cli/import-formats/otel-family.d.ts +5 -0
- package/dist/cli/import-formats/otel-family.js +17 -0
- package/dist/cli/import-formats/otel-spans.d.ts +94 -0
- package/dist/cli/import-formats/otel-spans.js +262 -0
- package/dist/cli/import-formats/otlp-protobuf.d.ts +35 -0
- package/dist/cli/import-formats/otlp-protobuf.js +486 -0
- package/dist/cli/import-formats/otlp.d.ts +4 -0
- package/dist/cli/import-formats/otlp.js +15 -0
- package/dist/cli/import-formats/phoenix.d.ts +3 -0
- package/dist/cli/import-formats/phoenix.js +256 -0
- package/dist/cli/import-formats/raindrop.d.ts +21 -0
- package/dist/cli/import-formats/raindrop.js +255 -0
- package/dist/cli/import-formats/reader.d.ts +56 -0
- package/dist/cli/import-formats/reader.js +364 -0
- package/dist/cli/import-formats/registry.d.ts +36 -0
- package/dist/cli/import-formats/registry.js +71 -0
- package/dist/cli/import-formats/row-platform-fields.d.ts +32 -0
- package/dist/cli/import-formats/row-platform-fields.js +76 -0
- package/dist/cli/import-formats/rows.d.ts +165 -0
- package/dist/cli/import-formats/rows.js +638 -0
- package/dist/cli/import-formats/transcripts.d.ts +25 -0
- package/dist/cli/import-formats/transcripts.js +498 -0
- package/dist/cli/import-formats/weave.d.ts +2 -0
- package/dist/cli/import-formats/weave.js +159 -0
- package/dist/cli/import-formats/zipkin.d.ts +2 -0
- package/dist/cli/import-formats/zipkin.js +219 -0
- package/dist/cli/import.d.ts +1 -1
- package/dist/cli/import.js +4 -2
- package/dist/cli/traces-import-convert.d.ts +65 -33
- package/dist/cli/traces-import-convert.js +211 -214
- package/dist/cli/traces-import-prepare.d.ts +26 -22
- package/dist/cli/traces-import-prepare.js +51 -122
- package/dist/cli/traces-import.d.ts +4 -4
- package/dist/cli/traces-import.js +63 -11
- package/dist/transport.d.ts +12 -0
- package/dist/transport.js +139 -24
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +2 -1
|
@@ -0,0 +1,364 @@
|
|
|
1
|
+
import { conversionFailure, MAX_SOURCE_NESTING, nestedTooDeep, sha256Hex, } from "../traces-import-convert.js";
|
|
2
|
+
import { decodeContainer, parseValues } from "./containers.js";
|
|
3
|
+
import { applyIdentityKeys, traceIdentity } from "./identity.js";
|
|
4
|
+
import { binaryValues } from "./otlp-protobuf.js";
|
|
5
|
+
import { adapterFor, importAdapters, unrecognizedReason } from "./registry.js";
|
|
6
|
+
import { RowTraceCollector, rowTraceId } from "./rows.js";
|
|
7
|
+
/**
|
|
8
|
+
* Reads source files through the containers and the format registry (`registry.ts`) and gathers
|
|
9
|
+
* every trace they hold, with what could not be read: files, records (lines of JSON Lines,
|
|
10
|
+
* records of a JSON array) and traces, each named with where it was found.
|
|
11
|
+
*/
|
|
12
|
+
/** Records of one file reported as not read, or notes on one file; past this many the rest are
|
|
13
|
+
* counted. */
|
|
14
|
+
const MAX_REPORTED_PER_FILE = 100;
|
|
15
|
+
/** Records of a JSON array no format reads, in a row from its start, after which the rest of the
|
|
16
|
+
* array is not looked at: it is not an export of any format. */
|
|
17
|
+
const MAX_UNCLAIMED_RUN = 1000;
|
|
18
|
+
/** One file's reported records: the first `MAX_REPORTED_PER_FILE` kept, the rest counted. */
|
|
19
|
+
class Reported {
|
|
20
|
+
file;
|
|
21
|
+
items = [];
|
|
22
|
+
more = 0;
|
|
23
|
+
constructor(file) {
|
|
24
|
+
this.file = file;
|
|
25
|
+
}
|
|
26
|
+
add(item) {
|
|
27
|
+
if (this.items.length < MAX_REPORTED_PER_FILE)
|
|
28
|
+
this.items.push({ file: this.file, ...item });
|
|
29
|
+
else
|
|
30
|
+
this.more++;
|
|
31
|
+
}
|
|
32
|
+
/** The kept items in the order of the file, and one counting the rest. */
|
|
33
|
+
finish(what) {
|
|
34
|
+
const items = this.items.sort((a, b) => (a.line ?? 0) - (b.line ?? 0) || (a.record ?? 0) - (b.record ?? 0));
|
|
35
|
+
return this.more ? [...items, { file: this.file, reason: `${this.more} more ${what}` }] : items;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
/** Where a duplicate was first read, as the report names it from `file`. */
|
|
39
|
+
function where(earlier, file) {
|
|
40
|
+
const position = earlier.line !== undefined
|
|
41
|
+
? `line ${earlier.line}`
|
|
42
|
+
: earlier.record !== undefined
|
|
43
|
+
? `record ${earlier.record}`
|
|
44
|
+
: "";
|
|
45
|
+
if (earlier.file === file)
|
|
46
|
+
return position || "earlier in the file";
|
|
47
|
+
return position ? `${earlier.file} ${position}` : earlier.file;
|
|
48
|
+
}
|
|
49
|
+
export class ImportReader {
|
|
50
|
+
options;
|
|
51
|
+
/** Run-wide collectors, by format id or by the key formats that share one name. */
|
|
52
|
+
collectors = new Map();
|
|
53
|
+
/** The shared collector key of each format that named one. */
|
|
54
|
+
sharedKeys = new Map();
|
|
55
|
+
rowCollectors = new Map();
|
|
56
|
+
whole = [];
|
|
57
|
+
/** Traces already refused and named; a collector's own refusal of one is not repeated. */
|
|
58
|
+
refusedTraces = new Set();
|
|
59
|
+
/** The file names each content digest was read under. */
|
|
60
|
+
fileNames = new Map();
|
|
61
|
+
supported = new Map();
|
|
62
|
+
result;
|
|
63
|
+
/** `unsupported` is where files that cannot be read are listed, in the order they are read. */
|
|
64
|
+
constructor(options) {
|
|
65
|
+
this.options = options;
|
|
66
|
+
this.result = {
|
|
67
|
+
unsupported: options.unsupported ?? [],
|
|
68
|
+
skipped: [],
|
|
69
|
+
notes: [],
|
|
70
|
+
refused: [],
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
/** Reads one file's bytes; returns the size of its decoded content. */
|
|
74
|
+
read(file, raw) {
|
|
75
|
+
const decoded = decodeContainer(raw, this.options.maxFileBytes);
|
|
76
|
+
if ("unsupported" in decoded) {
|
|
77
|
+
this.result.unsupported.push({ file, reason: decoded.unsupported });
|
|
78
|
+
return raw.byteLength;
|
|
79
|
+
}
|
|
80
|
+
const binary = binaryValues(decoded.bytes, this.options.maxFileBytes);
|
|
81
|
+
const parsed = binary ?? parseValues(decoded.bytes);
|
|
82
|
+
if ("unsupported" in parsed) {
|
|
83
|
+
this.result.unsupported.push({ file, reason: parsed.unsupported });
|
|
84
|
+
return decoded.bytes.byteLength;
|
|
85
|
+
}
|
|
86
|
+
const size = () => ("bytes" in parsed ? parsed.bytes() : decoded.bytes.byteLength);
|
|
87
|
+
const digest = sha256Hex(decoded.bytes);
|
|
88
|
+
const skipped = new Reported(file);
|
|
89
|
+
const notes = new Reported(file);
|
|
90
|
+
if (decoded.note)
|
|
91
|
+
notes.add({ reason: decoded.note });
|
|
92
|
+
const failures = [];
|
|
93
|
+
let claimed = false;
|
|
94
|
+
let firstReason;
|
|
95
|
+
/** Converts a value with the format that claims it; the reason it was not read otherwise. */
|
|
96
|
+
const offer = (value, source) => {
|
|
97
|
+
const adapter = adapterFor(value);
|
|
98
|
+
if (!adapter)
|
|
99
|
+
return unrecognizedReason(value);
|
|
100
|
+
try {
|
|
101
|
+
this.convert(adapter, value, file, source, notes);
|
|
102
|
+
}
|
|
103
|
+
catch (error) {
|
|
104
|
+
const failure = conversionFailure(error);
|
|
105
|
+
if (!failure)
|
|
106
|
+
throw error;
|
|
107
|
+
failures.push(failure.message);
|
|
108
|
+
return failure.message;
|
|
109
|
+
}
|
|
110
|
+
claimed = true;
|
|
111
|
+
const key = `${file}\u0000${adapter.id}`;
|
|
112
|
+
if (!this.supported.has(key))
|
|
113
|
+
this.supported.set(key, { file, format: adapter.id, traces: 0 });
|
|
114
|
+
return undefined;
|
|
115
|
+
};
|
|
116
|
+
const tooDeep = `JSON nested more than ${MAX_SOURCE_NESTING} levels deep`;
|
|
117
|
+
/** The records of an array no format reads whole, each offered on its own. */
|
|
118
|
+
const offerRecords = (records, source, line) => {
|
|
119
|
+
let run = 0;
|
|
120
|
+
let claimedAny = false;
|
|
121
|
+
for (const [index, element] of records.entries()) {
|
|
122
|
+
if (!claimedAny && run >= MAX_UNCLAIMED_RUN) {
|
|
123
|
+
skipped.add({
|
|
124
|
+
...line,
|
|
125
|
+
reason: `${records.length - index} more records not read: no format reads the first ${MAX_UNCLAIMED_RUN}`,
|
|
126
|
+
});
|
|
127
|
+
return;
|
|
128
|
+
}
|
|
129
|
+
let problem;
|
|
130
|
+
if (nestedTooDeep(element))
|
|
131
|
+
problem = tooDeep;
|
|
132
|
+
else {
|
|
133
|
+
let text;
|
|
134
|
+
// A record's text is made only when a format asks for it.
|
|
135
|
+
problem = offer(element, {
|
|
136
|
+
digest,
|
|
137
|
+
unsafeIntegers: source.unsafeIntegers,
|
|
138
|
+
get text() {
|
|
139
|
+
return (text ??= JSON.stringify(element));
|
|
140
|
+
},
|
|
141
|
+
get bytes() {
|
|
142
|
+
return Buffer.byteLength(this.text);
|
|
143
|
+
},
|
|
144
|
+
...line,
|
|
145
|
+
index,
|
|
146
|
+
});
|
|
147
|
+
}
|
|
148
|
+
if (problem === undefined)
|
|
149
|
+
claimedAny = true;
|
|
150
|
+
else {
|
|
151
|
+
run++;
|
|
152
|
+
skipped.add({ ...line, record: index + 1, reason: problem });
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
};
|
|
156
|
+
for (const item of parsed.values) {
|
|
157
|
+
const line = item.line !== undefined ? { line: item.line } : {};
|
|
158
|
+
const source = {
|
|
159
|
+
digest,
|
|
160
|
+
get text() {
|
|
161
|
+
return item.text;
|
|
162
|
+
},
|
|
163
|
+
get bytes() {
|
|
164
|
+
return Buffer.byteLength(item.text);
|
|
165
|
+
},
|
|
166
|
+
unsafeIntegers: item.unsafeIntegers,
|
|
167
|
+
...line,
|
|
168
|
+
};
|
|
169
|
+
if (item.note)
|
|
170
|
+
notes.add({ ...line, reason: item.note });
|
|
171
|
+
const deep = nestedTooDeep(item.value);
|
|
172
|
+
const reason = deep ? tooDeep : offer(item.value, source);
|
|
173
|
+
firstReason ??= reason;
|
|
174
|
+
if (reason === undefined)
|
|
175
|
+
continue;
|
|
176
|
+
// An array no format reads whole (or too deep to read whole) is read record by record.
|
|
177
|
+
if (Array.isArray(item.value) &&
|
|
178
|
+
item.value.length &&
|
|
179
|
+
(deep || adapterFor(item.value) === undefined)) {
|
|
180
|
+
offerRecords(item.value, source, line);
|
|
181
|
+
continue;
|
|
182
|
+
}
|
|
183
|
+
skipped.add({ ...line, reason });
|
|
184
|
+
}
|
|
185
|
+
for (const problem of parsed.problems)
|
|
186
|
+
skipped.add(problem);
|
|
187
|
+
if (!claimed) {
|
|
188
|
+
// Nothing in the file was read: the file is reported, not each of its records.
|
|
189
|
+
this.result.unsupported.push({
|
|
190
|
+
file,
|
|
191
|
+
reason: failures[0] ?? firstReason ?? parsed.problems[0]?.reason ?? "no value was read",
|
|
192
|
+
});
|
|
193
|
+
return size();
|
|
194
|
+
}
|
|
195
|
+
const names = this.fileNames.get(digest) ?? new Set();
|
|
196
|
+
this.fileNames.set(digest, names.add(file));
|
|
197
|
+
this.result.skipped.push(...skipped.finish("records were not read"));
|
|
198
|
+
this.result.notes.push(...notes.finish("notes"));
|
|
199
|
+
return size();
|
|
200
|
+
}
|
|
201
|
+
rows(format) {
|
|
202
|
+
let collector = this.rowCollectors.get(format);
|
|
203
|
+
if (!collector) {
|
|
204
|
+
collector = new RowTraceCollector(format, this.options.keys);
|
|
205
|
+
this.rowCollectors.set(format, collector);
|
|
206
|
+
}
|
|
207
|
+
return collector;
|
|
208
|
+
}
|
|
209
|
+
convert(adapter, value, file, source, notes) {
|
|
210
|
+
const at = {
|
|
211
|
+
file,
|
|
212
|
+
...(source.line !== undefined ? { line: source.line } : {}),
|
|
213
|
+
...(source.index !== undefined ? { record: source.index + 1 } : {}),
|
|
214
|
+
};
|
|
215
|
+
// Rows the value gave and those an identical copy of which was already read elsewhere: a
|
|
216
|
+
// value that gave nothing new is noted, so a collapsed duplicate is never silent.
|
|
217
|
+
let rows = 0;
|
|
218
|
+
let duplicates = 0;
|
|
219
|
+
let earlier;
|
|
220
|
+
const repeats = new Map();
|
|
221
|
+
const context = {
|
|
222
|
+
source,
|
|
223
|
+
keys: this.options.keys ?? {},
|
|
224
|
+
row: (row) => {
|
|
225
|
+
rows++;
|
|
226
|
+
const copy = this.rows(adapter.id).add(row, source.digest, at);
|
|
227
|
+
if (copy) {
|
|
228
|
+
duplicates++;
|
|
229
|
+
earlier ??= copy;
|
|
230
|
+
}
|
|
231
|
+
},
|
|
232
|
+
event: (event) => this.rows(adapter.id).addEvent(event, source.digest, repeats),
|
|
233
|
+
trace: (trace) => {
|
|
234
|
+
trace.format = adapter.id;
|
|
235
|
+
trace.sourceDigests.add(source.digest);
|
|
236
|
+
this.whole.push(trace);
|
|
237
|
+
},
|
|
238
|
+
refuse: (reason, traceKey) => {
|
|
239
|
+
const traceId = traceKey === undefined ? undefined : rowTraceId(adapter.id, traceKey);
|
|
240
|
+
if (traceId)
|
|
241
|
+
this.refusedTraces.add(traceId);
|
|
242
|
+
this.result.refused.push({ ...at, ...(traceId ? { trace_id: traceId } : {}), reason });
|
|
243
|
+
},
|
|
244
|
+
collector: (create, shared) => {
|
|
245
|
+
if (shared !== undefined)
|
|
246
|
+
this.sharedKeys.set(adapter.id, shared);
|
|
247
|
+
const key = shared ?? adapter.id;
|
|
248
|
+
// A format always creates the same kind of collector, and formats that share one create
|
|
249
|
+
// the same kind, so the one held is a T.
|
|
250
|
+
const held = this.collectors.get(key);
|
|
251
|
+
if (held)
|
|
252
|
+
return held;
|
|
253
|
+
const created = create();
|
|
254
|
+
this.collectors.set(key, created);
|
|
255
|
+
return created;
|
|
256
|
+
},
|
|
257
|
+
};
|
|
258
|
+
adapter.convert(value, context);
|
|
259
|
+
if (rows && duplicates === rows && earlier)
|
|
260
|
+
notes.add({
|
|
261
|
+
...(at.line !== undefined ? { line: at.line } : {}),
|
|
262
|
+
...(at.record !== undefined ? { record: at.record } : {}),
|
|
263
|
+
reason: `the same ${rows === 1 ? "record" : "records"} as ${where(earlier, file)}, read once`,
|
|
264
|
+
});
|
|
265
|
+
}
|
|
266
|
+
/** Every trace read, merged across files: the same trace from two sources is one trace when
|
|
267
|
+
* its spans agree, and refused when they differ. */
|
|
268
|
+
finish() {
|
|
269
|
+
const traces = new Map();
|
|
270
|
+
const conflicts = new Map();
|
|
271
|
+
const kept = new Set();
|
|
272
|
+
// A trace refused after another format's copy of it was merged is left out with it.
|
|
273
|
+
const refuse = (traceId, reason) => {
|
|
274
|
+
if (traceId !== undefined)
|
|
275
|
+
traces.delete(traceId);
|
|
276
|
+
if (traceId !== undefined && this.refusedTraces.has(traceId))
|
|
277
|
+
return;
|
|
278
|
+
if (traceId !== undefined)
|
|
279
|
+
this.refusedTraces.add(traceId);
|
|
280
|
+
this.result.refused.push({ ...(traceId !== undefined ? { trace_id: traceId } : {}), reason });
|
|
281
|
+
};
|
|
282
|
+
const merge = (trace) => {
|
|
283
|
+
if (this.refusedTraces.has(trace.traceId))
|
|
284
|
+
return;
|
|
285
|
+
const held = traces.get(trace.traceId);
|
|
286
|
+
if (!held)
|
|
287
|
+
traces.set(trace.traceId, trace);
|
|
288
|
+
else if (JSON.stringify(held.entries) !== JSON.stringify(trace.entries))
|
|
289
|
+
conflicts.set(trace.traceId, "the trace appears in two sources with different content");
|
|
290
|
+
else
|
|
291
|
+
for (const digest of trace.sourceDigests)
|
|
292
|
+
held.sourceDigests.add(digest);
|
|
293
|
+
};
|
|
294
|
+
const finishedShared = new Set();
|
|
295
|
+
for (const adapter of importAdapters) {
|
|
296
|
+
const shared = this.sharedKeys.get(adapter.id);
|
|
297
|
+
// A shared collector is finished once, under the first of its formats.
|
|
298
|
+
const done = shared !== undefined && finishedShared.has(shared);
|
|
299
|
+
if (shared !== undefined)
|
|
300
|
+
finishedShared.add(shared);
|
|
301
|
+
for (const [collector, rowFormat] of [
|
|
302
|
+
[done ? undefined : this.collectors.get(shared ?? adapter.id), false],
|
|
303
|
+
[this.rowCollectors.get(adapter.id), true],
|
|
304
|
+
]) {
|
|
305
|
+
if (!collector)
|
|
306
|
+
continue;
|
|
307
|
+
const finished = collector.finish();
|
|
308
|
+
for (const problem of finished.refused)
|
|
309
|
+
refuse(problem.traceId, problem.reason);
|
|
310
|
+
for (const trace of finished.traces) {
|
|
311
|
+
if (rowFormat || shared === undefined)
|
|
312
|
+
trace.format = adapter.id;
|
|
313
|
+
// Spans the shared emitter wrote already carry the identity keys' user and session.
|
|
314
|
+
if (!rowFormat && !collector.appliesKeys)
|
|
315
|
+
kept.add(trace);
|
|
316
|
+
merge(trace);
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
for (const trace of this.whole) {
|
|
321
|
+
kept.add(trace);
|
|
322
|
+
merge(trace);
|
|
323
|
+
}
|
|
324
|
+
for (const [traceId, reason] of conflicts)
|
|
325
|
+
refuse(traceId, reason);
|
|
326
|
+
// A trace a shared collector gathered from files of several formats counts for each file
|
|
327
|
+
// under the format it was read with.
|
|
328
|
+
const group = (format) => this.sharedKeys.get(format) ?? format;
|
|
329
|
+
const byGroup = new Map();
|
|
330
|
+
for (const entry of this.supported.values()) {
|
|
331
|
+
const key = `${entry.file}\u0000${group(entry.format)}`;
|
|
332
|
+
if (!byGroup.has(key))
|
|
333
|
+
byGroup.set(key, entry);
|
|
334
|
+
}
|
|
335
|
+
const ordered = [];
|
|
336
|
+
for (const trace of [...traces.values()].sort((a, b) => (a.traceId < b.traceId ? -1 : 1))) {
|
|
337
|
+
try {
|
|
338
|
+
if (kept.has(trace))
|
|
339
|
+
applyIdentityKeys(trace, this.options.keys ?? {});
|
|
340
|
+
const identity = traceIdentity(trace.entries);
|
|
341
|
+
if (identity)
|
|
342
|
+
trace.identity = identity;
|
|
343
|
+
else
|
|
344
|
+
delete trace.identity;
|
|
345
|
+
}
|
|
346
|
+
catch (error) {
|
|
347
|
+
const failure = conversionFailure(error);
|
|
348
|
+
if (!failure)
|
|
349
|
+
throw error;
|
|
350
|
+
refuse(trace.traceId, failure.message);
|
|
351
|
+
continue;
|
|
352
|
+
}
|
|
353
|
+
ordered.push(trace);
|
|
354
|
+
for (const digest of trace.sourceDigests)
|
|
355
|
+
for (const name of this.fileNames.get(digest) ?? []) {
|
|
356
|
+
const entry = this.supported.get(`${name}\u0000${trace.format}`) ??
|
|
357
|
+
byGroup.get(`${name}\u0000${group(trace.format)}`);
|
|
358
|
+
if (entry)
|
|
359
|
+
entry.traces++;
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
return { traces: ordered, supported: [...this.supported.values()], ...this.result };
|
|
363
|
+
}
|
|
364
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import type { ImportAdapter } from "./adapter.js";
|
|
2
|
+
/**
|
|
3
|
+
* The source formats `hue traces import prepare` reads, and how a new one is added.
|
|
4
|
+
*
|
|
5
|
+
* How a file is read (`reader.ts`):
|
|
6
|
+
* 1. Containers (`containers.ts`): gzip or zstd is undone, the text decoded, and the JSON read as
|
|
7
|
+
* one document, or as JSON Lines whose every line is read on its own. A line that is not JSON
|
|
8
|
+
* is reported with its number and the rest of the file is read.
|
|
9
|
+
* 2. Detection: each value (the document, or each line) is offered to the formats below in this
|
|
10
|
+
* order and the first whose `detect` claims it converts it. A JSON array no format claims
|
|
11
|
+
* whole is read record by record the same way. A value or record no format claims is
|
|
12
|
+
* reported with the reason {@link unrecognizedReason} gives, never guessed at.
|
|
13
|
+
* 3. Conversion: the format's `convert` maps the value to canonical span rows (`rows.ts`), which
|
|
14
|
+
* one emitter turns into the OTLP spans Hue reads, or hands whole traces or a collector of its
|
|
15
|
+
* own to the context. A malformed record refuses its trace, never the file.
|
|
16
|
+
* 4. Identity (`identity.ts`): each trace's user and session are read from the emitted spans the
|
|
17
|
+
* way Hue reads them, after `--user-key` and `--session-key` are applied.
|
|
18
|
+
*
|
|
19
|
+
* Adding a format:
|
|
20
|
+
* - Write `import-formats/<id>.ts` exporting an {@link ImportAdapter}: `detect` by distinctive
|
|
21
|
+
* top-level keys (never a file name, never a shape another format claims), and `convert` that
|
|
22
|
+
* maps each record to a `SpanRow` (`rows.ts`) with `context.row` (inside `readRecords`, so a bad
|
|
23
|
+
* record refuses only its trace). Map the vendor's ids, parent, times (`nanosFrom`), kind,
|
|
24
|
+
* model, content, tool call, usage, error, and the user and conversation the record names
|
|
25
|
+
* (never from a tool's arguments or result); keep the whole record in `record`. Leave a field
|
|
26
|
+
* the source does not record unset: the emitter's documented rules fill in instants and say so.
|
|
27
|
+
* - Add it to {@link importAdapters} below, and a fixture to the exclusivity test so no format
|
|
28
|
+
* claims another's records.
|
|
29
|
+
* - `manifestFormat` stays `otlp_json` until Hue's import registration lists the new id.
|
|
30
|
+
*/
|
|
31
|
+
export declare const importAdapters: readonly ImportAdapter[];
|
|
32
|
+
/** The format that claims a value, the first in registry order. */
|
|
33
|
+
export declare const adapterFor: (value: unknown) => ImportAdapter | undefined;
|
|
34
|
+
export declare const adapterById: (id: string) => ImportAdapter | undefined;
|
|
35
|
+
/** Why no format claims a value. */
|
|
36
|
+
export declare function unrecognizedReason(value: unknown): string;
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import { isObject } from "../traces-import-convert.js";
|
|
2
|
+
import { agentEventAdapters } from "./agent-events.js";
|
|
3
|
+
import { atif } from "./atif.js";
|
|
4
|
+
import { braintrust } from "./braintrust.js";
|
|
5
|
+
import { langfuse, langfuseObservations } from "./langfuse.js";
|
|
6
|
+
import { langsmithRuns } from "./langsmith.js";
|
|
7
|
+
import { mlflowTraces } from "./mlflow.js";
|
|
8
|
+
import { opikTraces } from "./opik.js";
|
|
9
|
+
import { otelFamilyAdapters } from "./otel-family.js";
|
|
10
|
+
import { otlpJson } from "./otlp.js";
|
|
11
|
+
import { weaveCalls } from "./weave.js";
|
|
12
|
+
/**
|
|
13
|
+
* The source formats `hue traces import prepare` reads, and how a new one is added.
|
|
14
|
+
*
|
|
15
|
+
* How a file is read (`reader.ts`):
|
|
16
|
+
* 1. Containers (`containers.ts`): gzip or zstd is undone, the text decoded, and the JSON read as
|
|
17
|
+
* one document, or as JSON Lines whose every line is read on its own. A line that is not JSON
|
|
18
|
+
* is reported with its number and the rest of the file is read.
|
|
19
|
+
* 2. Detection: each value (the document, or each line) is offered to the formats below in this
|
|
20
|
+
* order and the first whose `detect` claims it converts it. A JSON array no format claims
|
|
21
|
+
* whole is read record by record the same way. A value or record no format claims is
|
|
22
|
+
* reported with the reason {@link unrecognizedReason} gives, never guessed at.
|
|
23
|
+
* 3. Conversion: the format's `convert` maps the value to canonical span rows (`rows.ts`), which
|
|
24
|
+
* one emitter turns into the OTLP spans Hue reads, or hands whole traces or a collector of its
|
|
25
|
+
* own to the context. A malformed record refuses its trace, never the file.
|
|
26
|
+
* 4. Identity (`identity.ts`): each trace's user and session are read from the emitted spans the
|
|
27
|
+
* way Hue reads them, after `--user-key` and `--session-key` are applied.
|
|
28
|
+
*
|
|
29
|
+
* Adding a format:
|
|
30
|
+
* - Write `import-formats/<id>.ts` exporting an {@link ImportAdapter}: `detect` by distinctive
|
|
31
|
+
* top-level keys (never a file name, never a shape another format claims), and `convert` that
|
|
32
|
+
* maps each record to a `SpanRow` (`rows.ts`) with `context.row` (inside `readRecords`, so a bad
|
|
33
|
+
* record refuses only its trace). Map the vendor's ids, parent, times (`nanosFrom`), kind,
|
|
34
|
+
* model, content, tool call, usage, error, and the user and conversation the record names
|
|
35
|
+
* (never from a tool's arguments or result); keep the whole record in `record`. Leave a field
|
|
36
|
+
* the source does not record unset: the emitter's documented rules fill in instants and say so.
|
|
37
|
+
* - Add it to {@link importAdapters} below, and a fixture to the exclusivity test so no format
|
|
38
|
+
* claims another's records.
|
|
39
|
+
* - `manifestFormat` stays `otlp_json` until Hue's import registration lists the new id.
|
|
40
|
+
*/
|
|
41
|
+
export const importAdapters = [
|
|
42
|
+
otlpJson,
|
|
43
|
+
langfuseObservations,
|
|
44
|
+
langfuse,
|
|
45
|
+
atif,
|
|
46
|
+
braintrust,
|
|
47
|
+
...otelFamilyAdapters,
|
|
48
|
+
langsmithRuns,
|
|
49
|
+
mlflowTraces,
|
|
50
|
+
opikTraces,
|
|
51
|
+
weaveCalls,
|
|
52
|
+
...agentEventAdapters,
|
|
53
|
+
];
|
|
54
|
+
/** The format that claims a value, the first in registry order. */
|
|
55
|
+
export const adapterFor = (value) => importAdapters.find((adapter) => adapter.detect(value));
|
|
56
|
+
export const adapterById = (id) => importAdapters.find((adapter) => adapter.id === id);
|
|
57
|
+
/** Why no format claims a value. */
|
|
58
|
+
export function unrecognizedReason(value) {
|
|
59
|
+
if (Array.isArray(value))
|
|
60
|
+
return value.length ? "a JSON array of no recognized format" : "an empty JSON array";
|
|
61
|
+
if (!isObject(value))
|
|
62
|
+
return "not a JSON object";
|
|
63
|
+
if (Array.isArray(value.resourceLogs) || Array.isArray(value.resource_logs))
|
|
64
|
+
return "an OTLP logs export; only traces are imported";
|
|
65
|
+
if (Array.isArray(value.resourceMetrics) || Array.isArray(value.resource_metrics))
|
|
66
|
+
return "an OTLP metrics export; only traces are imported";
|
|
67
|
+
for (const key of ["data", "events", "observations"])
|
|
68
|
+
if (Array.isArray(value[key]) && !value[key].length)
|
|
69
|
+
return "an export page with no records";
|
|
70
|
+
return "a JSON document of no recognized format";
|
|
71
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { type JsonValue } from "../traces-import-convert.js";
|
|
2
|
+
/**
|
|
3
|
+
* Field readers the row-platform formats (LangSmith, MLflow, Opik, W&B Weave) share: their
|
|
4
|
+
* records are JSON objects whose instants are ISO 8601 text and whose metadata names the end user
|
|
5
|
+
* and conversation under the field names applications commonly give them.
|
|
6
|
+
*/
|
|
7
|
+
export type JsonObject = Record<string, JsonValue>;
|
|
8
|
+
export declare const object: (value: JsonValue | undefined) => JsonObject | undefined;
|
|
9
|
+
export declare const text: (value: JsonValue | undefined) => string | undefined;
|
|
10
|
+
/** A token count as recorded, when it is a nonnegative integer. */
|
|
11
|
+
export declare const count: (value: JsonValue | undefined) => number | undefined;
|
|
12
|
+
/** The first of `names` that holds an identifier in `fields`. */
|
|
13
|
+
export declare function first(fields: JsonObject | undefined, names: readonly string[]): string | undefined;
|
|
14
|
+
/** The end user a metadata map names: the fields Hue reads in a metadata map, in its order
|
|
15
|
+
* (`userId`, `user_id`, `langfuse_user_id`), then OpenTelemetry's `user.id` and `enduser.id`
|
|
16
|
+
* as an exporter copied them in. `user.id` beside `user.account_uuid` is a Claude Code
|
|
17
|
+
* installation, not a user, and is passed over as Hue passes it over. */
|
|
18
|
+
export declare function metadataUser(metadata: JsonObject | undefined): string | undefined;
|
|
19
|
+
/** The metadata fields applications name the conversation by. */
|
|
20
|
+
export declare const SESSION_FIELDS: string[];
|
|
21
|
+
/**
|
|
22
|
+
* An ISO 8601 instant as Unix nanoseconds: `T` or a space between date and time (Python's
|
|
23
|
+
* `str(datetime)` writes a space), up to nine fractional digits, and `Z` or an offset. An instant
|
|
24
|
+
* without a zone is UTC, as LangSmith documents its run times and its SDK reads them.
|
|
25
|
+
*/
|
|
26
|
+
export declare function isoNanos(value: JsonValue | undefined): bigint;
|
|
27
|
+
/** A JSON value recorded as itself or as its JSON text (a column a warehouse export stores as
|
|
28
|
+
* text); text that is not JSON, or nests too deep to read, is undefined. Integers beyond
|
|
29
|
+
* JavaScript's range stay decimal text. */
|
|
30
|
+
export declare function jsonOf(value: JsonValue | undefined): JsonValue | undefined;
|
|
31
|
+
/** A field that must be a JSON object when present, recorded as one or as its JSON text. */
|
|
32
|
+
export declare function objectField(value: JsonValue | undefined, what: string): JsonObject | undefined;
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { ConversionError, isObject, parseNestedJsonText, } from "../traces-import-convert.js";
|
|
2
|
+
import { identityText } from "./identity.js";
|
|
3
|
+
export const object = (value) => isObject(value) ? value : undefined;
|
|
4
|
+
export const text = (value) => typeof value === "string" && value ? value : undefined;
|
|
5
|
+
/** A token count as recorded, when it is a nonnegative integer. */
|
|
6
|
+
export const count = (value) => typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : undefined;
|
|
7
|
+
/** The first of `names` that holds an identifier in `fields`. */
|
|
8
|
+
export function first(fields, names) {
|
|
9
|
+
for (const name of names) {
|
|
10
|
+
const value = identityText(fields?.[name]);
|
|
11
|
+
if (value !== undefined)
|
|
12
|
+
return value;
|
|
13
|
+
}
|
|
14
|
+
return undefined;
|
|
15
|
+
}
|
|
16
|
+
/** The end user a metadata map names: the fields Hue reads in a metadata map, in its order
|
|
17
|
+
* (`userId`, `user_id`, `langfuse_user_id`), then OpenTelemetry's `user.id` and `enduser.id`
|
|
18
|
+
* as an exporter copied them in. `user.id` beside `user.account_uuid` is a Claude Code
|
|
19
|
+
* installation, not a user, and is passed over as Hue passes it over. */
|
|
20
|
+
export function metadataUser(metadata) {
|
|
21
|
+
const user = first(metadata, ["userId", "user_id", "langfuse_user_id"]);
|
|
22
|
+
if (user !== undefined)
|
|
23
|
+
return user;
|
|
24
|
+
const installation = metadata?.["user.account_uuid"] !== undefined;
|
|
25
|
+
return first(metadata, installation ? ["enduser.id"] : ["user.id", "enduser.id"]);
|
|
26
|
+
}
|
|
27
|
+
/** The metadata fields applications name the conversation by. */
|
|
28
|
+
export const SESSION_FIELDS = [
|
|
29
|
+
"session_id",
|
|
30
|
+
"sessionId",
|
|
31
|
+
"thread_id",
|
|
32
|
+
"threadId",
|
|
33
|
+
"conversation_id",
|
|
34
|
+
"conversationId",
|
|
35
|
+
"gen_ai.conversation.id",
|
|
36
|
+
];
|
|
37
|
+
const ISO = /^(\d{4}-\d{2}-\d{2})[Tt ](\d{2}:\d{2}:\d{2})(?:\.(\d{1,9}))?(?:([Zz])|([+-]\d{2}):?(\d{2}))?$/;
|
|
38
|
+
/**
|
|
39
|
+
* An ISO 8601 instant as Unix nanoseconds: `T` or a space between date and time (Python's
|
|
40
|
+
* `str(datetime)` writes a space), up to nine fractional digits, and `Z` or an offset. An instant
|
|
41
|
+
* without a zone is UTC, as LangSmith documents its run times and its SDK reads them.
|
|
42
|
+
*/
|
|
43
|
+
export function isoNanos(value) {
|
|
44
|
+
const match = typeof value === "string" ? ISO.exec(value) : null;
|
|
45
|
+
if (!match)
|
|
46
|
+
throw new ConversionError("Invalid source timestamp");
|
|
47
|
+
const zone = match[5] ? `${match[5]}:${match[6]}` : "Z";
|
|
48
|
+
const millis = Date.parse(`${match[1]}T${match[2]}${zone}`);
|
|
49
|
+
if (!Number.isSafeInteger(millis) || millis <= 0)
|
|
50
|
+
throw new ConversionError("Invalid source timestamp");
|
|
51
|
+
return BigInt(millis) * 1000000n + BigInt((match[3] ?? "").padEnd(9, "0"));
|
|
52
|
+
}
|
|
53
|
+
/** A JSON value recorded as itself or as its JSON text (a column a warehouse export stores as
|
|
54
|
+
* text); text that is not JSON, or nests too deep to read, is undefined. Integers beyond
|
|
55
|
+
* JavaScript's range stay decimal text. */
|
|
56
|
+
export function jsonOf(value) {
|
|
57
|
+
if (typeof value !== "string")
|
|
58
|
+
return value;
|
|
59
|
+
try {
|
|
60
|
+
return parseNestedJsonText(value).value;
|
|
61
|
+
}
|
|
62
|
+
catch {
|
|
63
|
+
return undefined;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
/** A field that must be a JSON object when present, recorded as one or as its JSON text. */
|
|
67
|
+
export function objectField(value, what) {
|
|
68
|
+
if (value === undefined || value === null)
|
|
69
|
+
return undefined;
|
|
70
|
+
const parsed = jsonOf(value);
|
|
71
|
+
if (parsed === null)
|
|
72
|
+
return undefined;
|
|
73
|
+
if (!isObject(parsed))
|
|
74
|
+
throw new ConversionError(`${what} is not a JSON object`);
|
|
75
|
+
return parsed;
|
|
76
|
+
}
|