@hue-run/sdk 0.21.1 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLI.md +246 -52
- package/README.md +69 -14
- package/WORLDS.md +6 -3
- package/dist/ai-sdk.d.ts +23 -1
- package/dist/ai-sdk.js +60 -2
- package/dist/cli/import-formats/adapter.d.ts +80 -0
- package/dist/cli/import-formats/adapter.js +1 -0
- package/dist/cli/import-formats/agent-events.d.ts +5 -0
- package/dist/cli/import-formats/agent-events.js +16 -0
- package/dist/cli/import-formats/atif.d.ts +3 -0
- package/dist/cli/import-formats/atif.js +15 -0
- package/dist/cli/import-formats/braintrust.d.ts +2 -0
- package/dist/cli/import-formats/braintrust.js +222 -0
- package/dist/cli/import-formats/containers.d.ts +32 -0
- package/dist/cli/import-formats/containers.js +399 -0
- package/dist/cli/import-formats/datadog.d.ts +2 -0
- package/dist/cli/import-formats/datadog.js +264 -0
- package/dist/cli/import-formats/helicone.d.ts +2 -0
- package/dist/cli/import-formats/helicone.js +161 -0
- package/dist/cli/import-formats/identity.d.ts +112 -0
- package/dist/cli/import-formats/identity.js +594 -0
- package/dist/cli/import-formats/jaeger.d.ts +3 -0
- package/dist/cli/import-formats/jaeger.js +266 -0
- package/dist/cli/import-formats/langfuse.d.ts +7 -0
- package/dist/cli/import-formats/langfuse.js +510 -0
- package/dist/cli/import-formats/langsmith.d.ts +2 -0
- package/dist/cli/import-formats/langsmith.js +244 -0
- package/dist/cli/import-formats/messages.d.ts +28 -0
- package/dist/cli/import-formats/messages.js +301 -0
- package/dist/cli/import-formats/mlflow.d.ts +2 -0
- package/dist/cli/import-formats/mlflow.js +260 -0
- package/dist/cli/import-formats/openai-agents.d.ts +2 -0
- package/dist/cli/import-formats/openai-agents.js +221 -0
- package/dist/cli/import-formats/opik.d.ts +2 -0
- package/dist/cli/import-formats/opik.js +193 -0
- package/dist/cli/import-formats/otel-family.d.ts +5 -0
- package/dist/cli/import-formats/otel-family.js +17 -0
- package/dist/cli/import-formats/otel-spans.d.ts +94 -0
- package/dist/cli/import-formats/otel-spans.js +262 -0
- package/dist/cli/import-formats/otlp-protobuf.d.ts +35 -0
- package/dist/cli/import-formats/otlp-protobuf.js +486 -0
- package/dist/cli/import-formats/otlp.d.ts +4 -0
- package/dist/cli/import-formats/otlp.js +15 -0
- package/dist/cli/import-formats/phoenix.d.ts +3 -0
- package/dist/cli/import-formats/phoenix.js +256 -0
- package/dist/cli/import-formats/raindrop.d.ts +21 -0
- package/dist/cli/import-formats/raindrop.js +255 -0
- package/dist/cli/import-formats/reader.d.ts +56 -0
- package/dist/cli/import-formats/reader.js +364 -0
- package/dist/cli/import-formats/registry.d.ts +36 -0
- package/dist/cli/import-formats/registry.js +71 -0
- package/dist/cli/import-formats/row-platform-fields.d.ts +32 -0
- package/dist/cli/import-formats/row-platform-fields.js +76 -0
- package/dist/cli/import-formats/rows.d.ts +165 -0
- package/dist/cli/import-formats/rows.js +638 -0
- package/dist/cli/import-formats/transcripts.d.ts +25 -0
- package/dist/cli/import-formats/transcripts.js +498 -0
- package/dist/cli/import-formats/weave.d.ts +2 -0
- package/dist/cli/import-formats/weave.js +159 -0
- package/dist/cli/import-formats/zipkin.d.ts +2 -0
- package/dist/cli/import-formats/zipkin.js +219 -0
- package/dist/cli/import.d.ts +1 -1
- package/dist/cli/import.js +4 -2
- package/dist/cli/traces-import-convert.d.ts +65 -33
- package/dist/cli/traces-import-convert.js +211 -214
- package/dist/cli/traces-import-prepare.d.ts +26 -22
- package/dist/cli/traces-import-prepare.js +51 -122
- package/dist/cli/traces-import.d.ts +4 -4
- package/dist/cli/traces-import.js +63 -11
- package/dist/transport.d.ts +12 -0
- package/dist/transport.js +139 -24
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +2 -1
|
@@ -1,15 +1,20 @@
|
|
|
1
|
-
import
|
|
1
|
+
import type { IdentityKeys } from "./import-formats/identity.js";
|
|
2
|
+
import { type ReadResult } from "./import-formats/reader.js";
|
|
3
|
+
import { type TraceImportManifest, type TraceImportOrigin, type TraceImportTotals } from "./traces-import-convert.js";
|
|
2
4
|
/**
|
|
3
5
|
* `hue traces import prepare`: reads a file, a directory or a ZIP archive, converts every trace
|
|
4
|
-
*
|
|
5
|
-
* batch, one file per large value, and
|
|
6
|
-
* `prepare_trace_import` is called with. Nothing is sent
|
|
6
|
+
* the format registry (`import-formats/registry.ts`) recognizes and writes the bundle Hue's
|
|
7
|
+
* import routes take: one gzip OTLP/JSON file per batch, one file per large value, and
|
|
8
|
+
* `manifest.json`, whose digest and totals `prepare_trace_import` is called with. Nothing is sent
|
|
9
|
+
* anywhere.
|
|
7
10
|
*/
|
|
8
11
|
/** Bounds on what one preparation reads. */
|
|
9
12
|
export declare const PREPARE_LIMITS: {
|
|
10
13
|
/** One source file, or one archive member. */
|
|
11
14
|
readonly fileBytes: number;
|
|
12
|
-
/** Every source file together.
|
|
15
|
+
/** Every source file together, decompressed. Every trace is held until all are read, at about
|
|
16
|
+
* seven times its source size, so this keeps a run inside a 2 GiB JavaScript heap (Node.js's
|
|
17
|
+
* default on a machine with 8 GiB of memory). */
|
|
13
18
|
readonly totalBytes: number;
|
|
14
19
|
/** Files one directory tree or archive may hold. */
|
|
15
20
|
readonly files: 50000;
|
|
@@ -23,18 +28,17 @@ export interface PrepareReport {
|
|
|
23
28
|
totals: TraceImportTotals;
|
|
24
29
|
source: TraceImportManifest["source"];
|
|
25
30
|
output_directory: string;
|
|
26
|
-
/** Traces
|
|
27
|
-
formats: Record<
|
|
31
|
+
/** Traces prepared per format id, for the formats that held any. */
|
|
32
|
+
formats: Record<string, number>;
|
|
28
33
|
files: {
|
|
29
|
-
supported:
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
}[];
|
|
34
|
+
supported: ReadResult["supported"];
|
|
35
|
+
/** Files of which nothing could be read, and why. */
|
|
36
|
+
unsupported: ReadResult["unsupported"];
|
|
37
|
+
/** Records of a read file (a line of JSON Lines, a record of a JSON array) that were not. */
|
|
38
|
+
skipped: ReadResult["skipped"];
|
|
39
|
+
/** What was read with a repair (a damaged gzip or zstd file read up to the damage, `NaN` read
|
|
40
|
+
* as null) or read once though the input held it more than once. */
|
|
41
|
+
notes: ReadResult["notes"];
|
|
38
42
|
};
|
|
39
43
|
traces: {
|
|
40
44
|
without_identity: number;
|
|
@@ -44,12 +48,10 @@ export interface PrepareReport {
|
|
|
44
48
|
traces: number;
|
|
45
49
|
spans: number;
|
|
46
50
|
};
|
|
47
|
-
/** Traces
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
reason: string;
|
|
52
|
-
}[];
|
|
51
|
+
/** Traces a documented rule supplied instants for, per rule (`hue.import.timing`). */
|
|
52
|
+
timing: Record<string, number>;
|
|
53
|
+
/** Traces and records the sources held that could not be converted. */
|
|
54
|
+
refused: ReadResult["refused"];
|
|
53
55
|
};
|
|
54
56
|
blobs: number;
|
|
55
57
|
}
|
|
@@ -62,6 +64,8 @@ export declare function defaultOutputDirectory(input: string): string;
|
|
|
62
64
|
export declare function manifestTooLarge(bytes: Uint8Array): string | undefined;
|
|
63
65
|
export interface PrepareOptions {
|
|
64
66
|
out?: string;
|
|
67
|
+
/** `--user-key` and `--session-key` (`import-formats/identity.ts`). */
|
|
68
|
+
keys?: IdentityKeys;
|
|
65
69
|
}
|
|
66
70
|
/** Prepares the bundle; resolves to the report, or throws a {@link PrepareError}. */
|
|
67
71
|
export declare function prepareTraceBundle(input: string, options?: PrepareOptions): Promise<PrepareReport>;
|
|
@@ -3,20 +3,25 @@ import { mkdir, open, readdir, readFile, rename, rm, stat, writeFile } from "nod
|
|
|
3
3
|
import { basename, join, relative, resolve, sep } from "node:path";
|
|
4
4
|
import { gzipSync } from "node:zlib";
|
|
5
5
|
import { sdkVersion } from "../version.js";
|
|
6
|
-
import {
|
|
6
|
+
import { ImportReader } from "./import-formats/reader.js";
|
|
7
|
+
import { adapterById } from "./import-formats/registry.js";
|
|
8
|
+
import { batchEntries, ConversionError, manifestTotals, manifestTraceHasIdentity, traceImportFormats, traceImportLimits, validateManifest, } from "./traces-import-convert.js";
|
|
7
9
|
import { isZipSignature, ZipArchive, ZipError } from "./traces-import-zip.js";
|
|
8
10
|
/**
|
|
9
11
|
* `hue traces import prepare`: reads a file, a directory or a ZIP archive, converts every trace
|
|
10
|
-
*
|
|
11
|
-
* batch, one file per large value, and
|
|
12
|
-
* `prepare_trace_import` is called with. Nothing is sent
|
|
12
|
+
* the format registry (`import-formats/registry.ts`) recognizes and writes the bundle Hue's
|
|
13
|
+
* import routes take: one gzip OTLP/JSON file per batch, one file per large value, and
|
|
14
|
+
* `manifest.json`, whose digest and totals `prepare_trace_import` is called with. Nothing is sent
|
|
15
|
+
* anywhere.
|
|
13
16
|
*/
|
|
14
17
|
/** Bounds on what one preparation reads. */
|
|
15
18
|
export const PREPARE_LIMITS = {
|
|
16
19
|
/** One source file, or one archive member. */
|
|
17
20
|
fileBytes: 64 * 1024 * 1024,
|
|
18
|
-
/** Every source file together.
|
|
19
|
-
|
|
21
|
+
/** Every source file together, decompressed. Every trace is held until all are read, at about
|
|
22
|
+
* seven times its source size, so this keeps a run inside a 2 GiB JavaScript heap (Node.js's
|
|
23
|
+
* default on a machine with 8 GiB of memory). */
|
|
24
|
+
totalBytes: 256 * 1024 * 1024,
|
|
20
25
|
/** Files one directory tree or archive may hold. */
|
|
21
26
|
files: 50_000,
|
|
22
27
|
/** How deep a directory tree is walked. */
|
|
@@ -211,117 +216,44 @@ async function writePrivate(path, bytes) {
|
|
|
211
216
|
await writeFile(temporary, bytes, { mode: 0o600, flag: "w" });
|
|
212
217
|
await rename(temporary, path);
|
|
213
218
|
}
|
|
214
|
-
/** Merges a converted trace into the collection: the same trace from two sources is one trace
|
|
215
|
-
* when its entries agree, a conflict otherwise. */
|
|
216
|
-
function merge(collected, conflicts, trace, sourceDigest) {
|
|
217
|
-
trace.sourceDigests.add(sourceDigest);
|
|
218
|
-
const held = collected.get(trace.traceId);
|
|
219
|
-
if (!held) {
|
|
220
|
-
collected.set(trace.traceId, trace);
|
|
221
|
-
return;
|
|
222
|
-
}
|
|
223
|
-
if (JSON.stringify(held.entries) !== JSON.stringify(trace.entries)) {
|
|
224
|
-
conflicts.set(trace.traceId, "the trace appears in two sources with different content");
|
|
225
|
-
return;
|
|
226
|
-
}
|
|
227
|
-
for (const digest of trace.sourceDigests)
|
|
228
|
-
held.sourceDigests.add(digest);
|
|
229
|
-
}
|
|
230
219
|
/** Prepares the bundle; resolves to the report, or throws a {@link PrepareError}. */
|
|
231
220
|
export async function prepareTraceBundle(input, options = {}) {
|
|
232
221
|
const outputDirectory = resolve(options.out ?? defaultOutputDirectory(input));
|
|
233
222
|
const unsupported = [];
|
|
234
|
-
const
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
const conflicts = new Map();
|
|
240
|
-
/** The file names each source digest was read under, so the traces an OTLP or Langfuse file
|
|
241
|
-
* held are counted once the traces are known: a trace may come from several files. */
|
|
242
|
-
const fileNames = new Map();
|
|
243
|
-
const tracesPerFile = new Map();
|
|
244
|
-
const nameFile = (digest, name) => {
|
|
245
|
-
const names = fileNames.get(digest) ?? new Set();
|
|
246
|
-
names.add(name);
|
|
247
|
-
fileNames.set(digest, names);
|
|
248
|
-
};
|
|
223
|
+
const reader = new ImportReader({
|
|
224
|
+
maxFileBytes: PREPARE_LIMITS.fileBytes,
|
|
225
|
+
unsupported,
|
|
226
|
+
...(options.keys ? { keys: options.keys } : {}),
|
|
227
|
+
});
|
|
249
228
|
let totalBytes = 0;
|
|
250
|
-
{
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
unsupported.push({
|
|
261
|
-
file: source.name,
|
|
262
|
-
reason: error instanceof ZipError ? error.message : "the file could not be read",
|
|
263
|
-
});
|
|
264
|
-
continue;
|
|
265
|
-
}
|
|
266
|
-
const digest = sha256Hex(bytes);
|
|
267
|
-
const detected = detectFormat(bytes);
|
|
268
|
-
if (detected.format === "unsupported") {
|
|
269
|
-
unsupported.push({ file: source.name, reason: detected.reason });
|
|
270
|
-
continue;
|
|
271
|
-
}
|
|
272
|
-
const note = (problems) => {
|
|
273
|
-
for (const problem of problems)
|
|
274
|
-
refused.push({
|
|
275
|
-
file: source.name,
|
|
276
|
-
...(problem.traceId ? { trace_id: problem.traceId } : {}),
|
|
277
|
-
reason: problem.reason,
|
|
278
|
-
});
|
|
279
|
-
};
|
|
280
|
-
try {
|
|
281
|
-
if (detected.format === "otlp_json") {
|
|
282
|
-
note(otlp.add(detected.envelopes, digest));
|
|
283
|
-
nameFile(digest, source.name);
|
|
284
|
-
supported.push({ file: source.name, format: "otlp_json", traces: 0 });
|
|
285
|
-
}
|
|
286
|
-
else if (detected.format === "langfuse_observations") {
|
|
287
|
-
if (detected.unsafeIntegers)
|
|
288
|
-
throw new ConversionError("Source JSON contains an unsafe number");
|
|
289
|
-
langfuse.add(detected.observations, detected.envelope, digest);
|
|
290
|
-
nameFile(digest, source.name);
|
|
291
|
-
supported.push({ file: source.name, format: "langfuse_observations", traces: 0 });
|
|
292
|
-
}
|
|
293
|
-
else {
|
|
294
|
-
const trace = convertAtifExport(detected, bytes.byteLength);
|
|
295
|
-
merge(whole, conflicts, trace, digest);
|
|
296
|
-
supported.push({ file: source.name, format: "atif", traces: 1 });
|
|
297
|
-
}
|
|
298
|
-
}
|
|
299
|
-
catch (error) {
|
|
300
|
-
if (!(error instanceof ConversionError))
|
|
301
|
-
throw error;
|
|
302
|
-
unsupported.push({ file: source.name, reason: error.message });
|
|
303
|
-
}
|
|
229
|
+
const countBytes = (bytes) => {
|
|
230
|
+
totalBytes += bytes;
|
|
231
|
+
if (totalBytes > PREPARE_LIMITS.totalBytes)
|
|
232
|
+
throw new PrepareError(`The input holds more than ${PREPARE_LIMITS.totalBytes} bytes decompressed; prepare it in parts of at most that size, each to its own --out`);
|
|
233
|
+
};
|
|
234
|
+
for await (const source of sourceFiles(input, unsupported)) {
|
|
235
|
+
countBytes(source.size);
|
|
236
|
+
let bytes;
|
|
237
|
+
try {
|
|
238
|
+
bytes = await source.read();
|
|
304
239
|
}
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
refused.push({
|
|
310
|
-
...(problem.traceId ? { trace_id: problem.traceId } : {}),
|
|
311
|
-
reason: problem.reason,
|
|
240
|
+
catch (error) {
|
|
241
|
+
unsupported.push({
|
|
242
|
+
file: source.name,
|
|
243
|
+
reason: error instanceof ZipError ? error.message : "the file could not be read",
|
|
312
244
|
});
|
|
313
|
-
|
|
314
|
-
const first = [...trace.sourceDigests][0] ?? "";
|
|
315
|
-
merge(whole, conflicts, trace, first);
|
|
245
|
+
continue;
|
|
316
246
|
}
|
|
247
|
+
// Decompressed content counts toward the bound too.
|
|
248
|
+
countBytes(Math.max(0, reader.read(source.name, bytes) - bytes.byteLength));
|
|
317
249
|
}
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
}
|
|
322
|
-
const traces = [...whole.values()].sort((a, b) => (a.traceId < b.traceId ? -1 : 1));
|
|
250
|
+
const read = reader.finish();
|
|
251
|
+
const refused = read.refused;
|
|
252
|
+
const traces = read.traces;
|
|
323
253
|
if (!traces.length) {
|
|
324
|
-
const problems = [...unsupported, ...
|
|
254
|
+
const problems = [...unsupported, ...read.skipped, ...refused]
|
|
255
|
+
.slice(0, 3)
|
|
256
|
+
.map((item) => item.reason);
|
|
325
257
|
throw new PrepareError(`No trace was recognized in ${input}${problems.length ? ` (${problems.join("; ")})` : ""}`);
|
|
326
258
|
}
|
|
327
259
|
if (traces.length > traceImportLimits.traces)
|
|
@@ -329,11 +261,8 @@ export async function prepareTraceBundle(input, options = {}) {
|
|
|
329
261
|
await prepareOutputDirectory(outputDirectory);
|
|
330
262
|
const manifestTraces = [];
|
|
331
263
|
const formats = new Set();
|
|
332
|
-
const formatCounts = {
|
|
333
|
-
|
|
334
|
-
langfuse_observations: 0,
|
|
335
|
-
atif: 0,
|
|
336
|
-
};
|
|
264
|
+
const formatCounts = {};
|
|
265
|
+
const timing = {};
|
|
337
266
|
const byOrigin = {
|
|
338
267
|
production: 0,
|
|
339
268
|
evaluation: 0,
|
|
@@ -403,8 +332,14 @@ export async function prepareTraceBundle(input, options = {}) {
|
|
|
403
332
|
...(sourceDigest ? { source_digest: sourceDigest } : {}),
|
|
404
333
|
};
|
|
405
334
|
manifestTraces.push(declared);
|
|
406
|
-
|
|
407
|
-
|
|
335
|
+
// What was written is let go: the run holds each trace's spans only until they are on disk.
|
|
336
|
+
trace.entries = [];
|
|
337
|
+
for (const blob of trace.blobs)
|
|
338
|
+
blob.bytes = new Uint8Array(0);
|
|
339
|
+
formats.add(adapterById(trace.format).manifestFormat);
|
|
340
|
+
formatCounts[trace.format] = (formatCounts[trace.format] ?? 0) + 1;
|
|
341
|
+
for (const rule of trace.inferredTiming ?? [])
|
|
342
|
+
timing[rule] = (timing[rule] ?? 0) + 1;
|
|
408
343
|
byOrigin[trace.origin]++;
|
|
409
344
|
if (!manifestTraceHasIdentity(declared))
|
|
410
345
|
withoutIdentity++;
|
|
@@ -412,16 +347,9 @@ export async function prepareTraceBundle(input, options = {}) {
|
|
|
412
347
|
remappedTraces++;
|
|
413
348
|
remappedSpans += trace.remappedSpans;
|
|
414
349
|
blobCount += trace.blobs.length;
|
|
415
|
-
if (trace.format !== "atif")
|
|
416
|
-
for (const digest of trace.sourceDigests)
|
|
417
|
-
for (const name of fileNames.get(digest) ?? [])
|
|
418
|
-
tracesPerFile.set(name, (tracesPerFile.get(name) ?? 0) + 1);
|
|
419
350
|
}
|
|
420
351
|
if (!manifestTraces.length)
|
|
421
352
|
throw new PrepareError(`No trace of ${input} could be prepared: ${refused[0]?.reason ?? ""}`);
|
|
422
|
-
for (const entry of supported)
|
|
423
|
-
if (entry.format !== "atif")
|
|
424
|
-
entry.traces = tracesPerFile.get(entry.file) ?? 0;
|
|
425
353
|
const manifest = {
|
|
426
354
|
version: 1,
|
|
427
355
|
source: {
|
|
@@ -445,11 +373,12 @@ export async function prepareTraceBundle(input, options = {}) {
|
|
|
445
373
|
source: manifest.source,
|
|
446
374
|
output_directory: outputDirectory,
|
|
447
375
|
formats: formatCounts,
|
|
448
|
-
files: { supported, unsupported },
|
|
376
|
+
files: { supported: read.supported, unsupported, skipped: read.skipped, notes: read.notes },
|
|
449
377
|
traces: {
|
|
450
378
|
without_identity: withoutIdentity,
|
|
451
379
|
by_origin: byOrigin,
|
|
452
380
|
remapped: { traces: remappedTraces, spans: remappedSpans },
|
|
381
|
+
timing,
|
|
453
382
|
refused,
|
|
454
383
|
},
|
|
455
384
|
blobs: blobCount,
|
|
@@ -2,9 +2,9 @@ import { type UploadIo } from "./traces-import-upload.js";
|
|
|
2
2
|
/**
|
|
3
3
|
* `hue traces import`: imports a bundle of existing traces into Hue in two steps that keep the
|
|
4
4
|
* payloads on this machine until the agent has authorized their delivery. `prepare` converts a
|
|
5
|
-
* file, directory or ZIP archive of
|
|
6
|
-
*
|
|
7
|
-
*
|
|
5
|
+
* file, directory or ZIP archive of trace exports (`import-formats/registry.ts`) into a bundle
|
|
6
|
+
* and prints its manifest digest and totals; `prepare_trace_import` (Hue MCP) takes those and
|
|
7
|
+
* answers an import authorization; `upload` delivers the bundle with it.
|
|
8
8
|
*/
|
|
9
9
|
/** Streams and seams for {@link runTracesCommand}; tests inject these. */
|
|
10
10
|
export interface TracesCommandIo extends UploadIo {
|
|
@@ -13,6 +13,6 @@ export interface TracesCommandIo extends UploadIo {
|
|
|
13
13
|
env?: NodeJS.ProcessEnv;
|
|
14
14
|
cwd?: string;
|
|
15
15
|
}
|
|
16
|
-
export declare const TRACES_USAGE
|
|
16
|
+
export declare const TRACES_USAGE: string;
|
|
17
17
|
/** Runs `hue traces import …`; resolves to the exit code. */
|
|
18
18
|
export declare function runTracesCommand(argv: string[], io?: TracesCommandIo): Promise<number>;
|
|
@@ -1,25 +1,48 @@
|
|
|
1
1
|
import { readFile } from "node:fs/promises";
|
|
2
2
|
import { resolve } from "node:path";
|
|
3
3
|
import { parseArgs } from "node:util";
|
|
4
|
+
import { parseKeyPath } from "./import-formats/identity.js";
|
|
5
|
+
import { importAdapters } from "./import-formats/registry.js";
|
|
4
6
|
import { parseHueOrigin } from "./login.js";
|
|
5
7
|
import { PrepareError, prepareTraceBundle } from "./traces-import-prepare.js";
|
|
6
8
|
import { parseAuthorizationFile, parseImportUrl, UploadStop, uploadTraceBundle, } from "./traces-import-upload.js";
|
|
7
|
-
|
|
9
|
+
/** The format ids, wrapped to the usage text's width under its indent. */
|
|
10
|
+
function formatList(indent, width = 98) {
|
|
11
|
+
const lines = [];
|
|
12
|
+
let line = "";
|
|
13
|
+
for (const id of importAdapters.map((adapter) => adapter.id)) {
|
|
14
|
+
const next = line ? `${line}, ${id}` : id;
|
|
15
|
+
if (line && indent.length + next.length + 1 > width) {
|
|
16
|
+
lines.push(`${line},`);
|
|
17
|
+
line = id;
|
|
18
|
+
}
|
|
19
|
+
else
|
|
20
|
+
line = next;
|
|
21
|
+
}
|
|
22
|
+
lines.push(line);
|
|
23
|
+
return lines.map((text) => indent + text).join("\n");
|
|
24
|
+
}
|
|
25
|
+
export const TRACES_USAGE = `Usage: hue traces import prepare <path> [--out <directory>] [--user-key <key>] [--session-key <key>] [--json]
|
|
8
26
|
hue traces import upload <directory> --authorization-file <file> [--import-url <url>] [--json]
|
|
9
27
|
|
|
10
28
|
Import a bundle of existing traces into Hue. Nothing leaves this machine until the upload step,
|
|
11
29
|
and no project key is involved: a Hue MCP connection authorizes the delivery.
|
|
12
30
|
|
|
13
|
-
prepare Reads a file, a directory or a ZIP archive and recognizes each file by its content
|
|
14
|
-
|
|
15
|
-
|
|
31
|
+
prepare Reads a file, a directory or a ZIP archive and recognizes each file by its content,
|
|
32
|
+
never its name: one JSON document, a JSON array of records, JSON Lines or binary OTLP,
|
|
33
|
+
gzip or zstd compressed or not, in these formats:
|
|
34
|
+
${formatList(" ")}
|
|
16
35
|
Every trace becomes gzip OTLP/JSON batches written to --out (default <path>.hue-import)
|
|
17
36
|
with manifest.json. The report names the manifest_digest and totals that
|
|
18
|
-
prepare_trace_import takes, the files it could not read, invalid
|
|
19
|
-
to OpenTelemetry ids, traces without a history owner and
|
|
20
|
-
recorded.
|
|
21
|
-
--out <directory>
|
|
22
|
-
--
|
|
37
|
+
prepare_trace_import takes, the files, lines and records it could not read, invalid
|
|
38
|
+
vendor ids it mapped to OpenTelemetry ids, traces without a declared history owner and
|
|
39
|
+
the origin each source recorded.
|
|
40
|
+
--out <directory> Where the bundle is written (created with mode 0700)
|
|
41
|
+
--user-key <key> Where the export records each trace's end user under a key of its own: an
|
|
42
|
+
attribute or metadata key (app.customer_id), or a JSON path into each
|
|
43
|
+
source record ($.metadata.customer.id or /metadata/customer/id)
|
|
44
|
+
--session-key <key> Where the export records each trace's conversation, the same way
|
|
45
|
+
--json Print the report as one JSON object
|
|
23
46
|
|
|
24
47
|
upload Registers the manifest, then sends each trace's values and batches in order, two
|
|
25
48
|
requests at a time, checks every span's receipt, reconciles and completes. Progress is
|
|
@@ -81,6 +104,8 @@ function parsePrepareArguments(argv) {
|
|
|
81
104
|
strict: true,
|
|
82
105
|
options: {
|
|
83
106
|
out: { type: "string" },
|
|
107
|
+
"user-key": { type: "string" },
|
|
108
|
+
"session-key": { type: "string" },
|
|
84
109
|
json: { type: "boolean", default: false },
|
|
85
110
|
},
|
|
86
111
|
});
|
|
@@ -96,10 +121,24 @@ async function prepare(argv, cwd, out, warn) {
|
|
|
96
121
|
? `Unexpected argument: ${positionals[1]}`
|
|
97
122
|
: "Name the file, directory or archive to prepare");
|
|
98
123
|
const input = resolve(cwd, positionals[0]);
|
|
124
|
+
const keys = {};
|
|
125
|
+
for (const [flag, kind] of [
|
|
126
|
+
["user-key", "user"],
|
|
127
|
+
["session-key", "session"],
|
|
128
|
+
]) {
|
|
129
|
+
const text = values[flag];
|
|
130
|
+
if (text === undefined)
|
|
131
|
+
continue;
|
|
132
|
+
const path = parseKeyPath(text);
|
|
133
|
+
if (!path)
|
|
134
|
+
throw new UsageError(`--${flag} must be an attribute or metadata key, or a JSON path such as $.metadata.user_id or /metadata/user_id`);
|
|
135
|
+
keys[kind] = path;
|
|
136
|
+
}
|
|
99
137
|
let report;
|
|
100
138
|
try {
|
|
101
139
|
report = await prepareTraceBundle(input, {
|
|
102
140
|
...(values.out !== undefined ? { out: resolve(cwd, values.out) } : {}),
|
|
141
|
+
...(keys.user || keys.session ? { keys } : {}),
|
|
103
142
|
});
|
|
104
143
|
}
|
|
105
144
|
catch (error) {
|
|
@@ -126,15 +165,28 @@ async function prepare(argv, cwd, out, warn) {
|
|
|
126
165
|
out(`Totals: ${JSON.stringify(report.totals)}`);
|
|
127
166
|
out(`Source: ${JSON.stringify(report.source)}`);
|
|
128
167
|
if (report.traces.without_identity)
|
|
129
|
-
out(`${plural(report.traces.without_identity, "trace
|
|
168
|
+
out(`${plural(report.traces.without_identity, "trace declares", "traces declare")} no history owner.`);
|
|
130
169
|
const origins = Object.entries(report.traces.by_origin).filter(([, count]) => count);
|
|
131
170
|
out(`Origin: ${origins.map(([origin, count]) => `${origin} ${count}`).join(", ")}.`);
|
|
132
171
|
if (report.traces.remapped.traces)
|
|
133
172
|
out(`Mapped invalid vendor ids to OpenTelemetry ids in ${plural(report.traces.remapped.traces, "trace")} (${plural(report.traces.remapped.spans, "span")}).`);
|
|
173
|
+
for (const [rule, count] of Object.entries(report.traces.timing))
|
|
174
|
+
out(`Instants the source did not record (${rule}): ${plural(count, "trace")}.`);
|
|
175
|
+
const where = (item) => [
|
|
176
|
+
item.file,
|
|
177
|
+
item.line !== undefined ? `line ${item.line}` : undefined,
|
|
178
|
+
item.record !== undefined ? `record ${item.record}` : undefined,
|
|
179
|
+
]
|
|
180
|
+
.filter(Boolean)
|
|
181
|
+
.join(" ");
|
|
134
182
|
for (const file of report.files.unsupported)
|
|
135
183
|
warn(`Not read: ${file.file}: ${file.reason}.`);
|
|
184
|
+
for (const record of report.files.skipped)
|
|
185
|
+
warn(`Not read: ${where(record)}: ${record.reason}.`);
|
|
186
|
+
for (const note of report.files.notes)
|
|
187
|
+
warn(`Note: ${where(note)}: ${note.reason}.`);
|
|
136
188
|
for (const refusal of report.traces.refused)
|
|
137
|
-
warn(`Not converted: ${[refusal
|
|
189
|
+
warn(`Not converted: ${[where(refusal), refusal.trace_id ? `trace ${refusal.trace_id}` : undefined].filter(Boolean).join(" ")}: ${refusal.reason}.`);
|
|
138
190
|
out("Next: call prepare_trace_import with manifest_digest, totals and source, write the authorization to a private file and run hue traces import upload.");
|
|
139
191
|
return 0;
|
|
140
192
|
}
|
package/dist/transport.d.ts
CHANGED
|
@@ -53,6 +53,9 @@ export declare class HueTransport {
|
|
|
53
53
|
private failed;
|
|
54
54
|
private spans;
|
|
55
55
|
private logs;
|
|
56
|
+
private deliveryCohort?;
|
|
57
|
+
private deliveries;
|
|
58
|
+
private deliveryUploads;
|
|
56
59
|
private pendingBytes;
|
|
57
60
|
private dropped;
|
|
58
61
|
/** Records of each trace the SDK never sent (dropped from the queue, refused as invalid or
|
|
@@ -81,6 +84,8 @@ export declare class HueTransport {
|
|
|
81
84
|
private placeholdersRejected;
|
|
82
85
|
private placeholders;
|
|
83
86
|
private placeholderSources;
|
|
87
|
+
private forcedSpanDrains;
|
|
88
|
+
private completedSpanDrains;
|
|
84
89
|
private batchSpans?;
|
|
85
90
|
private limits;
|
|
86
91
|
/** The rate-limit holds in progress, each ended early by calling it. */
|
|
@@ -120,11 +125,16 @@ export declare class HueTransport {
|
|
|
120
125
|
* skipped silently.
|
|
121
126
|
*/
|
|
122
127
|
private enqueue;
|
|
128
|
+
/** Register in the caller's request before it can return; outside Vercel this is a no-op. */
|
|
129
|
+
private protectDelivery;
|
|
130
|
+
private holdDelivery;
|
|
131
|
+
private finishDelivery;
|
|
123
132
|
private track;
|
|
124
133
|
/** Placeholders are built lazily, so input set right after a span starts is included. */
|
|
125
134
|
private announceLiveSpans;
|
|
126
135
|
private stopLiveTimer;
|
|
127
136
|
private stopLiveSpans;
|
|
137
|
+
private beginSpanDrain;
|
|
128
138
|
/** Holds the staged attachments of a log record being admitted with the queued record, within
|
|
129
139
|
* the held-value budget; those the budget refuses are counted as not uploaded. */
|
|
130
140
|
private admitLogParts;
|
|
@@ -144,6 +154,8 @@ export declare class HueTransport {
|
|
|
144
154
|
private protectMedia;
|
|
145
155
|
private releaseMediaIdentity;
|
|
146
156
|
private sweepMedia;
|
|
157
|
+
/** Keep an export's warning aggregation within each cohort's upload ownership. */
|
|
158
|
+
private collectDeliveryUploads;
|
|
147
159
|
/** Bounded: a trace whose root never arrives (an abandoned run) gives way to newer ones. */
|
|
148
160
|
private countDropped;
|
|
149
161
|
/** Cumulative counters and current queue gauges. */
|