@hue-run/sdk 0.21.1 → 0.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. package/CLI.md +246 -52
  2. package/README.md +69 -14
  3. package/WORLDS.md +6 -3
  4. package/dist/ai-sdk.d.ts +23 -1
  5. package/dist/ai-sdk.js +60 -2
  6. package/dist/cli/import-formats/adapter.d.ts +80 -0
  7. package/dist/cli/import-formats/adapter.js +1 -0
  8. package/dist/cli/import-formats/agent-events.d.ts +5 -0
  9. package/dist/cli/import-formats/agent-events.js +16 -0
  10. package/dist/cli/import-formats/atif.d.ts +3 -0
  11. package/dist/cli/import-formats/atif.js +15 -0
  12. package/dist/cli/import-formats/braintrust.d.ts +2 -0
  13. package/dist/cli/import-formats/braintrust.js +222 -0
  14. package/dist/cli/import-formats/containers.d.ts +32 -0
  15. package/dist/cli/import-formats/containers.js +399 -0
  16. package/dist/cli/import-formats/datadog.d.ts +2 -0
  17. package/dist/cli/import-formats/datadog.js +264 -0
  18. package/dist/cli/import-formats/helicone.d.ts +2 -0
  19. package/dist/cli/import-formats/helicone.js +161 -0
  20. package/dist/cli/import-formats/identity.d.ts +112 -0
  21. package/dist/cli/import-formats/identity.js +594 -0
  22. package/dist/cli/import-formats/jaeger.d.ts +3 -0
  23. package/dist/cli/import-formats/jaeger.js +266 -0
  24. package/dist/cli/import-formats/langfuse.d.ts +7 -0
  25. package/dist/cli/import-formats/langfuse.js +510 -0
  26. package/dist/cli/import-formats/langsmith.d.ts +2 -0
  27. package/dist/cli/import-formats/langsmith.js +244 -0
  28. package/dist/cli/import-formats/messages.d.ts +28 -0
  29. package/dist/cli/import-formats/messages.js +301 -0
  30. package/dist/cli/import-formats/mlflow.d.ts +2 -0
  31. package/dist/cli/import-formats/mlflow.js +260 -0
  32. package/dist/cli/import-formats/openai-agents.d.ts +2 -0
  33. package/dist/cli/import-formats/openai-agents.js +221 -0
  34. package/dist/cli/import-formats/opik.d.ts +2 -0
  35. package/dist/cli/import-formats/opik.js +193 -0
  36. package/dist/cli/import-formats/otel-family.d.ts +5 -0
  37. package/dist/cli/import-formats/otel-family.js +17 -0
  38. package/dist/cli/import-formats/otel-spans.d.ts +94 -0
  39. package/dist/cli/import-formats/otel-spans.js +262 -0
  40. package/dist/cli/import-formats/otlp-protobuf.d.ts +35 -0
  41. package/dist/cli/import-formats/otlp-protobuf.js +486 -0
  42. package/dist/cli/import-formats/otlp.d.ts +4 -0
  43. package/dist/cli/import-formats/otlp.js +15 -0
  44. package/dist/cli/import-formats/phoenix.d.ts +3 -0
  45. package/dist/cli/import-formats/phoenix.js +256 -0
  46. package/dist/cli/import-formats/raindrop.d.ts +21 -0
  47. package/dist/cli/import-formats/raindrop.js +255 -0
  48. package/dist/cli/import-formats/reader.d.ts +56 -0
  49. package/dist/cli/import-formats/reader.js +364 -0
  50. package/dist/cli/import-formats/registry.d.ts +36 -0
  51. package/dist/cli/import-formats/registry.js +71 -0
  52. package/dist/cli/import-formats/row-platform-fields.d.ts +32 -0
  53. package/dist/cli/import-formats/row-platform-fields.js +76 -0
  54. package/dist/cli/import-formats/rows.d.ts +165 -0
  55. package/dist/cli/import-formats/rows.js +638 -0
  56. package/dist/cli/import-formats/transcripts.d.ts +25 -0
  57. package/dist/cli/import-formats/transcripts.js +498 -0
  58. package/dist/cli/import-formats/weave.d.ts +2 -0
  59. package/dist/cli/import-formats/weave.js +159 -0
  60. package/dist/cli/import-formats/zipkin.d.ts +2 -0
  61. package/dist/cli/import-formats/zipkin.js +219 -0
  62. package/dist/cli/import.d.ts +1 -1
  63. package/dist/cli/import.js +4 -2
  64. package/dist/cli/traces-import-convert.d.ts +65 -33
  65. package/dist/cli/traces-import-convert.js +211 -214
  66. package/dist/cli/traces-import-prepare.d.ts +26 -22
  67. package/dist/cli/traces-import-prepare.js +51 -122
  68. package/dist/cli/traces-import.d.ts +4 -4
  69. package/dist/cli/traces-import.js +63 -11
  70. package/dist/transport.d.ts +12 -0
  71. package/dist/transport.js +139 -24
  72. package/dist/version.d.ts +1 -1
  73. package/dist/version.js +1 -1
  74. package/package.json +2 -1
@@ -1,15 +1,20 @@
1
- import { type TraceImportFormat, type TraceImportManifest, type TraceImportOrigin, type TraceImportTotals } from "./traces-import-convert.js";
1
+ import type { IdentityKeys } from "./import-formats/identity.js";
2
+ import { type ReadResult } from "./import-formats/reader.js";
3
+ import { type TraceImportManifest, type TraceImportOrigin, type TraceImportTotals } from "./traces-import-convert.js";
2
4
  /**
3
5
  * `hue traces import prepare`: reads a file, a directory or a ZIP archive, converts every trace
4
- * it recognizes and writes the bundle Hue's import routes take: one gzip OTLP/JSON file per
5
- * batch, one file per large value, and `manifest.json`, whose digest and totals
6
- * `prepare_trace_import` is called with. Nothing is sent anywhere.
6
+ * the format registry (`import-formats/registry.ts`) recognizes and writes the bundle Hue's
7
+ * import routes take: one gzip OTLP/JSON file per batch, one file per large value, and
8
+ * `manifest.json`, whose digest and totals `prepare_trace_import` is called with. Nothing is sent
9
+ * anywhere.
7
10
  */
8
11
  /** Bounds on what one preparation reads. */
9
12
  export declare const PREPARE_LIMITS: {
10
13
  /** One source file, or one archive member. */
11
14
  readonly fileBytes: number;
12
- /** Every source file together. */
15
+ /** Every source file together, decompressed. Every trace is held until all are read, at about
16
+ * seven times its source size, so this keeps a run inside a 2 GiB JavaScript heap (Node.js's
17
+ * default on a machine with 8 GiB of memory). */
13
18
  readonly totalBytes: number;
14
19
  /** Files one directory tree or archive may hold. */
15
20
  readonly files: 50000;
@@ -23,18 +28,17 @@ export interface PrepareReport {
23
28
  totals: TraceImportTotals;
24
29
  source: TraceImportManifest["source"];
25
30
  output_directory: string;
26
- /** Traces recognized per format. */
27
- formats: Record<TraceImportFormat, number>;
31
+ /** Traces prepared per format id, for the formats that held any. */
32
+ formats: Record<string, number>;
28
33
  files: {
29
- supported: {
30
- file: string;
31
- format: TraceImportFormat;
32
- traces: number;
33
- }[];
34
- unsupported: {
35
- file: string;
36
- reason: string;
37
- }[];
34
+ supported: ReadResult["supported"];
35
+ /** Files of which nothing could be read, and why. */
36
+ unsupported: ReadResult["unsupported"];
37
+ /** Records of a read file (a line of JSON Lines, a record of a JSON array) that were not. */
38
+ skipped: ReadResult["skipped"];
39
+ /** What was read with a repair (a damaged gzip or zstd file read up to the damage, `NaN` read
40
+ * as null) or read once though the input held it more than once. */
41
+ notes: ReadResult["notes"];
38
42
  };
39
43
  traces: {
40
44
  without_identity: number;
@@ -44,12 +48,10 @@ export interface PrepareReport {
44
48
  traces: number;
45
49
  spans: number;
46
50
  };
47
- /** Traces and spans the sources held that could not be converted. */
48
- refused: {
49
- file?: string;
50
- trace_id?: string;
51
- reason: string;
52
- }[];
51
+ /** Traces a documented rule supplied instants for, per rule (`hue.import.timing`). */
52
+ timing: Record<string, number>;
53
+ /** Traces and records the sources held that could not be converted. */
54
+ refused: ReadResult["refused"];
53
55
  };
54
56
  blobs: number;
55
57
  }
@@ -62,6 +64,8 @@ export declare function defaultOutputDirectory(input: string): string;
62
64
  export declare function manifestTooLarge(bytes: Uint8Array): string | undefined;
63
65
  export interface PrepareOptions {
64
66
  out?: string;
67
+ /** `--user-key` and `--session-key` (`import-formats/identity.ts`). */
68
+ keys?: IdentityKeys;
65
69
  }
66
70
  /** Prepares the bundle; resolves to the report, or throws a {@link PrepareError}. */
67
71
  export declare function prepareTraceBundle(input: string, options?: PrepareOptions): Promise<PrepareReport>;
@@ -3,20 +3,25 @@ import { mkdir, open, readdir, readFile, rename, rm, stat, writeFile } from "nod
3
3
  import { basename, join, relative, resolve, sep } from "node:path";
4
4
  import { gzipSync } from "node:zlib";
5
5
  import { sdkVersion } from "../version.js";
6
- import { batchEntries, ConversionError, convertAtifExport, detectFormat, LangfuseTraceCollector, manifestTotals, manifestTraceHasIdentity, OtlpTraceCollector, sha256Hex, traceImportFormats, traceImportLimits, validateManifest, } from "./traces-import-convert.js";
6
+ import { ImportReader } from "./import-formats/reader.js";
7
+ import { adapterById } from "./import-formats/registry.js";
8
+ import { batchEntries, ConversionError, manifestTotals, manifestTraceHasIdentity, traceImportFormats, traceImportLimits, validateManifest, } from "./traces-import-convert.js";
7
9
  import { isZipSignature, ZipArchive, ZipError } from "./traces-import-zip.js";
8
10
  /**
9
11
  * `hue traces import prepare`: reads a file, a directory or a ZIP archive, converts every trace
10
- * it recognizes and writes the bundle Hue's import routes take: one gzip OTLP/JSON file per
11
- * batch, one file per large value, and `manifest.json`, whose digest and totals
12
- * `prepare_trace_import` is called with. Nothing is sent anywhere.
12
+ * the format registry (`import-formats/registry.ts`) recognizes and writes the bundle Hue's
13
+ * import routes take: one gzip OTLP/JSON file per batch, one file per large value, and
14
+ * `manifest.json`, whose digest and totals `prepare_trace_import` is called with. Nothing is sent
15
+ * anywhere.
13
16
  */
14
17
  /** Bounds on what one preparation reads. */
15
18
  export const PREPARE_LIMITS = {
16
19
  /** One source file, or one archive member. */
17
20
  fileBytes: 64 * 1024 * 1024,
18
- /** Every source file together. */
19
- totalBytes: 1024 * 1024 * 1024,
21
+ /** Every source file together, decompressed. Every trace is held until all are read, at about
22
+ * seven times its source size, so this keeps a run inside a 2 GiB JavaScript heap (Node.js's
23
+ * default on a machine with 8 GiB of memory). */
24
+ totalBytes: 256 * 1024 * 1024,
20
25
  /** Files one directory tree or archive may hold. */
21
26
  files: 50_000,
22
27
  /** How deep a directory tree is walked. */
@@ -211,117 +216,44 @@ async function writePrivate(path, bytes) {
211
216
  await writeFile(temporary, bytes, { mode: 0o600, flag: "w" });
212
217
  await rename(temporary, path);
213
218
  }
214
- /** Merges a converted trace into the collection: the same trace from two sources is one trace
215
- * when its entries agree, a conflict otherwise. */
216
- function merge(collected, conflicts, trace, sourceDigest) {
217
- trace.sourceDigests.add(sourceDigest);
218
- const held = collected.get(trace.traceId);
219
- if (!held) {
220
- collected.set(trace.traceId, trace);
221
- return;
222
- }
223
- if (JSON.stringify(held.entries) !== JSON.stringify(trace.entries)) {
224
- conflicts.set(trace.traceId, "the trace appears in two sources with different content");
225
- return;
226
- }
227
- for (const digest of trace.sourceDigests)
228
- held.sourceDigests.add(digest);
229
- }
230
219
  /** Prepares the bundle; resolves to the report, or throws a {@link PrepareError}. */
231
220
  export async function prepareTraceBundle(input, options = {}) {
232
221
  const outputDirectory = resolve(options.out ?? defaultOutputDirectory(input));
233
222
  const unsupported = [];
234
- const supported = [];
235
- const refused = [];
236
- const otlp = new OtlpTraceCollector();
237
- const langfuse = new LangfuseTraceCollector();
238
- const whole = new Map();
239
- const conflicts = new Map();
240
- /** The file names each source digest was read under, so the traces an OTLP or Langfuse file
241
- * held are counted once the traces are known: a trace may come from several files. */
242
- const fileNames = new Map();
243
- const tracesPerFile = new Map();
244
- const nameFile = (digest, name) => {
245
- const names = fileNames.get(digest) ?? new Set();
246
- names.add(name);
247
- fileNames.set(digest, names);
248
- };
223
+ const reader = new ImportReader({
224
+ maxFileBytes: PREPARE_LIMITS.fileBytes,
225
+ unsupported,
226
+ ...(options.keys ? { keys: options.keys } : {}),
227
+ });
249
228
  let totalBytes = 0;
250
- {
251
- for await (const source of sourceFiles(input, unsupported)) {
252
- totalBytes += source.size;
253
- if (totalBytes > PREPARE_LIMITS.totalBytes)
254
- throw new PrepareError(`The input holds more than ${PREPARE_LIMITS.totalBytes} bytes`);
255
- let bytes;
256
- try {
257
- bytes = await source.read();
258
- }
259
- catch (error) {
260
- unsupported.push({
261
- file: source.name,
262
- reason: error instanceof ZipError ? error.message : "the file could not be read",
263
- });
264
- continue;
265
- }
266
- const digest = sha256Hex(bytes);
267
- const detected = detectFormat(bytes);
268
- if (detected.format === "unsupported") {
269
- unsupported.push({ file: source.name, reason: detected.reason });
270
- continue;
271
- }
272
- const note = (problems) => {
273
- for (const problem of problems)
274
- refused.push({
275
- file: source.name,
276
- ...(problem.traceId ? { trace_id: problem.traceId } : {}),
277
- reason: problem.reason,
278
- });
279
- };
280
- try {
281
- if (detected.format === "otlp_json") {
282
- note(otlp.add(detected.envelopes, digest));
283
- nameFile(digest, source.name);
284
- supported.push({ file: source.name, format: "otlp_json", traces: 0 });
285
- }
286
- else if (detected.format === "langfuse_observations") {
287
- if (detected.unsafeIntegers)
288
- throw new ConversionError("Source JSON contains an unsafe number");
289
- langfuse.add(detected.observations, detected.envelope, digest);
290
- nameFile(digest, source.name);
291
- supported.push({ file: source.name, format: "langfuse_observations", traces: 0 });
292
- }
293
- else {
294
- const trace = convertAtifExport(detected, bytes.byteLength);
295
- merge(whole, conflicts, trace, digest);
296
- supported.push({ file: source.name, format: "atif", traces: 1 });
297
- }
298
- }
299
- catch (error) {
300
- if (!(error instanceof ConversionError))
301
- throw error;
302
- unsupported.push({ file: source.name, reason: error.message });
303
- }
229
+ const countBytes = (bytes) => {
230
+ totalBytes += bytes;
231
+ if (totalBytes > PREPARE_LIMITS.totalBytes)
232
+ throw new PrepareError(`The input holds more than ${PREPARE_LIMITS.totalBytes} bytes decompressed; prepare it in parts of at most that size, each to its own --out`);
233
+ };
234
+ for await (const source of sourceFiles(input, unsupported)) {
235
+ countBytes(source.size);
236
+ let bytes;
237
+ try {
238
+ bytes = await source.read();
304
239
  }
305
- }
306
- for (const collector of [otlp, langfuse]) {
307
- const finished = collector.finish();
308
- for (const problem of finished.refused)
309
- refused.push({
310
- ...(problem.traceId ? { trace_id: problem.traceId } : {}),
311
- reason: problem.reason,
240
+ catch (error) {
241
+ unsupported.push({
242
+ file: source.name,
243
+ reason: error instanceof ZipError ? error.message : "the file could not be read",
312
244
  });
313
- for (const trace of finished.traces) {
314
- const first = [...trace.sourceDigests][0] ?? "";
315
- merge(whole, conflicts, trace, first);
245
+ continue;
316
246
  }
247
+ // Decompressed content counts toward the bound too.
248
+ countBytes(Math.max(0, reader.read(source.name, bytes) - bytes.byteLength));
317
249
  }
318
- for (const [traceId, reason] of conflicts) {
319
- whole.delete(traceId);
320
- refused.push({ trace_id: traceId, reason });
321
- }
322
- const traces = [...whole.values()].sort((a, b) => (a.traceId < b.traceId ? -1 : 1));
250
+ const read = reader.finish();
251
+ const refused = read.refused;
252
+ const traces = read.traces;
323
253
  if (!traces.length) {
324
- const problems = [...unsupported, ...refused].slice(0, 3).map((item) => item.reason);
254
+ const problems = [...unsupported, ...read.skipped, ...refused]
255
+ .slice(0, 3)
256
+ .map((item) => item.reason);
325
257
  throw new PrepareError(`No trace was recognized in ${input}${problems.length ? ` (${problems.join("; ")})` : ""}`);
326
258
  }
327
259
  if (traces.length > traceImportLimits.traces)
@@ -329,11 +261,8 @@ export async function prepareTraceBundle(input, options = {}) {
329
261
  await prepareOutputDirectory(outputDirectory);
330
262
  const manifestTraces = [];
331
263
  const formats = new Set();
332
- const formatCounts = {
333
- otlp_json: 0,
334
- langfuse_observations: 0,
335
- atif: 0,
336
- };
264
+ const formatCounts = {};
265
+ const timing = {};
337
266
  const byOrigin = {
338
267
  production: 0,
339
268
  evaluation: 0,
@@ -403,8 +332,14 @@ export async function prepareTraceBundle(input, options = {}) {
403
332
  ...(sourceDigest ? { source_digest: sourceDigest } : {}),
404
333
  };
405
334
  manifestTraces.push(declared);
406
- formats.add(trace.format);
407
- formatCounts[trace.format]++;
335
+ // What was written is let go: the run holds each trace's spans only until they are on disk.
336
+ trace.entries = [];
337
+ for (const blob of trace.blobs)
338
+ blob.bytes = new Uint8Array(0);
339
+ formats.add(adapterById(trace.format).manifestFormat);
340
+ formatCounts[trace.format] = (formatCounts[trace.format] ?? 0) + 1;
341
+ for (const rule of trace.inferredTiming ?? [])
342
+ timing[rule] = (timing[rule] ?? 0) + 1;
408
343
  byOrigin[trace.origin]++;
409
344
  if (!manifestTraceHasIdentity(declared))
410
345
  withoutIdentity++;
@@ -412,16 +347,9 @@ export async function prepareTraceBundle(input, options = {}) {
412
347
  remappedTraces++;
413
348
  remappedSpans += trace.remappedSpans;
414
349
  blobCount += trace.blobs.length;
415
- if (trace.format !== "atif")
416
- for (const digest of trace.sourceDigests)
417
- for (const name of fileNames.get(digest) ?? [])
418
- tracesPerFile.set(name, (tracesPerFile.get(name) ?? 0) + 1);
419
350
  }
420
351
  if (!manifestTraces.length)
421
352
  throw new PrepareError(`No trace of ${input} could be prepared: ${refused[0]?.reason ?? ""}`);
422
- for (const entry of supported)
423
- if (entry.format !== "atif")
424
- entry.traces = tracesPerFile.get(entry.file) ?? 0;
425
353
  const manifest = {
426
354
  version: 1,
427
355
  source: {
@@ -445,11 +373,12 @@ export async function prepareTraceBundle(input, options = {}) {
445
373
  source: manifest.source,
446
374
  output_directory: outputDirectory,
447
375
  formats: formatCounts,
448
- files: { supported, unsupported },
376
+ files: { supported: read.supported, unsupported, skipped: read.skipped, notes: read.notes },
449
377
  traces: {
450
378
  without_identity: withoutIdentity,
451
379
  by_origin: byOrigin,
452
380
  remapped: { traces: remappedTraces, spans: remappedSpans },
381
+ timing,
453
382
  refused,
454
383
  },
455
384
  blobs: blobCount,
@@ -2,9 +2,9 @@ import { type UploadIo } from "./traces-import-upload.js";
2
2
  /**
3
3
  * `hue traces import`: imports a bundle of existing traces into Hue in two steps that keep the
4
4
  * payloads on this machine until the agent has authorized their delivery. `prepare` converts a
5
- * file, directory or ZIP archive of OTLP/JSON, Langfuse observation or ATIF exports into a
6
- * bundle and prints its manifest digest and totals; `prepare_trace_import` (Hue MCP) takes
7
- * those and answers an import authorization; `upload` delivers the bundle with it.
5
+ * file, directory or ZIP archive of trace exports (`import-formats/registry.ts`) into a bundle
6
+ * and prints its manifest digest and totals; `prepare_trace_import` (Hue MCP) takes those and
7
+ * answers an import authorization; `upload` delivers the bundle with it.
8
8
  */
9
9
  /** Streams and seams for {@link runTracesCommand}; tests inject these. */
10
10
  export interface TracesCommandIo extends UploadIo {
@@ -13,6 +13,6 @@ export interface TracesCommandIo extends UploadIo {
13
13
  env?: NodeJS.ProcessEnv;
14
14
  cwd?: string;
15
15
  }
16
- export declare const TRACES_USAGE = "Usage: hue traces import prepare <path> [--out <directory>] [--json]\n hue traces import upload <directory> --authorization-file <file> [--import-url <url>] [--json]\n\nImport a bundle of existing traces into Hue. Nothing leaves this machine until the upload step,\nand no project key is involved: a Hue MCP connection authorizes the delivery.\n\nprepare Reads a file, a directory or a ZIP archive and recognizes each file by its content:\n OTLP/JSON trace exports ({\"resourceSpans\": [\u2026]}, also one per line), complete Langfuse\n observation exports (the data or observations envelope) and text ATIF v1.7/v1.8 exports.\n Every trace becomes gzip OTLP/JSON batches written to --out (default <path>.hue-import)\n with manifest.json. The report names the manifest_digest and totals that\n prepare_trace_import takes, the files it could not read, invalid vendor ids it mapped\n to OpenTelemetry ids, traces without a history owner and the origin each source\n recorded.\n --out <directory> Where the bundle is written (created with mode 0700)\n --json Print the report as one JSON object\n\nupload Registers the manifest, then sends each trace's values and batches in order, two\n requests at a time, checks every span's receipt, reconciles and completes. Progress is\n kept in <directory>/progress.json, so running the same command again resumes.\n --authorization-file <file> A private file holding what prepare_trace_import answered: its\n JSON (authorization and upload_url), or the authorization\n (hue_it_\u2026) alone on the first line, the upload_url on the second.\n The authorization is never taken from the command line or the\n environment, and never printed.\n --import-url <url> The upload_url prepare_trace_import returned (https://\u2026/api/v1/trace-imports/<id>),\n when the authorization file does not hold it\n --origin <url> Accept only an import URL on this Hue origin\n --json Print one JSON progress object per line on stderr and the report on stdout\n\nExit codes: 0 prepared or completed, 1 not completed (the report says what to do next),\n2 usage error, 130 stopped (run the same command again to resume).";
16
+ export declare const TRACES_USAGE: string;
17
17
  /** Runs `hue traces import …`; resolves to the exit code. */
18
18
  export declare function runTracesCommand(argv: string[], io?: TracesCommandIo): Promise<number>;
@@ -1,25 +1,48 @@
1
1
  import { readFile } from "node:fs/promises";
2
2
  import { resolve } from "node:path";
3
3
  import { parseArgs } from "node:util";
4
+ import { parseKeyPath } from "./import-formats/identity.js";
5
+ import { importAdapters } from "./import-formats/registry.js";
4
6
  import { parseHueOrigin } from "./login.js";
5
7
  import { PrepareError, prepareTraceBundle } from "./traces-import-prepare.js";
6
8
  import { parseAuthorizationFile, parseImportUrl, UploadStop, uploadTraceBundle, } from "./traces-import-upload.js";
7
- export const TRACES_USAGE = `Usage: hue traces import prepare <path> [--out <directory>] [--json]
9
+ /** The format ids, wrapped to the usage text's width under its indent. */
10
+ function formatList(indent, width = 98) {
11
+ const lines = [];
12
+ let line = "";
13
+ for (const id of importAdapters.map((adapter) => adapter.id)) {
14
+ const next = line ? `${line}, ${id}` : id;
15
+ if (line && indent.length + next.length + 1 > width) {
16
+ lines.push(`${line},`);
17
+ line = id;
18
+ }
19
+ else
20
+ line = next;
21
+ }
22
+ lines.push(line);
23
+ return lines.map((text) => indent + text).join("\n");
24
+ }
25
+ export const TRACES_USAGE = `Usage: hue traces import prepare <path> [--out <directory>] [--user-key <key>] [--session-key <key>] [--json]
8
26
  hue traces import upload <directory> --authorization-file <file> [--import-url <url>] [--json]
9
27
 
10
28
  Import a bundle of existing traces into Hue. Nothing leaves this machine until the upload step,
11
29
  and no project key is involved: a Hue MCP connection authorizes the delivery.
12
30
 
13
- prepare Reads a file, a directory or a ZIP archive and recognizes each file by its content:
14
- OTLP/JSON trace exports ({"resourceSpans": […]}, also one per line), complete Langfuse
15
- observation exports (the data or observations envelope) and text ATIF v1.7/v1.8 exports.
31
+ prepare Reads a file, a directory or a ZIP archive and recognizes each file by its content,
32
+ never its name: one JSON document, a JSON array of records, JSON Lines or binary OTLP,
33
+ gzip or zstd compressed or not, in these formats:
34
+ ${formatList(" ")}
16
35
  Every trace becomes gzip OTLP/JSON batches written to --out (default <path>.hue-import)
17
36
  with manifest.json. The report names the manifest_digest and totals that
18
- prepare_trace_import takes, the files it could not read, invalid vendor ids it mapped
19
- to OpenTelemetry ids, traces without a history owner and the origin each source
20
- recorded.
21
- --out <directory> Where the bundle is written (created with mode 0700)
22
- --json Print the report as one JSON object
37
+ prepare_trace_import takes, the files, lines and records it could not read, invalid
38
+ vendor ids it mapped to OpenTelemetry ids, traces without a declared history owner and
39
+ the origin each source recorded.
40
+ --out <directory> Where the bundle is written (created with mode 0700)
41
+ --user-key <key> Where the export records each trace's end user under a key of its own: an
42
+ attribute or metadata key (app.customer_id), or a JSON path into each
43
+ source record ($.metadata.customer.id or /metadata/customer/id)
44
+ --session-key <key> Where the export records each trace's conversation, the same way
45
+ --json Print the report as one JSON object
23
46
 
24
47
  upload Registers the manifest, then sends each trace's values and batches in order, two
25
48
  requests at a time, checks every span's receipt, reconciles and completes. Progress is
@@ -81,6 +104,8 @@ function parsePrepareArguments(argv) {
81
104
  strict: true,
82
105
  options: {
83
106
  out: { type: "string" },
107
+ "user-key": { type: "string" },
108
+ "session-key": { type: "string" },
84
109
  json: { type: "boolean", default: false },
85
110
  },
86
111
  });
@@ -96,10 +121,24 @@ async function prepare(argv, cwd, out, warn) {
96
121
  ? `Unexpected argument: ${positionals[1]}`
97
122
  : "Name the file, directory or archive to prepare");
98
123
  const input = resolve(cwd, positionals[0]);
124
+ const keys = {};
125
+ for (const [flag, kind] of [
126
+ ["user-key", "user"],
127
+ ["session-key", "session"],
128
+ ]) {
129
+ const text = values[flag];
130
+ if (text === undefined)
131
+ continue;
132
+ const path = parseKeyPath(text);
133
+ if (!path)
134
+ throw new UsageError(`--${flag} must be an attribute or metadata key, or a JSON path such as $.metadata.user_id or /metadata/user_id`);
135
+ keys[kind] = path;
136
+ }
99
137
  let report;
100
138
  try {
101
139
  report = await prepareTraceBundle(input, {
102
140
  ...(values.out !== undefined ? { out: resolve(cwd, values.out) } : {}),
141
+ ...(keys.user || keys.session ? { keys } : {}),
103
142
  });
104
143
  }
105
144
  catch (error) {
@@ -126,15 +165,28 @@ async function prepare(argv, cwd, out, warn) {
126
165
  out(`Totals: ${JSON.stringify(report.totals)}`);
127
166
  out(`Source: ${JSON.stringify(report.source)}`);
128
167
  if (report.traces.without_identity)
129
- out(`${plural(report.traces.without_identity, "trace names", "traces name")} no history owner.`);
168
+ out(`${plural(report.traces.without_identity, "trace declares", "traces declare")} no history owner.`);
130
169
  const origins = Object.entries(report.traces.by_origin).filter(([, count]) => count);
131
170
  out(`Origin: ${origins.map(([origin, count]) => `${origin} ${count}`).join(", ")}.`);
132
171
  if (report.traces.remapped.traces)
133
172
  out(`Mapped invalid vendor ids to OpenTelemetry ids in ${plural(report.traces.remapped.traces, "trace")} (${plural(report.traces.remapped.spans, "span")}).`);
173
+ for (const [rule, count] of Object.entries(report.traces.timing))
174
+ out(`Instants the source did not record (${rule}): ${plural(count, "trace")}.`);
175
+ const where = (item) => [
176
+ item.file,
177
+ item.line !== undefined ? `line ${item.line}` : undefined,
178
+ item.record !== undefined ? `record ${item.record}` : undefined,
179
+ ]
180
+ .filter(Boolean)
181
+ .join(" ");
134
182
  for (const file of report.files.unsupported)
135
183
  warn(`Not read: ${file.file}: ${file.reason}.`);
184
+ for (const record of report.files.skipped)
185
+ warn(`Not read: ${where(record)}: ${record.reason}.`);
186
+ for (const note of report.files.notes)
187
+ warn(`Note: ${where(note)}: ${note.reason}.`);
136
188
  for (const refusal of report.traces.refused)
137
- warn(`Not converted: ${[refusal.file, refusal.trace_id ? `trace ${refusal.trace_id}` : undefined].filter(Boolean).join(" ")}: ${refusal.reason}.`);
189
+ warn(`Not converted: ${[where(refusal), refusal.trace_id ? `trace ${refusal.trace_id}` : undefined].filter(Boolean).join(" ")}: ${refusal.reason}.`);
138
190
  out("Next: call prepare_trace_import with manifest_digest, totals and source, write the authorization to a private file and run hue traces import upload.");
139
191
  return 0;
140
192
  }
@@ -53,6 +53,9 @@ export declare class HueTransport {
53
53
  private failed;
54
54
  private spans;
55
55
  private logs;
56
+ private deliveryCohort?;
57
+ private deliveries;
58
+ private deliveryUploads;
56
59
  private pendingBytes;
57
60
  private dropped;
58
61
  /** Records of each trace the SDK never sent (dropped from the queue, refused as invalid or
@@ -81,6 +84,8 @@ export declare class HueTransport {
81
84
  private placeholdersRejected;
82
85
  private placeholders;
83
86
  private placeholderSources;
87
+ private forcedSpanDrains;
88
+ private completedSpanDrains;
84
89
  private batchSpans?;
85
90
  private limits;
86
91
  /** The rate-limit holds in progress, each ended early by calling it. */
@@ -120,11 +125,16 @@ export declare class HueTransport {
120
125
  * skipped silently.
121
126
  */
122
127
  private enqueue;
128
+ /** Register in the caller's request before it can return; outside Vercel this is a no-op. */
129
+ private protectDelivery;
130
+ private holdDelivery;
131
+ private finishDelivery;
123
132
  private track;
124
133
  /** Placeholders are built lazily, so input set right after a span starts is included. */
125
134
  private announceLiveSpans;
126
135
  private stopLiveTimer;
127
136
  private stopLiveSpans;
137
+ private beginSpanDrain;
128
138
  /** Holds the staged attachments of a log record being admitted with the queued record, within
129
139
  * the held-value budget; those the budget refuses are counted as not uploaded. */
130
140
  private admitLogParts;
@@ -144,6 +154,8 @@ export declare class HueTransport {
144
154
  private protectMedia;
145
155
  private releaseMediaIdentity;
146
156
  private sweepMedia;
157
+ /** Keep an export's warning aggregation within each cohort's upload ownership. */
158
+ private collectDeliveryUploads;
147
159
  /** Bounded: a trace whose root never arrives (an abandoned run) gives way to newer ones. */
148
160
  private countDropped;
149
161
  /** Cumulative counters and current queue gauges. */