@hue-run/sdk 0.18.0 → 0.18.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,692 @@
1
+ import { existsSync } from "node:fs";
2
+ import { readFile } from "node:fs/promises";
3
+ import { resolve } from "node:path";
4
+ import { createInterface } from "node:readline";
5
+ import { parseArgs, parseEnv } from "node:util";
6
+ import { resourceFromAttributes } from "@opentelemetry/resources";
7
+ import { BlobUploader, uploadBudgetMillis } from "../blobs.js";
8
+ import { createHue, HueConnectionError } from "../client.js";
9
+ import { isLoopbackHost } from "../config.js";
10
+ import { HueExportError } from "../transport.js";
11
+ import { sdkVersion } from "../version.js";
12
+ import { DEFAULT_ENV_FILE, envFileArgument, envFileOptions } from "./env-file.js";
13
+ import { hueTraceId, ImportStopped, LangfuseApi, LangfuseError, observationSpan, placeMedia, readableSpan, spanMedia, storeSpanContent, traceContext, TraceMedia, traceSpan, uploadableUses, widen, } from "./langfuse.js";
14
+ import { parseHueOrigin } from "./login.js";
15
+ const DEFAULT_ORIGIN = "https://app.hue.run";
16
+ const DEFAULT_LANGFUSE = "https://cloud.langfuse.com";
17
+ const DEFAULT_SERVICE_NAME = "langfuse";
18
+ /** The instrumentation scope of every imported span. */
19
+ const SCOPE = { name: "@hue-run/sdk/import/langfuse" };
20
+ /** Spans queued for export before the import waits for them to be sent. */
21
+ const DRAIN_SPANS = 1024;
22
+ /** Bytes queued for export before the import waits for them to be sent. */
23
+ const DRAIN_BYTES = 32 * 1024 * 1024;
24
+ /** The SDK queue's byte budget for an import: its largest. */
25
+ const QUEUE_BYTES = 64 * 1024 * 1024;
26
+ /** Traces whose Langfuse id is remembered to name them in Hue's export issues. */
27
+ const REMEMBERED_TRACES = 10_000;
28
+ /** Failed traces listed in the summary. */
29
+ const LISTED_FAILURES = 20;
30
+ const MAX_TRACE_IDS = 1000;
31
+ const PROGRESS_EVERY_TRACES = 100;
32
+ const PROGRESS_EVERY_MS = 10_000;
33
+ /** A stop request this soon after the first is the same one: `npx` forwards the terminal's
34
+ * SIGINT to a process group that already received it. */
35
+ const REPEATED_STOP_MS = 1_000;
36
+ /** How long before the command starts the default window ends: a trace still running, or still
37
+ * arriving in Langfuse, is left for a later run rather than sent half recorded (Hue would then
38
+ * refuse its finished spans). */
39
+ const SETTLE_MS = 60 * 60_000;
40
+ /** How long one value's upload may take: up to 25 MiB over a slow link. */
41
+ const UPLOAD_DEADLINE_MS = 300_000;
42
+ export const IMPORT_USAGE = `Usage: hue import langfuse [options]
43
+
44
+ Copy traces from a Langfuse project into Hue. The import runs on this machine with your own
45
+ Langfuse keys, which are sent only to Langfuse. Each observation becomes a span with the
46
+ langfuse.* attributes Hue reads; ids are kept, so running the same command again resumes an
47
+ import without duplicating what was already sent.
48
+
49
+ Langfuse (keys from the environment or --env-path; the secret key never from the command line):
50
+ LANGFUSE_PUBLIC_KEY, LANGFUSE_SECRET_KEY The Langfuse project's API keys
51
+ LANGFUSE_BASE_URL or LANGFUSE_HOST Langfuse host (default ${DEFAULT_LANGFUSE})
52
+ --langfuse-host URL Langfuse host, instead of the variables
53
+ --langfuse-public-key KEY Public key, instead of LANGFUSE_PUBLIC_KEY
54
+
55
+ Hue:
56
+ HUE_API_KEY The Hue project key the traces are sent with (environment, --env-path
57
+ or ./.env.hue); not needed with --dry-run
58
+ --origin URL Hue origin (default HUE_BASE_URL, then ${DEFAULT_ORIGIN})
59
+ --service-name NAME service.name of the imported spans (default ${DEFAULT_SERVICE_NAME})
60
+
61
+ Selection (newest first):
62
+ --from TIME Traces that started at or after TIME: an ISO 8601 date (midnight UTC)
63
+ or a date and time with a zone, such as 2026-01-01T09:00:00Z
64
+ --to TIME Traces that started before TIME (default: an hour before the command
65
+ starts, so traces still running are left for a later run)
66
+ --limit N At most N traces
67
+ --trace-id ID This Langfuse trace (repeatable); not with --from, --to or --limit
68
+
69
+ Content:
70
+ --media Download the files Langfuse media references (@@@langfuseMedia:…@@@)
71
+ point to and upload them to Hue; without it, references stay as text
72
+ --no-content Send metadata only: no inputs, outputs, status messages or model
73
+ parameters (content is sent by default, as the SDK records it)
74
+
75
+ Run:
76
+ --dry-run Read Langfuse and count what would be imported; send nothing to Hue
77
+ --yes Import without asking for confirmation (required without a terminal)
78
+ --env-path PATH Load a dotenv file first (--env-file also works when it exists);
79
+ ./.env.hue loads when HUE_API_KEY is not set. Variables already set win
80
+ -h, --help Show this help
81
+
82
+ Exit codes: 0 every trace was imported, 1 a trace, span or file was not, 2 usage error,
83
+ 130 stopped (run the same command again to resume).`;
84
+ class UsageError extends Error {
85
+ }
86
+ function parseImportArguments(argv) {
87
+ return parseArgs({
88
+ args: argv,
89
+ allowPositionals: true,
90
+ strict: true,
91
+ options: {
92
+ from: { type: "string" },
93
+ to: { type: "string" },
94
+ limit: { type: "string" },
95
+ "trace-id": { type: "string", multiple: true },
96
+ "dry-run": { type: "boolean", default: false },
97
+ media: { type: "boolean", default: false },
98
+ "no-content": { type: "boolean", default: false },
99
+ yes: { type: "boolean", default: false },
100
+ origin: { type: "string" },
101
+ "service-name": { type: "string" },
102
+ "langfuse-host": { type: "string" },
103
+ "langfuse-public-key": { type: "string" },
104
+ ...envFileOptions,
105
+ help: { type: "boolean", short: "h", default: false },
106
+ },
107
+ });
108
+ }
109
+ /** An instant of the window: an ISO 8601 date (midnight UTC) or a date and time with a zone. */
110
+ export function parseInstant(option, value) {
111
+ const match = /^(\d{4})-(\d{2})-(\d{2})(?:T(\d{2}):(\d{2})(?::(\d{2})(?:\.\d{1,9})?)?(?:Z|[+-](\d{2}):(\d{2})))?$/iu.exec(value);
112
+ const [year, month, day, hour, minute, second, zoneHour, zoneMinute] = (match ?? [])
113
+ .slice(1)
114
+ .map((part) => Number(part ?? 0));
115
+ // A calendar date and a time of day as written: JavaScript would roll 2026-02-30 into March.
116
+ const valid = match !== null &&
117
+ month >= 1 &&
118
+ month <= 12 &&
119
+ day >= 1 &&
120
+ day <= new Date(Date.UTC(year, month, 0)).getUTCDate() &&
121
+ hour <= 23 &&
122
+ minute <= 59 &&
123
+ second <= 59 &&
124
+ zoneHour <= 23 &&
125
+ zoneMinute <= 59;
126
+ const millis = valid ? Date.parse(match[4] === undefined ? `${value}T00:00:00Z` : value) : NaN;
127
+ if (!Number.isFinite(millis))
128
+ throw new UsageError(`${option} must be an ISO 8601 date or a date and time with a time zone, for example 2026-01-01 or 2026-01-01T09:00:00Z`);
129
+ return new Date(millis).toISOString();
130
+ }
131
+ /** The Langfuse host as a base URL without a trailing slash, or why it is refused. */
132
+ export function parseLangfuseHost(value) {
133
+ let url;
134
+ try {
135
+ url = new URL(value);
136
+ }
137
+ catch {
138
+ throw new UsageError("The Langfuse host must be a URL such as https://cloud.langfuse.com");
139
+ }
140
+ if (url.protocol !== "https:" && !(url.protocol === "http:" && isLoopbackHost(url.hostname)))
141
+ throw new UsageError("The Langfuse host must use https (plain http is accepted for this machine only), because the keys travel with every request");
142
+ if (url.username || url.password || url.search || url.hash)
143
+ throw new UsageError("The Langfuse host must not carry credentials, a query or a fragment");
144
+ return `${url.origin}${url.pathname.replace(/\/+$/u, "")}`;
145
+ }
146
+ /** A dotenv file's variables under `env`'s: as Node's --env-file does, a variable already set
147
+ * keeps its value. */
148
+ async function withEnvFile(env, cwd, file) {
149
+ let content;
150
+ try {
151
+ content = await readFile(resolve(cwd, file), "utf8");
152
+ }
153
+ catch (error) {
154
+ throw new UsageError(`Unable to load ${file}: ${error.code ?? "unreadable"}`);
155
+ }
156
+ return { ...parseEnv(content), ...env };
157
+ }
158
+ /** The environment with the named env file loaded, then, when the Hue key is needed and neither
159
+ * set `HUE_API_KEY`, `./.env.hue` (where `hue login` stores it): the Langfuse keys can sit in a
160
+ * file of their own. A dry run needs no Hue key and reads no `./.env.hue` beside a named file. */
161
+ async function loadEnv(env, cwd, file, needsHueKey) {
162
+ const named = file ? await withEnvFile(env, cwd, file) : env;
163
+ if (named.HUE_API_KEY?.trim() ||
164
+ (file && !needsHueKey) ||
165
+ !existsSync(resolve(cwd, DEFAULT_ENV_FILE)))
166
+ return named;
167
+ return withEnvFile(named, cwd, DEFAULT_ENV_FILE);
168
+ }
169
+ function defaultOnInterrupt(listener) {
170
+ process.on("SIGINT", listener);
171
+ process.on("SIGTERM", listener);
172
+ return () => {
173
+ process.removeListener("SIGINT", listener);
174
+ process.removeListener("SIGTERM", listener);
175
+ };
176
+ }
177
+ function defaultSleep(ms, signal) {
178
+ return new Promise((done) => {
179
+ const finish = () => {
180
+ clearTimeout(timer);
181
+ signal.removeEventListener("abort", finish);
182
+ done();
183
+ };
184
+ const timer = setTimeout(finish, ms);
185
+ signal.addEventListener("abort", finish, { once: true });
186
+ });
187
+ }
188
+ function plural(count, one, many = `${one}s`) {
189
+ return `${count} ${count === 1 ? one : many}`;
190
+ }
191
+ /** Runs `hue import langfuse`; resolves to the exit code. */
192
+ export async function runImportCommand(argv, io = {}) {
193
+ const stdout = io.stdout ?? process.stdout;
194
+ const stderr = io.stderr ?? process.stderr;
195
+ const stdin = io.stdin ?? process.stdin;
196
+ const cwd = io.cwd ?? process.cwd();
197
+ const secrets = [];
198
+ // Every line loses the keys and control characters; Langfuse ids and hosts are printed as text.
199
+ const clean = (line) => {
200
+ let out = line;
201
+ for (const secret of secrets)
202
+ if (secret)
203
+ out = out.replaceAll(secret, "[redacted]");
204
+ return out.replace(/[^\P{Cc}\n\t]|\p{Cf}/gu, " ");
205
+ };
206
+ const out = (line) => void stdout.write(`${clean(line)}\n`);
207
+ const warn = (line) => void stderr.write(`${clean(line)}\n`);
208
+ let values;
209
+ let window = {};
210
+ let limit;
211
+ let traceIds = [];
212
+ let langfuseHost;
213
+ let publicKey;
214
+ let secretKey;
215
+ let origin;
216
+ let apiKey = "";
217
+ let serviceName;
218
+ try {
219
+ let parsed;
220
+ try {
221
+ parsed = parseImportArguments(argv);
222
+ }
223
+ catch (error) {
224
+ throw new UsageError(error.message);
225
+ }
226
+ values = parsed.values;
227
+ if (values.help) {
228
+ stdout.write(`${IMPORT_USAGE}\n`);
229
+ return 0;
230
+ }
231
+ const positionals = parsed.positionals[0] === "import" ? parsed.positionals.slice(1) : parsed.positionals;
232
+ if (positionals.length === 0)
233
+ throw new UsageError("Name the source to import: langfuse");
234
+ if (positionals[0] !== "langfuse")
235
+ throw new UsageError(`Unknown source: ${positionals[0]}; hue import reads langfuse`);
236
+ if (positionals.length > 1)
237
+ throw new UsageError(`Unexpected argument: ${positionals[1]}`);
238
+ traceIds = values["trace-id"] ?? [];
239
+ if (traceIds.length && (values.from || values.to || values.limit))
240
+ throw new UsageError("--trace-id selects its traces: do not combine it with --from, --to or --limit");
241
+ if (traceIds.length > MAX_TRACE_IDS)
242
+ throw new UsageError(`Pass at most ${MAX_TRACE_IDS} --trace-id values`);
243
+ for (const id of traceIds)
244
+ if (!id || id.length > 512 || /[\s/?#]/u.test(id))
245
+ throw new UsageError("--trace-id must be a Langfuse trace id");
246
+ traceIds = [...new Set(traceIds)];
247
+ const to = values.to
248
+ ? parseInstant("--to", values.to)
249
+ : new Date((io.now?.() ?? new Date()).getTime() - SETTLE_MS).toISOString();
250
+ window = { ...(values.from ? { from: parseInstant("--from", values.from) } : {}), to };
251
+ if (window.from && window.from >= to)
252
+ throw new UsageError("--from must be before --to");
253
+ if (values.limit !== undefined) {
254
+ if (!/^\d{1,9}$/u.test(values.limit) || Number(values.limit) < 1)
255
+ throw new UsageError("--limit must be a positive integer");
256
+ limit = Number(values.limit);
257
+ }
258
+ if (values.media && values["no-content"])
259
+ throw new UsageError("--media uploads content; it cannot be combined with --no-content");
260
+ serviceName = values["service-name"] ?? DEFAULT_SERVICE_NAME;
261
+ if (!serviceName.trim() || serviceName.length > 256)
262
+ throw new UsageError("--service-name must be 1 to 256 characters");
263
+ let envFile;
264
+ try {
265
+ envFile = envFileArgument(values, cwd);
266
+ }
267
+ catch (error) {
268
+ throw new UsageError(error.message);
269
+ }
270
+ const env = await loadEnv(io.env ?? process.env, cwd, envFile, !values["dry-run"]);
271
+ langfuseHost = parseLangfuseHost(values["langfuse-host"] ??
272
+ (env.LANGFUSE_BASE_URL?.trim() || env.LANGFUSE_HOST?.trim() || DEFAULT_LANGFUSE));
273
+ publicKey = values["langfuse-public-key"] ?? env.LANGFUSE_PUBLIC_KEY?.trim() ?? "";
274
+ secretKey = env.LANGFUSE_SECRET_KEY?.trim() ?? "";
275
+ secrets.push(secretKey);
276
+ if (!publicKey || !secretKey)
277
+ throw new UsageError("Set LANGFUSE_PUBLIC_KEY and LANGFUSE_SECRET_KEY (the Langfuse project's API keys) in the environment or an --env-path file");
278
+ if (/\s/u.test(publicKey) || /\s/u.test(secretKey))
279
+ throw new UsageError("The Langfuse keys must not contain whitespace");
280
+ if (publicKey.startsWith("sk-lf-"))
281
+ throw new UsageError("The Langfuse public key is a secret key (sk-lf-…); the secret key goes in LANGFUSE_SECRET_KEY");
282
+ const parsedOrigin = parseHueOrigin(values.origin ?? (env.HUE_BASE_URL?.trim() || DEFAULT_ORIGIN));
283
+ if (!parsedOrigin)
284
+ throw new UsageError("--origin must be an HTTPS origin such as https://app.hue.run (plain HTTP is accepted for loopback test servers only)");
285
+ origin = parsedOrigin;
286
+ if (!values["dry-run"]) {
287
+ apiKey = env.HUE_API_KEY?.trim() ?? "";
288
+ secrets.push(apiKey);
289
+ if (!apiKey)
290
+ throw new UsageError("HUE_API_KEY is required: the Hue project key, set in the environment, an --env-path file or ./.env.hue");
291
+ if (!values.yes && stdin.isTTY !== true)
292
+ throw new UsageError("Pass --yes to import without a terminal to confirm in");
293
+ }
294
+ }
295
+ catch (error) {
296
+ if (!(error instanceof UsageError))
297
+ throw error;
298
+ warn(error.message);
299
+ stderr.write(`\n${IMPORT_USAGE}\n`);
300
+ return 2;
301
+ }
302
+ const dryRun = values["dry-run"];
303
+ const stopping = new AbortController();
304
+ const forced = new AbortController();
305
+ let stoppedAt = 0;
306
+ const removeInterrupt = (io.onInterrupt ?? defaultOnInterrupt)(() => {
307
+ if (!stopping.signal.aborted) {
308
+ stoppedAt = performance.now();
309
+ stopping.abort();
310
+ warn("Stopping: sending what was read (press Ctrl+C again to stop at once).");
311
+ }
312
+ else if (performance.now() - stoppedAt >= REPEATED_STOP_MS && !forced.signal.aborted) {
313
+ warn("Stopping at once; run the same command again to resume.");
314
+ forced.abort();
315
+ }
316
+ });
317
+ const sleep = (ms) => (io.sleep ?? defaultSleep)(ms, stopping.signal);
318
+ let hue;
319
+ let uploader;
320
+ // A second stop request ends every wait for Hue at once: uploads in flight are closed and the
321
+ // queue is left unsent.
322
+ const forcedStop = new Promise((done) => forced.signal.addEventListener("abort", () => {
323
+ uploader?.close();
324
+ done();
325
+ }, { once: true }));
326
+ try {
327
+ const api = new LangfuseApi({
328
+ baseUrl: langfuseHost,
329
+ publicKey,
330
+ secretKey,
331
+ ...(io.fetch ? { fetch: io.fetch } : {}),
332
+ sleep,
333
+ signal: stopping.signal,
334
+ onWait: (seconds, reason) => warn(`${reason}; waiting ${seconds} s.`),
335
+ });
336
+ let projectName = "";
337
+ if (!dryRun) {
338
+ hue = createHue({
339
+ apiKey,
340
+ serviceName,
341
+ baseUrl: origin,
342
+ captureContent: !values["no-content"],
343
+ liveSpans: false,
344
+ maxQueueBytes: QUEUE_BYTES,
345
+ });
346
+ try {
347
+ projectName = (await hue.checkConnection()).name;
348
+ }
349
+ catch (error) {
350
+ const status = error instanceof HueConnectionError ? error.status : undefined;
351
+ warn(status === 401 || status === 403
352
+ ? `Hue refused HUE_API_KEY (HTTP ${status}).`
353
+ : `${error instanceof Error ? error.message : "Hue could not be reached"}${status ? ` (HTTP ${status})` : ""}.`);
354
+ return 1;
355
+ }
356
+ }
357
+ let version;
358
+ try {
359
+ version = await api.version();
360
+ }
361
+ catch (error) {
362
+ if (error instanceof ImportStopped)
363
+ return 130;
364
+ if (!(error instanceof LangfuseError))
365
+ throw error;
366
+ warn(`${error.message}.`);
367
+ return 1;
368
+ }
369
+ if (!dryRun && !values.yes) {
370
+ const confirmed = await confirm(stdin, stdout, clean(`Import traces from ${langfuseHost} into the Hue project "${projectName}" at ${origin}? [y/N] `));
371
+ if (!confirmed) {
372
+ warn("Nothing was imported.");
373
+ return 1;
374
+ }
375
+ }
376
+ const scope = traceIds.length
377
+ ? plural(traceIds.length, "trace")
378
+ : `traces that started ${window.from ? `from ${window.from} ` : ""}before ${window.to}${limit ? `, at most ${limit}` : ""}`;
379
+ out(`Reading ${scope} from ${langfuseHost} (Langfuse API ${version}).`);
380
+ const transport = hue?.transport;
381
+ // A replay sends the same spans on every run: no trace's root records this run's losses.
382
+ transport?.omitDroppedRecords();
383
+ const resource = resourceFromAttributes({ "service.name": serviceName });
384
+ // Media and values over Hue's inline limit are stored before export, so a span is sent only
385
+ // once its files are; with --no-content the transport sends metadata only and stores nothing.
386
+ if (hue && !values["no-content"])
387
+ uploader = new BlobUploader({
388
+ baseUrl: origin,
389
+ apiKey,
390
+ timeoutMillis: hue.transport.options.timeoutMillis,
391
+ allowInsecureHttp: false,
392
+ userAgent: `hue-sdk-typescript/${sdkVersion} hue-import`,
393
+ });
394
+ const uploadContext = () => ({
395
+ deadline: Date.now() +
396
+ Math.max(UPLOAD_DEADLINE_MS, uploadBudgetMillis(hue.transport.options.timeoutMillis)),
397
+ wait: async (ms) => {
398
+ await sleep(ms);
399
+ return !stopping.signal.aborted;
400
+ },
401
+ });
402
+ const stats = {
403
+ traces: 0,
404
+ spans: 0,
405
+ references: 0,
406
+ uploaded: 0,
407
+ kept: 0,
408
+ fileFailures: 0,
409
+ stored: 0,
410
+ overLimits: 0,
411
+ };
412
+ // Traces not fully imported: counted, the first few kept to list, the newest remembered so
413
+ // Hue's export issues do not name one twice.
414
+ const failed = {
415
+ count: 0,
416
+ listed: [],
417
+ ids: new Set(),
418
+ };
419
+ /** Records a trace not fully imported; false when it already was. */
420
+ const fail = (id, reason) => {
421
+ if (failed.ids.has(id))
422
+ return false;
423
+ failed.count++;
424
+ if (failed.listed.length < LISTED_FAILURES)
425
+ failed.listed.push({ id, reason });
426
+ failed.ids.add(id);
427
+ if (failed.ids.size > REMEMBERED_TRACES)
428
+ failed.ids.delete(failed.ids.values().next().value);
429
+ return true;
430
+ };
431
+ // Hue's export issues name traces by their OpenTelemetry id; the newest are remembered.
432
+ const langfuseIds = new Map();
433
+ let ending;
434
+ let lastProgress = Date.now();
435
+ let progressAt = 0;
436
+ // Hue's refusals, read from the transport's issues after every drain: the traces they name
437
+ // were not imported, and a refusal that names none (or rolled out of the history) is noted.
438
+ let issueSequence = 0;
439
+ let unnamedRefusals = false;
440
+ const collectIssues = () => {
441
+ if (!transport)
442
+ return;
443
+ const issues = transport.getIssues();
444
+ if (issues.length && issues[0].sequence > issueSequence + 1)
445
+ unnamedRefusals = true;
446
+ for (const issue of issues) {
447
+ if (issue.sequence <= issueSequence)
448
+ continue;
449
+ issueSequence = issue.sequence;
450
+ if (issue.kind === "warning")
451
+ continue;
452
+ const ids = (issue.traceIds ?? []).flatMap((traceId) => langfuseIds.get(traceId) ?? []);
453
+ if (!ids.length)
454
+ unnamedRefusals = true;
455
+ for (const id of ids)
456
+ fail(id, "Hue did not accept all of its spans");
457
+ }
458
+ };
459
+ const drain = async () => {
460
+ if (!transport)
461
+ return;
462
+ const flushed = transport.flush().catch((error) => {
463
+ // Counted in the transport's report and named in its issues; the import continues.
464
+ if (!(error instanceof HueExportError))
465
+ throw error;
466
+ });
467
+ await Promise.race([flushed, forcedStop]);
468
+ collectIssues();
469
+ };
470
+ const send = async (span) => {
471
+ stats.spans++;
472
+ if (!transport)
473
+ return;
474
+ transport.spanProcessor.onEnd(readableSpan(span, resource, SCOPE));
475
+ const report = transport.getReport();
476
+ if (report.pendingSpans >= DRAIN_SPANS || report.pendingBytes >= DRAIN_BYTES)
477
+ await drain();
478
+ progress();
479
+ };
480
+ /** Places a span's Langfuse media files, then stores the rest of its content as the
481
+ * exporter would; the reason when a file or value could not be read or stored now, so the
482
+ * span is not sent. */
483
+ const prepare = async (span, files) => {
484
+ const found = spanMedia(span);
485
+ const located = found.attributes.reduce((sum, item) => sum + item.uses.length, 0);
486
+ stats.references += located + found.kept;
487
+ if (files) {
488
+ const { uses, kept } = uploadableUses(found);
489
+ const placed = new Map();
490
+ let left = found.kept + kept;
491
+ for (const use of uses) {
492
+ const outcome = await files.file(use);
493
+ if (outcome.kind === "uploaded")
494
+ placed.set(use, outcome.file);
495
+ else if (outcome.kind === "kept")
496
+ left++;
497
+ else {
498
+ stats.fileFailures++;
499
+ return `a media file could not be copied: ${outcome.reason}`;
500
+ }
501
+ }
502
+ stats.uploaded += placeMedia(span, found, placed);
503
+ stats.kept += left;
504
+ }
505
+ else
506
+ stats.kept += located + found.kept;
507
+ if (!uploader)
508
+ return undefined;
509
+ const content = await storeSpanContent(span, uploader, uploadContext());
510
+ if (content.heldBack)
511
+ return `a file or value could not be stored: ${content.heldBack}`;
512
+ stats.stored += content.stored;
513
+ stats.overLimits += content.overLimits;
514
+ return undefined;
515
+ };
516
+ const importTrace = async (id, trace) => {
517
+ const context = trace ? traceContext(trace) : { langfuseTraceId: id };
518
+ const traceId = hueTraceId(id);
519
+ langfuseIds.set(traceId, id);
520
+ if (langfuseIds.size > REMEMBERED_TRACES)
521
+ langfuseIds.delete(langfuseIds.keys().next().value);
522
+ // Files are read and uploaded only with --media, and never in a dry run.
523
+ const files = values.media && uploader
524
+ ? new TraceMedia(api, uploader, traceId, uploadContext)
525
+ : undefined;
526
+ let reason;
527
+ let observations = 0;
528
+ let observed;
529
+ const pages = version === "v2" ? api.v2Observations(id) : api.v1Observations(id);
530
+ for await (const page of pages) {
531
+ for (const observation of page) {
532
+ observations++;
533
+ let span;
534
+ try {
535
+ span = observationSpan(observation, context);
536
+ }
537
+ catch (error) {
538
+ reason ??= error.message;
539
+ continue;
540
+ }
541
+ observed = widen(observed, span);
542
+ const refused = await prepare(span, files);
543
+ if (refused) {
544
+ reason ??= refused;
545
+ continue;
546
+ }
547
+ await send(span);
548
+ }
549
+ }
550
+ if (trace) {
551
+ const root = traceSpan(trace, observed);
552
+ const refused = await prepare(root, files);
553
+ if (refused)
554
+ reason ??= refused;
555
+ else
556
+ await send(root);
557
+ }
558
+ else if (observations === 0)
559
+ reason = "Langfuse holds no observations for it";
560
+ return reason;
561
+ };
562
+ const progress = () => {
563
+ if (stats.traces - progressAt >= PROGRESS_EVERY_TRACES ||
564
+ Date.now() - lastProgress >= PROGRESS_EVERY_MS) {
565
+ progressAt = stats.traces;
566
+ lastProgress = Date.now();
567
+ out(`Read ${plural(stats.traces, "trace")} (${plural(stats.spans, "span")}).`);
568
+ }
569
+ };
570
+ /** The selected traces: by id, or listed in the window. A v1 trace named by id is read
571
+ * with its own trace. */
572
+ async function* selected() {
573
+ if (traceIds.length)
574
+ for (const id of traceIds)
575
+ yield { id };
576
+ else if (version === "v2")
577
+ for await (const id of api.rootTraceIds(window))
578
+ yield { id };
579
+ else
580
+ for await (const trace of api.v1Traces(window))
581
+ yield { id: trace.id, trace };
582
+ }
583
+ // The trace being read, when the import stops before it is read whole.
584
+ let reading;
585
+ try {
586
+ for await (const { id, trace } of selected()) {
587
+ stats.traces++;
588
+ reading = id;
589
+ let reason;
590
+ try {
591
+ const read = trace ?? (version === "v1" ? await api.v1Trace(id) : undefined);
592
+ reason =
593
+ version === "v1" && !read
594
+ ? "Langfuse holds no trace with this id"
595
+ : await importTrace(id, read);
596
+ }
597
+ catch (error) {
598
+ if (!(error instanceof LangfuseError) || error.fatal)
599
+ throw error;
600
+ reason = error.message;
601
+ }
602
+ reading = undefined;
603
+ if (reason) {
604
+ fail(id, reason);
605
+ warn(`Trace ${id} was not fully imported: ${reason}.`);
606
+ }
607
+ progress();
608
+ if (stopping.signal.aborted)
609
+ throw new ImportStopped();
610
+ if (limit !== undefined && stats.traces >= limit)
611
+ break;
612
+ }
613
+ }
614
+ catch (error) {
615
+ if (error instanceof ImportStopped)
616
+ ending = { kind: "stopped" };
617
+ else if (error instanceof LangfuseError)
618
+ ending = { kind: "failed", message: error.message };
619
+ else
620
+ throw error;
621
+ if (reading)
622
+ fail(reading, "the import ended before it was read whole");
623
+ }
624
+ if (dryRun) {
625
+ const line = `Dry run: ${plural(stats.traces, "trace")}, ${plural(stats.spans, "span")} and ${plural(stats.references, "media reference")} would be imported; nothing was sent to Hue.`;
626
+ out(line);
627
+ if (failed.count)
628
+ warn(`${plural(failed.count, "trace")} could not be read.`);
629
+ if (ending?.kind === "failed")
630
+ warn(`${ending.message}.`);
631
+ return ending?.kind === "stopped" ? 130 : ending || failed.count ? 1 : 0;
632
+ }
633
+ // Everything read is sent before the summary, unless a second stop request ends the wait.
634
+ await drain();
635
+ const report = transport.getReport();
636
+ const lost = report.rejectedSpans + report.failedSpans + report.droppedSpans;
637
+ // Every trace read is imported unless it was recorded as not fully imported.
638
+ const imported = Math.max(0, stats.traces - failed.count);
639
+ out(`Imported ${plural(imported, "trace")} of ${stats.traces} (${plural(stats.spans, "span")} sent, ${report.acceptedSpans} accepted) into the Hue project "${projectName}".`);
640
+ if (stats.references)
641
+ out(`Media: ${plural(stats.uploaded, "file")} uploaded, ${stats.kept} left in Langfuse${stats.fileFailures ? `, ${stats.fileFailures} not copied` : ""}.`);
642
+ if (lost)
643
+ warn(`Hue did not accept ${plural(lost, "span")} (${report.rejectedSpans} rejected, ${report.failedSpans} not delivered, ${report.droppedSpans} dropped).`);
644
+ if (stats.stored || stats.overLimits)
645
+ out(`Stored apart from their spans: ${plural(stats.stored, "inline file or large value", "inline files and large values")}${stats.overLimits ? `; ${stats.overLimits} over Hue's limits kept inline or cut` : ""}.`);
646
+ if (lost && unnamedRefusals)
647
+ warn("Some spans Hue did not accept could not be tied to a trace.");
648
+ if (report.uploadFallbacks)
649
+ warn(`${plural(report.uploadFallbacks, "inline file or value", "inline files or values")} could not be uploaded and kept the SDK's inline fallback; Hue keeps the first copy of a span, so a later run does not change ${report.uploadFallbacks === 1 ? "it" : "them"}.`);
650
+ if (failed.count) {
651
+ warn(`${plural(failed.count, "trace was", "traces were")} not fully imported:`);
652
+ for (const item of failed.listed)
653
+ warn(` ${item.id}: ${item.reason}`);
654
+ if (failed.count > failed.listed.length)
655
+ warn(` … and ${failed.count - failed.listed.length} more`);
656
+ }
657
+ if (ending?.kind === "failed")
658
+ warn(`The import stopped: ${ending.message}.`);
659
+ if (ending || failed.count || lost || stopping.signal.aborted)
660
+ warn("Run the same command again to resume; spans Hue already holds are not duplicated.");
661
+ if (ending?.kind === "stopped" || stopping.signal.aborted)
662
+ return 130;
663
+ return ending || failed.count || lost || report.uploadFallbacks ? 1 : 0;
664
+ }
665
+ catch (error) {
666
+ warn(`The import failed: ${error instanceof Error ? error.message : String(error)}`);
667
+ return 1;
668
+ }
669
+ finally {
670
+ removeInterrupt();
671
+ uploader?.close();
672
+ if (hue)
673
+ await hue.shutdownSafe({ timeoutMillis: forced.signal.aborted ? 100 : 5_000 });
674
+ // Exports still in flight are abandoned: a later run sends them again.
675
+ if (forced.signal.aborted)
676
+ (io.exit ?? process.exit)(130);
677
+ }
678
+ }
679
+ /** Asks for a yes on the terminal. */
680
+ async function confirm(stdin, stdout, question) {
681
+ const prompt = createInterface({ input: stdin, output: stdout, terminal: true, historySize: 0 });
682
+ try {
683
+ const answer = await new Promise((done) => {
684
+ prompt.question(question, done);
685
+ prompt.once("close", () => done(""));
686
+ });
687
+ return /^y(?:es)?$/iu.test(answer.trim());
688
+ }
689
+ finally {
690
+ prompt.close();
691
+ }
692
+ }