@hue-run/sdk 0.18.0 → 0.18.2

package/dist/blobs.js

@@ -3,6 +3,7 @@ import { context } from "@opentelemetry/api";
 import { suppressTracing } from "@opentelemetry/core";
 import { isLoopbackHost } from "./config.js";
 import { digestPart, hashInlineFiles, hashNestedInlineFiles, INLINE_FILE_LIMIT, INLINE_FILE_TEXT_PER_RECORD, isMessageKey, keepsPrefix, literalKeys, mediaBytes, mediaContentType, mediaIn, isTextMediaType, mayInlineFiles, isMediaContentType, MAX_INLINE_FILE_TEXT, placeable, placedMediaText, prefixPart, readMediaText, reencodeMedia, } from "./inline-files.js";
+import { isByteSource, JoinedText, longFileBytes, longHead, placedLargeText, readLargeMediaText, } from "./large-media.js";
 import { capLogBody, TRUNCATED_KEY, truncatedMarker, withTruncatedKeys } from "./privacy.js";
 import { truncateUtf8 } from "./safety.js";
 /**
@@ -99,40 +100,53 @@ export function blobPrefix(value) {
     return Buffer.from(truncateUtf8(value, BLOB_PREFIX_BYTES), "utf8").toString("utf8");
 }
 /** A value's size in bytes as it is uploaded: a string's UTF-8 encoding, counted, not made. */
-function sizeOf(value) {
-    return typeof value === "string" ? Buffer.byteLength(value, "utf8") : value.byteLength;
+export function sizeOf(value) {
+    if (typeof value === "string")
+        return Buffer.byteLength(value, "utf8");
+    return isByteSource(value) ? value.size : value.byteLength;
+}
+/** The bytes a value holds in memory: none for a source read as it is uploaded. */
+export function heldBytesOf(value) {
+    return isByteSource(value) ? 0 : value.byteLength;
 }
 /** A string's UTF-8 encoding a slice at a time, never splitting a surrogate pair. */
 function* utf8Chunks(text) {
+    // One buffer, reused: a reader is done with each slice before it asks for the next.
+    const buffer = Buffer.allocUnsafe(TEXT_CHUNK_UNITS * 3);
     for (let start = 0; start < text.length;) {
         let end = Math.min(text.length, start + TEXT_CHUNK_UNITS);
         if (end < text.length && /[\uD800-\uDBFF]/.test(text[end - 1]))
             end--;
-        yield Buffer.from(text.slice(start, end), "utf8");
+        yield buffer.subarray(0, buffer.write(text.slice(start, end), "utf8"));
         start = end;
     }
 }
 /** The bytes an upload carries, a slice at a time: no second copy of a large value is made. */
-function* uploadChunks(value) {
-    if (typeof value === "string") {
-        yield* utf8Chunks(value);
-        return;
-    }
-    for (let start = 0; start < value.byteLength; start += TEXT_CHUNK_UNITS)
-        yield value.subarray(start, Math.min(value.byteLength, start + TEXT_CHUNK_UNITS));
+function uploadChunks(value) {
+    if (isByteSource(value))
+        return value.chunks();
+    return (function* () {
+        if (typeof value === "string") {
+            yield* utf8Chunks(value);
+            return;
+        }
+        for (let start = 0; start < value.byteLength; start += TEXT_CHUNK_UNITS)
+            yield value.subarray(start, Math.min(value.byteLength, start + TEXT_CHUNK_UNITS));
+    })();
 }
 const nextTurn = () => new Promise((resolve) => setImmediate(resolve));
 /** The SHA-256 (lowercase hex) and byte size of what an upload of `value` carries, yielding to
- * the event loop between slices so a large value does not stall the application. */
-export async function digestOf(value) {
+ * the event loop between slices so a large value does not stall the application. Past
+ * `deadline`, it throws rather than read on. */
+export async function digestOf(value, deadline = Infinity) {
     const hash = createHash("sha256");
     let size = 0;
-    let slices = 0;
-    for (const chunk of uploadChunks(value)) {
+    for await (const chunk of uploadChunks(value)) {
         hash.update(chunk);
         size += chunk.byteLength;
-        if (++slices % 4 === 0)
-            await nextTurn();
+        await nextTurn();
+        if (Date.now() >= deadline)
+            throw new Error("The upload ran out of time");
     }
     return { sha256: hash.digest("hex"), size };
 }
@@ -258,9 +272,12 @@ export class BlobUploader {
         if (digest.size > MAX_BLOB_BYTES)
             return { ok: false, reason: "tooLarge" };
         const identity = `${traceId}:${digest.sha256}`;
-        const known = this.outcomes.get(identity);
-        if (known)
-            return known.promise;
+        for (let known = this.outcomes.get(identity); known; known = this.outcomes.get(identity)) {
+            const shared = await known.promise;
+            // Another reference's file changed since it was recorded: this one tries its own bytes.
+            if (shared.ok || !shared.changed)
+                return shared;
+        }
         if (this.outcomes.size >= 4096) {
             const oldest = [...this.outcomes].find(([, item]) => item.settled)?.[0];
             if (oldest === undefined)
@@ -308,6 +325,8 @@ export class BlobUploader {
             const put = await this.put(grant, value, digest.size, upload.deadline);
             if (put === "stored")
                 break;
+            if (put === "changed")
+                return { ok: false, reason: "failed", changed: true };
             if (this.closed)
                 return { ok: false, reason: "failed" };
             const expired = put === "expired" || Date.now() >= grant.expiresAt;
@@ -464,14 +483,21 @@ export class BlobUploader {
             grant: { url, headers, expiresAt: Number.isFinite(expiresAt) ? expiresAt : Infinity },
         };
     }
-    /** The PUT of the value's bytes, with exactly the headers Hue signed. */
+    /** The PUT of the value's bytes, with exactly the headers Hue signed. A source that no longer
+     * holds its bytes (a file changed since it was recorded) ends the request before its body is
+     * complete, and fails the value without a retry. */
     async put(grant, value, size, deadline) {
         const reply = await this.request("PUT", grant.url, grant.headers, value, deadline, MAX_ERROR_RESPONSE_BYTES, size);
+        if (isByteSource(value) && value.changed)
+            return "changed";
         if (!reply)
             return "transient";
-        // 412: the object exists already (another upload of the same value won).
-        if ((reply.status >= 200 && reply.status < 300) || reply.status === 412)
+        // 412: the object exists already (another upload of the same value won). A success answered
+        // before the whole body was written stores nothing that was sent.
+        if (reply.status === 412)
             return "stored";
+        if (reply.status >= 200 && reply.status < 300)
+            return reply.complete ? "stored" : "failed";
         if (reply.status === 403)
             return reply.body.toString("utf8").includes("Request has expired") ? "expired" : "failed";
         if (reply.status >= 500 || reply.status === 408 || reply.status === 429)
@@ -501,6 +527,8 @@ export class BlobUploader {
             return undefined;
         return new Promise((resolve) => {
             let settled = false;
+            // Whether the whole body was handed to the request.
+            let ended = false;
             let request;
             const finish = (reply) => {
                 if (settled)
@@ -509,16 +537,20 @@ export class BlobUploader {
                 clearTimeout(timer);
                 if (request) {
                     this.requests.delete(request);
-                    if (!reply)
+                    // An answer before the whole body was handed over ends the request: a store that
+                    // neither reads nor closes would otherwise hold it, and its source, open.
+                    if (!reply || !ended)
                         request.destroy();
                 }
                 resolve(reply);
             };
             const timer = setTimeout(() => finish(), Math.max(1, deadline - Date.now()));
+            // The bytes handed to the request: its body is complete once they are its length.
+            let sentBytes = 0;
+            const written = () => length === undefined || sentBytes === length;
             const requestHeaders = { ...headers };
             if (length === undefined)
-                requestHeaders["Content-Length"] =
-                    typeof body === "string" ? Buffer.byteLength(body, "utf8") : body.byteLength;
+                requestHeaders["Content-Length"] = sizeOf(body);
             try {
                 // Uploads are the SDK's own export traffic: an http instrumentation must not trace them.
                 request = context.with(suppressTracing(context.active()), () => http.request(url, { method, headers: requestHeaders, agent: false }, (response) => {
@@ -528,6 +560,7 @@ export class BlobUploader {
                         status: response.statusCode ?? 0,
                         headers: response.headers,
                         body: Buffer.concat(chunks),
+                        complete: written(),
                     });
                     response.on("data", (chunk) => {
                         size += chunk.byteLength;
@@ -552,26 +585,41 @@ export class BlobUploader {
             sent.on("error", () => finish());
             sent.setTimeout(this.options.timeoutMillis, () => finish());
             void (async () => {
-                if (length === undefined) {
+                if (length === undefined && !isByteSource(body)) {
+                    ended = true;
                     sent.end(body);
                     return;
                 }
-                for (const chunk of uploadChunks(body)) {
+                // A source or a text may reuse a slice's memory for the next: each slice is written out
+                // before the next is read. Bytes are views of memory that does not change.
+                const reused = !(body instanceof Uint8Array);
+                for await (const chunk of uploadChunks(body)) {
                     if (settled)
                         return;
-                    if (!sent.write(chunk))
-                        await new Promise((drained) => {
-                            const done = () => {
-                                sent.off("drain", done);
-                                sent.off("close", done);
-                                drained();
-                            };
+                    sentBytes += chunk.byteLength;
+                    await new Promise((proceed) => {
+                        const done = () => {
+                            sent.off("drain", done);
+                            sent.off("close", done);
+                            proceed();
+                        };
+                        sent.on("close", done);
+                        if (reused) {
+                            sent.write(chunk, () => done());
+                            return;
+                        }
+                        if (sent.write(chunk))
+                            done();
+                        else
                             sent.on("drain", done);
-                            sent.on("close", done);
-                        });
+                    });
+                    if (settled)
+                        return;
                 }
-                if (!settled)
-                    sent.end();
+                if (settled)
+                    return;
+                ended = true;
+                sent.end();
             })().catch(() => finish());
         });
     }
@@ -615,30 +663,46 @@ export async function offloadRecord(attributes, collector, options) {
      * uploaded keeps its bytes inline, except in a recorded message: a span's own message keeps its
      * digest, and a log record's keeps the digest of a file over the inline limit, as nested
      * messages kept them before log records uploaded. */
-    const placeFiles = async (index, key, files) => {
+    const placeFiles = async (index, key, files, long) => {
         const message = isMessageKey(key);
         let changed = false;
         await Promise.all(files.map((file, position) => {
-            const decoded = mediaBytes(file);
-            if (!placeable(file, decoded))
+            // A file whose content is a long string is read from its place in the recorded text, a
+            // slice at a time: never decoded or copied whole.
+            const inPlace = long?.get(file.content);
+            const decoded = inPlace ? longFileBytes(file, inPlace) : mediaBytes(file);
+            const bytes = "source" in decoded ? decoded.source : decoded.bytes;
+            const size = sizeOf(bytes);
+            if (!placeable(file, decoded, size))
                 return Promise.resolve();
             const contentType = mediaContentType(file, decoded);
             const textual = isTextMediaType(contentType);
-            if (textual && decoded.bytes.byteLength <= INLINE_FILE_LIMIT)
+            if (textual && size <= INLINE_FILE_LIMIT)
                 return Promise.resolve();
-            const prefix = keepsPrefix(file, textual);
+            const prefix = keepsPrefix(file, textual, decoded);
             const partKey = `${key}#${prefix ? file.pointer : file.partPointer}`;
             const slot = slots > 0 && isBlobKey(partKey) ? (slots--, true) : false;
             return (async () => {
-                const digest = options.memo
-                    ? options.memo.digest(file, decoded, traceId)
-                    : await digestOf(decoded.bytes);
+                // Hashing a long file takes an upload place, as hashing a large value does; one the
+                // export's upload budget cannot hash stays as recorded and is not uploaded.
+                let digest;
+                try {
+                    digest = isByteSource(bytes)
+                        ? await uploader.limit(() => digestOf(bytes, upload.deadline))
+                        : options.memo
+                            ? options.memo.digest(file, decoded, traceId)
+                            : await digestOf(bytes);
+                }
+                catch {
+                    fallback("failed");
+                    return;
+                }
                 const outcome = slot
-                    ? await uploader.store(traceId, partKey, decoded.bytes, contentType, digest, upload)
+                    ? await uploader.store(traceId, partKey, bytes, contentType, digest, upload)
                     : { ok: false, reason: "budget" };
                 if (outcome.ok) {
                     if (prefix)
-                        prefixPart(file, blobPrefix(file.content));
+                        prefixPart(file, blobPrefix(inPlace ? longHead(inPlace, BLOB_PREFIX_BYTES + 1) : file.content));
                     else
                         digestPart(file, digest);
                     changed = true;
@@ -649,7 +713,7 @@ export async function offloadRecord(attributes, collector, options) {
                     tally.uploaded++;
                 }
                 else {
-                    if (message && (!collector.log || decoded.bytes.byteLength > INLINE_FILE_LIMIT)) {
+                    if (message && (!collector.log || size > INLINE_FILE_LIMIT)) {
                         digestPart(file, digest);
                         changed = true;
                     }
@@ -661,30 +725,59 @@ export async function offloadRecord(attributes, collector, options) {
     };
     const literal = literalKeys(attributes);
     const parts = async (index, key, text) => {
-        // Both bounds count UTF-16 code units, as admission does. Literal text holds no file.
-        if (!mayInlineFiles(key, text) || text.length > MAX_INLINE_FILE_TEXT || literal.has(key))
+        // Literal text holds no file.
+        if (!mayInlineFiles(key, text) || literal.has(key))
             return text;
-        if (collector.inspected) {
-            // A value admission skipped stays as recorded.
-            if (!collector.inspected.has(key))
-                return text;
+        let media;
+        let large;
+        if (collector.large?.has(key)) {
+            // Held whole past the inspection bounds: its long strings were read in place, the JSON
+            // around them within the per-attribute bound.
+            media = large = largeReads.get(index);
         }
         else {
-            if (inspected + text.length > INLINE_FILE_TEXT_PER_RECORD)
+            // Both bounds count UTF-16 code units, as admission does.
+            if (text.length > MAX_INLINE_FILE_TEXT)
                 return text;
-            inspected += text.length;
+            if (collector.inspected) {
+                // A value admission skipped stays as recorded.
+                if (!collector.inspected.has(key))
+                    return text;
+            }
+            else {
+                if (inspected + text.length > INLINE_FILE_TEXT_PER_RECORD)
+                    return text;
+                inspected += text.length;
+            }
+            try {
+                media = readMediaText(key, text);
+            }
+            catch {
+                return text;
+            }
         }
-        let media;
+        if (!media?.files.length)
+            return text;
+        if (!(await placeFiles(index, key, media.files, large?.long)))
+            return text;
+        return large ? await placedLargeText(large) : placedMediaText(media);
+    };
+    // The values read in place are scanned first, in candidate order, between turns of the event
+    // loop: places in `hue.blobs` are then taken in candidate order, as for any value.
+    const largeReads = new Map();
+    for (const [index, { key, value }] of collector.candidates.entries()) {
+        if (typeof value !== "string" ||
+            !collector.large?.has(key) ||
+            !mayInlineFiles(key, value) ||
+            literal.has(key))
+            continue;
         try {
-            media = readMediaText(key, text);
+            largeReads.set(index, await readLargeMediaText(key, value, options.longUnits));
         }
         catch {
-            return text;
+            // JSON that does not parse holds no file: the value stays as recorded.
         }
-        if (!media?.files.length)
-            return text;
-        return (await placeFiles(index, key, media.files)) ? placedMediaText(media) : text;
-    };
+    }
     /** A log record's body with its files placed. Each file's content counts toward the record's
      * inspection bounds, as an attribute's text does. */
     const placeBody = async (body) => {
@@ -710,14 +803,22 @@ export async function offloadRecord(attributes, collector, options) {
         return media.box.value;
     };
     const place = async (index, { key, value }) => {
-        const placed = typeof value === "string" ? await parts(index, key, value) : value;
-        const size = sizeOf(placed);
+        const read = typeof value === "string" ? await parts(index, key, value) : value;
+        // Text joined from pieces of the recorded text is made one string only when it is kept
+        // inline; otherwise it is measured, hashed, uploaded and cut from its pieces.
+        const joined = read instanceof JoinedText ? read : undefined;
+        const size = joined ? joined.utf8Size() : sizeOf(read);
         if (size <= valueBytes) {
-            result[key] = placed;
+            result[key] = joined ? joined.text() : read;
             return;
         }
+        const placed = joined ? joined.source(size) : read;
         listed.push({ order: index, key });
-        const cut = () => typeof placed === "string" ? truncateUtf8(placed, valueBytes) : truncatedMarker(size);
+        const cut = () => joined
+            ? truncateUtf8(joined.head(valueBytes + 1), valueBytes)
+            : typeof placed === "string"
+                ? truncateUtf8(placed, valueBytes)
+                : truncatedMarker(size);
         // A log record's value is cut, as no log reader pages an uploaded text value.
         if (collector.log) {
             result[key] = cut();
@@ -733,13 +834,27 @@ export async function offloadRecord(attributes, collector, options) {
             if (size > MAX_BLOB_BYTES)
                 return { ok: false, reason: "tooLarge" };
             // Hashing a large value takes a slot as its upload does; the upload takes its own.
-            const digest = await uploader.limit(() => digestOf(placed));
-            const contentType = typeof placed === "string" ? textContentType(placed) : "application/octet-stream";
+            let digest;
+            try {
+                digest = await uploader.limit(() => digestOf(placed, upload.deadline));
+            }
+            catch {
+                return { ok: false, reason: "failed" };
+            }
+            const contentType = joined
+                ? textContentType(joined.head(256) + joined.tail(256))
+                : typeof placed === "string"
+                    ? textContentType(placed)
+                    : "application/octet-stream";
             const outcome = await uploader.store(traceId, key, placed, contentType, digest, upload);
             return outcome.ok ? { ...outcome, digest } : outcome;
         })();
         if (stored.ok) {
-            result[key] = typeof placed === "string" ? blobPrefix(placed) : blobPrefix(placed);
+            result[key] = joined
+                ? blobPrefix(joined.head(BLOB_PREFIX_BYTES + 1))
+                : typeof placed === "string"
+                    ? blobPrefix(placed)
+                    : blobPrefix(placed);
             entries.push({
                 order: [index, Infinity],
                 entry: blobEntry(key, stored.digest.sha256, stored.digest.size, stored.contentType),
@@ -754,7 +869,7 @@ export async function offloadRecord(attributes, collector, options) {
     const body = collector.body ? placeBody(collector.body.value) : undefined;
     await Promise.all(collector.candidates.map((candidate, index) => place(index, candidate)));
     await Promise.all((collector.attachments ?? []).map((file, position) => {
-        if (!file.bytes.byteLength)
+        if (!sizeOf(file.bytes))
             return Promise.resolve();
         const slot = slots > 0 ? (slots--, true) : false;
         return (async () => {

package/dist/cli/import.d.ts

@@ -0,0 +1,35 @@
+/**
+ * `hue import langfuse`: copies a Langfuse project's traces into Hue from this machine. Langfuse
+ * is read with the project's own keys through its public API; each observation becomes the span
+ * the Langfuse OpenTelemetry SDKs would have recorded and is exported through the SDK's own
+ * transport, so batching, retries, limits, content capture and large-value uploads are the SDK's.
+ * Ids are kept or derived from Langfuse's, so a second run sends the same spans again and Hue
+ * keeps one copy of each: an interrupted or partly failed import is resumed by running it again.
+ */
+/** Streams, environment and timing seams for {@link runImportCommand}; tests inject these. */
+export interface ImportCommandIo {
+    stdout?: NodeJS.WritableStream;
+    stderr?: NodeJS.WritableStream;
+    stdin?: NodeJS.ReadableStream & {
+        isTTY?: boolean;
+    };
+    env?: NodeJS.ProcessEnv;
+    cwd?: string;
+    fetch?: typeof fetch;
+    /** Waits `ms`, resolving early when the import is stopped. */
+    sleep?: (ms: number, signal: AbortSignal) => Promise<void>;
+    /** Registers the stop request (SIGINT and SIGTERM by default) and returns its removal. */
+    onInterrupt?: (listener: () => void) => () => void;
+    /** The current time, for the default end of the window. */
+    now?: () => Date;
+    /** Ends the process after a second stop request (`process.exit` by default), so exports still
+     * in flight do not hold it until their requests time out. */
+    exit?: (code: number) => void;
+}
+export declare const IMPORT_USAGE = "Usage: hue import langfuse [options]\n\nCopy traces from a Langfuse project into Hue. The import runs on this machine with your own\nLangfuse keys, which are sent only to Langfuse. Each observation becomes a span with the\nlangfuse.* attributes Hue reads; ids are kept, so running the same command again resumes an\nimport without duplicating what was already sent.\n\nLangfuse (keys from the environment or --env-path; the secret key never from the command line):\n  LANGFUSE_PUBLIC_KEY, LANGFUSE_SECRET_KEY  The Langfuse project's API keys\n  LANGFUSE_BASE_URL or LANGFUSE_HOST        Langfuse host (default https://cloud.langfuse.com)\n  --langfuse-host URL                       Langfuse host, instead of the variables\n  --langfuse-public-key KEY                 Public key, instead of LANGFUSE_PUBLIC_KEY\n\nHue:\n  HUE_API_KEY            The Hue project key the traces are sent with (environment, --env-path\n                         or ./.env.hue); not needed with --dry-run\n  --origin URL           Hue origin (default HUE_BASE_URL, then https://app.hue.run)\n  --service-name NAME    service.name of the imported spans (default langfuse)\n\nSelection (newest first):\n  --from TIME            Traces that started at or after TIME: an ISO 8601 date (midnight UTC)\n                         or a date and time with a zone, such as 2026-01-01T09:00:00Z\n  --to TIME              Traces that started before TIME (default: an hour before the command\n                         starts, so traces still running are left for a later run)\n  --limit N              At most N traces\n  --trace-id ID          This Langfuse trace (repeatable); not with --from, --to or --limit\n\nContent:\n  --media                Download the files Langfuse media references (@@@langfuseMedia:\u2026@@@)\n                         point to and upload them to Hue; without it, references stay as text\n  --no-content           Send metadata only: no inputs, outputs, status messages or model\n                         parameters (content is sent by default, as the SDK records it)\n\nRun:\n  --dry-run              Read Langfuse and count what would be imported; send nothing to Hue\n  --yes                  Import without asking for confirmation (required without a terminal)\n  --env-path PATH        Load a dotenv file first (--env-file also works when it exists);\n                         ./.env.hue loads when HUE_API_KEY is not set. Variables already set win\n  -h, --help             Show this help\n\nExit codes: 0 every trace was imported, 1 a trace, span or file was not, 2 usage error,\n130 stopped (run the same command again to resume).";
+/** An instant of the window: an ISO 8601 date (midnight UTC) or a date and time with a zone. */
+export declare function parseInstant(option: string, value: string): string;
+/** The Langfuse host as a base URL without a trailing slash, or why it is refused. */
+export declare function parseLangfuseHost(value: string): string;
+/** Runs `hue import langfuse`; resolves to the exit code. */
+export declare function runImportCommand(argv: string[], io?: ImportCommandIo): Promise<number>;