@hue-run/sdk 0.18.0 → 0.18.2

package/dist/attachment.js

@@ -1,6 +1,9 @@
 import { createHash } from "node:crypto";
-import { closeSync, fstatSync, openSync, readSync } from "node:fs";
+import { closeSync, constants, fstatSync, openSync, readSync } from "node:fs";
+import { open } from "node:fs/promises";
+import { resolve } from "node:path";
 import { types as utilTypes } from "node:util";
+import { MAX_BLOB_BYTES } from "./blobs.js";
 import { MAX_FILE_DATA_BYTES } from "./config.js";
 import { inlineFileBytes, isMediaContentType, isMediaName } from "./inline-files.js";
 /** How deep in a value an attachment's part may sit and still be uploaded: Hue reads media parts
@@ -19,41 +22,137 @@ function stringBytes(text) {
         return undefined;
     return Buffer.from(text, "base64");
 }
-/** A file's bytes, read through one descriptor and never past the limit. */
-function fileBytes(path) {
-    const descriptor = openSync(path, "r");
+/** How much of a file given by path is read at once, to hash it or to upload it. */
+const FILE_SLICE = 1024 * 1024;
+/**
+ * A regular file given by path: its bytes when it is no larger than 25 MiB, and otherwise, up to
+ * Hue's 1 GB upload limit, its size and SHA-256 from one reading pass a slice at a time, never
+ * holding it whole. Undefined for a file over its limit and for anything that is no regular file
+ * (a named pipe, a device), which is opened without waiting and never read; throws for one that
+ * cannot be opened.
+ */
+export function readPath(path) {
+    // Resolved first: the file read now is the one read again when it is uploaded.
+    const absolute = resolve(path);
+    // Without waiting: a named pipe would block the application until something wrote to it.
+    const descriptor = openSync(absolute, constants.O_RDONLY | (constants.O_NONBLOCK ?? 0));
     try {
-        if (fstatSync(descriptor).size > MAX_FILE_DATA_BYTES)
+        const info = fstatSync(descriptor);
+        if (!info.isFile())
+            return undefined;
+        if (info.size <= MAX_FILE_DATA_BYTES) {
+            const bytes = fileBytes(descriptor);
+            return bytes ? { bytes } : undefined;
+        }
+        if (info.size > MAX_BLOB_BYTES)
             return undefined;
-        // Read to its end, which may lie past its size as stated (a file that grew, a pipe), but
-        // never more than one byte past the limit.
-        const parts = [];
-        let total = 0;
+        const hash = createHash("sha256");
+        const slice = Buffer.allocUnsafe(FILE_SLICE);
+        let size = 0;
         for (;;) {
-            const room = Math.min(1 << 20, MAX_FILE_DATA_BYTES + 1 - total);
-            if (room <= 0)
-                return undefined;
-            const buffer = Buffer.alloc(room);
-            const read = readSync(descriptor, buffer, 0, room, null);
+            const read = readSync(descriptor, slice, 0, FILE_SLICE, null);
             if (read === 0)
                 break;
-            parts.push(buffer.subarray(0, read));
-            total += read;
+            size += read;
+            // A file that grew past the limit while it was read is refused, not read on.
+            if (size > MAX_BLOB_BYTES)
+                return undefined;
+            hash.update(slice.subarray(0, read));
         }
-        return Buffer.concat(parts, total);
+        return { file: { path: absolute, size, sha256: hash.digest("hex") } };
     }
     finally {
         closeSync(descriptor);
     }
 }
+/**
+ * A file given by path as an upload body, read from disk a slice at a time. Each slice is hashed as
+ * it is read and sent once the next one is read; the last is sent only once the file ended at its
+ * recorded size with its recorded SHA-256. A file that changed since it was recorded, or can no
+ * longer be read, ends the read with an error before its body is complete and sets `changed`, so
+ * the upload is abandoned and the value is counted as not uploaded; so does a path that is no
+ * longer a regular file, which is opened without waiting (a named pipe in its place would wait for
+ * a writer) and never read. Hue's presigned PUT also signs the size and SHA-256, so the store
+ * refuses a body that differs.
+ */
+export function fileSource(file) {
+    const source = {
+        size: file.size,
+        async *chunks() {
+            // Each read starts fresh: a retried upload reads the file again.
+            source.changed = false;
+            const changed = () => {
+                source.changed = true;
+                return new Error("The file changed after it was recorded");
+            };
+            const handle = await open(file.path, constants.O_RDONLY | (constants.O_NONBLOCK ?? 0)).catch(() => {
+                throw changed();
+            });
+            try {
+                const info = await handle.stat().catch(() => undefined);
+                if (!info?.isFile())
+                    throw changed();
+                const hash = createHash("sha256");
+                let size = 0;
+                let pending;
+                // Two buffers in turn: the slice withheld until the next is read, and the next. A reader
+                // is done with a slice before it asks for the next, so each buffer is free when reused.
+                const buffers = [Buffer.allocUnsafe(FILE_SLICE), Buffer.allocUnsafe(FILE_SLICE)];
+                for (let turn = 0;; turn ^= 1) {
+                    const slice = buffers[turn];
+                    const { bytesRead } = await handle.read(slice, 0, FILE_SLICE, null).catch(() => {
+                        throw changed();
+                    });
+                    if (bytesRead === 0)
+                        break;
+                    size += bytesRead;
+                    if (size > file.size)
+                        throw changed();
+                    const read = slice.subarray(0, bytesRead);
+                    hash.update(read);
+                    if (pending)
+                        yield pending;
+                    pending = read;
+                }
+                if (size !== file.size || hash.digest("hex") !== file.sha256)
+                    throw changed();
+                if (pending)
+                    yield pending;
+            }
+            finally {
+                await handle.close().catch(() => undefined);
+            }
+        },
+    };
+    return source;
+}
+/** A file's bytes, read through one descriptor and never past the in-memory limit. */
+function fileBytes(descriptor) {
+    // Read to its end, which may lie past its size as stated (a file that grew), but never more
+    // than one byte past the limit.
+    const parts = [];
+    let total = 0;
+    for (;;) {
+        const room = Math.min(1 << 20, MAX_FILE_DATA_BYTES + 1 - total);
+        if (room <= 0)
+            return undefined;
+        const buffer = Buffer.alloc(room);
+        const read = readSync(descriptor, buffer, 0, room, null);
+        if (read === 0)
+            break;
+        parts.push(buffer.subarray(0, read));
+        total += read;
+    }
+    return Buffer.concat(parts, total);
+}
 function attachmentBytes(options) {
     const { data, path } = options;
     if ((data === undefined) === (path === undefined))
         return undefined;
+    if (path !== undefined)
+        return typeof path === "string" && path ? readPath(path) : undefined;
     let bytes;
-    if (typeof path === "string")
-        bytes = path ? fileBytes(path) : undefined;
-    else if (typeof data === "string") {
+    if (typeof data === "string") {
         if (data.length > 4 * MAX_FILE_DATA_BYTES)
             return undefined;
         bytes = stringBytes(data);
@@ -70,7 +169,7 @@ function attachmentBytes(options) {
             return undefined;
         bytes = new Uint8Array(data.slice(0));
     }
-    return bytes && bytes.byteLength <= MAX_FILE_DATA_BYTES ? bytes : undefined;
+    return bytes && bytes.byteLength <= MAX_FILE_DATA_BYTES ? { bytes } : undefined;
 }
 /**
  * A file placed in a recorded value, the equivalent of Langfuse's `LangfuseMedia` and
@@ -80,10 +179,16 @@ function attachmentBytes(options) {
  * names the file by digest (`{ type: "blob", mime_type, modality, sha256, size, name }`) and
  * uploads its bytes during export, listed under the record's `hue.blobs`.
  *
- * The bytes are read and hashed when the attachment is created; `data` and `path` are limited to
- * 25 MiB. Creating one never throws: an attachment without exactly one of `data` and `path`, with
- * unreadable data, larger data or an invalid `contentType` records `null` where it is placed and
- * counts an instrumentation failure; an invalid `name` alone is omitted and counted.
+ * The bytes are read and hashed when the attachment is created; `data` is limited to 25 MiB, as
+ * the bytes are held in memory until they are uploaded. A `path` names a regular file, opened
+ * without waiting: anything else, such as a named pipe, is invalid. One up to 25 MiB is read then
+ * too; a larger regular file, up to Hue's 1 GB upload limit, is hashed then a slice at a time and read
+ * again from disk when it is uploaded, so it must stay in place and unchanged until the client
+ * flushes: a file whose size or SHA-256 then differs is not uploaded and counts as an upload
+ * fallback, its part keeping the digest it was recorded with. Creating one never throws: an
+ * attachment without exactly one of `data` and `path`, with unreadable data, larger data or an
+ * invalid `contentType` records `null` where it is placed and counts an instrumentation failure;
+ * an invalid `name` alone is omitted and counted.
  */
 export class Attachment {
     /** The media type given. */
@@ -97,21 +202,24 @@ export class Attachment {
     constructor(options) {
         const contentType = options?.contentType;
         const name = options?.name;
-        let bytes;
+        let read;
         try {
-            bytes = isMediaContentType(contentType) ? attachmentBytes(options) : undefined;
+            read = isMediaContentType(contentType) ? attachmentBytes(options) : undefined;
         }
         catch {
             // An unreadable path, or data that is not bytes: the attachment is invalid.
-            bytes = undefined;
+            read = undefined;
         }
-        const sha256 = bytes ? createHash("sha256").update(bytes).digest("hex") : undefined;
+        const bytes = read && "bytes" in read ? read.bytes : undefined;
+        const file = read && "file" in read ? read.file : undefined;
+        const sha256 = bytes ? createHash("sha256").update(bytes).digest("hex") : file?.sha256;
         this.contentType = typeof contentType === "string" ? contentType : "";
         this.name = isMediaName(name) ? name : undefined;
         this.sha256 = sha256;
-        this.size = bytes?.byteLength;
+        this.size = bytes?.byteLength ?? file?.size;
         states.set(this, {
-            ...(bytes ? { bytes, sha256 } : {}),
+            ...(bytes ? { bytes, sha256, size: bytes.byteLength } : {}),
+            ...(file ? { file, sha256, size: file.size } : {}),
             contentType: this.contentType,
             ...(this.name === undefined ? {} : { name: this.name }),
             invalidName: name !== undefined && name !== null && this.name === undefined,
@@ -144,14 +252,14 @@ export function modalityOf(contentType) {
 /** An attachment's digest-form GenAI part, the name only with content capture; undefined for an
  * invalid attachment. */
 export function attachmentPart(state, captureContent) {
-    if (!state.bytes || !state.sha256)
+    if (state.size === undefined || !state.sha256)
         return undefined;
     return {
         type: "blob",
         mime_type: state.contentType,
         modality: modalityOf(state.contentType),
         sha256: state.sha256,
-        size: state.bytes.byteLength,
+        size: state.size,
         ...(captureContent && state.name !== undefined ? { name: state.name } : {}),
     };
 }

package/dist/blobs.d.ts

@@ -1,5 +1,6 @@
 import { type HrTime } from "@opentelemetry/api";
 import { type MediaDigestMemo, type FileDigest } from "./inline-files.js";
+import { type ByteSource } from "./large-media.js";
 /**
  * Large values and binary media travel apart from their span through reservation, presigned PUT,
  * and completion. Text keeps a 16 KiB prefix. Binary media keeps digest metadata on its part.
@@ -43,6 +44,9 @@ export interface OffloadCollector {
      * export inspects only these. Absent for a record admission did not copy, and for a log record,
      * which export inspects within its own allowance. */
     inspected?: ReadonlySet<string> | undefined;
+    /** The span's own attributes admission held whole for upload past the inspection bounds: export
+     * reads their media in place (`readLargeMediaText`). */
+    large?: ReadonlySet<string> | undefined;
     /** Set for a log record: its values over the inline limit are cut, never uploaded whole; only
      * the media in its inspected attributes and its body upload, with the attachments placed in
      * its body (`attachments`, keyed `body#<pointer>`). */
@@ -56,7 +60,8 @@ export interface OffloadCollector {
     bodyCut?: Set<string>;
 }
 export interface MediaAttachment {
-    bytes: Uint8Array;
+    /** The file's bytes, or for a file given by path over the in-memory limit, its source on disk. */
+    bytes: Uint8Array | ByteSource;
     digest: FileDigest;
     contentType: string;
     name?: string;
@@ -88,9 +93,14 @@ export declare function textContentType(text: string): string;
  * as its own copy so the whole value is not kept alive by it. */
 export declare function blobPrefix(value: string): string;
 export declare function blobPrefix(value: Uint8Array): Uint8Array;
+/** A value's size in bytes as it is uploaded: a string's UTF-8 encoding, counted, not made. */
+export declare function sizeOf(value: string | Uint8Array | ByteSource): number;
+/** The bytes a value holds in memory: none for a source read as it is uploaded. */
+export declare function heldBytesOf(value: Uint8Array | ByteSource): number;
 /** The SHA-256 (lowercase hex) and byte size of what an upload of `value` carries, yielding to
- * the event loop between slices so a large value does not stall the application. */
-export declare function digestOf(value: string | Uint8Array): Promise<{
+ * the event loop between slices so a large value does not stall the application. Past
+ * `deadline`, it throws rather than read on. */
+export declare function digestOf(value: string | Uint8Array | ByteSource, deadline?: number): Promise<{
     sha256: string;
     size: number;
 }>;
@@ -102,12 +112,15 @@ export interface UploadContext {
     deadline: number;
     wait(millis: number): Promise<boolean>;
 }
+/** An upload's outcome; `changed` when the value's own source no longer held its bytes (a file
+ * that changed since it was recorded), which says nothing of another reference's bytes. */
 type Outcome = {
     ok: true;
     contentType: string;
 } | {
     ok: false;
     reason: FallbackReason;
+    changed?: boolean;
 };
 /** The upload route's options, from the transport's validated options. */
 export interface UploaderOptions {
@@ -153,7 +166,7 @@ export declare class BlobUploader {
      * trace is already storing shares that upload's outcome without one, so repeated files never
      * hold the slots new files need. The caller must not hold a slot.
      */
-    store(traceId: string, key: string, value: string | Uint8Array, contentType: string, digest: FileDigest, upload: UploadContext): Promise<Outcome>;
+    store(traceId: string, key: string, value: string | Uint8Array | ByteSource, contentType: string, digest: FileDigest, upload: UploadContext): Promise<Outcome>;
     private storeFresh;
     private jsonHeaders;
     /** Reserves the value: Hue answers that it holds it already, or a presigned PUT. A rate limit
@@ -162,7 +175,9 @@ export declare class BlobUploader {
     private delay;
     /** A 200 answer read: `exists`, or an upload whose URL and headers are usable as given. */
     private reservation;
-    /** The PUT of the value's bytes, with exactly the headers Hue signed. */
+    /** The PUT of the value's bytes, with exactly the headers Hue signed. A source that no longer
+     * holds its bytes (a file changed since it was recorded) ends the request before its body is
+     * complete, and fails the value without a retry. */
     private put;
     /**
      * One HTTP request with a deadline and an idle timeout, never following a redirect, its
@@ -181,6 +196,8 @@ export interface OffloadOptions {
     upload: UploadContext;
     tally: OffloadTally;
     memo?: MediaDigestMemo;
+    /** For tests: the length past which a long string is read in place (`LONG_STRING_UNITS`). */
+    longUnits?: number;
 }
 /**
  * A span's own attributes with each candidate placed: binary media becomes a digest, while