@juspay/neurolink 13.4.9 → 13.5.0

package/dist/utils/decisionMedia.js

@@ -7,11 +7,13 @@ const MAX_ECHOED_PATH_CHARS = 200;
 const DATA_URL = {
     image: /^data:image\/[a-z0-9.+-]+;base64,/i,
     video: /^data:video\/[a-z0-9.+-]+;base64,/i,
+    audio: /^data:audio\/[a-z0-9.+-]+;base64,/i,
 };
 const WEBM_HEADER = [0x1a, 0x45, 0xdf, 0xa3];
 const NOT_RECOGNISED = {
     image: "is not a recognised image format",
     video: "is not a recognised video format (MP4, MOV or WebM)",
+    audio: "is not a recognised audio format (WAV, FLAC, Ogg or MP3)",
 };
 /**
  * MP4 and MOV share the ISO-BMFF box. An AVIF or HEIC image shares it too, and
@@ -26,6 +28,153 @@ function sniffVideoMimeType(buffer) {
         ? "video/webm"
         : null;
 }
+const ascii = (buffer, start, end) => buffer.subarray(start, end).toString("latin1");
+/**
+ * The audio containers a decision model is known to decode. MP4 audio (M4A,
+ * AAC) is told apart so it can be refused by name: it was measured decoding to
+ * silence, with the request still answered as if it had been heard.
+ */
+function sniffAudioFormat(buffer) {
+    if (ascii(buffer, 0, 4) === "RIFF" && ascii(buffer, 8, 12) === "WAVE") {
+        return "wav";
+    }
+    if (ascii(buffer, 0, 4) === "fLaC") {
+        return "flac";
+    }
+    if (ascii(buffer, 0, 4) === "OggS") {
+        return "ogg";
+    }
+    if (hasFtypBoxSignature(buffer)) {
+        return "aac";
+    }
+    if (ascii(buffer, 0, 3) === "ID3") {
+        return "mp3";
+    }
+    // An MPEG frame sync. Layer bits 00 mark an ADTS (AAC) stream instead.
+    if (buffer.length >= 2 && buffer[0] === 0xff && (buffer[1] & 0xe0) === 0xe0) {
+        return (buffer[1] & 0x06) === 0 ? "aac" : "mp3";
+    }
+    return null;
+}
+/** A WAV clip's length from its `fmt ` byte rate and `data` size, or undefined. */
+function wavSeconds(buffer) {
+    let byteRate;
+    let offset = 12;
+    while (offset + 8 <= buffer.length) {
+        const id = ascii(buffer, offset, offset + 4);
+        const size = buffer.readUInt32LE(offset + 4);
+        const body = offset + 8;
+        if (id === "fmt " && body + 12 <= buffer.length) {
+            byteRate = buffer.readUInt32LE(body + 8);
+        }
+        else if (id === "data") {
+            // A streamed WAV leaves the size unset; what follows is the data.
+            const dataBytes = size === 0 || size === 0xffffffff
+                ? buffer.length - body
+                : Math.min(size, buffer.length - body);
+            return byteRate ? dataBytes / byteRate : undefined;
+        }
+        offset = body + size + (size % 2);
+    }
+    return undefined;
+}
+/** A FLAC clip's length from its STREAMINFO block, or undefined when unset. */
+function flacSeconds(buffer) {
+    // "fLaC", a 4-byte block header, then STREAMINFO.
+    if (buffer.length < 26 || (buffer[4] & 0x7f) !== 0) {
+        return undefined;
+    }
+    const info = 8;
+    const sampleRate = (buffer[info + 10] << 12) |
+        (buffer[info + 11] << 4) |
+        (buffer[info + 12] >> 4);
+    const totalSamples = (buffer[info + 13] & 0x0f) * 2 ** 32 + buffer.readUInt32BE(info + 14);
+    return sampleRate > 0 && totalSamples > 0
+        ? totalSamples / sampleRate
+        : undefined;
+}
+/**
+ * An Ogg clip's length: the granule position of its last page over the stream's
+ * sample rate. Opus counts at 48 kHz from its pre-skip; Vorbis at the rate in
+ * its identification header. Undefined for any other codec, or a stream whose
+ * pages cannot be read.
+ */
+function oggSeconds(buffer) {
+    // The first page carries the codec's identification packet.
+    const segments = buffer[26];
+    if (buffer.length < 28 || segments === undefined) {
+        return undefined;
+    }
+    const packet = 27 + segments;
+    let rate;
+    let preSkip = 0;
+    if (ascii(buffer, packet, packet + 8) === "OpusHead") {
+        rate = 48000;
+        preSkip =
+            buffer.length >= packet + 12 ? buffer.readUInt16LE(packet + 10) : 0;
+    }
+    else if (ascii(buffer, packet, packet + 7) === "\x01vorbis") {
+        rate =
+            buffer.length >= packet + 16
+                ? buffer.readUInt32LE(packet + 12)
+                : undefined;
+    }
+    const last = buffer.lastIndexOf("OggS");
+    if (!rate ||
+        last < 0 ||
+        last + 14 > buffer.length ||
+        buffer[last + 4] !== 0) {
+        return undefined;
+    }
+    const granule = buffer.readBigInt64LE(last + 6);
+    if (granule <= 0n) {
+        return undefined;
+    }
+    return Math.max(0, Number(granule) - preSkip) / rate;
+}
+/**
+ * The clip in wire form, or why it cannot be sent. A clip whose length can be
+ * read (WAV, FLAC, Ogg Opus or Vorbis) and that runs past what the model hears is
+ * refused; the model would answer from its start alone. An MP3's length is not
+ * read: it needs every frame header walked.
+ */
+function encodeAudio(buffer, limits, label) {
+    const format = sniffAudioFormat(buffer);
+    if (format === null) {
+        return { status: "error", message: `${label} ${NOT_RECOGNISED.audio}.` };
+    }
+    if (format === "aac") {
+        return {
+            status: "error",
+            message: `${label} is AAC or M4A audio, which the model does not decode. Convert it to WAV, FLAC, Ogg or MP3.`,
+        };
+    }
+    const seconds = format === "wav"
+        ? wavSeconds(buffer)
+        : format === "flac"
+            ? flacSeconds(buffer)
+            : format === "ogg"
+                ? oggSeconds(buffer)
+                : undefined;
+    // Every WAV the model can open has both chunks; the server answers one
+    // without them with an engine error, which reads as a fault worth retrying.
+    if (format === "wav" && seconds === undefined) {
+        return {
+            status: "error",
+            message: `${label} is a WAV file without a complete fmt and data chunk.`,
+        };
+    }
+    if (seconds !== undefined && seconds > limits.maxSeconds) {
+        return {
+            status: "error",
+            message: `${label} runs ${seconds.toFixed(1)} seconds; the model hears at most ${limits.maxSeconds}. Trim it, or split it across requests.`,
+        };
+    }
+    return {
+        status: "ok",
+        audio: { data: buffer.toString("base64"), format },
+    };
+}
 function encode(buffer, kind, label) {
     const mime = kind === "image" ? sniffImageMimeType(buffer) : sniffVideoMimeType(buffer);
     return mime === null
@@ -73,14 +222,15 @@ async function readMediaFile(path, label, maxBytes) {
     }
 }
 /**
- * Nothing here echoes a `data:` URL or a remote URL: either can carry a
+ * A source resolved for one kind of media, with every refusal the three kinds
+ * share. Nothing here echoes a `data:` URL or a remote URL: either can carry a
  * credential or megabytes of base64, and neither belongs in an error message.
  */
-async function toDataUrl(source, kind, label, maxBytes) {
+async function readMediaSource(source, kind, label, maxBytes) {
     if (typeof source !== "string") {
         return source.length === 0
             ? { status: "error", message: `${label} is an empty Buffer.` }
-            : encode(source, kind, label);
+            : { status: "buffer", buffer: source };
     }
     const value = source.trim();
     if (value === "") {
@@ -94,7 +244,7 @@ async function toDataUrl(source, kind, label, maxBytes) {
     }
     if (/^data:/i.test(value)) {
         return DATA_URL[kind].test(value)
-            ? { status: "ok", dataUrl: value }
+            ? { status: "dataUrl", dataUrl: value }
             : {
                 status: "error",
                 message: `${label} is a data: URL that is not base64 ${kind} content.`,
@@ -107,24 +257,59 @@ async function toDataUrl(source, kind, label, maxBytes) {
         };
     }
     const file = await readMediaFile(value, label, maxBytes);
-    return file.status === "ok" ? encode(file.buffer, kind, label) : file;
+    return file.status === "ok"
+        ? { status: "buffer", buffer: file.buffer }
+        : file;
+}
+/** An image or video as a `data:` URL: bytes are identified first, a URL of the right kind is sent as given. */
+async function toDataUrl(source, kind, label, maxBytes) {
+    const read = await readMediaSource(source, kind, label, maxBytes);
+    if (read.status === "buffer") {
+        return encode(read.buffer, kind, label);
+    }
+    return read.status === "dataUrl"
+        ? { status: "ok", dataUrl: read.dataUrl }
+        : read;
+}
+/** An audio clip as bytes: a `data:audio/…` URL is decoded, since the wire takes raw base64 and the container is sniffed. */
+async function readAudioSource(source, label, maxBytes) {
+    const read = await readMediaSource(source, "audio", label, maxBytes);
+    if (read.status === "dataUrl") {
+        const base64 = read.dataUrl.slice(read.dataUrl.indexOf(",") + 1);
+        return { status: "ok", buffer: Buffer.from(base64, "base64") };
+    }
+    return read.status === "buffer"
+        ? { status: "ok", buffer: read.buffer }
+        : read;
 }
 /**
- * Turn the caller's images and video into `data:` URLs, or say why not.
- * Runs before any request, so every refusal costs nothing and none is retried.
+ * Turn the caller's images and video into `data:` URLs, and an audio clip into
+ * `{ data, format }`, or say why not. Runs before any request, so every
+ * refusal costs nothing and none is retried.
  */
 export async function prepareDecisionMedia(request, limits, vendor, alternatives) {
     const images = request.images ?? [];
-    if (images.length === 0 && request.video === undefined) {
+    if (images.length === 0 &&
+        request.video === undefined &&
+        request.audio === undefined) {
         return { status: "prepared", media: undefined };
     }
+    const use = alternatives.length > 0
+        ? ` Use a decision provider that does: ${alternatives.join(", ")}.`
+        : "";
     if (!limits) {
-        const use = alternatives.length > 0
-            ? ` Use a decision provider that does: ${alternatives.join(", ")}.`
-            : "";
+        const what = images.length === 0 && request.video === undefined
+            ? "audio"
+            : "images or video";
         return {
             status: "refused",
-            message: `${vendor} does not accept images or video.${use}`,
+            message: `${vendor} does not accept ${what}.${use}`,
+        };
+    }
+    if (request.audio !== undefined && limits.audio === undefined) {
+        return {
+            status: "refused",
+            message: `${vendor} does not accept audio.${use}`,
         };
     }
     if (images.length > limits.maxImages) {
@@ -152,9 +337,26 @@ export async function prepareDecisionMedia(request, limits, vendor, alternatives
         }
         video = converted.dataUrl;
     }
-    const bytes = prepared.reduce((sum, url) => sum + url.length, video?.length ?? 0);
+    let audio;
+    if (request.audio !== undefined && limits.audio !== undefined) {
+        const read = await readAudioSource(request.audio, "The audio", limits.maxRequestBytes);
+        if (read.status === "error") {
+            return { status: "refused", message: read.message };
+        }
+        const encoded = encodeAudio(read.buffer, limits.audio, "The audio");
+        if (encoded.status === "error") {
+            return { status: "refused", message: encoded.message };
+        }
+        audio = encoded.audio;
+    }
+    const bytes = prepared.reduce((sum, url) => sum + url.length, (video?.length ?? 0) + (audio?.data.length ?? 0));
     return {
         status: "prepared",
-        media: { images: prepared, ...(video ? { video } : {}), bytes },
+        media: {
+            images: prepared,
+            ...(video ? { video } : {}),
+            ...(audio ? { audio } : {}),
+            bytes,
+        },
     };
 }