functionalscript 0.32.4 → 0.34.0

package/fs/mime/module.f.d.ts

@@ -1,10 +1,15 @@
 /**
  * Magic-byte MIME type detection.
  *
- * A pure table lookup over the leading bytes of a `Vec`: no I/O, no
- * dependencies beyond `fs/types/bit_vec`. `detect` returns a MIME type string
- * for the container formats whose signatures it knows, or `null` for anything
- * else — text, unknown binary, or a `Vec` too short to match.
+ * `detect` is a pure table lookup over the leading bytes of a `Vec`: it returns a
+ * MIME type string for the container formats whose signatures it knows, or `null`
+ * for anything else — text, unknown binary, or a `Vec` too short to match.
+ *
+ * Beside it, `detectStream` is the **streaming counterpart**: a byte-accepting
+ * state machine (length × magic-byte eliminator × UTF-8 validity DFA) that derives
+ * `{ length, mime_type, type }` by folding a CAS read stream in O(1) space, without
+ * ever buffering the blob into a single `maxLength`-bounded `Vec`. See the README
+ * for the factored design.
  *
  * The CAS store is type-agnostic and keeps raw bytes only, so type is never
  * stored; it is recovered on read by sniffing the content. Callers decide what
@@ -29,11 +34,102 @@
  */
 import { type Vec } from '../types/bit_vec/module.f.ts';
 import type { Nullable } from '../types/nullable/module.f.ts';
+import { type Effect, type Operation } from '../effects/module.f.ts';
+import type { List } from '../effects/list/module.f.ts';
+import type { IoResult } from '../effects/node/module.f.ts';
+import { type Utf8State } from '../text/utf8/module.f.ts';
 /**
  * Detects the MIME type of `bytes` from its leading magic-byte signature.
  *
- * @returns the MIME type string for a recognised format, or `null` when the
+ * @returns the MIME type string for a recognized format, or `null` when the
  *   leading bytes match no known signature (including any `Vec` shorter than
  *   the signature it might otherwise match).
  */
 export declare const detect: (bytes: Vec) => Nullable<string>;
+/**
+ * A magic-byte signature as a byte pattern. `null` entries are wildcards (the
+ * four little-endian size bytes of WebP, between its `RIFF` and `WEBP` markers).
+ */
+type Signature = {
+    readonly pattern: readonly Nullable<number>[];
+    readonly mime: string;
+};
+/**
+ * `A_magic`: signature elimination. `scan` holds the byte offset and the still-viable
+ * signatures; a fully matched signature absorbs into `matched`, an empty viable set
+ * into `dead`. Settles within 12 bytes — `matched`/`dead` are absorbing.
+ */
+type MagicState = {
+    readonly tag: 'scan';
+    readonly pos: number;
+    readonly viable: readonly Signature[];
+} | {
+    readonly tag: 'matched';
+    readonly mime: string;
+} | {
+    readonly tag: 'dead';
+};
+/**
+ * `A_utf8`: a streaming UTF-8 validity-and-text check riding the shared
+ * `utf8ByteToCodePointOp` decoder. `st` is the decoder's mid-sequence state;
+ * `valid` is `false` once an illegal byte, surrogate, or out-of-range code point
+ * is seen — `valid: false` is absorbing. A non-null `st` at EOF (a truncated
+ * multi-byte sequence) is invalid. `text` is the orthogonal text-ness verdict: it
+ * is `false` once a non-text (control) code point is decoded, even though that
+ * code point is perfectly well-formed UTF-8 — `text: false` is absorbing too.
+ * Keeping the two distinct lets a valid-but-control blob (e.g. NUL) decode
+ * cleanly yet still classify as binary.
+ */
+type Utf8Detect = {
+    readonly st: Utf8State;
+    readonly valid: boolean;
+    readonly text: boolean;
+};
+/**
+ * The product state: running bit length × magic eliminator × UTF-8 validator.
+ * The factors never read each other; they meet only in {@link finish}.
+ */
+export type DetectState = {
+    readonly length: bigint;
+    readonly magic: MagicState;
+    readonly utf8: Utf8Detect;
+};
+/** The initial detector state `q₀`. */
+export declare const detectInit: DetectState;
+/**
+ * Folds one `Vec` chunk into the detector state (`δ` over a whole chunk). Length
+ * always advances by the chunk's bit length; per-byte iteration stops as soon as
+ * the verdict is fixed (see {@link isSettled}), so large blobs — including large
+ * magic-matched ones — cost ≈ length counting.
+ */
+export declare const push: (s: DetectState) => (chunk: Vec) => DetectState;
+/** The metadata read off the detector at end-of-stream. */
+export type DetectMeta = {
+    readonly length: bigint;
+    readonly mime_type: string;
+    readonly type: 'text' | 'base64';
+};
+/**
+ * Reads the answer off the final state (`λ`). Reproduces the three-way result of
+ * the pure path: magic hit → `base64` + detected mime; else whole-blob-valid UTF-8
+ * that is also all-text (byte-aligned, no invalidity, no control bytes) → `text` +
+ * `text/plain`; else → `base64` + `application/octet-stream`. A valid-but-control
+ * blob (NUL, other controls) is well-formed UTF-8 yet falls through to the binary
+ * branch.
+ */
+export declare const finish: (s: DetectState) => DetectMeta;
+/**
+ * Classifies a whole `Vec` with the same state machine as {@link detectStream}.
+ * The single-buffer counterpart for callers that already hold the bytes (the
+ * `cas_get` `content: true` path materializes the blob anyway): both paths read
+ * the three-way `{ length, mime_type, type }` verdict from one machine instead of
+ * re-deriving it from `detect` + a separate UTF-8 check.
+ */
+export declare const detectVec: (bytes: Vec) => DetectMeta;
+/**
+ * Folds a CAS read stream through {@link push} and reads {@link finish} at EOF,
+ * deriving `cas_get` metadata without ever materializing the blob. A read `error`
+ * item short-circuits into the `IoResult` error.
+ */
+export declare const detectStream: <O extends Operation>(stream: List<O, IoResult<Vec>>) => Effect<O, IoResult<DetectMeta>>;
+export {};

package/fs/mime/module.f.js

@@ -1,10 +1,15 @@
 /**
  * Magic-byte MIME type detection.
  *
- * A pure table lookup over the leading bytes of a `Vec`: no I/O, no
- * dependencies beyond `fs/types/bit_vec`. `detect` returns a MIME type string
- * for the container formats whose signatures it knows, or `null` for anything
- * else — text, unknown binary, or a `Vec` too short to match.
+ * `detect` is a pure table lookup over the leading bytes of a `Vec`: it returns a
+ * MIME type string for the container formats whose signatures it knows, or `null`
+ * for anything else — text, unknown binary, or a `Vec` too short to match.
+ *
+ * Beside it, `detectStream` is the **streaming counterpart**: a byte-accepting
+ * state machine (length × magic-byte eliminator × UTF-8 validity DFA) that derives
+ * `{ length, mime_type, type }` by folding a CAS read stream in O(1) space, without
+ * ever buffering the blob into a single `maxLength`-bounded `Vec`. See the README
+ * for the factored design.
  *
  * The CAS store is type-agnostic and keeps raw bytes only, so type is never
  * stored; it is recovered on read by sniffing the content. Callers decide what
@@ -27,7 +32,12 @@
  *
  * @module
  */
-import { msb, fromSentinel, length } from "../types/bit_vec/module.f.js";
+import { msb, fromSentinel, length, u8List } from "../types/bit_vec/module.f.js";
+import { iterable } from "../types/list/module.f.js";
+import { pure } from "../effects/module.f.js";
+import { ok, error } from "../types/result/module.f.js";
+import { isValidCodePoint, isTextCodePoint } from "../text/code_point/module.f.js";
+import { utf8ByteToCodePointOp } from "../text/utf8/module.f.js";
 const { startsWith, removeFront } = msb;
 // Each signature is written as a hex literal whose leading `1` nibble is a
 // sentinel marking the start of the byte run (so leading zero bytes survive)
@@ -61,7 +71,7 @@ const isWebp = (bytes) => length(bytes) >= 96n
 /**
  * Detects the MIME type of `bytes` from its leading magic-byte signature.
  *
- * @returns the MIME type string for a recognised format, or `null` when the
+ * @returns the MIME type string for a recognized format, or `null` when the
  *   leading bytes match no known signature (including any `Vec` shorter than
  *   the signature it might otherwise match).
  */
@@ -76,3 +86,146 @@ export const detect = (bytes) => {
     }
     return null;
 };
+// The streaming counterpart of `table`/`isWebp`: the same signatures expressed as
+// byte patterns the eliminator can consume one byte at a time. WebP's gap is the
+// only wildcard run.
+const signatures = [
+    { pattern: [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a], mime: 'image/png' },
+    { pattern: [0xff, 0xd8, 0xff], mime: 'image/jpeg' },
+    { pattern: [0x47, 0x49, 0x46, 0x38, 0x37, 0x61], mime: 'image/gif' },
+    { pattern: [0x47, 0x49, 0x46, 0x38, 0x39, 0x61], mime: 'image/gif' },
+    { pattern: [0x25, 0x50, 0x44, 0x46, 0x2d], mime: 'application/pdf' },
+    { pattern: [0x50, 0x4b, 0x03, 0x04], mime: 'application/zip' },
+    { pattern: [0x50, 0x4b, 0x05, 0x06], mime: 'application/zip' },
+    { pattern: [0x50, 0x4b, 0x07, 0x08], mime: 'application/zip' },
+    {
+        pattern: [0x52, 0x49, 0x46, 0x46, null, null, null, null, 0x57, 0x45, 0x42, 0x50],
+        mime: 'image/webp',
+    },
+];
+const magicInit = { tag: 'scan', pos: 0, viable: signatures };
+const magicStep = (m, byte) => {
+    if (m.tag !== 'scan') {
+        return m;
+    }
+    const { pos } = m;
+    const viable = m.viable.filter(s => {
+        const p = s.pattern[pos];
+        return p === null || p === byte;
+    });
+    for (const s of viable) {
+        if (s.pattern.length === pos + 1) {
+            return { tag: 'matched', mime: s.mime };
+        }
+    }
+    return viable.length === 0 ? { tag: 'dead' } : { tag: 'scan', pos: pos + 1, viable };
+};
+const magicMime = (m) => m.tag === 'matched' ? m.mime : null;
+const utf8Init = { st: null, valid: true, text: true };
+const utf8Step = (u, byte) => {
+    if (!u.valid) {
+        return u;
+    }
+    const [cps, st] = utf8ByteToCodePointOp(byte, u.st);
+    let text = u.text;
+    for (const cp of cps) {
+        if (!isValidCodePoint(cp)) {
+            return { st, valid: false, text };
+        }
+        if (!isTextCodePoint(cp)) {
+            text = false;
+        }
+    }
+    return { st, valid: true, text };
+};
+const utf8Valid = (u) => u.valid && u.st === null;
+// A blob is text only when it is whole-blob-valid UTF-8 *and* every decoded code
+// point is a text code point (no NUL/other controls).
+const utf8Text = (u) => utf8Valid(u) && u.text;
+/** The initial detector state `q₀`. */
+export const detectInit = {
+    length: 0n,
+    magic: magicInit,
+    utf8: utf8Init,
+};
+// The outcome can no longer change — `push` may stop decoding and only count
+// length — once `finish` is pinned down. A magic `matched` pins it on its own
+// (`finish` returns the detected mime and ignores the utf8 verdict), so we must
+// not wait for utf8 to go invalid (it may stay valid forever, e.g. an ASCII PDF).
+// A magic `dead` leaves text-vs-octet open, so it settles only once utf8 can no
+// longer be text — either invalid or a control byte seen (both absorbing); `scan`
+// is never settled.
+const isSettled = (magic, utf8) => {
+    switch (magic.tag) {
+        case 'matched': return true;
+        case 'dead': return !utf8.valid || !utf8.text;
+        case 'scan': return false;
+    }
+};
+/**
+ * Folds one `Vec` chunk into the detector state (`δ` over a whole chunk). Length
+ * always advances by the chunk's bit length; per-byte iteration stops as soon as
+ * the verdict is fixed (see {@link isSettled}), so large blobs — including large
+ * magic-matched ones — cost ≈ length counting.
+ */
+export const push = (s) => (chunk) => {
+    const bits = length(chunk);
+    let magic = s.magic;
+    let utf8 = s.utf8;
+    if (!isSettled(magic, utf8)) {
+        for (const byte of iterable(u8List(msb)(chunk))) {
+            magic = magicStep(magic, byte);
+            utf8 = utf8Step(utf8, byte);
+            if (isSettled(magic, utf8)) {
+                break;
+            }
+        }
+    }
+    return { length: s.length + bits, magic, utf8 };
+};
+/**
+ * Reads the answer off the final state (`λ`). Reproduces the three-way result of
+ * the pure path: magic hit → `base64` + detected mime; else whole-blob-valid UTF-8
+ * that is also all-text (byte-aligned, no invalidity, no control bytes) → `text` +
+ * `text/plain`; else → `base64` + `application/octet-stream`. A valid-but-control
+ * blob (NUL, other controls) is well-formed UTF-8 yet falls through to the binary
+ * branch.
+ */
+export const finish = (s) => {
+    const byteLength = s.length >> 3n;
+    const mime = magicMime(s.magic);
+    if (mime !== null) {
+        return { length: byteLength, mime_type: mime, type: 'base64' };
+    }
+    if (utf8Text(s.utf8) && (s.length & 7n) === 0n) {
+        return { length: byteLength, mime_type: 'text/plain', type: 'text' };
+    }
+    return { length: byteLength, mime_type: 'application/octet-stream', type: 'base64' };
+};
+/**
+ * Classifies a whole `Vec` with the same state machine as {@link detectStream}.
+ * The single-buffer counterpart for callers that already hold the bytes (the
+ * `cas_get` `content: true` path materializes the blob anyway): both paths read
+ * the three-way `{ length, mime_type, type }` verdict from one machine instead of
+ * re-deriving it from `detect` + a separate UTF-8 check.
+ */
+export const detectVec = (bytes) => finish(push(detectInit)(bytes));
+/**
+ * Folds a CAS read stream through {@link push} and reads {@link finish} at EOF,
+ * deriving `cas_get` metadata without ever materializing the blob. A read `error`
+ * item short-circuits into the `IoResult` error.
+ */
+export const detectStream = (stream) => {
+    const loop = (s) => (l) => l.step((node) => {
+        if (node === undefined) {
+            return pure(ok(finish(s)));
+        }
+        const { first, tail } = node;
+        const [t, v] = first;
+        if (t === 'error') {
+            return pure(error(v));
+        }
+        return loop(push(s)(v))(tail);
+    });
+    return loop(detectInit)(stream);
+};

package/fs/mime/proof.f.d.ts

@@ -12,4 +12,29 @@ export declare const proof: {
     textIsNull: () => void;
     shortIsNull: () => void;
     emptyIsNull: () => void;
+    stream: {
+        text: () => void;
+        png: () => void;
+        octetStream: () => void;
+        nul: () => void;
+        controlByte: () => void;
+        c1Control: () => void;
+        whitespaceControlsStayText: () => void;
+        magicAcrossChunks: () => void;
+        webp: () => void;
+        utf8AcrossChunks: () => void;
+        validUntilTrailingInvalidByte: () => void;
+        truncatedSequence: () => void;
+        largeMultiChunkBlob: () => void;
+        pdfThenLargeTextTail: () => void;
+        pdfThenBinaryTail: () => void;
+        readErrorSurfaces: () => void;
+        empty: () => void;
+    };
+    vec: {
+        text: () => void;
+        png: () => void;
+        octetStream: () => void;
+        nul: () => void;
+    };
 };

package/fs/mime/proof.f.js

@@ -1,9 +1,31 @@
-import { assertEq } from "../asserts/module.f.js";
-import { msb, u8ListToVec, empty } from "../types/bit_vec/module.f.js";
-import { detect } from "./module.f.js";
+import { assert, assertEq } from "../asserts/module.f.js";
+import { msb, u8ListToVec, vec8, repeat, empty } from "../types/bit_vec/module.f.js";
+import { decode } from "../effects/module.f.js";
+import { nonEmpty, empty as emptyList } from "../effects/list/module.f.js";
+import { ok } from "../types/result/module.f.js";
+import { detect, detectStream, detectVec } from "./module.f.js";
 // Builds a big-endian `Vec` from a list of byte values — mirrors how the CAS
 // store would hold the leading bytes of a stored blob.
 const bytes = (...b) => u8ListToVec(msb)(b);
+// ── Streaming detector helpers ──────────────────────────────────────────────────
+// Evaluates a fully pure effect (no operations) to its result.
+const runPure = (e) => {
+    const d = decode(e);
+    if (!d.done) {
+        throw 'effect is not pure';
+    }
+    return d.result;
+};
+// Builds a CAS-style read stream from a sequence of ok(chunk) items.
+const stream = (...chunks) => chunks.reduceRight((tail, c) => nonEmpty(ok(c), tail), emptyList());
+// Runs the streaming detector over the given chunks and unwraps the metadata.
+const detectChunks = (...chunks) => {
+    const r = runPure(detectStream(stream(...chunks)));
+    if (r[0] === 'error') {
+        throw r[1];
+    }
+    return r[1];
+};
 export const proof = {
     png: () => assertEq(detect(bytes(0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a)), 'image/png'),
     jpeg: () => assertEq(detect(bytes(0xff, 0xd8, 0xff, 0xe0)), 'image/jpeg'),
@@ -27,4 +49,172 @@ export const proof = {
     // A prefix shorter than any signature falls through to null, not a partial match.
     shortIsNull: () => assertEq(detect(bytes(0x89, 0x50)), null),
     emptyIsNull: () => assertEq(detect(empty), null),
+    // ── Streaming detector (detectStream) ───────────────────────────────────────
+    stream: {
+        // Plain UTF-8 text classifies as text/plain with the correct byte length.
+        text: () => {
+            const m = detectChunks(bytes(0x68, 0x65, 0x6c, 0x6c, 0x6f)); // "hello"
+            assertEq(m.type, 'text');
+            assertEq(m.mime_type, 'text/plain');
+            assertEq(m.length, 5n);
+        },
+        // A magic-byte signature classifies as base64 with the detected mime type.
+        png: () => {
+            const m = detectChunks(bytes(0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x01));
+            assertEq(m.type, 'base64');
+            assertEq(m.mime_type, 'image/png');
+            assertEq(m.length, 10n);
+        },
+        // Unknown binary (no signature, not UTF-8) falls back to octet-stream/base64.
+        octetStream: () => {
+            const m = detectChunks(bytes(0xff, 0xfe, 0x00, 0x01));
+            assertEq(m.type, 'base64');
+            assertEq(m.mime_type, 'application/octet-stream');
+            assertEq(m.length, 4n);
+        },
+        // A NUL-byte blob is valid single-byte UTF-8 (all U+0000) but is binary:
+        // NUL is the sharpest binary marker, so it classifies as octet-stream.
+        nul: () => {
+            const m = detectChunks(bytes(0x00, 0x00, 0x00));
+            assertEq(m.type, 'base64');
+            assertEq(m.mime_type, 'application/octet-stream');
+            assertEq(m.length, 3n);
+        },
+        // A control byte embedded in otherwise-valid ASCII (here ESC) is still
+        // binary — text/plain must not imply terminal escapes.
+        controlByte: () => {
+            const m = detectChunks(bytes(0x68, 0x69, 0x1b, 0x5b, 0x30, 0x6d)); // "hi\x1b[0m"
+            assertEq(m.type, 'base64');
+            assertEq(m.mime_type, 'application/octet-stream');
+            assertEq(m.length, 6n);
+        },
+        // A C1 control arrives as 2-byte UTF-8 (C2 85 = U+0085, NEL); invisible at
+        // the byte level, it is caught at the code-point level and reads as binary.
+        c1Control: () => {
+            const m = detectChunks(bytes(0x41, 0xc2, 0x85)); // "A" + U+0085
+            assertEq(m.type, 'base64');
+            assertEq(m.mime_type, 'application/octet-stream');
+            assertEq(m.length, 3n);
+        },
+        // The text whitespace controls (TAB, LF, VT, FF, CR) are legitimate in text.
+        whitespaceControlsStayText: () => {
+            const m = detectChunks(bytes(0x61, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x62)); // "a\t\n\v\f\rb"
+            assertEq(m.type, 'text');
+            assertEq(m.mime_type, 'text/plain');
+            assertEq(m.length, 7n);
+        },
+        // Magic-byte detection threads across a chunk boundary mid-signature.
+        magicAcrossChunks: () => {
+            const m = detectChunks(bytes(0x89, 0x50, 0x4e), // first 3 bytes of the PNG signature
+            bytes(0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00)); // the rest, in a second chunk
+            assertEq(m.type, 'base64');
+            assertEq(m.mime_type, 'image/png');
+            assertEq(m.length, 9n);
+        },
+        // WebP's wildcard size gap is handled in the streaming form too.
+        webp: () => {
+            const m = detectChunks(bytes(0x52, 0x49, 0x46, 0x46), // "RIFF"
+            bytes(0x1a, 0x00, 0x00, 0x00), // 4-byte size (wildcards)
+            bytes(0x57, 0x45, 0x42, 0x50)); // "WEBP"
+            assertEq(m.type, 'base64');
+            assertEq(m.mime_type, 'image/webp');
+            assertEq(m.length, 12n);
+        },
+        // A multi-byte UTF-8 sequence split across chunks stays valid text.
+        utf8AcrossChunks: () => {
+            const m = detectChunks(bytes(0xc3), bytes(0xa9)); // "é" = C3 A9
+            assertEq(m.type, 'text');
+            assertEq(m.mime_type, 'text/plain');
+            assertEq(m.length, 2n);
+        },
+        // The decisive case: valid UTF-8 for every chunk but the last, which adds a
+        // lone invalid byte. A leading-bytes buffer would wrongly say "text"; the
+        // streaming validator sees the whole blob and classifies it as base64.
+        validUntilTrailingInvalidByte: () => {
+            const m = detectChunks(bytes(0x68, 0x65, 0x6c, 0x6c, 0x6f), // "hello" — valid UTF-8
+            bytes(0xff)); // trailing invalid byte
+            assertEq(m.type, 'base64');
+            assertEq(m.mime_type, 'application/octet-stream');
+            assertEq(m.length, 6n);
+        },
+        // A truncated multi-byte sequence at EOF is invalid UTF-8, hence base64.
+        truncatedSequence: () => {
+            const m = detectChunks(bytes(0xc3)); // lead byte with no continuation
+            assertEq(m.type, 'base64');
+            assertEq(m.mime_type, 'application/octet-stream');
+            assertEq(m.length, 1n);
+        },
+        // A blob larger than `maxLength` (1,048,576 bits) split across chunks — the
+        // exact case where `collectRead` would error — returns correct metadata.
+        largeMultiChunkBlob: () => {
+            const a = repeat(70000n)(vec8(0x61n)); // 70,000 bytes of 'a' (560,000 bits)
+            const m = detectChunks(a, a); // 140,000 bytes = 1,120,000 bits > maxLength
+            assertEq(m.type, 'text');
+            assertEq(m.mime_type, 'text/plain');
+            assertEq(m.length, 140000n);
+        },
+        // A magic match settles the verdict on its own: a large magic-matched blob
+        // whose tail stays valid UTF-8 (an ASCII PDF) classifies by the signature.
+        // `push` stops decoding the tail once `magic` is matched — `finish` ignores
+        // the utf8 verdict here — so this is the magic-matched fast path.
+        pdfThenLargeTextTail: () => {
+            const header = bytes(0x25, 0x50, 0x44, 0x46, 0x2d); // "%PDF-"
+            const tail = repeat(70000n)(vec8(0x61n)); // valid ASCII, never invalidates utf8
+            const m = detectChunks(header, tail);
+            assertEq(m.type, 'base64');
+            assertEq(m.mime_type, 'application/pdf');
+            assertEq(m.length, 70005n);
+        },
+        // A magic match wins regardless of the tail: a binary (non-UTF-8) tail
+        // after the signature is still classified by the signature.
+        pdfThenBinaryTail: () => {
+            const header = bytes(0x25, 0x50, 0x44, 0x46, 0x2d); // "%PDF-"
+            const m = detectChunks(header, bytes(0xff, 0xfe, 0x00));
+            assertEq(m.type, 'base64');
+            assertEq(m.mime_type, 'application/pdf');
+            assertEq(m.length, 8n);
+        },
+        // A read `error` item short-circuits into the IoResult error.
+        readErrorSurfaces: () => {
+            const errStream = nonEmpty(['error', 'boom'], emptyList());
+            const r = runPure(detectStream(errStream));
+            assert(r[0] === 'error');
+        },
+        // Empty stream: zero-length text/plain.
+        empty: () => {
+            const m = detectChunks();
+            assertEq(m.type, 'text');
+            assertEq(m.length, 0n);
+        },
+    },
+    // ── Single-Vec detector (detectVec) ─────────────────────────────────────────
+    // The whole-Vec form used by the `cas_get` content:true path; same machine,
+    // same three-way verdict as the streaming form.
+    vec: {
+        text: () => {
+            const m = detectVec(bytes(0x68, 0x65, 0x6c, 0x6c, 0x6f)); // "hello"
+            assertEq(m.type, 'text');
+            assertEq(m.mime_type, 'text/plain');
+            assertEq(m.length, 5n);
+        },
+        png: () => {
+            const m = detectVec(bytes(0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x01));
+            assertEq(m.type, 'base64');
+            assertEq(m.mime_type, 'image/png');
+            assertEq(m.length, 10n);
+        },
+        octetStream: () => {
+            const m = detectVec(bytes(0xff, 0xfe, 0x00, 0x01));
+            assertEq(m.type, 'base64');
+            assertEq(m.mime_type, 'application/octet-stream');
+            assertEq(m.length, 4n);
+        },
+        // Valid UTF-8 NUL run is binary in the single-Vec path too.
+        nul: () => {
+            const m = detectVec(bytes(0x00, 0x00, 0x00));
+            assertEq(m.type, 'base64');
+            assertEq(m.mime_type, 'application/octet-stream');
+            assertEq(m.length, 3n);
+        },
+    },
 };

package/fs/text/code_point/module.f.d.ts

@@ -1,8 +1,10 @@
 /**
  * Shared Unicode code-point contract for the UTF-8 and UTF-16 decoders: the
- * error-tag mask used to flag invalid sequences, and a streaming `decoder`
+ * error-tag mask used to flag invalid sequences, the streaming `decoder`
  * factory that wraps a per-unit step and an end-of-input step into a single
- * `List`-to-`List` conversion.
+ * `List`-to-`List` conversion, and the code-point classification predicates
+ * (BMP / surrogate / supplementary-plane / overall validity) that both codecs
+ * share.
  *
  * @module
  */
@@ -26,3 +28,38 @@ export declare const errorMask = 2147483648;
  * @returns A function converting a list of code units into a list of code points.
  */
 export declare const decoder: <Unit, S, Cp>(byteOp: StateScan<Unit, S | null, List<Cp>>, eofOp: (state: S | null) => readonly [List<Cp>, S | null]) => (input: List<Unit>) => List<Cp>;
+/**
+ * Checks whether the 16-bit word (U16) is a high surrogate.
+ * Range: 0xD800 - 0xDBFF.
+ */
+export declare const isHighSurrogate: (i: number) => boolean;
+/**
+ * Checks whether the 16-bit word (U16) is a low surrogate.
+ * Range: 0xDC00 - 0xDFFF.
+ */
+export declare const isLowSurrogate: (i: number) => boolean;
+/**
+ * Checks whether the code point is in the BMP range.
+ * BMP is the main Unicode plane that covers code points 0x0000 - 0xFFFF, except
+ * for the range of surrogates.
+ */
+export declare const isBmpCodePoint: (codePoint: number) => boolean;
+/**
+ * Checks whether the code point belongs to a supplementary (additional) Unicode
+ * plane. Supplementary planes cover code points from 0x010000 to 0x10FFFF.
+ */
+export declare const isSupplementaryPlane: (i: number) => boolean;
+/**
+ * Checks whether the code point is a valid scalar value: within the assignable
+ * Unicode range (0x0000 - 0x10FFFF) and not a surrogate.
+ */
+export declare const isValidCodePoint: (c: number) => boolean;
+/**
+ * Checks whether the code point is a *text* code point — distinct from
+ * {@link isValidCodePoint}, which gates well-formedness for decoding. A code
+ * point is text unless it is a control character: the controls are 0x0000 -
+ * 0x001F, 0x007F (DEL), and 0x0080 - 0x009F (C1), minus the whitespace block
+ * 0x0009 - 0x000D (TAB, LF, VT, FF, CR), which is legitimate in text. Every
+ * code point at or above 0x0020 that is not DEL or a C1 control is text.
+ */
+export declare const isTextCodePoint: (c: number) => boolean;