functionalscript 0.33.0 → 0.35.0

package/fs/cas/mcp/proof.f.js

@@ -1,18 +1,18 @@
 import { assert, assertEq } from "../../asserts/module.f.js";
 import { pure } from "../../effects/module.f.js";
 import { run } from "../../effects/mock/module.f.js";
-import { asBase, asNominal, create, read, write } from "../../effects/memory/module.f.js";
-import { msb, u8ListToVec, vec8 } from "../../types/bit_vec/module.f.js";
+import { asBase, asNominal, create } from "../../effects/memory/module.f.js";
+import { msb, u8ListToVec, vec8, repeat, length, maxLengthBytes } from "../../types/bit_vec/module.f.js";
 import { vecToCBase32 } from "../../cbase32/module.f.js";
 import { encode as base64Encode } from "../../base64/module.f.js";
-import { computeSync, sha256 } from "../../crypto/sha2/module.f.js";
 import { utf8 } from "../../text/module.f.js";
-import { fileCas } from "../module.f.js";
+import {} from "../module.f.js";
 import { mcpStep, uninitializedState, } from "../../mcp/module.f.js";
-import {} from "../../effects/node/module.f.js";
 import { emptyState, virtual } from "../../effects/node/virtual/module.f.js";
 import { casConfig, casMcpHandlers } from "./module.f.js";
 import { ok as resultOk } from "../../types/result/module.f.js";
+import { stdioTransport } from "../../mcp/stdio/module.f.js";
+import { fromVec } from "../../types/uint8array/module.f.js";
 const initialTestState = { memory: { next: 0, values: {} } };
 const mock = {
     memCreate: value => state => {
@@ -32,18 +32,15 @@ const mock = {
     rename: (_src, _dst) => _ => { throw new Error('rename not supported in memory mock'); },
     readBytes: (_path, _offset, _size) => _ => { throw new Error('readBytes not supported in memory mock'); },
     randomInt: () => _ => { throw new Error('randomInt not supported in memory mock'); },
+    now: () => state => [state, 0],
+    access: (_path) => _ => { throw new Error('access not supported in memory mock'); },
+    createExclusive: (_path) => _ => { throw new Error('createExclusive not supported in memory mock'); },
+    readdir: (_path, _opts) => _ => { throw new Error('readdir not supported in memory mock'); },
+    rm: (_path) => _ => { throw new Error('rm not supported in memory mock'); },
+    stat: (_path) => _ => { throw new Error('stat not supported in memory mock'); },
+    writeBytes: (_path, _offset, _data) => _ => { throw new Error('writeBytes not supported in memory mock'); },
 };
 const runMem = (effect) => run(mock)(initialTestState)(effect)[1];
-const memCas = (mapKey) => ({
-    read: (key) => read(mapKey).step(m => pure(m[vecToCBase32(key)]?.[1])),
-    write: (value) => {
-        const key = computeSync(sha256)([value]);
-        return read(mapKey)
-            .step(m => write(mapKey, { ...m, [vecToCBase32(key)]: [key, value] }))
-            .step(() => pure(key));
-    },
-    list: () => read(mapKey).step(m => pure(Object.values(m).map(([k]) => k))),
-});
 // ── Session driver ──────────────────────────────────────────────────────────────
 // Feeds each message to `step` in order, collecting every response.
 const feed = (step) => (msgs) => {
@@ -52,19 +49,12 @@ const feed = (step) => (msgs) => {
         : step(msgs[i]).step(r => go(i + 1, [...acc, r]));
     return go(0, []);
 };
-// Runs a full session over a fresh in-memory CAS, returning all responses.
-const runSession = (msgs, home = '/home/user') => runMem(create({}).step(mapKey => create(uninitializedState).step(sessionKey => {
-    const c = memCas(mapKey);
-    const step = mcpStep(casConfig)(casMcpHandlers(c, home))(sessionKey);
-    return feed(step)(msgs);
-})));
 // Runs a session backed by the virtual node runner (for cas_upload which uses
 // Rename/ReadBytes/RandomInt/Mkdir). Uses fileKvStore so upload and get share
 // the same filesystem-backed CAS.
 const runSessionVirtual = (root, home = '/home/user') => (msgs) => {
     const effect = create(uninitializedState).step(sessionKey => {
-        const c = fileCas(sha256)(home);
-        const step = mcpStep(casConfig)(casMcpHandlers(c, home))(sessionKey);
+        const step = mcpStep(casConfig)(casMcpHandlers(home))(sessionKey);
         return feed(step)(msgs);
     });
     return virtual({ ...emptyState, root })(effect)[1];
@@ -76,7 +66,24 @@ const initialized = { jsonrpc: '2.0', method: 'notifications/initialized' };
 const call = (id, name, args) => ({ jsonrpc: '2.0', method: 'tools/call', id, params: { name, arguments: args } });
 const list = (id) => ({ jsonrpc: '2.0', method: 'tools/list', id });
 // Runs `init`, `notifications/initialized`, then `msgs`; returns the tool responses.
-const session = (...msgs) => runSession([init, initialized, ...msgs]).slice(2);
+const session = (...msgs) => runSessionVirtual({}, '/home/user')([init, initialized, ...msgs]).slice(2);
+// UTF-8 bytes of `s` as a plain array — the virtual stdin byte stream.
+const toBytes = (s) => [...fromVec(utf8(s))];
+// Runs `init`, `notifications/initialized`, then `msgs` through the *real*
+// stdio pipeline — `mcpStep` wrapped in `stdioTransport`, driving
+// `writeResponse`'s `tryUtf8` encode over actual stdin/stdout bytes. Unlike
+// `session`/`runSessionVirtual`, which collect `mcpStep`'s `Response` values
+// as plain JS objects and never serialize/encode them, this is the only
+// helper that can observe the transport's oversized-response fallback (see
+// `fs/mcp/stdio/module.f.ts` `writeResponse`).
+const runStdio = (root, home = '/home/user') => (msgs) => {
+    const input = [init, initialized, ...msgs].map(m => JSON.stringify(m)).join('\n') + '\n';
+    const effect = create(uninitializedState).step(sessionKey => stdioTransport(mcpStep(casConfig)(casMcpHandlers(home))(sessionKey)));
+    const stdout = virtual({ ...emptyState, root, stdin: toBytes(input) })(effect)[0].stdout;
+    // Only requests get a written line (notifications, like `initialized`,
+    // write nothing) — drop the `init` response, keep one line per `msgs` entry.
+    return stdout.split('\n').filter(line => line.length > 0).slice(1).map(line => JSON.parse(line));
+};
 const resultOf = (resp) => resp.result;
 const item0 = (resp) => resultOf(resp).content[0];
 const textOf = (resp) => item0(resp).text;
@@ -87,10 +94,123 @@ const binarySample = base64Encode(vec8(0x2an));
 // A base64 blob whose leading bytes are the PNG magic-byte signature, so
 // `cas_get` detects its type and returns base64 with mime_type image/png.
 const pngSample = base64Encode(u8ListToVec(msb)([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x01]));
+// Returns the RFC 4648 base64 encoding of `n` zero bytes, computed directly
+// without bigint arithmetic — safe near maxLengthBytes where base64Encode on a
+// padded Vec would trigger the encode-padding-overflow bug. Handles every n%3
+// residue: rem=0 → no padding, rem=1 → 'AA==', rem=2 → 'AAA='.
+const base64OfA = (n) => {
+    const groups = Number(n / 3n);
+    const rem = Number(n % 3n);
+    const body = 'AAAA'.repeat(groups);
+    if (rem === 0) {
+        return body;
+    }
+    if (rem === 1) {
+        return body + 'AA==';
+    }
+    return body + 'AAA=';
+};
 // ── Tests ───────────────────────────────────────────────────────────────────────
+// A blob larger than one read chunk (> 128 KiB) used to fail metadata-only
+// cas_get because `collectRead` overflowed `maxLength`. With `detectStream`,
+// content:false returns correct metadata regardless of size. Each chunk is
+// exactly `maxLengthBytes` (the largest single `Vec` the runtime allows), so the
+// two-chunk blob spans two read chunks without any one `Vec` exceeding the cap.
+const largeMultiChunkBlobMeta = (chunk0, chunk1, expectedType, expectedMime) => () => {
+    const root = { 'home': { 'user': { 'cas_upload': { 'big': [chunk0, chunk1] } } } };
+    const [addResp] = runSessionVirtual(root)([
+        init, initialized,
+        call(2, 'cas_add', { content: '/home/user/cas_upload/big', type: 'url' }),
+    ]).slice(2);
+    assert(!resultOf(addResp).isError);
+    const hash = textOf(addResp);
+    const [, metaResp] = runSessionVirtual(root)([
+        init, initialized,
+        call(2, 'cas_add', { content: '/home/user/cas_upload/big', type: 'url' }),
+        call(3, 'cas_get', { hash }),
+    ]).slice(2);
+    assert(!resultOf(metaResp).isError);
+    const meta = JSON.parse(textOf(metaResp));
+    assertEq(meta.type, expectedType);
+    assertEq(meta.mime_type, expectedMime);
+    assertEq(meta.length, Number((length(chunk0) + length(chunk1)) / 8n));
+    assertEq(meta.content, undefined);
+};
+// A full `maxLengthBytes`-long chunk of repeated ASCII 'a' — valid UTF-8.
+const asciiChunk = repeat(maxLengthBytes)(vec8(0x61n));
+// The same length, but repeated 2-byte "é" (C3 A9) — valid UTF-8 of multi-byte symbols.
+const symbolChunk = repeat(maxLengthBytes / 2n)(u8ListToVec(msb)([0xc3, 0xa9]));
+// A full chunk of 0xFF — an invalid UTF-8 lead byte, so binary (no magic match).
+const binaryChunk = repeat(maxLengthBytes)(vec8(0xffn));
+// 33,000 `"` bytes (0x22): valid UTF-8, so `cas_get` classifies it as text and
+// takes the double-JSON-escaping path (`toJson` builds the inner CAS JSON, then
+// the outer `stringifyJson` embeds that as the `text` field). Each quote costs
+// 4 final bytes (2 on each escaping pass), so 33,000 of them alone push the
+// encoded line past `maxLength` (131,072 bytes) — well above the strict 32,769
+// minimum needed, without approaching any other size-related edge case.
+const quotesChunk = repeat(33000n)(vec8(0x22n));
 export const proof = {
+    // Both chunks repeated ASCII → text/plain, no error on a > 128 KiB blob.
+    getMetaLargeMultiChunkBlobNoError: largeMultiChunkBlobMeta(asciiChunk, asciiChunk, 'text', 'text/plain'),
+    // Both chunks valid UTF-8 sequences of multi-byte symbols → text/plain.
+    getMetaLargeMultiChunkUtf8Symbols: largeMultiChunkBlobMeta(symbolChunk, symbolChunk, 'text', 'text/plain'),
+    // First chunk valid UTF-8, second chunk binary → base64/octet-stream. The
+    // streaming validator must see the trailing binary chunk to reject text.
+    getMetaLargeMultiChunkTextThenBinary: largeMultiChunkBlobMeta(asciiChunk, binaryChunk, 'base64', 'application/octet-stream'),
+    // content:true on a present blob larger than `maxLength` reports a distinct
+    // "too large" error, not "no such hash" — the blob exists, it just can't be
+    // buffered inline. The metadata-only path (above) still returns its size/type.
+    getContentLargeBlobTooLargeError: () => {
+        const root = { 'home': { 'user': { 'cas_upload': { 'big': [asciiChunk, asciiChunk] } } } };
+        const [addResp] = runSessionVirtual(root)([
+            init, initialized,
+            call(2, 'cas_add', { content: '/home/user/cas_upload/big', type: 'url' }),
+        ]).slice(2);
+        const hash = textOf(addResp);
+        const [, getResp] = runSessionVirtual(root)([
+            init, initialized,
+            call(2, 'cas_add', { content: '/home/user/cas_upload/big', type: 'url' }),
+            call(3, 'cas_get', { hash, content: true }),
+        ]).slice(2);
+        assertEq(resultOf(getResp).isError, true);
+        const text = textOf(getResp);
+        assert(text.includes('too large'));
+        assert(!text.includes('no such hash'));
+    },
+    // content:true on a genuinely absent hash still reports "no such hash" — the
+    // oversized branch above must not absorb the absent case.
+    getContentMissingHashIsError: () => {
+        const absent = vecToCBase32(vec8(0x55n));
+        const [resp] = session(call(2, 'cas_get', { hash: absent, content: true }));
+        assertEq(resultOf(resp).isError, true);
+        assert(textOf(resp).includes('no such hash'));
+    },
+    // A text blob made entirely of `"` characters keeps the *raw* content
+    // (33,000 bytes) well under `maxLengthBytes`, but each quote costs 2 bytes on
+    // the inner CAS-JSON escape and 4 bytes once the outer JSON-RPC envelope
+    // re-escapes that backslash-quote pair — no `cas_get`-level size guard could
+    // see this coming, only the transport's real encode attempt can. Uploaded via
+    // `type: 'url'` (a file, not an inlined JSON string) so the test exercises
+    // `cas_get`'s response-side escaping without also paying for parsing a huge
+    // inline `content` argument on the request side. Confirms `writeResponse`'s
+    // `tryUtf8` fallback (not a crash) handles double-escaping overflow, again
+    // preserving the request's `id`.
+    getContentDoubleEscapedOverflowWritesInternalError: () => {
+        const root = { 'home': { 'user': { 'cas_upload': { 'q': [quotesChunk] } } } };
+        const [addResp] = runStdio(root)([
+            call(2, 'cas_add', { content: '/home/user/cas_upload/q', type: 'url' }),
+        ]);
+        const hash = textOf(addResp);
+        const [, getResp] = runStdio(root)([
+            call(2, 'cas_add', { content: '/home/user/cas_upload/q', type: 'url' }),
+            call(3, 'cas_get', { hash, content: true }),
+        ]);
+        const err = getResp;
+        assertEq(err.error?.code, -32603);
+        assertEq(err.id, 3);
+    },
     toolsListAdvertisesThreeTools: () => {
-        const [resp] = runSession([init, initialized, list(2)]).slice(2);
+        const [resp] = runSessionVirtual({})([init, initialized, list(2)]).slice(2);
         const tools = resp.result.tools;
         assertEq(tools.length, 3);
         assertEq(tools.map(t => t.name).join(','), 'cas_add,cas_get,cas_list');
@@ -190,6 +310,37 @@ export const proof = {
         const [resp] = session(call(2, 'cas_add', { content: 'A', type: 'base64' }));
         assertEq(resultOf(resp).isError, true);
     },
+    // 174_764 'A' chars × 6 bits = 1_048_584 bits, 8 over maxLength — base64Decode
+    // returns null and cas_add surfaces an error.
+    addBase64OverLimitIsError: () => {
+        const [resp] = session(call(2, 'cas_add', { content: 'A'.repeat(174_764), type: 'base64' }));
+        assertEq(resultOf(resp).isError, true);
+        assert(textOf(resp).includes('too large or malformed'));
+    },
+    // One ASCII byte past maxLengthBytes — tryUtf8 returns null and cas_add
+    // surfaces an error.
+    addTextOverLimitIsError: () => {
+        const [resp] = session(call(2, 'cas_add', { content: 'a'.repeat(Number(maxLengthBytes) + 1) }));
+        assertEq(resultOf(resp).isError, true);
+        assert(textOf(resp).includes('too large or malformed'));
+    },
+    // Exactly maxLengthBytes (131,072) ASCII bytes — tryUtf8 returns a non-null
+    // Vec (len === maxLength, not over), so cas_add stores cleanly.
+    addTextAtLimitSucceeds: () => {
+        const [resp] = session(call(2, 'cas_add', { content: 'a'.repeat(Number(maxLengthBytes)) }));
+        assert(!resultOf(resp).isError);
+        assert(textOf(resp).length > 0);
+    },
+    // base64 of exactly maxLengthBytes — currently returns isError because
+    // base64Decode measures the raw pre-trim body (1_048_578 bits) against
+    // maxLength before stripping the 2 padding bits that would land it exactly at
+    // maxLength. Assert the current failing behavior; flip to a success assertion
+    // once `fs/base64/todo/decode-rejects-max-size-input.md` is fixed.
+    addBase64AtLimitIsError: () => {
+        const [resp] = session(call(2, 'cas_add', { content: base64OfA(maxLengthBytes), type: 'base64' }));
+        assertEq(resultOf(resp).isError, true);
+        assert(textOf(resp).includes('too large or malformed'));
+    },
     getUnterminatedHashIsError: () => {
         const [resp] = session(call(2, 'cas_get', { hash: '0' }));
         assertEq(resultOf(resp).isError, true);
@@ -278,7 +429,7 @@ export const proof = {
         const meta = JSON.parse(textOf(metaResp));
         assertEq(meta.mime_type, 'text/plain');
         assertEq(meta.type, 'text');
-        assertEq(meta.length, Number(BigInt(/* 'text content'.length */ 12)));
+        assertEq(meta.length, 12);
         assertEq(meta.content, undefined);
     },
     getMetaBinaryBlob: () => {
@@ -304,15 +455,18 @@ export const proof = {
         assertEq(meta.type, 'base64');
         assertEq(meta.content, undefined);
     },
-    getMetaNoUrlWhenToUrlAbsent: () => {
-        const content = utf8('no url');
-        const binaryB64 = base64Encode(content);
-        const [addResp] = session(call(2, 'cas_add', { content: binaryB64 }));
+    // A NUL-bearing blob is valid UTF-8 yet binary: cas_get must report
+    // base64/octet-stream, not text/plain.
+    getMetaOctetStreamForNulBlob: () => {
+        const nulContent = u8ListToVec(msb)([0x00, 0x00, 0x00]);
+        const nulB64 = base64Encode(nulContent);
+        const [addResp] = session(call(2, 'cas_add', { content: nulB64, type: 'base64' }));
         const hash = textOf(addResp);
-        const [, metaResp] = session(call(2, 'cas_add', { content: binaryB64 }), call(3, 'cas_get', { hash }));
+        const [, metaResp] = session(call(2, 'cas_add', { content: nulB64, type: 'base64' }), call(3, 'cas_get', { hash }));
         assert(!resultOf(metaResp).isError);
         const meta = JSON.parse(textOf(metaResp));
-        assertEq(meta.url, undefined);
+        assertEq(meta.mime_type, 'application/octet-stream');
+        assertEq(meta.type, 'base64');
     },
     getMetaMissingHashIsError: () => {
         const absent = vecToCBase32(vec8(0x77n));
@@ -357,4 +511,17 @@ export const proof = {
         assert(resultOf(resp).isError === true);
         assert(textOf(resp).includes('/home/user/cas_upload/'));
     },
+    // cas_get with content:true on octet-stream (no magic bytes, not UTF-8) returns inline base64.
+    getOctetStreamWithContentIncludesBase64: () => {
+        const binaryContent = u8ListToVec(msb)([0xFF, 0xFE, 0x00, 0x01]);
+        const binaryB64 = base64Encode(binaryContent);
+        const [addResp] = session(call(2, 'cas_add', { content: binaryB64, type: 'base64' }));
+        const hash = textOf(addResp);
+        const [, getResp] = session(call(2, 'cas_add', { content: binaryB64, type: 'base64' }), call(3, 'cas_get', { hash, content: true }));
+        assert(!resultOf(getResp).isError);
+        const result = JSON.parse(textOf(getResp));
+        assertEq(result.mime_type, 'application/octet-stream');
+        assertEq(result.type, 'base64');
+        assertEq(result.content, binaryB64);
+    },
 };

package/fs/cas/module.f.d.ts

@@ -6,29 +6,49 @@
 import { type Sha2 } from '../crypto/sha2/module.f.ts';
 import { type Vec } from '../types/bit_vec/module.f.ts';
 import { type Effect, type Operation } from '../effects/module.f.ts';
-import { type Access, type All, type IoResult, type Mkdir, type NodeProgramOptions, type RandomInt, type Read, type ReadBytes, type Readdir, type ReadFile, type Rename, type Write, type WriteFile } from '../effects/node/module.f.ts';
-import { type Commands } from '../cli/module.f.ts';
-import type { MemOp } from '../effects/memory/module.f.ts';
+import { type Access, type CreateExclusive, type IoResult, type Mkdir, type Now, type RandomInt, type ReadBytes, type Readdir, type Rename, type Rm, type Stat, type WriteBytes } from '../effects/node/module.f.ts';
+import { type List } from '../effects/list/module.f.ts';
 /** Converts a content key to its sharded relative CAS file path. */
 export declare const toPath: (key: Vec) => string;
-export type FileCasOperation = ReadFile | Mkdir | WriteFile | Access | Readdir;
+/**
+ * The filesystem effects the streaming CAS performs: `read` pulls shards
+ * (`ReadBytes`); `list` walks the store (`Access`/`Readdir`); `write` runs the
+ * lock-free staging upload (`Mkdir`/`CreateExclusive`/`WriteBytes`/`Rename`/`Rm`/
+ * `Stat`, lease deadlines from `Now`, staging names from `RandomInt`) and GC's
+ * expired staging files (`Readdir`/`Rm`).
+ */
+export type FileCasOperation = ReadBytes | Mkdir | Readdir | Access | Rename | Rm | RandomInt | Now | CreateExclusive | WriteBytes | Stat | Now | Readdir | Rm;
 export type Cas<O extends Operation> = {
-    /** Reads content by hash; returns `undefined` when not found. */
-    readonly read: (key: Vec) => Effect<O, Vec | undefined>;
-    /** Stores content and returns its computed hash. */
-    readonly write: (value: Vec) => Effect<O, Vec | undefined>;
+    /**
+     * Streams the content for `hash` out in `<=128 KiB` chunks. Every pull yields an
+     * explicit `ok(chunk)` or `error` item, so a missing shard or I/O error is a distinct
+     * error *item* in the stream, never collapsed into end-of-stream (`undefined`).
+     */
+    readonly read: (hash: Vec) => List<O, IoResult<Vec>>;
+    /**
+     * Consumes a chunk stream — each item `ok(chunk)` or `error` — hashing incrementally,
+     * and returns the content address. An error item aborts the upload.
+     */
+    readonly write: <O1 extends Operation>(payload: List<O1, IoResult<Vec>>) => Effect<O | O1, IoResult<Vec>>;
     /** Lists all stored content hashes. */
     readonly list: () => Effect<O, readonly Vec[]>;
 };
+export type FileCas = Cas<FileCasOperation> & {
+    url: (v: Vec) => string;
+};
 /**
  * Builds a content-addressable storage facade from a SHA-2 implementation.
  */
-export declare const fileCas: (sha2: Sha2) => (path: string) => Cas<FileCasOperation>;
+export declare const fileCas: (sha2: Sha2) => (path: string) => FileCas;
+/**
+ * Streams the file at `path` through `cas.write`, returning the content hash.
+ * Both the CLI `cas add` and the MCP `add` delegate to this; the MCP layer
+ * additionally deletes the source file on success.
+ */
+export declare const casAddFile: <O extends Operation>(cas: Cas<O>) => (path: string) => Effect<O | ReadBytes, IoResult<Vec>>;
 /**
- * Move-hash-move upload pipeline: moves `fileName` from `~/cas_upload/` to a
- * random staging path, stream-hashes it, then renames it to its final CAS shard
- * path. Returns `ok(hash)` on success or `error(reason)` on any I/O failure.
+ * Upload pipeline: streams `fileName` from `~/cas_upload/` through `casAddFile`,
+ * then deletes the source on success. On failure the source is left in place so
+ * the upload can be retried; `write` already cleans up its own partial staging file.
  */
-export declare const casUpload: (home: string) => (fileName: string) => Effect<Mkdir | Rename | RandomInt | ReadBytes, IoResult<Vec>>;
-export declare const commands: Commands<FileCasOperation | Write | All | MemOp | Read>;
-export declare const main: (options: NodeProgramOptions) => Effect<All | Read | Write | FileCasOperation | MemOp, number>;
+export declare const casUpload: (home: string) => (fileName: string) => Effect<FileCasOperation, IoResult<Vec>>;