functionalscript 0.32.4 → 0.34.0

package/fs/cas/cli/proof.f.js

@@ -0,0 +1,149 @@
+import { commands } from "./module.f.js";
+import { computeSync, sha256 } from "../../crypto/sha2/module.f.js";
+import { maxLength, vec, vec8 } from "../../types/bit_vec/module.f.js";
+import { defaultNodeProgramOptions, emptyState, virtual } from "../../effects/node/virtual/module.f.js";
+import {} from "../../effects/node/module.f.js";
+import { dispatch } from "../../cli/module.f.js";
+import { vecToCBase32 } from "../../cbase32/module.f.js";
+import { assert, assertEq } from "../../asserts/module.f.js";
+const makeOptions = (args) => ({ ...defaultNodeProgramOptions, args });
+const main = dispatch(commands);
+export const proof = {
+    mainAdd: () => {
+        const content = vec8(0x2an);
+        const state = { ...emptyState, root: { myfile: [content] } };
+        const [finalState, exitCode] = virtual(state)(main(makeOptions(['add', 'myfile'])));
+        if (exitCode !== 0) {
+            throw ['expected exit 0', exitCode];
+        }
+        if (finalState.stdout.length === 0) {
+            throw 'expected hash in stdout';
+        }
+    },
+    mainAddGetBig: () => {
+        const chunk = vec(maxLength)(1n);
+        const content = [chunk, chunk];
+        const state = { ...emptyState, root: { myfile: content } };
+        //
+        const [finalState, exitCode] = virtual(state)(main(makeOptions(['add', 'myfile'])));
+        assertEq(exitCode, 0);
+        const stdout = finalState.stdout;
+        assert(stdout.length !== 0);
+        //
+        const h = computeSync(sha256)(content);
+        const hs = vecToCBase32(h);
+        assertEq(stdout, `${hs}\n`);
+        //
+        const [finalState2, exitCode2] = virtual(finalState)(main(makeOptions(['get', hs, 'myfile2'])));
+        // console.log(finalState2.stderr)
+        assertEq(exitCode2, 0, 'e2');
+        const { myfile2 } = finalState2.root;
+        assert(myfile2 instanceof Array);
+        const h2 = computeSync(sha256)(myfile2);
+        assertEq(h, h2, 'h');
+    },
+    mainAddWrongArgs: () => {
+        const [finalState, exitCode] = virtual(emptyState)(main(makeOptions(['add'])));
+        assertEq(exitCode, 1);
+        assert(finalState.stderr.length !== 0);
+    },
+    mainGetFound: () => {
+        const content = vec8(0x2an);
+        const state = { ...emptyState, root: { myfile: [content] } };
+        const [state1, exitCode1] = virtual(state)(main(makeOptions(['add', 'myfile'])));
+        if (exitCode1 !== 0) {
+            throw ['expected add exit 0', exitCode1];
+        }
+        const hashStr = state1.stdout.trim();
+        const [, exitCode2] = virtual(state1)(main(makeOptions(['get', hashStr, 'output'])));
+        if (exitCode2 !== 0) {
+            throw ['expected get exit 0', exitCode2];
+        }
+    },
+    mainGetNotFound: () => {
+        // valid cBase32 hash that has not been stored
+        const content = vec8(0x2an);
+        const state = { ...emptyState, root: { myfile: [content] } };
+        const [state1] = virtual(state)(main(makeOptions(['add', 'myfile'])));
+        const hashStr = state1.stdout.trim();
+        // use an empty store so the hash is not found
+        const [finalState, exitCode] = virtual(emptyState)(main(makeOptions(['get', hashStr, 'output'])));
+        if (exitCode !== 1) {
+            throw ['expected exit 1', exitCode];
+        }
+        if (finalState.stderr.length === 0) {
+            throw 'expected error in stderr';
+        }
+    },
+    mainGetWrongArgs: () => {
+        const [finalState, exitCode] = virtual(emptyState)(main(makeOptions(['get'])));
+        if (exitCode !== 1) {
+            throw ['expected exit 1', exitCode];
+        }
+        if (finalState.stderr.length === 0) {
+            throw 'expected error in stderr';
+        }
+    },
+    mainGetInvalidHash: () => {
+        const [finalState, exitCode] = virtual(emptyState)(main(makeOptions(['get', 'not-a-valid-hash', 'output'])));
+        if (exitCode !== 1) {
+            throw ['expected exit 1', exitCode];
+        }
+        if (finalState.stderr.length === 0) {
+            throw 'expected error in stderr';
+        }
+    },
+    mainList: () => {
+        const content = vec8(0x2an);
+        const state = { ...emptyState, root: { myfile: [content] } };
+        const [state1] = virtual(state)(main(makeOptions(['add', 'myfile'])));
+        const [, exitCode] = virtual(state1)(main(makeOptions(['list'])));
+        if (exitCode !== 0) {
+            throw ['expected exit 0', exitCode];
+        }
+    },
+    mainListEmptyStore: () => {
+        // A fresh directory has no `.cas` yet; listing must succeed (empty),
+        // not crash unwrapping a readdir ENOENT.
+        const [finalState, exitCode] = virtual(emptyState)(main(makeOptions(['list'])));
+        if (exitCode !== 0) {
+            throw ['expected exit 0', exitCode];
+        }
+        if (finalState.stdout !== '') {
+            throw ['expected empty stdout', finalState.stdout];
+        }
+    },
+    mainListCorruptStore: () => {
+        // `.cas` exists but is a file, not a directory: a real storage error
+        // that must surface, not be masked as an empty list.
+        const state = { ...emptyState, root: { '.cas': [vec8(0x2an)] } };
+        let threw = false;
+        try {
+            virtual(state)(main(makeOptions(['list'])));
+        }
+        catch {
+            threw = true;
+        }
+        if (!threw) {
+            throw 'expected list to surface the storage error';
+        }
+    },
+    mainNoCmd: () => {
+        const [finalState, exitCode] = virtual(emptyState)(main(makeOptions([])));
+        if (exitCode !== 1) {
+            throw ['expected exit 1', exitCode];
+        }
+        if (finalState.stderr.length === 0) {
+            throw 'expected error in stderr';
+        }
+    },
+    mainUnknownCmd: () => {
+        const [finalState, exitCode] = virtual(emptyState)(main(makeOptions(['bogus'])));
+        if (exitCode !== 1) {
+            throw ['expected exit 1', exitCode];
+        }
+        if (finalState.stderr.length === 0) {
+            throw 'expected error in stderr';
+        }
+    },
+};

package/fs/cas/mcp/module.f.d.ts

@@ -1,13 +1,12 @@
-import { type Effect, type Operation } from '../../effects/module.f.ts';
+import { type Effect } from '../../effects/module.f.ts';
 import { type MemOp } from '../../effects/memory/module.f.ts';
-import { type Vec } from '../../types/bit_vec/module.f.ts';
-import { type Mkdir, type RandomInt, type Read, type ReadBytes, type Rename, type Write } from '../../effects/node/module.f.ts';
+import { type Read, type Rm, type Write } from '../../effects/node/module.f.ts';
 import { type McpConfig, type McpHandlers } from '../../mcp/module.f.ts';
-import { type Cas } from '../module.f.ts';
+import { type FileCasOperation } from '../module.f.ts';
 /** Arguments for `cas_add`: content to store, with optional encoding type. */
 export declare const casAddArgs: {
     readonly content: import("../../types/rtti/module.f.ts").String;
-    readonly type: import("../../types/rtti/module.f.ts").Or<readonly [import("../../types/rtti/module.f.ts").Or<[import("../../types/rtti/module.f.ts").Or<["text", "base64"]>, "url"]>, undefined]>;
+    readonly type: import("../../types/rtti/module.f.ts").Or<["text", "base64", "url", undefined]>;
 };
 /** Arguments for `cas_get`: the cBase32 hash to look up; optionally request inline content. */
 export declare const casGetArgs: {
@@ -17,24 +16,17 @@ export declare const casGetArgs: {
 /** Arguments for `cas_list`: none. */
 export declare const casListArgs: {};
 /**
- * MCP handlers for an injected `Cas<O>` — generic in `O` exactly like `Cas`
- * itself, so the same handlers run over `Fs` (production) or memory (tests).
- *
- * When `toUrl` is provided, `cas_get` includes the `url` field pointing to
- * the blob on the local filesystem. When absent (e.g. memory-backed tests),
- * `url` is omitted.
+ * MCP handlers for `FileCas`.
  */
-export declare const casMcpHandlers: <O extends Operation>(c: Cas<O>, home: string, toUrl?: (hash: Vec) => string) => McpHandlers<Mkdir | Rename | RandomInt | ReadBytes | O>;
+export declare const casMcpHandlers: (home: string) => McpHandlers<FileCasOperation | Rm>;
 /**
  * Static MCP configuration for the CAS server: advertises the `tools`
  * capability, identifies the server, and pins the protocol version.
  */
 export declare const casConfig: McpConfig;
 /**
- * Runs the CAS MCP server over stdio: allocates the session-state slot, builds
+ * Runs the file CAS MCP server over stdio: allocates the session-state slot, builds
  * the `mcpStep` for `c`, and drives the read → parse → dispatch → write loop
- * until stdin EOF. Generic in `O` so it composes with any `Cas<O>` backing.
- *
- * When `toUrl` is provided, `cas_get` includes the blob's filesystem URL.
+ * until stdin EOF.
  */
-export declare const casMcpServer: <O extends Operation>(c: Cas<O>, home: string, toUrl?: (hash: Vec) => string) => Effect<Read | Write | MemOp | Mkdir | Rename | RandomInt | ReadBytes | O, void>;
+export declare const casMcpServer: (home: string) => Effect<Read | Write | MemOp | FileCasOperation | Rm, void>;

package/fs/cas/mcp/module.f.js

@@ -30,18 +30,28 @@
  * ## `cas_get` output
  *
  * Always returns a JSON object `{ length, mime_type, type[, url][, content] }`.
- * `type` is always present (`'text'` or `'base64'`). When `content: true` is
- * requested, the inline payload is also included. Two-phase MIME detection
- * determines the encoding:
- *
- * 1. **Magic-byte sniffing** (`fs/mime` `detect`): PNG/JPEG/GIF/WebP/PDF/ZIP →
- *    `type: 'base64'` with the detected `mime_type`.
- * 2. **UTF-8 validation** (`fs/text/utf8` `fromVec`): valid UTF-8 →
- *    `type: 'text'`, `mime_type: 'text/plain'`.
- * 3. **Fallback**: `type: 'base64'`, `mime_type: 'application/octet-stream'`.
- *
- * When `content: false` (default), only `{ length, mime_type, type[, url] }` is
- * returned — no content transfer.
+ * `type` is always present (`'text'` or `'base64'`). A single classifier — the
+ * `fs/mime` detector state machine (length × magic-byte eliminator × UTF-8 DFA) —
+ * produces the same three-way verdict on both paths, so there is no second,
+ * divergent copy of the rules:
+ *
+ * 1. **Magic-byte hit** (PNG/JPEG/GIF/WebP/PDF/ZIP) → `type: 'base64'` with the
+ *    detected `mime_type`.
+ * 2. **Whole-blob-valid UTF-8** → `type: 'text'`, `mime_type: 'text/plain'`.
+ * 3. **Fallback** → `type: 'base64'`, `mime_type: 'application/octet-stream'`.
+ *
+ * **Metadata-only (`content: false`, the default) is size-independent.** It folds
+ * the read stream through `fs/mime` `detectStream` and never buffers the blob.
+ * Because a single `Vec` caps at `maxLength` bits (128 KiB), the old
+ * drain-into-one-`Vec` approach failed on any blob larger than one chunk even when
+ * only metadata was asked for; streaming detection returns correct
+ * `{ length, mime_type, type[, url] }` regardless of size.
+ *
+ * **`content: true`** materializes the bytes (via `collectRead`, bounded by
+ * `maxLength`), classifies them with the same machine via `fs/mime` `detectVec`,
+ * then encodes the inline payload by `type` — a `fs/text/utf8` `fromVec` string for
+ * `text`, base64 for `base64`. A blob larger than `maxLength` is unsupported here
+ * and should be fetched via `url`.
  *
  * ## Encoding split: hashes vs. content
  *
@@ -64,40 +74,80 @@
  * @module
  */
 import { string, option, or, boolean } from "../../types/rtti/module.f.js";
+import { stringify } from "../../json/module.f.js";
 import { pure } from "../../effects/module.f.js";
 import { create } from "../../effects/memory/module.f.js";
 import { cBase32ToVec, vecToCBase32 } from "../../cbase32/module.f.js";
 import { decode as base64Decode, encode as base64Encode } from "../../base64/module.f.js";
 import { utf8 } from "../../text/module.f.js";
-import { detect } from "../../mime/module.f.js";
-import { length as bitVecLength } from "../../types/bit_vec/module.f.js";
-import {} from "../../effects/node/module.f.js";
+import { detectVec, detectStream } from "../../mime/module.f.js";
+import { empty, length as bitVecLength, maxLength, msb } from "../../types/bit_vec/module.f.js";
+import { ok, error } from "../../types/result/module.f.js";
+import { rm } from "../../effects/node/module.f.js";
 import { stdioTransport } from "../../mcp/stdio/module.f.js";
 import { mcpStep, uninitializedState, toolEntry, fromRegistry, errorResult, } from "../../mcp/module.f.js";
-import { casUpload } from "../module.f.js";
+import { casAddFile, fileCas } from "../module.f.js";
 import { fromVec } from "../../text/utf8/module.f.js";
+import { identity } from "../../types/function/module.f.js";
+import { sha256 } from "../../crypto/sha2/module.f.js";
+import { nonEmpty, empty as elEmpty } from "../../effects/list/module.f.js";
 // ── Argument schemas (declared once, used for both inputSchema and validate) ─────
 /** Arguments for `cas_add`: content to store, with optional encoding type. */
-export const casAddArgs = { content: string, type: option(or(or('text', 'base64'), 'url')) };
+export const casAddArgs = {
+    content: string,
+    type: or('text', 'base64', 'url', undefined)
+};
 /** Arguments for `cas_get`: the cBase32 hash to look up; optionally request inline content. */
-export const casGetArgs = { hash: string, content: option(boolean) };
+export const casGetArgs = {
+    hash: string,
+    content: option(boolean)
+};
 /** Arguments for `cas_list`: none. */
 export const casListArgs = {};
+// ── Stream helper ────────────────────────────────────────────────────────────────
+/**
+ * Drains a CAS read stream into a single `Vec`. Used only on the `content: true`
+ * path, where the whole blob is needed for inline base64 / UTF-8 transfer; the
+ * chunk stream is concatenated and an error item is surfaced as the result.
+ * Metadata-only `cas_get` avoids this entirely via `fs/mime` `detectStream`,
+ * which is why it is not bound by the `maxLength` ceiling below.
+ */
+const collectRead = (stream) => {
+    const loop = (acc) => (s) => s.step((node) => {
+        if (node === undefined) {
+            return pure(ok(acc));
+        }
+        const { first, tail } = node;
+        const [t, v] = first;
+        if (t === 'error') {
+            return pure(first);
+        }
+        // A single `Vec` cannot exceed `maxLength` bits; concatenating past it would
+        // overflow the runtime's `bigint` constraint. Surface that as an error item
+        // so the tool reports a failure rather than crashing the process.
+        if (bitVecLength(acc) + bitVecLength(v) > maxLength) {
+            return pure(error(`cas blob exceeds maximum vector length of ${maxLength} bits`));
+        }
+        return loop(msb.concat(acc)(v))(tail);
+    });
+    return loop(empty)(stream);
+};
 // ── Tool registry ──────────────────────────────────────────────────────────────
+const toJson = stringify(identity);
 /** Registry of all CAS tools. */
-const casToolRegistry = (c, home, toUrl) => {
+const casToolRegistry = (home) => {
+    const c = fileCas(sha256)(home);
     const casUploadDir = `${home}/cas_upload`;
     return [
         toolEntry('cas_add', 'Store content and return its hash (cBase32). Pass type:"base64" for binary; type:"url" to stream a file from $HOME/cas_upload/ (no size limit); omit or pass type:"text" for UTF-8 text (default).', casAddArgs, ({ type, content }) => {
-            // type:'url' — streaming move-hash-move pipeline (no size limit)
+            // type:'url' — stream the file into cas, then delete the source on success
             if (type === 'url') {
                 if (!content.startsWith(`${casUploadDir}/`) || content.includes('..')) {
                     return pure(errorResult(`cas_add type:url paths must be within ${casUploadDir}/ — got: ${content}`));
                 }
-                const fileName = content.slice(`${casUploadDir}/`.length);
-                return casUpload(home)(fileName).step(result => pure(result[0] === 'error'
-                    ? errorResult(`upload failed: ${result[1]}`)
-                    : okResult(vecToCBase32(result[1]))));
+                return casAddFile(c)(content).step(([t, v]) => t === 'error'
+                    ? pure(errorResult(`upload failed: ${v}`))
+                    : rm(content).step(() => pure(okResult(vecToCBase32(v)))));
             }
             // type:'text' or 'base64' — resolve content to Vec, store via c.write()
             let x;
@@ -112,63 +162,54 @@ const casToolRegistry = (c, home, toUrl) => {
             }
             return x.step(value => typeof value === 'string'
                 ? pure(errorResult(value))
-                : c.write(value).step(hash => pure(okResult(vecToCBase32(hash)))));
+                // The resolved content fits in one chunk; feed it as a single-item stream.
+                : c.write(nonEmpty(ok(value), elEmpty())).step(hashResult => pure(hashResult[0] === 'error'
+                    ? errorResult('write')
+                    : okResult(vecToCBase32(hashResult[1])))));
         }),
         toolEntry('cas_get', 'Inspect a blob by hash. Always returns JSON {length,mime_type,type[,url]} where type is "text" or "base64". Pass content:true to also include the inline content string.', casGetArgs, r => {
             const key = cBase32ToVec(r.hash);
             if (key === null) {
                 return pure(errorResult(`invalid cBase32 hash: ${r.hash}`));
             }
-            return c.read(key).step(value => {
-                if (value === undefined) {
-                    return pure(errorResult(`no such hash: ${r.hash}`));
-                }
-                const byteLength = Number(bitVecLength(value) / 8n);
-                // Phase 1: magic-byte sniffing for known binary formats.
-                const detectedMime = detect(value);
-                if (detectedMime !== null) {
-                    const url = toUrl?.(key);
-                    const meta = {
-                        length: byteLength,
-                        mime_type: detectedMime,
-                        type: 'base64',
-                        ...(url !== undefined && { url })
-                    };
-                    if (r.content === true) {
-                        const blob = base64Encode(value);
-                        return pure(blob === null
-                            ? errorResult(`content is not byte-aligned: ${r.hash}`)
-                            : okResult(JSON.stringify({ ...meta, content: blob })));
+            const url = c.url(key);
+            // Metadata-only (the default): derive `{ length, mime_type, type }` by
+            // streaming the blob through the detector state machine. Never buffers
+            // the blob, so this is size-independent — large blobs that overflow a
+            // single `Vec` still return correct metadata instead of an error.
+            if (r.content !== true) {
+                return detectStream(c.read(key)).step(result => {
+                    if (result[0] === 'error') {
+                        return pure(errorResult(`no such hash: ${r.hash}`));
                     }
-                    return pure(okResult(JSON.stringify(meta)));
-                }
-                // Phase 2: UTF-8 validation — text if valid, octet-stream otherwise.
-                const str = fromVec(value);
-                const url = toUrl?.(key);
-                if (str !== null) {
-                    const meta = {
-                        length: byteLength,
-                        mime_type: 'text/plain',
-                        type: 'text',
-                        ...(url !== undefined && { url })
-                    };
-                    return pure(r.content === true
-                        ? okResult(JSON.stringify({ ...meta, content: str }))
-                        : okResult(JSON.stringify(meta)));
+                    const { length, mime_type, type } = result[1];
+                    const meta = { length: Number(length), mime_type, type, url };
+                    return pure(okResult(toJson(meta)));
+                });
+            }
+            // content:true — collect the bytes (bounded by `maxLength`) so they can
+            // be encoded inline. Classification uses the *same* `detectVec` state
+            // machine as the metadata-only path; `type` then selects the encoding —
+            // a UTF-8 string for `text`, base64 for `base64`.
+            return collectRead(c.read(key)).step(result => {
+                if (result[0] === 'error') {
+                    return pure(errorResult(`no such hash: ${r.hash}`));
                 }
-                const meta = {
-                    length: byteLength,
-                    mime_type: 'application/octet-stream',
-                    type: 'base64',
-                    ...(url !== undefined && { url })
-                };
-                if (r.content === true) {
-                    const blob = base64Encode(value);
-                    return pure(blob === null
+                const value = result[1];
+                const { length, mime_type, type } = detectVec(value);
+                const meta = { length: Number(length), mime_type, type, url };
+                if (type === 'text') {
+                    // `type: 'text'` means the detector validated `value` as UTF-8, so
+                    // `fromVec` is non-null here; guard defensively regardless.
+                    const str = fromVec(value);
+                    return pure(str === null
                         ? errorResult(`content is not byte-aligned: ${r.hash}`)
-                        : okResult(JSON.stringify({ ...meta, content: blob })));
+                        : okResult(toJson({ ...meta, content: str })));
                 }
-                return pure(okResult(JSON.stringify(meta)));
+                const blob = base64Encode(value);
+                return pure(blob === null
+                    ? errorResult(`content is not byte-aligned: ${r.hash}`)
+                    : okResult(toJson({ ...meta, content: blob })));
             });
         }),
         toolEntry('cas_list', 'List all stored content hashes (cBase32), one per line.', casListArgs, () => c.list().step(hashes => pure(okResult(hashes.map(vecToCBase32).join('\n'))))),
@@ -179,14 +220,9 @@ const casToolRegistry = (c, home, toUrl) => {
 const okResult = (text) => ({ content: [{ type: 'text', text }] });
 // ── Handlers ────────────────────────────────────────────────────────────────────
 /**
- * MCP handlers for an injected `Cas<O>` — generic in `O` exactly like `Cas`
- * itself, so the same handlers run over `Fs` (production) or memory (tests).
- *
- * When `toUrl` is provided, `cas_get` includes the `url` field pointing to
- * the blob on the local filesystem. When absent (e.g. memory-backed tests),
- * `url` is omitted.
+ * MCP handlers for `FileCas`.
  */
-export const casMcpHandlers = (c, home, toUrl) => fromRegistry(casToolRegistry(c, home, toUrl));
+export const casMcpHandlers = (home) => fromRegistry(casToolRegistry(home));
 // ── Session configuration ───────────────────────────────────────────────────────
 /**
  * Static MCP configuration for the CAS server: advertises the `tools`
@@ -199,10 +235,8 @@ export const casConfig = {
 };
 // ── Server ──────────────────────────────────────────────────────────────────────
 /**
- * Runs the CAS MCP server over stdio: allocates the session-state slot, builds
+ * Runs the file CAS MCP server over stdio: allocates the session-state slot, builds
  * the `mcpStep` for `c`, and drives the read → parse → dispatch → write loop
- * until stdin EOF. Generic in `O` so it composes with any `Cas<O>` backing.
- *
- * When `toUrl` is provided, `cas_get` includes the blob's filesystem URL.
+ * until stdin EOF.
  */
-export const casMcpServer = (c, home, toUrl) => create(uninitializedState).step(key => stdioTransport(mcpStep(casConfig)(casMcpHandlers(c, home, toUrl))(key)));
+export const casMcpServer = (home) => create(uninitializedState).step(key => stdioTransport(mcpStep(casConfig)(casMcpHandlers(home))(key)));

package/fs/cas/mcp/proof.f.d.ts

@@ -1,4 +1,7 @@
 export declare const proof: {
+    getMetaLargeMultiChunkBlobNoError: () => void;
+    getMetaLargeMultiChunkUtf8Symbols: () => void;
+    getMetaLargeMultiChunkTextThenBinary: () => void;
     toolsListAdvertisesThreeTools: () => void;
     addReturnsHash: () => void;
     textAddGetMetaRoundTrips: () => void;
@@ -24,11 +27,12 @@ export declare const proof: {
     getMetaReturnsLengthAndMimeType: () => void;
     getMetaBinaryBlob: () => void;
     getMetaOctetStreamForUnknownBinary: () => void;
-    getMetaNoUrlWhenToUrlAbsent: () => void;
+    getMetaOctetStreamForNulBlob: () => void;
     getMetaMissingHashIsError: () => void;
     getMetaInvalidHashIsError: () => void;
     addUrlFromSubdirectorySucceeds: () => void;
     addUrlFromApprovedDirectorySucceeds: () => void;
     addUrlFromRandomDirectoryIsRejected: () => void;
     addUrlWithPathTraversalIsRejected: () => void;
+    getOctetStreamWithContentIncludesBase64: () => void;
 };