functionalscript 0.35.2 → 0.37.0

package/fs/cas/mcp/module.f.js

@@ -11,8 +11,8 @@
  *
  * | Tool       | args                          | action           | result                              |
  * |------------|-------------------------------|------------------|-------------------------------------|
- * | `cas_add`  | `{ content, type? }`          | write/upload     | hash (cBase32)                      |
- * | `cas_get`  | `{ hash, content?: boolean }` | `c.read(key)`    | JSON `{length,mime_type,type[,content]}` |
+ * | `cas_add`  | `{ content, type? }`          | `c.write(...)`   | hash (cBase32)                      |
+ * | `cas_get`  | `{ hash, content?: boolean }` | `c.read(key)`    | JSON `{length,mimeType,type[,uri][,text\|blob]}` |
  * | `cas_list` | `{}`                          | `c.list()`       | hashes, one per line                |
  *
  * ## `cas_add` input encoding
@@ -23,43 +23,64 @@
  *   without any encoding step.
  * - `'base64'`: `content` is RFC 4648 base64, decoded to bytes before storage.
  *   Use this for pre-encoded binary payloads.
- * - `'url'`: `content` is a filesystem path within `$HOME/cas_upload/`; the server
- *   moves the file via the streaming move-hash-move pipeline (no size limit).
- *   Paths outside `$HOME/cas_upload/` or containing `..` are rejected for security.
+ *
+ * Inline content (either encoding) is capped at 128 KiB (`maxLength`). There is
+ * no MCP route for larger content — the server never opens a local path named by
+ * the client (see the invariant in `fs/cas/mcp/README.md`); large files go
+ * through the `cas` CLI (`cas add <path>`) instead, run directly by the user.
  *
  * ## `cas_get` output
  *
- * Always returns a JSON object `{ length, mime_type, type[, url][, content] }`.
- * `type` is always present (`'text'` or `'base64'`). A single classifier — the
- * `fs/mime` detector state machine (length × magic-byte eliminator × UTF-8 DFA) —
- * produces the same three-way verdict on both paths, so there is no second,
- * divergent copy of the rules:
+ * Always returns a JSON object `{ length, mimeType, type[, uri][, text | blob] }`.
+ * These field names mirror the MCP resource-contents shape (`resources/read`
+ * results), so the tool view and the future resource view of a blob use the same
+ * vocabulary: `mimeType` (not `mime_type`), `uri` (not `url`) — the resource URI
+ * under which the same blob will be readable via `resources/read` — and `text` /
+ * `blob` (not a `type`-tagged `content`) for inline payloads. `type` is always
+ * present (`'text'` or `'base64'`) as the discriminator for which of `text` /
+ * `blob` a later `content: true` fetch would populate — MCP allows extra fields,
+ * so `type` and `length` simply stay alongside the required resource-contents
+ * fields. A single classifier — the `fs/media/type` detector state machine (length ×
+ * magic-byte eliminator × UTF-8 DFA) — produces the same three-way verdict on
+ * both paths, so there is no second, divergent copy of the rules:
  *
  * 1. **Magic-byte hit** (PNG/JPEG/GIF/WebP/PDF/ZIP) → `type: 'base64'` with the
- *    detected `mime_type`.
- * 2. **Whole-blob-valid UTF-8** → `type: 'text'`, `mime_type: 'text/plain'`.
- * 3. **Fallback** → `type: 'base64'`, `mime_type: 'application/octet-stream'`.
- *
- * **Metadata-only (`content: false`, the default) is size-independent.** It folds
- * the read stream through `fs/mime` `detectStream` and never buffers the blob.
- * Because a single `Vec` caps at `maxLength` bits (128 KiB), the old
- * drain-into-one-`Vec` approach failed on any blob larger than one chunk even when
- * only metadata was asked for; streaming detection returns correct
- * `{ length, mime_type, type[, url] }` regardless of size.
- *
- * **`content: true`** first derives `{ length, mime_type, type }` with the same
- * size-independent `fs/mime` `detectStream` machine, then materializes the bytes
- * (via `collectRead`, bounded by `maxLength`) and encodes the inline payload by
- * `type` — a `fs/text/utf8` `fromVec` string for `text`, base64 for `base64`. A
- * blob larger than `maxLength` (128 KiB) cannot be buffered into one `Vec`, so it
- * is rejected here with a descriptive *"too large"* error (carrying the byte size
- * and `url`) rather than being misreported as absent; it should be fetched via
- * `url` or inspected with metadata-only `cas_get`.
+ *    detected `mimeType`.
+ * 2. **Whole-blob-valid UTF-8** → `type: 'text'`, `mimeType: 'text/plain'`.
+ * 3. **Fallback** → `type: 'base64'`, `mimeType: 'application/octet-stream'`.
+ *
+ * **Metadata-only (`content: false`, the default) is size-independent for
+ * non-text and oversized blobs.** It folds the read stream through
+ * `fs/media/type` `detectStream` and never buffers the blob. Because a single
+ * `Vec` caps at `maxLength` bits (128 KiB), the old drain-into-one-`Vec`
+ * approach failed on any blob larger than one chunk even when only metadata
+ * was asked for; streaming detection returns correct
+ * `{ length, mimeType, type[, uri] }` regardless of size. A dialect (e.g.
+ * `vnd.fjs.revision`, see `fs/media/module.f.ts`) can only be recognized by
+ * parsing the whole blob as JSON, so when the streaming verdict is
+ * whole-blob-valid text within the same bounded inline cap as `content: true`
+ * (`maxLengthBytes`), one extra bounded read materializes the blob purely to
+ * refine `mimeType` through the dialect-aware detector; a larger or non-text
+ * blob is never a dialect match and is returned from the streaming verdict
+ * alone, unbuffered.
+ *
+ * **`content: true`** first derives `{ length, mimeType, type }` with the same
+ * size-independent `fs/media/type` `detectStream` machine (to decide whether the
+ * blob even fits inline), then materializes the bytes (via `collectRead`, bounded
+ * by `maxLength`) and re-derives the verdict from the dialect-aware detector
+ * (`fs/media/module.f.ts` `detect`) — now that the whole blob is in hand, a
+ * dialect match is reflected in the result — before encoding the inline payload
+ * by `type` — a `fs/text/utf8` `fromVec` string on `text` (for `type: 'text'`),
+ * base64 on `blob` (for `type: 'base64'`). A blob larger than `maxLength` (128
+ * KiB) cannot be buffered into one `Vec`, so it is rejected here with a
+ * descriptive *"too large"* error (carrying the byte size and `uri`) rather than
+ * being misreported as absent; it should be fetched via `uri` or inspected with
+ * metadata-only `cas_get`.
  *
  * ## Encoding split: hashes vs. content
  *
  * `Cas<O>` deals in `Vec` (bit vectors); MCP models only `textContent` today.
- * **Hashes** travel as cBase32 (`fs/cbase32`) — the canonical CAS hash format,
+ * **Hashes** travel as cBase32 (`fs/basen/cbase32`) — the canonical CAS hash format,
  * shared with the CLI and the on-disk store layout. **Content** encoding is
  * determined at read time as described above.
  *
@@ -78,29 +99,30 @@
  *
  * @module
  */
-import { string, option, or, boolean } from "../../types/rtti/module.f.js";
-import { stringify } from "../../json/module.f.js";
-import { pure } from "../../effects/module.f.js";
-import { create } from "../../effects/memory/module.f.js";
-import { cBase32ToVec, vecToCBase32 } from "../../cbase32/module.f.js";
-import { decode as base64Decode, encode as base64Encode } from "../../base64/module.f.js";
-import { tryUtf8 } from "../../text/module.f.js";
-import { detectStream } from "../../mime/module.f.js";
-import { empty, length as bitVecLength, maxLength, maxLengthBytes, msb } from "../../types/bit_vec/module.f.js";
-import { ok, error } from "../../types/result/module.f.js";
-import { rm } from "../../effects/node/module.f.js";
-import { stdioTransport } from "../../mcp/stdio/module.f.js";
-import { mcpStep, uninitializedState, toolEntry, fromRegistry, errorResult, } from "../../mcp/module.f.js";
-import { casAddFile, fileCas } from "../module.f.js";
-import { fromVec } from "../../text/utf8/module.f.js";
-import { identity } from "../../types/function/module.f.js";
-import { sha256 } from "../../crypto/sha2/module.f.js";
-import { nonEmpty, empty as elEmpty } from "../../effects/list/module.f.js";
+import { string, option, or, boolean } from '../../types/rtti/module.f.js';
+import { stringify } from '../../media/json/module.f.js';
+import { pure, decode } from '../../effects/module.f.js';
+import { create } from '../../effects/memory/module.f.js';
+import { cBase32ToVec, vecToCBase32 } from '../../basen/cbase32/module.f.js';
+import { decode as base64Decode, encode as base64Encode } from '../../basen/base64/module.f.js';
+import { tryUtf8 } from '../../text/module.f.js';
+import { detectStream } from '../../media/type/module.f.js';
+import { detect as detectDialect } from '../../media/module.f.js';
+import { empty, length as bitVecLength, maxLength, maxLengthBytes, msb, vec } from '../../types/bit_vec/module.f.js';
+import { ok, error } from '../../types/result/module.f.js';
+import {} from '../../effects/node/module.f.js';
+import { stdioTransport } from '../../mcp/stdio/module.f.js';
+import { mcpStep, uninitializedState, toolEntry, fromRegistry, errorResult, } from '../../mcp/module.f.js';
+import { fileCas } from '../module.f.js';
+import { fromVec } from '../../text/utf8/module.f.js';
+import { identity } from '../../types/function/module.f.js';
+import { sha256 } from '../../crypto/sha2/module.f.js';
+import { nonEmpty, empty as elEmpty } from '../../effects/list/module.f.js';
 // ── Argument schemas (declared once, used for both inputSchema and validate) ─────
 /** Arguments for `cas_add`: content to store, with optional encoding type. */
 export const casAddArgs = {
     content: string,
-    type: or('text', 'base64', 'url', undefined)
+    type: or('text', 'base64', undefined)
 };
 /** Arguments for `cas_get`: the cBase32 hash to look up; optionally request inline content. */
 export const casGetArgs = {
@@ -114,7 +136,7 @@ export const casListArgs = {};
  * Drains a CAS read stream into a single `Vec`. Used only on the `content: true`
  * path, where the whole blob is needed for inline base64 / UTF-8 transfer; the
  * chunk stream is concatenated and an error item is surfaced as the result.
- * Metadata-only `cas_get` avoids this entirely via `fs/mime` `detectStream`,
+ * Metadata-only `cas_get` avoids this entirely via `fs/media/type` `detectStream`,
  * which is why it is not bound by the `maxLength` ceiling below.
  */
 const collectRead = (stream) => {
@@ -142,68 +164,78 @@ const toJson = stringify(identity);
 /** Registry of all CAS tools. */
 const casToolRegistry = (home) => {
     const c = fileCas(sha256)(home);
-    const casUploadDir = `${home}/cas_upload`;
     return [
-        toolEntry('cas_add', 'Store content and return its hash (cBase32). Pass type:"base64" for binary; omit or pass type:"text" for UTF-8 text (default). Inline content (text/base64) is capped at 128 KiB (131072 bytes) — larger content is rejected. For larger content use type:"url" to stream a file from $HOME/cas_upload/ (no size limit).', casAddArgs, ({ type, content }) => {
-            // type:'url' — stream the file into cas, then delete the source on success
-            if (type === 'url') {
-                if (!content.startsWith(`${casUploadDir}/`) || content.includes('..')) {
-                    return pure(errorResult(`cas_add type:url paths must be within ${casUploadDir}/ — got: ${content}`));
-                }
-                return casAddFile(c)(content).step(([t, v]) => t === 'error'
-                    ? pure(errorResult(`upload failed: ${v}`))
-                    : rm(content).step(() => pure(okResult(vecToCBase32(v)))));
-            }
+        toolEntry('cas_add', 'Store content and return its hash (cBase32). Pass type:"base64" for binary; omit or pass type:"text" for UTF-8 text (default). Inline content is capped at 128 KiB (131072 bytes) — larger content is rejected. For larger content, store the file with the `cas` CLI instead: run `npx functionalscript cas add <path>` yourself if you have shell access, or give the user that exact command to run — it prints the resulting hash on stdout.', casAddArgs, ({ type, content }) => {
             // type:'text' or 'base64' — resolve content to Vec, store via c.write()
             let x = type === 'base64'
                 ? base64Decode(content)
                 : tryUtf8(content);
             return x === null
-                ? pure(errorResult('too large or malformed — use type:"url" for large content'))
+                ? pure(errorResult('too large or malformed — for large content, run `npx functionalscript cas add <path>` (or have the user run it) instead'))
                 // The resolved content fits in one chunk; feed it as a single-item stream.
                 : c.write(nonEmpty(ok(x), elEmpty())).step(([tag, hash]) => pure(tag === 'error'
                     ? errorResult('write')
                     : okResult(vecToCBase32(hash))));
         }),
-        toolEntry('cas_get', 'Inspect a blob by hash. Always returns JSON {length,mime_type,type[,url]} where type is "text" or "base64". Pass content:true to also include the inline content string, but content is capped at 128 KiB (131072 bytes) — a larger blob is rejected with an error. To download a blob, prefer the url field returned in the result instead of requesting inline content.', casGetArgs, r => {
+        toolEntry('cas_get', 'Inspect a blob by hash. Always returns JSON {length,mimeType,type[,uri]} where type is "text" or "base64". Pass content:true to also include the inline payload as text (type:"text") or blob (type:"base64"), but content is capped at 128 KiB (131072 bytes) — a larger blob is rejected with an error. To download a blob, prefer the uri field returned in the result instead of requesting inline content.', casGetArgs, r => {
             const key = cBase32ToVec(r.hash);
             if (key === null) {
                 return pure(errorResult(`invalid cBase32 hash: ${r.hash}`));
             }
-            const url = c.url(key);
+            const uri = c.url(key);
             return detectStream(c.read(key)).step(([tag, detected]) => {
                 if (tag === 'error') {
                     return pure(errorResult(`no such hash: ${r.hash}`));
                 }
-                const { length, mime_type, type } = detected;
-                const meta = { length: Number(length), mime_type, type, url };
+                const { length, mime_type: mimeType, type } = detected;
+                const meta = { length: Number(length), mimeType, type, uri };
                 if (r.content !== true) {
-                    // content:true path continues below; this is just the metadata step.
-                    return pure(okResult(toJson(meta)));
+                    // A dialect match can only be decided from the whole parsed blob;
+                    // only attempt the extra bounded read when it stands a chance
+                    // (whole-blob text within the same cap `content: true` allows).
+                    if (type !== 'text' || length > maxLengthBytes) {
+                        return pure(okResult(toJson(meta)));
+                    }
+                    return collectRead(c.read(key)).step(([collectTag, value]) => {
+                        // Already known to fit from the streaming pass above, so an
+                        // error here means the hash vanished between reads; fall back
+                        // to the streaming verdict rather than fail the whole request.
+                        if (collectTag === 'error') {
+                            return pure(okResult(toJson(meta)));
+                        }
+                        const refined = detectDialect(value);
+                        return pure(okResult(toJson({ ...meta, mimeType: refined.mime_type })));
+                    });
                 }
                 // A single `Vec` caps at `maxLength` bits (`maxLengthBytes` bytes), so
                 // a larger blob cannot be buffered for inline transfer. Report the
                 // size and point at the size-independent alternatives instead of
                 // misreporting an existing blob as `no such hash`.
                 if (length > maxLengthBytes) {
-                    return pure(errorResult(`blob too large to fetch inline (${length} bytes, limit ${maxLengthBytes} bytes); use the url field (${url}) or omit content for metadata`));
+                    return pure(errorResult(`blob too large to fetch inline (${length} bytes, limit ${maxLengthBytes} bytes); use the uri field (${uri}) or omit content for metadata`));
                 }
                 return collectRead(c.read(key)).step(([collectTag, value]) => {
                     if (collectTag === 'error') {
                         return pure(errorResult(`no such hash: ${r.hash}`));
                     }
-                    if (type === 'text') {
+                    // Re-derive the verdict from the now-materialized blob so a
+                    // dialect match — only decidable with the whole parsed JSON in
+                    // hand — is reflected in the inline result too, not just the
+                    // streaming guess.
+                    const refined = detectDialect(value);
+                    const refinedMeta = { length: Number(refined.length), mimeType: refined.mime_type, type: refined.type, uri };
+                    if (refined.type === 'text') {
                         // `type: 'text'` means the detector validated `value` as UTF-8,
                         // so `fromVec` is non-null here; guard defensively regardless.
                         const str = fromVec(value);
                         return pure(str === null
                             ? errorResult(`content is not byte-aligned: ${r.hash}`)
-                            : okResult(toJson({ ...meta, content: str })));
+                            : okResult(toJson({ ...refinedMeta, text: str })));
                     }
                     const blob = base64Encode(value);
                     return pure(blob === null
                         ? errorResult(`content is not byte-aligned: ${r.hash}`)
-                        : okResult(toJson({ ...meta, content: blob })));
+                        : okResult(toJson({ ...refinedMeta, blob })));
                 });
             });
         }),
@@ -235,3 +267,23 @@ export const casConfig = {
  * until stdin EOF.
  */
 export const casMcpServer = (home) => create(uninitializedState).step(key => stdioTransport(mcpStep(casConfig)(casMcpHandlers(home))(key)));
+// ── Tests ────────────────────────────────────────────────────────────────────
+export const proof = {
+    // casMcpServer is never called in integration tests because it drives a
+    // real stdio server; call it here to cover its Effect-building body.
+    casMcpServer: () => { casMcpServer('/'); },
+    // The overflow guard in collectRead (lines 125-126) is only reached when
+    // the running total of two stream chunks would exceed maxLength.  Feed a
+    // pure stream whose second chunk pushes it just over the limit so the
+    // error branch executes without any real I/O.
+    collectReadOverflow: () => {
+        const half = maxLength / 2n;
+        const v1 = vec(half)(0n);
+        const v2 = vec(half + 1n)(0n);
+        const stream = nonEmpty(ok(v1), nonEmpty(ok(v2), elEmpty()));
+        const d = decode(collectRead(stream));
+        if (!d.done || d.result[0] !== 'error') {
+            throw 'expected overflow error';
+        }
+    },
+};

package/fs/cas/mcp/proof.f.d.ts

@@ -16,13 +16,15 @@ export declare const proof: {
     addGetRoundTrips: () => void;
     getUntypedReturnsText: () => void;
     getTypedReturnsBinaryJson: () => void;
+    getMetaRecognizesRevisionDialect: () => void;
+    getContentRecognizesRevisionDialect: () => void;
     listEnumeratesStoredHashes: () => void;
     addInvalidContentIsError: () => void;
     addBadLengthContentIsError: () => void;
     addBase64OverLimitIsError: () => void;
     addTextOverLimitIsError: () => void;
     addTextAtLimitSucceeds: () => void;
-    addBase64AtLimitIsError: () => void;
+    addBase64AtLimitSucceeds: () => void;
     getUnterminatedHashIsError: () => void;
     addMissingContentIsError: () => void;
     getMissingHashArgumentIsError: () => void;
@@ -30,18 +32,11 @@ export declare const proof: {
     getMissingHashIsError: () => void;
     unknownToolIsError: () => void;
     toolErrorIsNotJsonRpcError: () => void;
-    addUrlStoresFileAndReturnsHash: () => void;
-    addUrlRoundTrips: () => void;
-    addUrlMissingFileIsError: () => void;
     getMetaReturnsLengthAndMimeType: () => void;
     getMetaBinaryBlob: () => void;
     getMetaOctetStreamForUnknownBinary: () => void;
     getMetaOctetStreamForNulBlob: () => void;
     getMetaMissingHashIsError: () => void;
     getMetaInvalidHashIsError: () => void;
-    addUrlFromSubdirectorySucceeds: () => void;
-    addUrlFromApprovedDirectorySucceeds: () => void;
-    addUrlFromRandomDirectoryIsRejected: () => void;
-    addUrlWithPathTraversalIsRejected: () => void;
     getOctetStreamWithContentIncludesBase64: () => void;
 };