functionalscript 0.33.0 → 0.35.0

package/fs/cas/module.f.js

@@ -3,16 +3,16 @@
  *
  * @module
  */
-import { computeSync, sha256 } from "../crypto/sha2/module.f.js";
+import { sha256 } from "../crypto/sha2/module.f.js";
 import { join, normalize, parse } from "../path/module.f.js";
 import { empty, length, maxLengthBytes, msb, vec } from "../types/bit_vec/module.f.js";
 import { cBase32ToVec, vecToCBase32 } from "../cbase32/module.f.js";
 import { foldStep, forEachStep, pure } from "../effects/module.f.js";
-import { access, errorExit, isNotFound, log, mkdir, randomInt, readBytes, readdir, readFile, rename, writeFile } from "../effects/node/module.f.js";
-import { dispatch } from "../cli/module.f.js";
+import { access, createExclusive, isNotFound, mkdir, now, randomInt, readBytes, readdir, rename, rm, stat, writeBytes, } from "../effects/node/module.f.js";
 import { toOption } from "../types/nullable/module.f.js";
 import { error, ok, unwrap } from "../types/result/module.f.js";
 import { splitAt } from "../types/string/module.f.js";
+import { nonEmpty, empty as elEmpty } from "../effects/list/module.f.js";
 const split2 = splitAt(2);
 const prefix = '.cas';
 /** Converts a content key to its sharded relative CAS file path. */
@@ -22,151 +22,188 @@ export const toPath = (key) => {
     const [b, c] = split2(bc);
     return join(prefix, a, b, c);
 };
+/** Maximum chunk size for streaming reads: the largest `Vec` the runtime allows. */
+const chunkBytes = Number(maxLengthBytes);
+/** Staging directory under the store root; GC and every uploader share it. */
+const stageRel = '_stage';
+/**
+ * Lease duration in ms: a staging file's deadline is `now() + leaseDelta`.
+ * Renewed after every chunk, so it only has to cover the gap between two
+ * consecutive chunks (see [staging-lease.md](../../issues/cas/staging-lease.md)).
+ */
+const leaseDelta = 30_000;
+/**
+ * Fixed width for the zero-padded epoch-ms deadline embedded in a staging name,
+ * so names sort lexically exactly as they sort chronologically. 19 digits keeps
+ * epoch-ms (13 digits today) padded with headroom well past year 2286.
+ */
+const deadlineWidth = 19;
+/** Builds a `<deadline>-<random256>` staging file name. */
+const stageName = (deadline, rnd) => `${String(deadline).padStart(deadlineWidth, '0')}-${rnd}`;
+/** Recovers the deadline (epoch ms) from a `<deadline>-<random256>` name. */
+const deadlineOf = (name) => Number(name.slice(0, name.indexOf('-')));
+/**
+ * Reclaims expired staging files: any `_stage/<deadline>-<rand>` whose deadline
+ * is already in the past. Lazy and piggy-backed on `write`. Best-effort — a
+ * missing `_stage/` (fresh store) or any `readdir`/`rm` error is ignored, and a
+ * still-live lease is left alone; the fencing rename keeps even a misjudged
+ * reclaim fail-safe (worst case: that upload restarts).
+ */
+const gcStage = (stageDir) => now().step(t => readdir(stageDir, {}).step(([k, v]) => {
+    if (k === 'error') {
+        return pure(undefined);
+    }
+    const expired = v.flatMap(d => d.isFile && deadlineOf(d.name) < t ? [d.name] : []);
+    return forEachStep((name) => rm(join(stageDir, name)).step(() => pure(undefined)))(expired);
+}));
 /**
  * Builds a content-addressable storage facade from a SHA-2 implementation.
  */
-export const fileCas = (sha2) => {
-    const compute = computeSync(sha2);
-    return (path) => {
-        const storePrefix = join(path, prefix);
-        const normalizedStorePrefix = normalize(storePrefix);
-        return {
-            // TODO: extend the interface with `Result<Vec, unknown>` instead of `Vec|undefined`.
-            read: (key) => readFile(join(path, toPath(key))).step(([t, v]) => pure(t === 'ok' ? v : undefined)),
-            write: (value) => {
-                const hash = compute([value]);
-                const p = toPath(hash);
-                const parts = parse(p);
-                const dir = join(path, ...parts.slice(0, -1));
-                // TODO: error handling
-                return mkdir(dir, { recursive: true })
-                    .step(() => writeFile(join(path, p), value))
-                    .step(([t]) => pure(t === 'ok' ? hash : undefined));
-            },
-            list: () => 
-            // A fresh store has no `.cas` directory yet. Treat *only* that case as an
-            // empty store, mirroring how `read` maps a missing file to `undefined`.
-            // A `.cas` that exists but cannot be read (permissions, corruption) is a
-            // genuine storage error and is surfaced, not masked as "no hashes".
-            access(storePrefix).step(a => {
-                if (a[0] === 'error') {
-                    if (isNotFound(a[1])) {
-                        return pure([]);
+export const fileCas = (sha2) => (path) => {
+    const storePrefix = join(path, prefix);
+    const normalizedStorePrefix = normalize(storePrefix);
+    const stageDir = join(storePrefix, stageRel);
+    return {
+        read: (hash) => {
+            const p = join(path, toPath(hash));
+            const loop = (offset) => readBytes(p, offset, chunkBytes)
+                .step((result) => {
+                const [t, v] = result;
+                // A missing shard or read error is an explicit error item, never EOF.
+                if (t === 'error') {
+                    return nonEmpty(result, elEmpty());
+                }
+                // End the stream only on an empty read; every non-empty read — including a
+                // final short (`< CHUNK_BYTES`) chunk — is emitted as an `ok` item.
+                return length(v) === 0n
+                    ? elEmpty()
+                    : nonEmpty(ok(v), loop(offset + chunkBytes));
+            });
+            return loop(0);
+        },
+        // Lock-free staging upload (issues/cas/staging-lease.md): stream each chunk straight
+        // to a `_stage/<deadline>-<rand>` file via `writeBytes` while folding it into the
+        // running SHA-2 state — the payload never lives in memory as a whole. The lease is
+        // renewed (rename to a fresh deadline) after every chunk; any error deletes the
+        // partial file and fails. On end-of-stream the file is published to its hash-derived
+        // shard path by a replace-`rename` (which also dedups/repairs a same-content shard),
+        // and success is confirmed by a `stat` size check. GC of expired staging files is
+        // piggy-backed at the start.
+        write: (payload) => {
+            // Publish the finished staging file to its content-addressed shard. The three
+            // filesystem steps run best-effort with their results ignored; success is decided
+            // afterward by observing the target's size (see staging-lease.md "Publish ignores
+            // results and checks the end state").
+            const publish = (state, offset, curPath) => {
+                const hash = sha2.end(state);
+                const rel = toPath(hash);
+                const dst = join(path, rel);
+                const dstDir = join(path, ...parse(rel).slice(0, -1));
+                return mkdir(dstDir, { recursive: true })
+                    .step(() => rename(curPath, dst))
+                    .step(() => rm(curPath))
+                    .step(() => stat(dst))
+                    .step(st => pure(st[0] === 'ok' && st[1].size === offset ? ok(hash) : error('publish size mismatch')));
+            };
+            // Any streaming error fails closed: delete the partial file, return the error.
+            const fail = (curPath, e) => rm(curPath).step(() => pure(error(e)));
+            return gcStage(stageDir).step(() => random256.step(rnd => {
+                const rndStr = vecToCBase32(rnd);
+                const loop = (state, offset, curPath) => (stream) => stream
+                    .step((node) => {
+                    if (node === undefined) {
+                        return publish(state, offset, curPath);
                     }
-                    throw a[1];
+                    const { first, tail } = node;
+                    if (first[0] === 'error') {
+                        return fail(curPath, first[1]);
+                    }
+                    const chunk = first[1];
+                    return writeBytes(curPath, offset, chunk)
+                        .step(wb => {
+                        if (wb[0] === 'error') {
+                            return fail(curPath, wb[1]);
+                        }
+                        const newState = sha2.append(chunk)(state);
+                        const newOffset = offset + Number(length(chunk) / 8n);
+                        // Renew the lease: rename to a fresh deadline (keeps `delta` constant).
+                        return now()
+                            .step(t => {
+                            const next = join(stageDir, stageName(t + leaseDelta, rndStr));
+                            return rename(curPath, next).step(([t, v]) => t === 'error'
+                                ? fail(curPath, v)
+                                : loop(newState, newOffset, next)(tail));
+                        });
+                    });
+                });
+                return mkdir(stageDir, { recursive: true })
+                    .step(() => now()
+                    .step(t0 => {
+                    const path0 = join(stageDir, stageName(t0 + leaseDelta, rndStr));
+                    return createExclusive(path0)
+                        .step(([c, e]) => c === 'error'
+                        ? pure(error(e))
+                        : loop(sha2.init, 0, path0)(payload));
+                }));
+            }));
+        },
+        list: () => 
+        // A fresh store has no `.cas` directory yet. Treat *only* that case as an
+        // empty store, mirroring how `read` maps a missing shard to an error item.
+        // A `.cas` that exists but cannot be read (permissions, corruption) is a
+        // genuine storage error and is surfaced, not masked as "no hashes".
+        access(storePrefix).step(a => {
+            if (a[0] === 'error') {
+                if (isNotFound(a[1])) {
+                    return pure([]);
                 }
-                return readdir(storePrefix, { recursive: true })
-                    .step(r => pure(unwrap(r).flatMap(({ name, parentPath, isFile }) => toOption(isFile
-                    ? cBase32ToVec(normalize(parentPath).substring(normalizedStorePrefix.length).replaceAll('/', '') + name)
-                    : null))));
-            }),
-        };
+                throw a[1];
+            }
+            return readdir(storePrefix, { recursive: true })
+                .step(r => pure(unwrap(r).flatMap(({ name, parentPath, isFile }) => toOption(isFile
+                ? cBase32ToVec(normalize(parentPath).substring(normalizedStorePrefix.length).replaceAll('/', '') + name)
+                : null))));
+        }),
+        url: (hash) => join(path, toPath(hash))
     };
 };
-/** Maximum chunk size for streaming reads: the largest `Vec` the runtime allows. */
-const CHUNK_BYTES = Number(maxLengthBytes);
 /** 256-bit random `Vec` built from 8 sequential `randomInt` (32-bit) calls. */
 const random256 = foldStep((_) => (acc) => randomInt().step(r => pure(msb.concat(acc)(vec(32n)(BigInt(r))))))(empty)([0, 1, 2, 3, 4, 5, 6, 7]);
-/**
- * Streams a file in `CHUNK_BYTES` chunks, feeding each into the SHA-2 state,
- * and returns the final hash without loading the whole file into memory.
- */
-const streamHash = (sha2) => (path) => {
-    const chunkBits = BigInt(CHUNK_BYTES) * 8n;
-    const loop = (state, offset) => readBytes(path, offset, CHUNK_BYTES).step(result => {
+/** Streams any file at `filePath` in `<=128 KiB` chunks as a `ListEffect` of `ok` items. */
+const streamFile = (filePath) => {
+    const loop = (offset) => readBytes(filePath, offset, chunkBytes).step((result) => {
         if (result[0] === 'error') {
-            return pure(error(result[1]));
+            return nonEmpty(result, elEmpty());
         }
         const chunk = result[1];
-        const newState = sha2.append(chunk)(state);
-        if (length(chunk) < chunkBits) {
-            return pure(ok(sha2.end(newState)));
-        }
-        return loop(newState, offset + CHUNK_BYTES);
+        return length(chunk) === 0n
+            ? elEmpty()
+            : nonEmpty(ok(chunk), loop(offset + chunkBytes));
     });
-    return loop(sha2.init, 0);
+    return loop(0);
 };
 /**
- * Move-hash-move upload pipeline: moves `fileName` from `~/cas_upload/` to a
- * random staging path, stream-hashes it, then renames it to its final CAS shard
- * path. Returns `ok(hash)` on success or `error(reason)` on any I/O failure.
+ * Streams the file at `path` through `cas.write`, returning the content hash.
+ * Both the CLI `cas add` and the MCP `add` delegate to this; the MCP layer
+ * additionally deletes the source file on success.
+ */
+export const casAddFile = (cas) => (path) => 
+// streamFile produces only ReadBytes effects. TypeScript can't prove ListEffect<ReadBytes,T>
+// ≤ ListEffect<O,T> for generic O (recursive type), but the cast is sound: every concrete
+// caller passes a Cas<O> where ReadBytes ⊆ O (e.g. FileCasOperation).
+cas.write(streamFile(path));
+/**
+ * Upload pipeline: streams `fileName` from `~/cas_upload/` through `casAddFile`,
+ * then deletes the source on success. On failure the source is left in place so
+ * the upload can be retried; `write` already cleans up its own partial staging file.
  */
 export const casUpload = (home) => (fileName) => {
     const src = join(home, 'cas_upload', fileName);
-    const stageDir = join(home, prefix, '.stage');
-    return random256.step(rnd => {
-        const rndStr = vecToCBase32(rnd);
-        const stagePath = join(stageDir, `${rndStr}-${fileName.replaceAll('/', '-')}`);
-        return mkdir(stageDir, { recursive: true })
-            .step(() => rename(src, stagePath))
-            .step(r => {
-            if (r[0] === 'error') {
-                return pure(error(r[1]));
-            }
-            return streamHash(sha256)(stagePath)
-                .step(hashResult => {
-                if (hashResult[0] === 'error') {
-                    return pure(hashResult);
-                }
-                const hash = hashResult[1];
-                const p = toPath(hash);
-                const parts = parse(p);
-                const finalDir = join(home, ...parts.slice(0, -1));
-                const finalPath = join(home, p);
-                return mkdir(finalDir, { recursive: true })
-                    .step(() => rename(stagePath, finalPath))
-                    .step(r2 => pure(r2[0] === 'error' ? error(r2[1]) : ok(hash)));
-            });
-        });
+    const c = fileCas(sha256)(home);
+    return casAddFile(c)(src).step((result) => {
+        if (result[0] === 'error') {
+            return pure(result);
+        }
+        return rm(src).step(() => pure(result));
     });
 };
-export const commands = [
-    {
-        names: ['add'],
-        description: 'Store file content and print its hash',
-        handler: ({ home, args: [path, ...rest] }) => {
-            if (path === undefined || rest.length !== 0) {
-                return errorExit("'cas add' expects one parameter");
-            }
-            const c = fileCas(sha256)(home);
-            return readFile(path)
-                .step(v => c.write(unwrap(v)))
-                .step(hash => hash === undefined
-                ? pure(1)
-                : log(vecToCBase32(hash)).step(() => pure(0)));
-        },
-    },
-    {
-        names: ['get'],
-        description: 'Restore content by hash into a file',
-        handler: ({ home, args: [hashCBase32, path, ...rest] }) => {
-            if (hashCBase32 === undefined || path === undefined || rest.length !== 0) {
-                return errorExit("'cas get' expects two parameters");
-            }
-            const hash = cBase32ToVec(hashCBase32);
-            if (hash === null) {
-                return errorExit(`invalid hash format: ${hashCBase32}`);
-            }
-            const c = fileCas(sha256)(home);
-            return c.read(hash)
-                .step(v => {
-                const result = v === undefined
-                    ? errorExit(`no such hash: ${hashCBase32}`)
-                    : writeFile(path, v)
-                        .step(() => pure(0));
-                return result;
-            });
-        },
-    },
-    {
-        names: ['list'],
-        description: 'List all stored content hashes',
-        handler: ({ home }) => {
-            const c = fileCas(sha256)(home);
-            return c.list()
-                .step(forEachStep(j => log(vecToCBase32(j))))
-                .step(() => pure(0));
-        },
-    },
-];
-export const main = dispatch(commands);

package/fs/cas/proof.f.d.ts

@@ -1,15 +1,18 @@
+import { type FileCasOperation } from './module.f.ts';
+import { type Effect } from '../effects/module.f.ts';
+import { type ReadFile, type WriteFile, type Rm, type Mkdir } from '../effects/node/module.f.ts';
+type TestOp = FileCasOperation | WriteFile | ReadFile | Rm | Mkdir;
 export declare const proof: {
-    mainAdd: () => void;
-    mainAddWrongArgs: () => void;
-    mainGetFound: () => void;
-    mainGetNotFound: () => void;
-    mainGetWrongArgs: () => void;
-    mainGetInvalidHash: () => void;
-    mainList: () => void;
-    mainListEmptyStore: () => void;
-    mainListCorruptStore: () => void;
-    mainNoCmd: () => void;
-    mainUnknownCmd: () => void;
-    casWrite: () => void;
-    casReadPassthrough: () => void;
+    addBigFile: () => Effect<TestOp, void>;
+    addAndGetBigFile: () => Effect<TestOp, void>;
+    casWriteRead: () => void;
+    casReadMissingShard: () => void;
+    casWriteMultiChunk: () => void;
+    casWriteDedup: () => void;
+    casWriteErrorItemAborts: () => void;
+    casWriteReadExceedsMaxLength: () => void;
+    casWriteGcReclaimsExpired: () => void;
+    casUploadSuccess: () => void;
+    casUploadFailureKeepsSource: () => void;
 };
+export {};