@ai-ecoverse/slicc-kernel 1.15.1 → 1.16.0

package/dist/node.js

@@ -1067,9 +1067,9 @@ var OpfsFs = class {
 var NAME = /^[A-Za-z0-9._+-]+$/;
 var ENV_KEY = /^[A-Za-z_][A-Za-z0-9_]*$/;
 var PACKAGE = /\$\{package\}/g;
-function inside(pkg, rel) {
-  if (typeof rel !== "string" || rel === "") return void 0;
-  const clean = rel.replace(/^\.\//, "");
+function inside(pkg, rel2) {
+  if (typeof rel2 !== "string" || rel2 === "") return void 0;
+  const clean = rel2.replace(/^\.\//, "");
   if (clean.startsWith("/") || clean.split("/").includes("..")) return void 0;
   return `${pkg}/${clean}`;
 }
@@ -5911,6 +5911,57 @@ function serveFilesystem(port, handlers, capabilities = {}) {
   return { invalidate: (paths) => reply({ invalidate: paths }) };
 }
 
+// src/mount/medium.ts
+var EMPTY_ROOT = { kind: "directory", size: 0, mtime: 0, mode: 16877 };
+var absent = () => fsError2("ENOMEDIUM", "No medium found");
+function mediumSlot(lost, onLost) {
+  let current;
+  function wrap(pick) {
+    return async (...args) => {
+      const medium = current;
+      if (!medium) throw absent();
+      try {
+        return await pick(medium.handlers)(...args);
+      } catch (err) {
+        if (!await lost(err, medium.key)) throw err;
+        if (current === medium) {
+          current = void 0;
+          onLost(medium.key);
+        }
+        throw absent();
+      }
+    };
+  }
+  const getattr = wrap((h) => h.getattr);
+  const readdir = wrap((h) => h.readdir);
+  const statfs = wrap((h) => async () => await h.statfs?.() ?? null);
+  return {
+    handlers: {
+      getattr: (path) => current || path !== "/" ? getattr(path) : Promise.resolve(EMPTY_ROOT),
+      readdir: (path) => current || path !== "/" ? readdir(path) : Promise.resolve([]),
+      open: wrap((h) => h.open),
+      read: wrap((h) => h.read),
+      write: wrap((h) => h.write),
+      release: wrap((h) => h.release),
+      mkdir: wrap((h) => h.mkdir),
+      rmdir: wrap((h) => h.rmdir),
+      unlink: wrap((h) => h.unlink),
+      rename: wrap((h) => h.rename),
+      symlink: wrap((h) => h.symlink),
+      readlink: wrap((h) => h.readlink),
+      setattr: wrap((h) => h.setattr),
+      statfs: () => current ? statfs() : Promise.resolve(null)
+    },
+    present: () => current !== void 0,
+    insert(key, handlers) {
+      current = { key, handlers };
+    },
+    eject() {
+      current = void 0;
+    }
+  };
+}
+
 // src/mount/fsa.ts
 var FSA_CAPABILITIES = { listingStats: true };
 var codeOf = (err) => err?.code;
@@ -5998,7 +6049,6 @@ function kernelFsHandlers(fs) {
     }
   };
 }
-var EMPTY_ROOT = { kind: "directory", size: 0, mtime: 0, mode: 16877 };
 function granted(handle) {
   if (!handle.queryPermission) return Promise.resolve(true);
   return handle.queryPermission({ mode: "readwrite" }).then(
@@ -6007,48 +6057,345 @@ function granted(handle) {
   );
 }
 function removableMedium(onLost, open = (handle) => new OpfsFs(handle)) {
-  let current;
-  const absent = () => fsError2("ENOMEDIUM", "No medium found");
-  function wrap(pick) {
-    return async (...args) => {
-      const medium = current;
-      if (!medium) throw absent();
-      try {
-        return await pick(medium.handlers)(...args);
-      } catch (err) {
-        if (!denied(err) || await granted(medium.handle)) throw err;
-        if (current === medium) {
-          current = void 0;
-          onLost(medium.handle);
+  const slot = mediumSlot(
+    async (err, handle) => denied(err) && !await granted(handle),
+    onLost
+  );
+  return {
+    handlers: slot.handlers,
+    present: slot.present,
+    insert: (handle) => slot.insert(handle, kernelFsHandlers(open(handle))),
+    eject: slot.eject
+  };
+}
+
+// src/mount/hostfs.ts
+var HOSTFS_IO = 16 * 1024 * 1024;
+var codeOf2 = (err) => err?.code;
+var unreachable = (err) => codeOf2(err) === "ENOTCONN";
+function rel(path) {
+  return path.replace(/^\/+/, "");
+}
+var HostfsClient = class {
+  grant;
+  regrant;
+  fetch;
+  constructor(grant, regrant, fetch2) {
+    this.grant = grant;
+    this.regrant = regrant;
+    this.fetch = fetch2;
+  }
+  get capabilities() {
+    return this.grant.capabilities ?? {};
+  }
+  get token() {
+    return this.grant.token;
+  }
+  get maxIo() {
+    return this.capabilities.maxIo ?? HOSTFS_IO;
+  }
+  async send(path, init, handle = false, retried = false) {
+    let response;
+    try {
+      response = await this.fetch(`${this.grant.url}${path}`, {
+        ...init,
+        credentials: "omit",
+        headers: {
+          ...init.headers,
+          "X-Hostfs-Token": this.grant.token
         }
-        throw absent();
+      });
+    } catch (err) {
+      throw fsError2("ENOTCONN", `the host proxy is unreachable: ${err.message}`);
+    }
+    if (response.ok) return response;
+    if (response.headers.get("X-Proxy-Error")) {
+      await response.body?.cancel();
+      if (retried || response.status !== 403) {
+        throw fsError2("EACCES", `the host proxy refused this folder (${response.status})`);
       }
-    };
+      this.grant = await this.regrant();
+      if (handle)
+        throw fsError2("ESTALE", "the folder was granted again, so its open files are gone");
+      return this.send(path, init, false, true);
+    }
+    const errno = response.headers.get("X-Hostfs-Errno") ?? "EIO";
+    const body = await response.json().catch(() => ({}));
+    throw fsError2(errno, typeof body.message === "string" ? body.message : errno);
+  }
+  async call(op, body = {}, handle = false) {
+    const response = await this.send(
+      "/api/hostfs",
+      {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ op, ...body })
+      },
+      handle
+    );
+    return await response.json();
+  }
+  async read(fh, offset, size, ifMatch) {
+    const response = await this.send(
+      "/api/hostfs",
+      {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ op: "read", fh, offset, size, ...ifMatch ? { ifMatch } : {} })
+      },
+      true
+    );
+    return new Uint8Array(await response.arrayBuffer());
+  }
+  async write(fh, offset, bytes) {
+    const response = await this.send(
+      "/api/hostfs/write",
+      {
+        method: "PUT",
+        headers: {
+          "Content-Type": "application/octet-stream",
+          "X-Hostfs-Request": JSON.stringify({ fh, offset })
+        },
+        body: bytes
+      },
+      true
+    );
+    await response.body?.cancel();
+  }
+  async watch(signal, onEvent, onOpen = () => void 0) {
+    const response = await this.send("/api/hostfs/watch", { method: "POST", signal });
+    onOpen();
+    const reader = response.body.getReader();
+    const decoder2 = new TextDecoder();
+    let buffered = "";
+    try {
+      for (; ; ) {
+        const { done: done4, value } = await reader.read();
+        if (done4) return;
+        buffered += decoder2.decode(value, { stream: true });
+        let end = buffered.indexOf("\n");
+        while (end >= 0) {
+          const line = buffered.slice(0, end).trim();
+          buffered = buffered.slice(end + 1);
+          if (line) onEvent(JSON.parse(line));
+          end = buffered.indexOf("\n");
+        }
+      }
+    } catch (err) {
+      if (signal.aborted) return;
+      throw fsError2("ENOTCONN", `the host proxy's watch broke: ${err.message}`);
+    } finally {
+      reader.releaseLock();
+    }
+  }
+};
+function hostfsHandlers(current) {
+  const handles = /* @__PURE__ */ new Map();
+  let nextFh = 0;
+  const fresh = (remote) => {
+    if (remote.client.token === remote.token) return remote;
+    throw fsError2("ESTALE", "the folder was granted again, so this open file is gone");
+  };
+  const live = (fh) => {
+    const remote = handles.get(fh);
+    if (!remote) throw fsError2("EBADF", `handle ${fh}`);
+    return fresh(remote);
+  };
+  const call = (op, body = {}) => current().call(op, body);
+  async function writable(client, path, flags) {
+    const opening = () => client.call("open", { path: rel(path), ...flags });
+    try {
+      return { opened: await opening(), restore: void 0 };
+    } catch (err) {
+      if (!flags.write || codeOf2(err) !== "EACCES") throw err;
+      const mode = (await client.call("stat", { path: rel(path) })).mode ?? 0;
+      if (mode & 128) throw err;
+      const restore = { path: rel(path), mode };
+      await client.call("setattr", { path: rel(path), mode: mode | 128 });
+      const opened2 = await opening().catch(async (again) => {
+        await client.call("setattr", restore);
+        throw again;
+      });
+      return { opened: opened2, restore };
+    }
   }
-  const getattr = wrap((h) => h.getattr);
-  const readdir = wrap((h) => h.readdir);
   return {
-    handlers: {
-      getattr: (path) => current || path !== "/" ? getattr(path) : Promise.resolve(EMPTY_ROOT),
-      readdir: (path) => current || path !== "/" ? readdir(path) : Promise.resolve([]),
-      open: wrap((h) => h.open),
-      read: wrap((h) => h.read),
-      write: wrap((h) => h.write),
-      release: wrap((h) => h.release),
-      mkdir: wrap((h) => h.mkdir),
-      rmdir: wrap((h) => h.rmdir),
-      unlink: wrap((h) => h.unlink),
-      rename: wrap((h) => h.rename),
-      symlink: wrap((h) => h.symlink),
-      readlink: wrap((h) => h.readlink),
-      setattr: wrap((h) => h.setattr)
+    getattr: async (path) => await call("stat", { path: rel(path) }),
+    async readdir(path) {
+      const { entries } = await call(
+        "list",
+        { path: rel(path) }
+      );
+      return entries.map(({ name: name2, attr }) => ({ name: name2, kind: attr.kind, attr }));
     },
-    present: () => current !== void 0,
-    insert(handle) {
-      current = { handle, handlers: kernelFsHandlers(open(handle)) };
+    async open(path, flags) {
+      const client = current();
+      const token = client.token;
+      const { opened: opened2, restore } = await writable(client, path, flags);
+      handles.set(++nextFh, {
+        client,
+        token,
+        fh: opened2.fh,
+        etag: flags.write ? void 0 : opened2.attr?.etag,
+        restore
+      });
+      return nextFh;
     },
-    eject() {
-      current = void 0;
+    async read(fh, offset, size) {
+      const remote = live(fh);
+      return remote.client.read(
+        remote.fh,
+        offset,
+        Math.min(size, remote.client.maxIo),
+        remote.etag
+      );
+    },
+    async write(fh, offset, bytes) {
+      const remote = live(fh);
+      const most = remote.client.maxIo;
+      for (let at = 0; at < bytes.length; at += most) {
+        await remote.client.write(remote.fh, offset + at, bytes.subarray(at, at + most));
+      }
+    },
+    async release(fh) {
+      const remote = handles.get(fh);
+      if (!remote) throw fsError2("EBADF", `handle ${fh}`);
+      handles.delete(fh);
+      try {
+        await fresh(remote).client.call("release", { fh: remote.fh }, true);
+      } finally {
+        if (remote.restore) await current().call("setattr", remote.restore);
+      }
+    },
+    mkdir: async (path) => void await call("mkdir", { path: rel(path) }),
+    rmdir: async (path) => void await call("rmdir", { path: rel(path) }),
+    unlink: async (path) => void await call("unlink", { path: rel(path) }),
+    rename: async (from, to) => void await call("rename", { from: rel(from), to: rel(to) }),
+    symlink: async (target, path) => void await call("symlink", { target, path: rel(path) }),
+    readlink: async (path) => (await call("readlink", { path: rel(path) })).target,
+    setattr: async (path, change) => void await call("setattr", { path: rel(path), ...change }),
+    statfs: () => call("statfs")
+  };
+}
+function sleep(ms, signal) {
+  return new Promise((resolve) => {
+    const done4 = () => {
+      clearTimeout(timer);
+      signal.removeEventListener("abort", done4);
+      resolve();
+    };
+    const timer = setTimeout(done4, ms);
+    signal.addEventListener("abort", done4);
+  });
+}
+function keepWatching(loop, signal) {
+  const silenceMs = loop.silenceMs ?? 45e3;
+  const [first, most] = loop.backoffMs ?? [1e3, 3e4];
+  const ready2 = Promise.withResolvers();
+  let wait = first;
+  let up = false;
+  let current;
+  async function session() {
+    const session2 = new AbortController();
+    current = session2;
+    const stop = () => session2.abort();
+    signal.addEventListener("abort", stop);
+    let timer;
+    const quiet = () => {
+      clearTimeout(timer);
+      timer = setTimeout(stop, silenceMs);
+    };
+    let opened2 = false;
+    try {
+      const client = await loop.connect();
+      await client.watch(
+        session2.signal,
+        (event) => {
+          quiet();
+          if (!event.ping) loop.event(event);
+        },
+        () => {
+          quiet();
+          opened2 = true;
+          up = true;
+          loop.up(client);
+          ready2.resolve();
+        }
+      );
+      return opened2;
+    } catch {
+      return opened2;
+    } finally {
+      clearTimeout(timer);
+      signal.removeEventListener("abort", stop);
+    }
+  }
+  void (async () => {
+    while (!signal.aborted) {
+      const began = Date.now();
+      const opened2 = await session();
+      if (signal.aborted) return;
+      if (opened2 && Date.now() - began >= first) {
+        wait = first;
+        continue;
+      }
+      if (!opened2 && up) {
+        up = false;
+        loop.down();
+      }
+      ready2.resolve();
+      await sleep(wait, signal);
+      wait = Math.min(wait * 2, most);
+    }
+  })();
+  return { ready: ready2.promise, kick: () => current?.abort() };
+}
+function invalidation(event) {
+  return event.paths ?? true;
+}
+async function openHostfs(spec, hook, fetch2, timing = {}) {
+  const readonly = spec.options?.ro !== void 0;
+  const { port1, port2 } = new MessageChannel();
+  const slot = mediumSlot(
+    async (err) => unreachable(err),
+    () => watching.kick()
+  );
+  const stop = new AbortController();
+  const grant = () => hook(spec.source, { readonly });
+  let active;
+  const handlers = hostfsHandlers(() => active);
+  const served = serveFilesystem(port2, slot.handlers, {
+    symlinks: true,
+    chmod: true,
+    listingStats: true,
+    maxIo: HOSTFS_IO,
+    ...readonly ? { readonly: true } : {}
+  });
+  const watching = keepWatching(
+    {
+      connect: async () => new HostfsClient(await grant(), grant, fetch2),
+      up(client) {
+        active = client;
+        slot.insert(client, handlers);
+        served.invalidate(true);
+      },
+      event: (event) => served.invalidate(invalidation(event)),
+      down() {
+        slot.eject();
+        served.invalidate(true);
+      },
+      ...timing
+    },
+    stop.signal
+  );
+  await watching.ready;
+  return {
+    port: port1,
+    present: slot.present,
+    dispose() {
+      stop.abort();
+      port1.close();
+      port2.close();
     }
   };
 }
@@ -6157,6 +6504,7 @@ var DriverConnection = class _DriverConnection {
 
 // src/mount/mount-fs.ts
 var MAX_LINKS4 = 40;
+var STALE_RETRIES = 3;
 var DEFAULT_IO = 8 * 1024 * 1024;
 var UNITS = { "": 1, k: 1024, m: 1024 ** 2, g: 1024 ** 3, t: 1024 ** 4 };
 function parseSize(text) {
@@ -6223,8 +6571,8 @@ var MountTable = class {
         best = mount;
     }
     if (!best) return void 0;
-    const rel = best.target === "/" ? path : path.slice(best.target.length);
-    return { mount: best, rel: rel || "/" };
+    const rel2 = best.target === "/" ? path : path.slice(best.target.length);
+    return { mount: best, rel: rel2 || "/" };
   }
   async mount(spec, fs) {
     const target = normalizePath(spec.target);
@@ -6329,20 +6677,20 @@ var MountTable = class {
     mount.lists.delete(parentOf4(path));
     for (const key of [...mount.lists.keys()]) if (within3(key, path)) mount.lists.delete(key);
   }
-  async getattr(mount, path, rel) {
+  async getattr(mount, path, rel2) {
     if (mount.conn.failure) throw mount.conn.failure;
     const cached = mount.attrs.get(path);
     if (cached && cached.expires > Date.now()) return cached.value;
-    const attr = await mount.conn.call({ op: "getattr", path: rel });
+    const attr = await mount.conn.call({ op: "getattr", path: rel2 });
     const ttl = mount.caps.attrTtl ?? 1e3;
     if (ttl > 0) mount.attrs.set(path, { value: attr, expires: Date.now() + ttl });
     return attr;
   }
-  async readdir(mount, path, rel) {
+  async readdir(mount, path, rel2) {
     if (mount.conn.failure) throw mount.conn.failure;
     const cached = mount.lists.get(path);
     if (cached && cached.expires > Date.now()) return cached.value;
-    const entries = await mount.conn.call({ op: "readdir", path: rel });
+    const entries = await mount.conn.call({ op: "readdir", path: rel2 });
     const ttl = mount.caps.entryTtl ?? mount.caps.attrTtl ?? 1e3;
     if (ttl > 0) mount.lists.set(path, { value: entries, expires: Date.now() + ttl });
     return entries;
@@ -6387,10 +6735,10 @@ var MountFs = class {
     const target = await found.mount.conn.call({ op: "readlink", path: found.rel });
     return this.follow(this.base.resolvePath(parentOf4(real), target), hops + 1, create);
   }
-  async open(mount, rel, write) {
+  async open(mount, rel2, write) {
     return await mount.conn.call({
       op: "open",
-      path: rel,
+      path: rel2,
       write,
       create: write,
       truncate: write,
@@ -6407,13 +6755,22 @@ var MountFs = class {
     const real = await this.follow(path);
     const found = this.at(real);
     if (!found) return this.base.readFileBuffer(real);
-    const { mount, rel } = found;
-    const attr = await this.table.getattr(mount, real, rel);
+    for (let attempt = 1; ; attempt++) {
+      try {
+        return await this.readWhole(found.mount, real, found.rel);
+      } catch (err) {
+        if (err.code !== "ESTALE" || attempt >= STALE_RETRIES) throw err;
+        this.table.forget(found.mount, real);
+      }
+    }
+  }
+  async readWhole(mount, real, rel2) {
+    const attr = await this.table.getattr(mount, real, rel2);
     if (attr.kind === "directory") throw errnoError("EISDIR", real);
     if (mount.maxFile > 0 && attr.size > mount.maxFile) {
       throw errnoError("EFBIG", `${real} is larger than this mount's maxfile (${mount.maxFile})`);
     }
-    const fh = await this.open(mount, rel, false);
+    const fh = await this.open(mount, rel2, false);
     const chunk2 = mount.caps.maxIo ?? DEFAULT_IO;
     const chunks2 = [];
     let total = 0;
@@ -6439,7 +6796,7 @@ var MountFs = class {
     const real = await this.follow(path, 0, true);
     const found = this.at(real);
     if (!found) return this.base.writeFile(real, content);
-    const { mount, rel } = found;
+    const { mount, rel: rel2 } = found;
     this.writable(mount, real);
     const bytes = typeof content === "string" ? new TextEncoder().encode(content) : content;
     if (mount.maxFile > 0 && bytes.length > mount.maxFile) {
@@ -6448,8 +6805,17 @@ var MountFs = class {
         `${real} would be larger than this mount's maxfile (${mount.maxFile})`
       );
     }
+    for (let attempt = 1; ; attempt++) {
+      try {
+        return await this.writeWhole(mount, real, rel2, bytes);
+      } catch (err) {
+        if (err.code !== "ESTALE" || attempt >= STALE_RETRIES) throw err;
+      }
+    }
+  }
+  async writeWhole(mount, real, rel2, bytes) {
     this.table.forget(mount, real);
-    const fh = await this.open(mount, rel, true);
+    const fh = await this.open(mount, rel2, true);
     const chunk2 = mount.caps.maxIo ?? DEFAULT_IO;
     try {
       for (let offset = 0; offset < bytes.length; offset += chunk2) {
@@ -6516,16 +6882,16 @@ var MountFs = class {
   async mkdir(path, options = {}) {
     const found = this.at(path);
     if (!found) return this.base.mkdir(path, options);
-    const { mount, rel } = found;
+    const { mount, rel: rel2 } = found;
     const real = normalizePath(path);
     this.writable(mount, real);
     if (!options.recursive) {
-      if (rel === "/") throw errnoError("EEXIST", real);
+      if (rel2 === "/") throw errnoError("EEXIST", real);
       this.table.forget(mount, real);
-      await mount.conn.call({ op: "mkdir", path: rel });
+      await mount.conn.call({ op: "mkdir", path: rel2 });
       return;
     }
-    const parts = rel.split("/").filter(Boolean);
+    const parts = rel2.split("/").filter(Boolean);
     for (let n = 1; n <= parts.length; n++) {
       const sub = `/${parts.slice(0, n).join("/")}`;
       const full = normalizePath(`${mount.target}${sub}`);
@@ -6539,25 +6905,25 @@ var MountFs = class {
   async rm(path, options = {}) {
     const found = this.at(path);
     if (!found) return this.base.rm(path, options);
-    const { mount, rel } = found;
+    const { mount, rel: rel2 } = found;
     const real = normalizePath(path);
-    if (rel === "/") throw errnoError("EBUSY", `${real} is a mount point`);
+    if (rel2 === "/") throw errnoError("EBUSY", `${real} is a mount point`);
     this.writable(mount, real);
     let attr;
     try {
-      attr = await this.table.getattr(mount, real, rel);
+      attr = await this.table.getattr(mount, real, rel2);
     } catch (err) {
       if (options.force && err.code === "ENOENT") return;
       throw err;
     }
     this.table.forget(mount, real);
     if (attr.kind === "directory" && options.recursive) {
-      for (const entry of await this.table.readdir(mount, real, rel)) {
+      for (const entry of await this.table.readdir(mount, real, rel2)) {
         await this.rm(`${real}/${entry.name}`, { recursive: true, force: options.force ?? false });
       }
       this.table.forget(mount, real);
     }
-    await mount.conn.call({ op: attr.kind === "directory" ? "rmdir" : "unlink", path: rel });
+    await mount.conn.call({ op: attr.kind === "directory" ? "rmdir" : "unlink", path: rel2 });
     this.table.forget(mount, real);
   }
   async rename(from, to) {
@@ -7019,6 +7385,7 @@ var Launcher = class {
   media;
   onMountPending;
   removable = /* @__PURE__ */ new Map();
+  hostfs;
   inserted = /* @__PURE__ */ new Map();
   openFiles = /* @__PURE__ */ new Set();
   held = /* @__PURE__ */ new Set();
@@ -7036,6 +7403,7 @@ var Launcher = class {
     this.createDriverWorker = options.createDriverWorker;
     this.media = options.media ?? memoryMedia();
     this.onMountPending = options.onMountPending;
+    this.hostfs = options;
     this.pnpmHome = options.env?.PNPM_HOME ?? PNPM_HOME;
     this.modulesDir = options.modules ?? "/node_modules";
     this.env = { ...networkEnv(), ...options.env };
@@ -7301,6 +7669,17 @@ var Launcher = class {
   }
   async driver(type, spec) {
     if (type === "fsa") return this.fsaDriver(spec);
+    if (type === "hostfs") {
+      const { hostfs, hostfsFetch, hostfsTiming } = this.hostfs;
+      if (!hostfs)
+        throw fsError("ENODEV", "this kernel has no hostfs hook (createKernel({ hostfs }))");
+      return openHostfs(
+        spec,
+        hostfs,
+        hostfsFetch ?? ((url, init) => fetch(url, init)),
+        hostfsTiming
+      );
+    }
     if (type === "tmpfs") {
       const { port1, port2 } = new MessageChannel();
       serveFilesystem(port2, tmpfs(), { symlinks: true, chmod: true, attrTtl: 0 });