@statewalker/webrun-files-rpc 0.1.0 → 0.1.2

package/src/server.ts

@@ -1,7 +1,16 @@
 import type { FileInfo, FilesApi } from "@statewalker/webrun-files";
 import { FilesRpcError } from "./errors.js";
 import { Keyed } from "./keyed.js";
-import type { FilesRpc, ListPage, OpenOptions, OpenReadOptions, ReadResult } from "./types.js";
+import type {
+  AppendableFilesApi,
+  FilesRpc,
+  ListPage,
+  OpenOptions,
+  OpenReadOptions,
+  OpenWriteOptions,
+  ReadResult,
+  RequestOptions,
+} from "./types.js";
 
 export interface FilesRpcServerOptions {
   /** The file system served. */
@@ -30,6 +39,8 @@ export interface FilesRpcServerOptions {
   maxStreams?: number;
   /** Largest listing page. Default 256. */
   maxPageSize?: number;
+  /** Answers of metadata calls kept by their `requestId` at most (default 1024). */
+  maxRequests?: number;
   /** Sweeps idle streams every this many ms (default: only when a stream is opened). */
   sweepInterval?: number;
   /** The clock (tests). Default `Date.now`. */
@@ -89,10 +100,33 @@ interface WriteStream {
   room: { chunkId: number; resolve: () => void; reject: (e: unknown) => void }[];
   failure?: unknown;
   done: Promise<void>;
+  /** An append the server emulates; `committing` once it writes the file back. */
+  emulated?: boolean;
+  committing?: boolean;
 }
 
 type Stream = ReadStream | WriteStream;
 
+/**
+ * Whether a write's outcome is the FilesApi's alone: every chunk was handed to it (an emulated
+ * append, until its write-back starts, can still be cancelled).
+ */
+const committed = (s: WriteStream): boolean =>
+  s.total !== undefined && s.next >= s.total && (!s.emulated || s.committing === true);
+
+/** `path` in normal form: one leading slash, no empty, "." or ".." segments, no trailing slash. */
+function normalPath(path: string): string {
+  const out: string[] = [];
+  for (const seg of String(path).split("/")) {
+    if (!seg || seg === ".") continue;
+    if (seg === "..") out.pop();
+    else out.push(seg);
+  }
+  return `/${out.join("/")}`;
+}
+
+const messageOf = (e: unknown) => String((e as Error)?.message ?? e);
+
 const EMPTY = new Uint8Array();
 const STREAM_ID = /^[A-Za-z0-9_-]{16,128}$/;
 
@@ -165,6 +199,44 @@ export function newFilesRpcServer(options: FilesRpcServerOptions): FilesRpcServe
    */
   const ended = new Map<string, { at: number; failure?: unknown }>();
 
+  /** Answers of metadata calls, by method and requestId, oldest first. */
+  const requests = new Map<string, { at: number; settled: boolean; answer: Promise<unknown> }>();
+  const maxRequests = Math.max(1, options.maxRequests ?? 1024);
+  /** Runs `call` once per requestId (within idleTimeout): a call delivered again gets its answer. */
+  const once = <T>(method: string, opts: RequestOptions | undefined, call: () => Promise<T>) => {
+    const id = opts?.requestId;
+    if (id === undefined) return call();
+    if (typeof id !== "string" || !STREAM_ID.test(id))
+      return Promise.reject(
+        new FilesRpcError("INVALID", "a requestId is 16 to 128 characters of [A-Za-z0-9_-]"),
+      );
+    const key = `${method} ${id}`;
+    const kept = requests.get(key);
+    if (kept) return kept.answer as Promise<T>;
+    const answer = call();
+    answer.catch(() => {});
+    const entry = { at: now(), settled: false, answer };
+    requests.set(key, entry);
+    // Kept for idleTimeout after it settled (a call still running is not forgotten).
+    const settle = () => {
+      entry.at = now();
+      entry.settled = true;
+    };
+    void answer.then(settle, settle);
+    // Past the cap, the oldest settled answer goes: forgetting a call still running would let a
+    // retry run it again. Only when every call is running does the oldest of them go.
+    while (requests.size > maxRequests) {
+      let victim: string | undefined;
+      for (const [k, r] of requests)
+        if (r.settled) {
+          victim = k;
+          break;
+        }
+      requests.delete(victim ?? (requests.keys().next().value as string));
+    }
+    return answer;
+  };
+
   const finish = (id: string, failure?: unknown) => {
     streams.delete(id);
     ended.set(id, { at: now(), failure });
@@ -311,6 +383,71 @@ export function newFilesRpcServer(options: FilesRpcServerOptions): FilesRpcServe
     }
   }
 
+  /** Emulated appends in progress, by normal path: one at a time per file. */
+  const appending = new Map<string, Promise<void>>();
+
+  /**
+   * Appends `content` to `path`: through the FilesApi's own `append`, or with read, write and
+   * remove only. The emulation never reads and writes one path at once (a FilesApi may empty a
+   * file when a write starts):
+   *  1. the file's content and the chunks go to a temporary sibling;
+   *  2. that sibling is written back over the file, then removed.
+   * A failure or an abort in step 1 leaves the file as it was and removes the sibling. A failure
+   * in step 2 keeps the sibling (the only complete copy) and names it in the error. Appends to
+   * one file through this server run one after the other; other writers are not excluded.
+   */
+  const append = async (
+    path: string,
+    content: AsyncIterable<Uint8Array>,
+    id: string,
+    s: WriteStream,
+  ): Promise<void> => {
+    const own = (fs as Partial<AppendableFilesApi>).append;
+    if (typeof own === "function") return own.call(fs, path, content);
+    s.emulated = true;
+    const file = normalPath(path);
+    const previous = appending.get(file) ?? Promise.resolve();
+    const run = previous.then(async () => {
+      // Only a directory is refused: whether the file exists is not trusted (reading a missing
+      // file yields nothing, so a missing file is created all the same).
+      if ((await fs.stats(file))?.kind === "directory")
+        throw new FilesRpcError("INVALID", `cannot append to a directory: ${file}`);
+      const slash = file.lastIndexOf("/");
+      const temp = `${file.slice(0, slash + 1)}.${file.slice(slash + 1, slash + 65)}.append-${id.slice(0, 16)}`;
+      try {
+        await fs.write(
+          temp,
+          (async function* () {
+            yield* fs.read(file);
+            yield* content;
+          })(),
+        );
+        if (s.failure !== undefined) throw s.failure; // aborted before the write-back
+      } catch (e) {
+        await fs.remove(temp).catch(() => {});
+        throw e;
+      }
+      s.committing = true;
+      try {
+        await fs.write(file, fs.read(temp));
+      } catch (e) {
+        throw Object.assign(
+          new Error(
+            `${messageOf(e)} (the append failed while writing ${file} back: it may be incomplete; its content and the appended chunks are kept in ${temp})`,
+          ),
+          { cause: e },
+        );
+      }
+      await fs.remove(temp).catch(() => {});
+    });
+    const settled = run.catch(() => {});
+    appending.set(file, settled);
+    void settled.then(() => {
+      if (appending.get(file) === settled) appending.delete(file);
+    });
+    return run;
+  };
+
   const failWrite = (s: WriteStream, failure: unknown) => {
     s.failure ??= failure;
     for (const r of s.room.splice(0)) r.reject(s.failure);
@@ -324,17 +461,17 @@ export function newFilesRpcServer(options: FilesRpcServerOptions): FilesRpcServe
     async exists(path) {
       return fs.exists(path);
     },
-    async mkdir(path) {
-      await fs.mkdir(path);
+    async mkdir(path, opts?: RequestOptions) {
+      await once("mkdir", opts, () => fs.mkdir(path));
     },
-    async remove(path) {
-      return fs.remove(path);
+    async remove(path, opts?: RequestOptions) {
+      return once("remove", opts, () => fs.remove(path));
     },
-    async move(source, target) {
-      return fs.move(source, target);
+    async move(source, target, opts?: RequestOptions) {
+      return once("move", opts, () => fs.move(source, target));
     },
-    async copy(source, target) {
-      return fs.copy(source, target);
+    async copy(source, target, opts?: RequestOptions) {
+      return once("copy", opts, () => fs.copy(source, target));
     },
     async list(path, opts = {}): Promise<ListPage> {
       const limit = Math.min(
@@ -429,7 +566,7 @@ export function newFilesRpcServer(options: FilesRpcServerOptions): FilesRpcServe
       await closeSource(s);
     },
 
-    async openWrite(path, opts: OpenOptions = {}) {
+    async openWrite(path, opts: OpenWriteOptions = {}) {
       const { id, existing } = claim(opts, "write");
       if (existing) return { streamId: id };
       const s: WriteStream = {
@@ -441,7 +578,7 @@ export function newFilesRpcServer(options: FilesRpcServerOptions): FilesRpcServe
         room: [],
         done: Promise.resolve(),
       };
-      s.done = fs.write(path, ordered(s)).then(
+      s.done = (opts.append ? append(path, ordered(s), id, s) : fs.write(path, ordered(s))).then(
         () => {
           if (s.failure !== undefined) throw s.failure;
         },
@@ -530,8 +667,9 @@ export function newFilesRpcServer(options: FilesRpcServerOptions): FilesRpcServe
       const id = String(streamId);
       const s = streams.get(id);
       if (s?.kind !== "write") return;
-      // Every chunk was handed to the FilesApi: the commit is under way; an abort is too late.
-      if (s.total !== undefined && s.next >= s.total) return;
+      // The outcome is the FilesApi's now (every chunk handed over; an emulated append already
+      // writing back): an abort is too late.
+      if (committed(s)) return;
       const error = new FilesRpcError("ABORTED", reason ? String(reason) : "the write was aborted");
       finish(id, error);
       failWrite(s, error);
@@ -557,6 +695,8 @@ export function newFilesRpcServer(options: FilesRpcServerOptions): FilesRpcServe
         else failWrite(s, error);
       }
       for (const [id, e] of [...ended]) if (t - e.at > idleTimeout) ended.delete(id);
+      for (const [key, r] of [...requests])
+        if (r.settled && t - r.at > idleTimeout) requests.delete(key);
       return dropped;
     },
 
@@ -568,6 +708,11 @@ export function newFilesRpcServer(options: FilesRpcServerOptions): FilesRpcServe
       if (timer) clearInterval(timer);
       const sources: Promise<unknown>[] = [];
       for (const [id, s] of [...streams]) {
+        // A write whose outcome is the FilesApi's keeps it: a closeWrite waiting on it gets that.
+        if (s.kind === "write" && committed(s)) {
+          streams.delete(id);
+          continue;
+        }
         const error = new FilesRpcError("ABORTED", "the server closed");
         finish(id, error);
         if (s.kind === "read") sources.push(closeSource(s));

package/src/types.ts

@@ -1,4 +1,4 @@
-import type { FileInfo, FileStats } from "@statewalker/webrun-files";
+import type { FileInfo, FileStats, FilesApi } from "@statewalker/webrun-files";
 
 /** One chunk of a read stream: its bytes, and whether it is the last. */
 export interface ReadResult {
@@ -29,6 +29,34 @@ export interface OpenOptions {
   streamId?: string;
 }
 
+/** Options of the calls that change metadata (`mkdir`, `remove`, `move`, `copy`). */
+export interface RequestOptions {
+  /**
+   * The call's id, chosen by the caller (the same form as a stream id). The server keeps each
+   * id's answer for `idleTimeout` (at most `maxRequests` of them): the same call delivered again
+   * gets that answer and does not run again.
+   */
+  requestId?: string;
+}
+
+export interface OpenWriteOptions extends OpenOptions {
+  /**
+   * Add the stream's chunks to the end of the file instead of replacing it (a file that does not
+   * exist is created). See `AppendableFilesApi` for how the server appends.
+   */
+  append?: boolean;
+}
+
+/**
+ * A FilesApi that can append itself. `FilesApi` has no append; a served FilesApi with this method
+ * gets appends through it. Any other is appended to with read, write and remove only: the old
+ * content and the new chunks go to a temporary sibling (`.<name>.append-<streamId>`), which is
+ * then written back over the file and removed.
+ */
+export interface AppendableFilesApi extends FilesApi {
+  append(path: string, content: AsyncIterable<Uint8Array>): Promise<void>;
+}
+
 export interface OpenReadOptions extends OpenOptions {
   /** First byte (default 0). */
   start?: number;
@@ -50,10 +78,10 @@ export interface OpenReadOptions extends OpenOptions {
 export interface FilesRpc {
   stats(path: string): Promise<FileStats | undefined>;
   exists(path: string): Promise<boolean>;
-  mkdir(path: string): Promise<void>;
-  remove(path: string): Promise<boolean>;
-  move(source: string, target: string): Promise<boolean>;
-  copy(source: string, target: string): Promise<boolean>;
+  mkdir(path: string, options?: RequestOptions): Promise<void>;
+  remove(path: string, options?: RequestOptions): Promise<boolean>;
+  move(source: string, target: string, options?: RequestOptions): Promise<boolean>;
+  copy(source: string, target: string, options?: RequestOptions): Promise<boolean>;
   list(path: string, options?: ListPageOptions): Promise<ListPage>;
 
   /** Opens `path` for reading. A file that does not exist reads as nothing. */
@@ -66,8 +94,11 @@ export interface FilesRpc {
   /** Ends the stream (unknown or closed: nothing happens). */
   closeRead(streamId: string): Promise<void>;
 
-  /** Opens `path` for writing; the FilesApi's `write` consumes the chunks in order. */
-  openWrite(path: string, options?: OpenOptions): Promise<{ streamId: string }>;
+  /**
+   * Opens `path` for writing (or, with `append`, for appending); the FilesApi consumes the chunks
+   * in order.
+   */
+  openWrite(path: string, options?: OpenWriteOptions): Promise<{ streamId: string }>;
   /**
    * Hands over chunk `chunkId`. Resolves once it is kept (not yet written): at once if it is
    * within `writeWindow` chunks of the next to write, else when it is. A chunk already received