@galaxyproject/orbit 0.0.0 → 0.9.0

package/extensions/orbit/provenance-fs-file.ts

@@ -0,0 +1,59 @@
+/**
+ * The ProvenanceFs over the real file system: today's bytes for the Node
+ * shells, moved here from galaxy-provenance.ts so that module imports no fs.
+ */
+
+import * as fs from "fs";
+import * as fsp from "fs/promises";
+import * as path from "path";
+import { randomHex } from "../../shared/random.js";
+import type { ProvenanceFs, ProvenanceStat } from "./provenance-fs";
+
+const code = (err: unknown) => (err as NodeJS.ErrnoException)?.code;
+
+export const fileProvenanceFs: ProvenanceFs = {
+  async lstat(p): Promise<ProvenanceStat | null> {
+    try {
+      return await fsp.lstat(p);
+    } catch (err) {
+      if (code(err) === "ENOENT") return null;
+      throw err;
+    }
+  },
+  lstatSync(p): ProvenanceStat | null {
+    try {
+      return fs.lstatSync(p);
+    } catch (err) {
+      if (code(err) === "ENOENT") return null;
+      throw err;
+    }
+  },
+  async mkdir(p) {
+    try {
+      await fsp.mkdir(p);
+    } catch (err) {
+      if (code(err) !== "EEXIST") throw err;
+    }
+  },
+  readFile: (p) => fsp.readFile(p, "utf-8"),
+  readFileSync: (p) => fs.readFileSync(p, "utf-8"),
+  // `wx`: a planted file or symlink at the path fails the open instead of
+  // being written through.
+  writeExclusive: (p, text) => fsp.writeFile(p, text, { encoding: "utf-8", flag: "wx" }),
+  async replace(file, text) {
+    const temp = path.join(
+      path.dirname(file),
+      `.${path.basename(file)}.${process.pid}.${randomHex(6)}.tmp`,
+    );
+    await fsp.writeFile(temp, text, { encoding: "utf-8", flag: "wx" });
+    try {
+      // rename() replaces a symlink at the target rather than following it, and
+      // readers see the old record or the new one, never half of either.
+      await fsp.rename(temp, file);
+    } catch (err) {
+      await fsp.rm(temp, { force: true });
+      throw err;
+    }
+  },
+  readdir: (dir) => fsp.readdir(dir),
+};

package/extensions/orbit/provenance-fs.ts

@@ -0,0 +1,32 @@
+/**
+ * The byte layer under galaxy-provenance.ts: exactly the file operations the
+ * provenance records and the reconcile cursor make, so a shell with no
+ * directory beside its notebook can keep them somewhere else. The ownership,
+ * symlink and validation logic stays in galaxy-provenance.ts; a backend only
+ * stores bytes.
+ *
+ * Types only, no Node imports: the browser's conformance test counts Node
+ * edges, and this file sits in every backend's import graph.
+ */
+
+export interface ProvenanceStat {
+  isFile(): boolean;
+  isDirectory(): boolean;
+  isSymbolicLink(): boolean;
+}
+
+export interface ProvenanceFs {
+  /** Of the entry itself (a symlink reports as one); null when absent. */
+  lstat(p: string): Promise<ProvenanceStat | null>;
+  lstatSync(p: string): ProvenanceStat | null;
+  /** One level; an existing directory is not an error. */
+  mkdir(p: string): Promise<void>;
+  readFile(p: string): Promise<string>;
+  readFileSync(p: string): string;
+  /** Create, never overwrite: rejects with { code: "EEXIST" } when p exists, a planted symlink included. */
+  writeExclusive(p: string, text: string): Promise<void>;
+  /** Create or replace so a reader sees the old text or the new, never half of either. */
+  replace(p: string, text: string): Promise<void>;
+  /** Entry names directly under dir. */
+  readdir(dir: string): Promise<string[]>;
+}

package/extensions/orbit/provenance-memory-fs.ts

@@ -0,0 +1,59 @@
+/**
+ * A ProvenanceFs over a Map, for a shell with no directory beside its
+ * notebook: the browser worker keeps attempt records and the reconcile cursor
+ * for the life of the process. The paths carry the analysis dir, so two
+ * records hosted by one worker keep separate provenance on the one Map.
+ *
+ * No Node imports here: the browser's conformance test counts Node edges.
+ */
+
+import type { ProvenanceFs, ProvenanceStat } from "./provenance-fs";
+
+const stat = (kind: "file" | "dir"): ProvenanceStat => ({
+  isFile: () => kind === "file",
+  isDirectory: () => kind === "dir",
+  isSymbolicLink: () => false,
+});
+
+// Exactly one separator and one name past the directory.
+const CHILD = /^[\\/][^\\/]+$/;
+
+export function memoryProvenanceFs(
+  files = new Map<string, string>(),
+): ProvenanceFs & { files: Map<string, string> } {
+  const dirs = new Set<string>();
+  // galaxy-provenance.ts reads `err.code` the way it reads Node's, so the codes match.
+  const errno = (code: string, p: string) => Object.assign(new Error(`${code}: ${p}`), { code });
+  const lstatSync = (p: string) => (files.has(p) ? stat("file") : dirs.has(p) ? stat("dir") : null);
+  const readFileSync = (p: string) => {
+    const text = files.get(p);
+    if (text === undefined) throw errno("ENOENT", p);
+    return text;
+  };
+  return {
+    files,
+    lstat: async (p) => lstatSync(p),
+    lstatSync,
+    mkdir: async (p) => {
+      dirs.add(p);
+    },
+    readFile: async (p) => readFileSync(p),
+    readFileSync,
+    writeExclusive: async (p, text) => {
+      if (files.has(p)) throw errno("EEXIST", p);
+      files.set(p, text);
+    },
+    replace: async (p, text) => {
+      files.set(p, text);
+    },
+    readdir: async (dir) => {
+      const names: string[] = [];
+      for (const key of [...files.keys(), ...dirs]) {
+        if (!key.startsWith(dir)) continue;
+        const rest = key.slice(dir.length);
+        if (CHILD.test(rest)) names.push(rest.slice(1));
+      }
+      return names;
+    },
+  };
+}

package/extensions/orbit/record-store-file.ts

@@ -0,0 +1,177 @@
+/**
+ * The file backend of the record store: `notebook.md` on disk, written
+ * atomically, versioned by a stat fingerprint, watched by chokidar.
+ *
+ * This is the only record module with fs/chokidar edges; record-store.ts
+ * resolves to it for every key the shell has not claimed.
+ */
+
+import { randomBytes } from "crypto";
+import { existsSync } from "fs";
+import * as fs from "fs/promises";
+import chokidar, { type FSWatcher } from "chokidar";
+import { NotebookChangedError, type RecordStore, type RecordVersion } from "./record-store";
+
+// Per-write scratch name. A fixed `<file>.tmp` is shared by every writer, so
+// two of them (two Orbit processes on one notebook, or one process that abandons
+// a guarded write while another is staging one) can overwrite or delete each
+// other's staging file and rename the wrong bytes into place. Random names cost
+// nothing and take that off the table; paired with the `wx` flag below they also
+// mean we never write through a path someone else got to first.
+function tmpPathFor(filePath: string): string {
+  return `${filePath}.tmp.${randomBytes(8).toString("hex")}`;
+}
+
+// Windows refuses to replace a file another rename (or a scanner) is touching
+// at that instant, with EPERM/EACCES/EBUSY, where POSIX would just swap it in.
+// Retrying briefly gives the same last-writer-wins outcome POSIX gets.
+async function renameReplacing(from: string, to: string): Promise<void> {
+  for (let attempt = 0; ; attempt++) {
+    try {
+      await fs.rename(from, to);
+      return;
+    } catch (err) {
+      const code = (err as NodeJS.ErrnoException).code;
+      const transient = code === "EPERM" || code === "EACCES" || code === "EBUSY";
+      if (process.platform !== "win32" || !transient || attempt >= 10) {
+        await fs.rm(from, { force: true });
+        throw err;
+      }
+      await new Promise((r) => setTimeout(r, 10 * (attempt + 1)));
+    }
+  }
+}
+
+export function createFileRecordStore(filePath: string): RecordStore {
+  const subscribers = new Set<(content: string) => void>();
+  let watcher: FSWatcher | null = null;
+
+  // Include dev/ino and ctimeMs, not just mtime+size: an atomic temp-and-rename
+  // save swaps the inode (changing ino), and ctime moves on any write -- so an
+  // inode-replacing or same-size rewrite with a preserved/coarse mtime is still
+  // seen as changed. #253
+  // This is the CAS key too, not only the reemit fingerprint, so a ctime-only
+  // touch (chmod, utimes, a sync client tagging xattrs) between a caller's read
+  // and its guarded write costs one retry of the loop rather than passing.
+  async function stat(): Promise<RecordVersion | null> {
+    try {
+      const st = await fs.stat(filePath);
+      return `${st.dev}:${st.ino}:${st.size}:${st.mtimeMs}:${st.ctimeMs}`;
+    } catch {
+      return null;
+    }
+  }
+
+  function startWatcher(): void {
+    // chokidar on a missing single file never emits `change`; the after-tool
+    // reemit in state.ts is the recovery path once the file appears.
+    if (!existsSync(filePath)) return;
+    try {
+      watcher = chokidar.watch(filePath, {
+        ignoreInitial: true,
+        awaitWriteFinish: {
+          stabilityThreshold: 500,
+          pollInterval: 100,
+        },
+      });
+      watcher.on("change", async () => {
+        try {
+          const content = await fs.readFile(filePath, "utf-8");
+          for (const subscriber of [...subscribers]) subscriber(content);
+        } catch (err) {
+          // A file deleted between the event and the read is nothing to report.
+          if ((err as NodeJS.ErrnoException).code !== "ENOENT") {
+            console.error("notebook watcher read failed:", err);
+          }
+        }
+      });
+    } catch (err) {
+      console.error("failed to start notebook watcher:", err);
+    }
+  }
+
+  function stopWatcher(): void {
+    if (!watcher) return;
+    try {
+      watcher.close();
+    } catch {
+      /* ignore */
+    }
+    watcher = null;
+  }
+
+  return {
+    kind: "file",
+    key: filePath,
+    label: filePath,
+    stat,
+
+    async read() {
+      const version = await stat();
+      const content = await fs.readFile(filePath, "utf-8");
+      return { content, version };
+    },
+
+    /**
+     * Atomic write: render to a scratch sibling then rename. The rename is
+     * atomic on POSIX, so the destination either has the old or the new
+     * content -- never partial. The file watcher may still fire on the rename,
+     * but it can no longer observe a half-written file.
+     *
+     * With `expect` (a version taken *before* the read this content was derived
+     * from) the write is a compare-and-swap: if the file changed in the meantime
+     * -- an agent `edit`/`bash` append, or another writer entirely -- the write
+     * is abandoned with `NotebookChangedError` instead of overwriting a
+     * stranger's update (#391). Callers are expected to re-read and retry.
+     *
+     * The check sits between the staging write and the rename, which narrows
+     * the exposure to a single `rename` syscall but does not close it:
+     * stat-then-rename is not atomic, and a writer that lands in that gap is
+     * still overwritten. Closing it for real needs an OS-level lock (or routing
+     * every notebook write through one owner), which is a bigger change than
+     * #391.
+     */
+    async write(content, expect) {
+      const tmp = tmpPathFor(filePath);
+      // `wx` is O_CREAT | O_EXCL: create the scratch file or fail. Never follow
+      // an existing path -- a symlink planted at a scratch name would otherwise
+      // let a write land on whatever it points at.
+      await fs.writeFile(tmp, content, { encoding: "utf-8", flag: "wx" });
+      if (expect) {
+        const current = await stat();
+        if (!current || current !== expect) {
+          await fs.rm(tmp, { force: true });
+          throw new NotebookChangedError(filePath);
+        }
+      }
+      await renameReplacing(tmp, filePath);
+    },
+
+    async ensure() {
+      try {
+        await fs.writeFile(filePath, "", { flag: "wx" });
+        return true;
+      } catch (err) {
+        if ((err as NodeJS.ErrnoException).code !== "EEXIST") throw err;
+      }
+      // `wx` refuses to follow a symlink, so a notebook.md that links to a
+      // not-yet-created target reads as EEXIST here while stat() says absent.
+      // Creating through the link keeps that layout working as the old
+      // existsSync + writeFileSync did.
+      if ((await stat()) !== null) return false;
+      const link = await fs.lstat(filePath).catch(() => null);
+      if (!link?.isSymbolicLink()) return false;
+      await fs.writeFile(filePath, "", { flag: "w" });
+      return true;
+    },
+
+    onChange(cb) {
+      subscribers.add(cb);
+      if (subscribers.size === 1) startWatcher();
+      return () => {
+        if (!subscribers.delete(cb)) return;
+        if (subscribers.size === 0) stopWatcher();
+      };
+    },
+  };
+}

package/extensions/orbit/record-store-page.ts

@@ -0,0 +1,77 @@
+/**
+ * The page backend of the record store: a Galaxy page is the record. The
+ * browser shell installs one with `setRecordStore` and supplies the transport
+ * (galaxy-api.ts) it talks through; nothing here touches Node.
+ *
+ * Galaxy's page PUT has no precondition, so the compare-and-swap is a GET,
+ * a hash compare and a PUT: the same stat-then-rename window the file backend
+ * has, closed in-process by `withRecordLock` exactly as there.
+ */
+
+import { getPage, updatePage, type GalaxyPage } from "./galaxy-pages-api";
+import { pageContentHash } from "../../shared/page-content-hash.js";
+import { NotebookChangedError, type RecordStore, type RecordVersion } from "./record-store";
+
+/** `content` is the embed-expanded render; `content_editor` is the source the agent edits. */
+const body = (page: GalaxyPage): string => page.content_editor || page.content || "";
+
+export function createPageRecordStore(opts: {
+  key: string;
+  pageId: string;
+  label?: string;
+}): RecordStore {
+  const { key, pageId } = opts;
+  const subscribers = new Set<(content: string) => void>();
+
+  async function stat(): Promise<RecordVersion | null> {
+    try {
+      return pageContentHash(body(await getPage(pageId)));
+    } catch {
+      return null;
+    }
+  }
+
+  return {
+    kind: "page",
+    key,
+    label: opts.label ?? `Galaxy page ${pageId}`,
+    pageId,
+    stat,
+
+    async read() {
+      const content = body(await getPage(pageId));
+      return { content, version: pageContentHash(content) };
+    },
+
+    async write(content, expect) {
+      if (expect !== undefined) {
+        const current = await stat();
+        if (current === null || current !== expect) throw new NotebookChangedError(key);
+      }
+      await updatePage(pageId, { content, content_format: "markdown", edit_source: "agent" });
+      // Own writes are the whole feed here: chokidar reports Orbit's own file
+      // writes on the desktop, and the subscribers (widget, proposal drift,
+      // auto-push) expect to see a poller write right away. Out-of-band page
+      // edits are caught by the after-tool reemit in state.ts.
+      for (const subscriber of [...subscribers]) {
+        try {
+          subscriber(content);
+        } catch (err) {
+          console.error("record change listener failed:", err);
+        }
+      }
+    },
+
+    // The shell binds the store to an existing page; the store never creates one.
+    async ensure() {
+      return false;
+    },
+
+    onChange(cb) {
+      subscribers.add(cb);
+      return () => {
+        subscribers.delete(cb);
+      };
+    },
+  };
+}

package/extensions/orbit/record-store-seam.ts

@@ -0,0 +1,47 @@
+/**
+ * The record store's process-wide state: the store a shell installed and the per-key write lock.
+ *
+ * Kept apart from record-store.ts, which imports the fs-backed file store, so the in-process
+ * Galaxy tool policy (galaxy-op-policy.ts) -- which runs in the browser worker too -- can wait its
+ * turn behind the record's writers without pulling a Node module into its graph. record-store.ts
+ * re-exports all of it, so every other caller still imports from there.
+ */
+
+import type { RecordStore } from "./record-store";
+
+// The shell seam. Deliberately outside state.ts so resetState() (which every
+// session_start runs before initSessionArtifacts) leaves it alone.
+let installed: RecordStore | null = null;
+
+export function setRecordStore(store: RecordStore | null): void {
+  installed = store;
+}
+
+export function installedRecordStore(): RecordStore | null {
+  return installed;
+}
+
+/**
+ * Per-key mutex chain. Two parallel `upsertInvocationBlock` calls (e.g.
+ * `galaxy_invocation_check_all` polling several invocations concurrently)
+ * would race read-modify-write on the same record: each reads the
+ * pre-update content, applies its block, and the second writer overwrites
+ * the first. Serializing via a per-key Promise chain prevents the lost
+ * update without paying for an OS-level lock.
+ */
+const writeLocks = new Map<string, Promise<void>>();
+
+export function withRecordLock<T>(key: string, work: () => Promise<T>): Promise<T> {
+  const prev = writeLocks.get(key) ?? Promise.resolve();
+  const next = prev.then(work, work);
+  // Always clear so completed locks don't pin memory; the chain is preserved
+  // through the Promise we just created.
+  writeLocks.set(
+    key,
+    next.then(
+      () => undefined,
+      () => undefined,
+    ),
+  );
+  return next;
+}

package/extensions/orbit/record-store.ts

@@ -0,0 +1,149 @@
+/**
+ * The record store: one interface over the durable record.
+ *
+ * On the desktop the record is `notebook.md` in the session cwd and the backend
+ * is the file store (record-store-file.ts). Under the browser shell the record
+ * is a Galaxy page and the shell installs the page store (record-store-page.ts)
+ * with `setRecordStore`. Every reader and writer of the record resolves its
+ * store from the key `getNotebookPath()` hands out, so the callers do not know
+ * which backend they are on.
+ *
+ * This file has no Node imports of its own: the file backend is the only record
+ * module with fs/chokidar edges, and the browser's conformance test counts them.
+ */
+
+import { GalaxyApiError } from "./galaxy-api";
+import { createFileRecordStore } from "./record-store-file";
+import { installedRecordStore } from "./record-store-seam";
+
+export { installedRecordStore, setRecordStore, withRecordLock } from "./record-store-seam";
+
+/**
+ * Opaque, never empty; equal strings mean the record did not change between two
+ * looks. File: `dev:ino:size:mtimeMs:ctimeMs`. Page: the content hash.
+ */
+export type RecordVersion = string;
+
+export interface RecordSnapshot {
+  content: string;
+  version: RecordVersion | null;
+}
+
+export interface RecordStore {
+  readonly kind: "file" | "page";
+  /**
+   * `getNotebookPath()`'s string: the lock key and the anchor the side stores
+   * (activity log, template replay) hang their `dirname` off. Path-shaped under
+   * both backends.
+   */
+  readonly key: string;
+  /** What a human or the model sees: the path, or `Galaxy page <id>`. */
+  readonly label: string;
+  /**
+   * The Galaxy page that is the record, under the page backend. The brain's update_page
+   * policy and the evidence gate compare it with a tool call's page_id to tell a write to
+   * the record from a write to some other page.
+   */
+  readonly pageId?: string;
+  /** null when absent or unstattable; never throws. `(await stat()) !== null` is the existence check. */
+  stat(): Promise<RecordVersion | null>;
+  /** Version taken before the body. Throws the backend's own error when missing (ENOENT / GalaxyApiError 404). */
+  read(): Promise<RecordSnapshot>;
+  /**
+   * Unconditional unless `expect` is given, then a compare-and-swap: throws
+   * NotebookChangedError and leaves the record untouched when the version moved.
+   */
+  write(content: string, expect?: RecordVersion): Promise<void>;
+  /** Create an empty record if absent; true when it created one. */
+  ensure(): Promise<boolean>;
+  /**
+   * Backend change feed (file: chokidar, own writes included; page: own writes
+   * only). Returns unsubscribe; the last unsubscribe releases the watcher.
+   */
+  onChange(cb: (content: string) => void): () => void;
+}
+
+/**
+ * Whether `read()` failed because the record is not there, as opposed to not reachable right
+ * now. Only the first means "no record yet"; a 5xx, a timeout or a dropped connection says
+ * nothing about what the record holds.
+ */
+export function isRecordMissing(error: unknown): boolean {
+  if (error instanceof GalaxyApiError) return error.status === 404;
+  return (error as { code?: unknown } | null)?.code === "ENOENT";
+}
+
+/** Thrown by a guarded `write` when the record moved under the caller. */
+export class NotebookChangedError extends Error {
+  constructor(key: string) {
+    super(`Notebook changed on disk since it was read: ${key}`);
+    this.name = "NotebookChangedError";
+  }
+}
+
+/**
+ * The store for a record key: the installed one when the key is its key, else
+ * a file store for that path. Resolving by key rather than by "is a store
+ * installed" keeps every test that points getNotebookPath at a real tmp file
+ * on the file backend.
+ */
+export function recordStore(key: string): RecordStore {
+  const installed = installedRecordStore();
+  if (installed && installed.key === key) return installed;
+  return createFileRecordStore(key);
+}
+
+/**
+ * How many times a guarded update re-reads and retries before giving up. A
+ * write only loses the version check when someone else wrote in the
+ * microseconds between our read and our write; three attempts is far more than
+ * convergence needs, and bounding it keeps a pathological writer from spinning
+ * us.
+ */
+export const MAX_CAS_ATTEMPTS = 3;
+
+/**
+ * Read -> apply -> write as a compare-and-swap, retrying against fresh content
+ * when the record moved under us.
+ *
+ * This is the discipline the poller already writes under (#391), packaged for
+ * the callers that mutate a block from outside a poll. An unguarded whole-record
+ * write renders the record from content captured before whatever landed in
+ * between -- a poll advancing a block, an agent `edit`, a `bash` append -- and
+ * the in-process lock cannot help, because it only orders Orbit's own writers.
+ *
+ * Call with the record lock held. `read()` takes the version *before* the body
+ * on purpose: a version taken afterwards would let a write that landed in
+ * between look unchanged, which is the exact clobber this prevents, while
+ * taking it first can only ever cost a spurious retry.
+ *
+ * `apply` runs against each attempt's fresh content and may throw to abandon
+ * the update outright -- a validation that depends on what the record says now
+ * belongs inside it, not before the loop.
+ */
+export async function withRecordCas<T>(
+  store: RecordStore,
+  apply: (content: string) => { content: string; result: T },
+): Promise<T> {
+  let lastError: unknown;
+  for (let attempt = 0; attempt < MAX_CAS_ATTEMPTS; attempt++) {
+    // A record that's actually gone or unreadable fails here with its own
+    // ENOENT/EACCES/404 instead of being dressed up as a race.
+    const { content: fresh, version } = await store.read();
+    // Readable but unversioned: without a version there's no compare-and-swap,
+    // and an unguarded whole-record write is the thing this exists to stop.
+    if (!version) {
+      lastError = new NotebookChangedError(store.key);
+      continue;
+    }
+    const { content, result } = apply(fresh);
+    try {
+      await store.write(content, version);
+      return result;
+    } catch (error) {
+      if (!(error instanceof NotebookChangedError)) throw error;
+      lastError = error;
+    }
+  }
+  throw lastError;
+}