@scelar/nodepod 1.0.0

package/src/threading/shared-vfs.ts

@@ -0,0 +1,352 @@
+// SharedVFS — SharedArrayBuffer-backed VFS for synchronous cross-thread reads.
+//
+// When SAB is available (requires COOP/COEP headers), workers can do
+// fs.readFileSync() directly from shared memory without async IPC.
+// Falls back to snapshot-based approach when SAB is unavailable.
+//
+// Layout: file table (paths -> offset+length) + data region, all in one SAB.
+// Main thread owns the canonical table; workers read from it.
+
+/* ------------------------------------------------------------------ */
+/*  Feature detection                                                  */
+/* ------------------------------------------------------------------ */
+export function isSharedArrayBufferAvailable(): boolean {
+  try {
+    return (
+      typeof SharedArrayBuffer !== 'undefined' &&
+      typeof Atomics !== 'undefined'
+    );
+  } catch {
+    return false;
+  }
+}
+
+/* ------------------------------------------------------------------ */
+/*  Shared memory layout                                               */
+/* ------------------------------------------------------------------ */
+
+// Per entry, 264 bytes:
+//   [0..3] flags  [4..7] contentOffset  [8..11] contentLength
+//   [12..15] modified  [16..263] null-terminated path
+const ENTRY_SIZE = 264;
+const ENTRY_FLAGS_OFFSET = 0;
+const ENTRY_CONTENT_OFFSET = 4;
+const ENTRY_CONTENT_LENGTH = 8;
+const ENTRY_MODIFIED_OFFSET = 12;
+const ENTRY_PATH_OFFSET = 16;
+const ENTRY_PATH_MAX = 248;
+
+const FLAG_ACTIVE = 1;
+const FLAG_DIRECTORY = 2;
+const FLAG_SYMLINK = 4;
+
+// Header: [0] version, [1] entry count, [2] data used, [3] lock
+const HEADER_SIZE = 16;
+const MAX_ENTRIES = 16384;
+const TABLE_SIZE = MAX_ENTRIES * ENTRY_SIZE;
+const DATA_OFFSET = HEADER_SIZE + TABLE_SIZE;
+
+const DEFAULT_BUFFER_SIZE = 64 * 1024 * 1024; // 64MB
+
+/* ------------------------------------------------------------------ */
+/*  FNV-1a hash                                                        */
+/* ------------------------------------------------------------------ */
+
+function fnv1a(str: string): number {
+  let hash = 0x811c9dc5;
+  for (let i = 0; i < str.length; i++) {
+    hash ^= str.charCodeAt(i);
+    hash = (hash * 0x01000193) >>> 0;
+  }
+  return hash;
+}
+
+/* ------------------------------------------------------------------ */
+/*  SharedVFSController (main thread)                                  */
+/* ------------------------------------------------------------------ */
+
+// Main-thread controller. Owns the SAB and manages the file table.
+export class SharedVFSController {
+  private _buffer: SharedArrayBuffer;
+  private _view: DataView;
+  private _int32: Int32Array;
+  private _uint8: Uint8Array;
+  private _pathEncoder = new TextEncoder();
+  private _pathDecoder = new TextDecoder();
+
+  constructor(bufferSize: number = DEFAULT_BUFFER_SIZE) {
+    if (!isSharedArrayBufferAvailable()) {
+      throw new Error('SharedArrayBuffer not available. Ensure COOP/COEP headers are set.');
+    }
+
+    this._buffer = new SharedArrayBuffer(bufferSize);
+    this._view = new DataView(this._buffer);
+    this._int32 = new Int32Array(this._buffer);
+    this._uint8 = new Uint8Array(this._buffer);
+
+    Atomics.store(this._int32, 0, 0);
+    Atomics.store(this._int32, 1, 0);
+    this._view.setUint32(8, 0);
+    Atomics.store(this._int32, 3, 0);
+  }
+
+  get buffer(): SharedArrayBuffer {
+    return this._buffer;
+  }
+
+  writeFile(path: string, content: Uint8Array): boolean {
+    this._lock();
+    try {
+      const entryCount = Atomics.load(this._int32, 1);
+      const dataUsed = this._view.getUint32(8);
+
+      const existingIdx = this._findEntry(path);
+      if (existingIdx !== -1) {
+        return this._updateEntry(existingIdx, content, dataUsed);
+      }
+
+      if (entryCount >= MAX_ENTRIES) return false;
+      if (DATA_OFFSET + dataUsed + content.byteLength > this._buffer.byteLength) return false;
+
+      const contentOffset = dataUsed;
+      this._uint8.set(content, DATA_OFFSET + contentOffset);
+
+      const entryOffset = HEADER_SIZE + entryCount * ENTRY_SIZE;
+      this._view.setUint32(entryOffset + ENTRY_FLAGS_OFFSET, FLAG_ACTIVE);
+      this._view.setUint32(entryOffset + ENTRY_CONTENT_OFFSET, contentOffset);
+      this._view.setUint32(entryOffset + ENTRY_CONTENT_LENGTH, content.byteLength);
+      this._view.setUint32(entryOffset + ENTRY_MODIFIED_OFFSET, (Date.now() / 1000) | 0);
+
+      const pathBytes = this._pathEncoder.encode(path);
+      const pathLen = Math.min(pathBytes.byteLength, ENTRY_PATH_MAX - 1);
+      this._uint8.set(pathBytes.subarray(0, pathLen), entryOffset + ENTRY_PATH_OFFSET);
+      this._uint8[entryOffset + ENTRY_PATH_OFFSET + pathLen] = 0;
+
+      Atomics.store(this._int32, 1, entryCount + 1);
+      this._view.setUint32(8, dataUsed + content.byteLength);
+
+      Atomics.add(this._int32, 0, 1);
+      Atomics.notify(this._int32, 0);
+
+      return true;
+    } finally {
+      this._unlock();
+    }
+  }
+
+  writeDirectory(path: string): boolean {
+    this._lock();
+    try {
+      const entryCount = Atomics.load(this._int32, 1);
+      if (entryCount >= MAX_ENTRIES) return false;
+      if (this._findEntry(path) !== -1) return true;
+
+      const entryOffset = HEADER_SIZE + entryCount * ENTRY_SIZE;
+      this._view.setUint32(entryOffset + ENTRY_FLAGS_OFFSET, FLAG_ACTIVE | FLAG_DIRECTORY);
+      this._view.setUint32(entryOffset + ENTRY_CONTENT_OFFSET, 0);
+      this._view.setUint32(entryOffset + ENTRY_CONTENT_LENGTH, 0);
+      this._view.setUint32(entryOffset + ENTRY_MODIFIED_OFFSET, (Date.now() / 1000) | 0);
+
+      const pathBytes = this._pathEncoder.encode(path);
+      const pathLen = Math.min(pathBytes.byteLength, ENTRY_PATH_MAX - 1);
+      this._uint8.set(pathBytes.subarray(0, pathLen), entryOffset + ENTRY_PATH_OFFSET);
+      this._uint8[entryOffset + ENTRY_PATH_OFFSET + pathLen] = 0;
+
+      Atomics.store(this._int32, 1, entryCount + 1);
+      Atomics.add(this._int32, 0, 1);
+      Atomics.notify(this._int32, 0);
+
+      return true;
+    } finally {
+      this._unlock();
+    }
+  }
+
+  deleteFile(path: string): boolean {
+    this._lock();
+    try {
+      const idx = this._findEntry(path);
+      if (idx === -1) return false;
+
+      const entryOffset = HEADER_SIZE + idx * ENTRY_SIZE;
+      this._view.setUint32(entryOffset + ENTRY_FLAGS_OFFSET, 0);
+
+      Atomics.add(this._int32, 0, 1);
+      Atomics.notify(this._int32, 0);
+
+      return true;
+    } finally {
+      this._unlock();
+    }
+  }
+
+  readFile(path: string): Uint8Array | null {
+    const idx = this._findEntry(path);
+    if (idx === -1) return null;
+
+    const entryOffset = HEADER_SIZE + idx * ENTRY_SIZE;
+    const flags = this._view.getUint32(entryOffset + ENTRY_FLAGS_OFFSET);
+    if (!(flags & FLAG_ACTIVE) || (flags & FLAG_DIRECTORY)) return null;
+
+    const contentOffset = this._view.getUint32(entryOffset + ENTRY_CONTENT_OFFSET);
+    const contentLength = this._view.getUint32(entryOffset + ENTRY_CONTENT_LENGTH);
+
+    // Copy, not view, to avoid races
+    return new Uint8Array(this._uint8.slice(
+      DATA_OFFSET + contentOffset,
+      DATA_OFFSET + contentOffset + contentLength,
+    ));
+  }
+
+  exists(path: string): boolean {
+    return this._findEntry(path) !== -1;
+  }
+
+  get version(): number {
+    return Atomics.load(this._int32, 0);
+  }
+
+  /* ---- Internal ---- */
+
+  private _findEntry(path: string): number {
+    const entryCount = Atomics.load(this._int32, 1);
+    const pathBytes = this._pathEncoder.encode(path);
+
+    for (let i = 0; i < entryCount; i++) {
+      const entryOffset = HEADER_SIZE + i * ENTRY_SIZE;
+      const flags = this._view.getUint32(entryOffset + ENTRY_FLAGS_OFFSET);
+      if (!(flags & FLAG_ACTIVE)) continue;
+
+      let match = true;
+      for (let j = 0; j < pathBytes.byteLength; j++) {
+        if (this._uint8[entryOffset + ENTRY_PATH_OFFSET + j] !== pathBytes[j]) {
+          match = false;
+          break;
+        }
+      }
+      if (match && this._uint8[entryOffset + ENTRY_PATH_OFFSET + pathBytes.byteLength] === 0) {
+        return i;
+      }
+    }
+    return -1;
+  }
+
+  private _updateEntry(idx: number, content: Uint8Array, dataUsed: number): boolean {
+    if (DATA_OFFSET + dataUsed + content.byteLength > this._buffer.byteLength) return false;
+
+    const entryOffset = HEADER_SIZE + idx * ENTRY_SIZE;
+
+    // Append-only — don't reuse old space to avoid races
+    const contentOffset = dataUsed;
+    this._uint8.set(content, DATA_OFFSET + contentOffset);
+
+    this._view.setUint32(entryOffset + ENTRY_CONTENT_OFFSET, contentOffset);
+    this._view.setUint32(entryOffset + ENTRY_CONTENT_LENGTH, content.byteLength);
+    this._view.setUint32(entryOffset + ENTRY_MODIFIED_OFFSET, (Date.now() / 1000) | 0);
+
+    this._view.setUint32(8, dataUsed + content.byteLength);
+    Atomics.add(this._int32, 0, 1);
+    Atomics.notify(this._int32, 0);
+
+    return true;
+  }
+
+  private _lock(): void {
+    while (Atomics.compareExchange(this._int32, 3, 0, 1) !== 0) {
+      Atomics.wait(this._int32, 3, 1, 1);
+    }
+  }
+
+  private _unlock(): void {
+    Atomics.store(this._int32, 3, 0);
+    Atomics.notify(this._int32, 3);
+  }
+}
+
+/* ------------------------------------------------------------------ */
+/*  SharedVFSReader (worker thread)                                    */
+/* ------------------------------------------------------------------ */
+
+// Worker-thread reader. Synchronous read access to shared memory.
+export class SharedVFSReader {
+  private _buffer: SharedArrayBuffer;
+  private _view: DataView;
+  private _int32: Int32Array;
+  private _uint8: Uint8Array;
+  private _pathEncoder = new TextEncoder();
+  private _pathDecoder = new TextDecoder();
+
+  constructor(buffer: SharedArrayBuffer) {
+    this._buffer = buffer;
+    this._view = new DataView(buffer);
+    this._int32 = new Int32Array(buffer);
+    this._uint8 = new Uint8Array(buffer);
+  }
+
+  readFileSync(path: string): Uint8Array | null {
+    const idx = this._findEntry(path);
+    if (idx === -1) return null;
+
+    const entryOffset = HEADER_SIZE + idx * ENTRY_SIZE;
+    const flags = this._view.getUint32(entryOffset + ENTRY_FLAGS_OFFSET);
+    if (!(flags & FLAG_ACTIVE) || (flags & FLAG_DIRECTORY)) return null;
+
+    const contentOffset = this._view.getUint32(entryOffset + ENTRY_CONTENT_OFFSET);
+    const contentLength = this._view.getUint32(entryOffset + ENTRY_CONTENT_LENGTH);
+
+    const result = new Uint8Array(contentLength);
+    result.set(this._uint8.subarray(
+      DATA_OFFSET + contentOffset,
+      DATA_OFFSET + contentOffset + contentLength,
+    ));
+    return result;
+  }
+
+  existsSync(path: string): boolean {
+    return this._findEntry(path) !== -1;
+  }
+
+  isDirectorySync(path: string): boolean {
+    const idx = this._findEntry(path);
+    if (idx === -1) return false;
+    const entryOffset = HEADER_SIZE + idx * ENTRY_SIZE;
+    const flags = this._view.getUint32(entryOffset + ENTRY_FLAGS_OFFSET);
+    return (flags & FLAG_ACTIVE) !== 0 && (flags & FLAG_DIRECTORY) !== 0;
+  }
+
+  get version(): number {
+    return Atomics.load(this._int32, 0);
+  }
+
+  // Blocks until version changes. Returns new version or -1 on timeout.
+  waitForChange(currentVersion: number, timeoutMs: number = 5000): number {
+    const result = Atomics.wait(this._int32, 0, currentVersion, timeoutMs);
+    if (result === 'timed-out') return -1;
+    return Atomics.load(this._int32, 0);
+  }
+
+  /* ---- Internal ---- */
+
+  private _findEntry(path: string): number {
+    const entryCount = Atomics.load(this._int32, 1);
+    const pathBytes = this._pathEncoder.encode(path);
+
+    for (let i = 0; i < entryCount; i++) {
+      const entryOffset = HEADER_SIZE + i * ENTRY_SIZE;
+      const flags = this._view.getUint32(entryOffset + ENTRY_FLAGS_OFFSET);
+      if (!(flags & FLAG_ACTIVE)) continue;
+
+      let match = true;
+      for (let j = 0; j < pathBytes.byteLength; j++) {
+        if (this._uint8[entryOffset + ENTRY_PATH_OFFSET + j] !== pathBytes[j]) {
+          match = false;
+          break;
+        }
+      }
+      if (match && this._uint8[entryOffset + ENTRY_PATH_OFFSET + pathBytes.byteLength] === 0) {
+        return i;
+      }
+    }
+    return -1;
+  }
+}

package/src/threading/sync-channel.ts

@@ -0,0 +1,135 @@
+// SyncChannel — true blocking execSync/spawnSync via SharedArrayBuffer + Atomics.
+// Worker allocates a slot, posts spawn-sync, blocks on Atomics.wait().
+// Main thread runs the child, writes result to the slot, calls Atomics.notify().
+
+import { isSharedArrayBufferAvailable } from "./shared-vfs";
+
+// --- Shared memory layout ---
+//
+// Per-slot (16KB = 4096 Int32s):
+//   [0] status (0=pending, 1=complete, 2=error)
+//   [1] exit code
+//   [2] stdout byte length
+//   [3..4095] stdout data
+//
+// Last Int32 at MAX_SLOTS * SLOT_SIZE is the atomic allocation counter.
+export const SLOT_SIZE = 4096; // 4096 Int32 values = 16KB per slot
+export const MAX_SLOTS = 64;
+const STATUS_PENDING = 0;
+const STATUS_COMPLETE = 1;
+const STATUS_ERROR = 2;
+
+const DEFAULT_SYNC_BUFFER_SIZE = MAX_SLOTS * SLOT_SIZE * 4 + 4; // ~1MB
+const COUNTER_INDEX = MAX_SLOTS * SLOT_SIZE;
+
+// --- SyncChannelController (main thread) ---
+export class SyncChannelController {
+  private _buffer: SharedArrayBuffer;
+  private _int32: Int32Array;
+  private _uint8: Uint8Array;
+
+  constructor(bufferSize: number = DEFAULT_SYNC_BUFFER_SIZE) {
+    if (!isSharedArrayBufferAvailable()) {
+      throw new Error("SharedArrayBuffer not available. Ensure COOP/COEP headers are set.");
+    }
+
+    this._buffer = new SharedArrayBuffer(bufferSize);
+    this._int32 = new Int32Array(this._buffer);
+    this._uint8 = new Uint8Array(this._buffer);
+
+    for (let i = 0; i < MAX_SLOTS; i++) {
+      Atomics.store(this._int32, i * SLOT_SIZE, STATUS_PENDING);
+    }
+    Atomics.store(this._int32, COUNTER_INDEX, 0);
+  }
+
+  get buffer(): SharedArrayBuffer {
+    return this._buffer;
+  }
+
+  writeResult(syncSlot: number, exitCode: number, stdout: string): void {
+    const base = syncSlot * SLOT_SIZE;
+
+    Atomics.store(this._int32, base + 1, exitCode);
+
+    const encoder = new TextEncoder();
+    const stdoutBytes = encoder.encode(stdout);
+    const maxStdoutLen = (SLOT_SIZE - 3) * 4;
+    const truncatedLen = Math.min(stdoutBytes.byteLength, maxStdoutLen);
+
+    Atomics.store(this._int32, base + 2, truncatedLen);
+
+    const dataOffset = (base + 3) * 4;
+    this._uint8.set(stdoutBytes.subarray(0, truncatedLen), dataOffset);
+
+    // Must be last — wakes the waiting worker
+    Atomics.store(this._int32, base, STATUS_COMPLETE);
+    Atomics.notify(this._int32, base);
+  }
+
+  writeError(syncSlot: number, exitCode: number, errorMessage: string): void {
+    const base = syncSlot * SLOT_SIZE;
+
+    Atomics.store(this._int32, base + 1, exitCode);
+
+    const encoder = new TextEncoder();
+    const errorBytes = encoder.encode(errorMessage);
+    const maxLen = (SLOT_SIZE - 3) * 4;
+    const truncatedLen = Math.min(errorBytes.byteLength, maxLen);
+
+    Atomics.store(this._int32, base + 2, truncatedLen);
+    const dataOffset = (base + 3) * 4;
+    this._uint8.set(errorBytes.subarray(0, truncatedLen), dataOffset);
+
+    Atomics.store(this._int32, base, STATUS_ERROR);
+    Atomics.notify(this._int32, base);
+  }
+}
+
+// --- SyncChannelWorker (worker thread) ---
+export class SyncChannelWorker {
+  private _int32: Int32Array;
+  private _uint8: Uint8Array;
+
+  constructor(buffer: SharedArrayBuffer) {
+    this._int32 = new Int32Array(buffer);
+    this._uint8 = new Uint8Array(buffer);
+  }
+
+  // Atomic counter prevents slot collisions across workers
+  allocateSlot(): number {
+    const raw = Atomics.add(this._int32, COUNTER_INDEX, 1);
+    const slot = raw % MAX_SLOTS;
+    Atomics.store(this._int32, slot * SLOT_SIZE, STATUS_PENDING);
+    return slot;
+  }
+
+  // Blocks the worker thread until main writes the result
+  waitForResult(syncSlot: number, timeoutMs: number = 120_000): { exitCode: number; stdout: string } {
+    const base = syncSlot * SLOT_SIZE;
+
+    const result = Atomics.wait(this._int32, base, STATUS_PENDING, timeoutMs);
+
+    if (result === "timed-out") {
+      throw new Error("execSync timed out");
+    }
+
+    const status = Atomics.load(this._int32, base);
+    const exitCode = Atomics.load(this._int32, base + 1);
+    const stdoutLen = Atomics.load(this._int32, base + 2);
+
+    const decoder = new TextDecoder();
+    const dataOffset = (base + 3) * 4;
+    const stdoutBytes = this._uint8.slice(dataOffset, dataOffset + stdoutLen);
+    const stdout = decoder.decode(stdoutBytes);
+
+    if (status === STATUS_ERROR) {
+      const err = new Error(`Command failed with exit code ${exitCode}\n${stdout}`);
+      (err as any).status = exitCode;
+      (err as any).stdout = stdout;
+      throw err;
+    }
+
+    return { exitCode, stdout };
+  }
+}

package/src/threading/task-queue.ts

@@ -0,0 +1,177 @@
+// TaskQueue — priority-sorted queue that dispatches tasks to the WorkerPool.
+
+import type { WorkerPool } from "./worker-pool";
+import type {
+  OffloadTask,
+  OffloadResult,
+  TransformTask,
+  TransformResult,
+  ExtractTask,
+  ExtractResult,
+  BuildTask,
+  BuildResult,
+} from "./offload-types";
+
+// --- Internal types ---
+
+interface QueuedTask {
+  task: OffloadTask;
+  resolve: (result: any) => void;
+  reject: (error: Error) => void;
+  cancelled: boolean;
+}
+
+// --- TaskQueue ---
+
+export class TaskQueue {
+  private queue: QueuedTask[] = [];
+  private pool: WorkerPool;
+  private dispatching = false;
+
+  constructor(pool: WorkerPool) {
+    this.pool = pool;
+  }
+
+  // --- Public API ---
+
+  submit<T extends OffloadTask>(
+    task: T,
+  ): Promise<
+    T extends TransformTask
+      ? TransformResult
+      : T extends ExtractTask
+        ? ExtractResult
+        : T extends BuildTask
+          ? BuildResult
+          : OffloadResult
+  > {
+    return new Promise((resolve, reject) => {
+      const queued: QueuedTask = {
+        task,
+        resolve,
+        reject,
+        cancelled: false,
+      };
+
+      // Lower number = higher priority
+      const idx = this.queue.findIndex(
+        (q) => q.task.priority > task.priority,
+      );
+      if (idx === -1) {
+        this.queue.push(queued);
+      } else {
+        this.queue.splice(idx, 0, queued);
+      }
+
+      this.dispatch();
+    });
+  }
+
+  submitBatch(tasks: OffloadTask[]): Promise<OffloadResult[]> {
+    return Promise.all(tasks.map((t) => this.submit(t)));
+  }
+
+  cancel(taskId: string): boolean {
+    const idx = this.queue.findIndex(
+      (q) => q.task.id === taskId && !q.cancelled,
+    );
+    if (idx >= 0) {
+      const queued = this.queue[idx];
+      queued.cancelled = true;
+      queued.reject(new Error(`Task ${taskId} cancelled`));
+      this.queue.splice(idx, 1);
+      return true;
+    }
+    return false;
+  }
+
+  get pending(): number {
+    return this.queue.length;
+  }
+
+  // --- Internal ---
+
+  private async dispatch(): Promise<void> {
+    if (this.dispatching) return;
+    this.dispatching = true;
+
+    try {
+      while (this.queue.length > 0) {
+        const queued = this.queue.shift()!;
+        if (queued.cancelled) continue;
+
+        let worker: any;
+        let release: (() => void) | undefined;
+        try {
+          const acquired = await this.pool.acquire();
+          worker = acquired.worker;
+          release = acquired.release;
+        } catch (err) {
+          // Pool broken — reject everything so offload() can fall back to main thread
+          const reason =
+            err instanceof Error ? err : new Error(String(err));
+          queued.reject(reason);
+          this.rejectAll(reason);
+          return;
+        }
+
+        // Don't block the dispatch loop — fire and re-trigger on completion
+        this.executeTask(worker.endpoint, queued).finally(() => {
+          release!();
+          // More tasks may have arrived while we were busy
+          if (this.queue.length > 0) {
+            this.dispatching = false;
+            this.dispatch();
+          }
+        });
+      }
+    } finally {
+      this.dispatching = false;
+    }
+  }
+
+  private rejectAll(reason: Error): void {
+    const pending = this.queue.splice(0);
+    for (const q of pending) {
+      if (!q.cancelled) {
+        q.reject(reason);
+      }
+    }
+  }
+
+  private async executeTask(
+    endpoint: any,
+    queued: QueuedTask,
+  ): Promise<void> {
+    if (queued.cancelled) return;
+
+    try {
+      let result: OffloadResult;
+      switch (queued.task.type) {
+        case "transform":
+          result = await endpoint.transform(queued.task);
+          break;
+        case "extract":
+          result = await endpoint.extract(queued.task);
+          break;
+        case "build":
+          result = await endpoint.build(queued.task);
+          break;
+        default:
+          throw new Error(
+            `Unknown task type: ${(queued.task as any).type}`,
+          );
+      }
+
+      if (!queued.cancelled) {
+        queued.resolve(result);
+      }
+    } catch (err) {
+      if (!queued.cancelled) {
+        queued.reject(
+          err instanceof Error ? err : new Error(String(err)),
+        );
+      }
+    }
+  }
+}