@atolljs/core 0.1.0

package/src/contract/types.ts

@@ -0,0 +1,123 @@
+import type { SharedAccess, SharedMemory, SharedSpec } from './sharedMemory';
+
+export type ViewType = 'Int32' | 'Float64' | 'BigInt64' | 'Uint8';
+
+/**
+ * Minimal validation contract — compatible with zod schemas (`z.object(...)`)
+ * and any validator exposing `parse`.
+ */
+export interface Schema<T> {
+  parse(value: unknown): T;
+}
+
+export type Logger = (...args: unknown[]) => void;
+
+/**
+ * Flattens a computed object type into its concrete shape so IDE hovers show
+ * `{ id: number; site: string }` instead of the mapped-type machinery that
+ * produced it. Applied to synthetic types (e.g. list records); user-named
+ * types keep their names since the name is the better display.
+ */
+export type Prettify<T> = { [K in keyof T]: T[K] } & {};
+
+export interface MemoryConfig {
+  initialPages?: number; // 1 page = 64KB
+  maximumPages?: number; // default 16384 (1GB)
+  growthFactor?: number;
+}
+
+export interface WorkerPoolConfig<S extends SharedSpec = SharedSpec, T extends TaskMap = TaskMap> {
+  /** Worker entry as a URL — e.g. `new URL('./worker.ts', import.meta.url)`. */
+  workerUrl?: URL;
+  /**
+   * Bundler-detectable worker factory. Most bundlers only emit a worker chunk
+   * when the entry appears inline, so prefer this form:
+   * `createWorker: () => new Worker(new URL('./worker.ts', import.meta.url), { type: 'module' })`.
+   * One of `workerUrl` / `createWorker` is required.
+   */
+  createWorker?: () => Worker;
+  /**
+   * The contract the pool binds and hands to each worker via INIT_MEMORY.
+   * Omit for message-only workers — the SharedArrayBuffer/isolation check is
+   * skipped and workers receive a plain `{ type: 'INIT' }` handshake.
+   */
+  sharedMemory?: SharedMemory<S> & Prettify<SharedAccess<S>>;
+  /** Worker count — a number, or 'auto' (the default) for navigator.hardwareConcurrency ?? 4. */
+  poolSize?: number | 'auto';
+  memory?: MemoryConfig;
+  /** Max in-flight tasks per worker (default 1); excess tasks queue FIFO. */
+  concurrency?: number;
+  /** Max queued tasks (default Infinity); a full queue rejects with PoolQueueFullError. */
+  maxQueue?: number;
+  /** Default true — a worker 'error' rejects its in-flight tasks and spawns a replacement. */
+  respawn?: boolean;
+  /**
+   * Default per-task timeout in ms (per-call `options.timeout` overrides).
+   * The timer starts at enqueue, so it bounds queue-wait + execution together.
+   */
+  taskTimeout?: number;
+  /**
+   * Named task contracts. Keys become first-class pool methods:
+   * `tasks: { queryIncidents: QueryIncidents }` → `pool.queryIncidents(q)`.
+   */
+  tasks?: T;
+}
+
+export interface SharedWorkerConfig<S extends SharedSpec = SharedSpec, T extends TaskMap = TaskMap> {
+  /** SharedWorker entry as a URL — e.g. `new URL('./worker.ts', import.meta.url)`. */
+  workerUrl?: URL;
+  /**
+   * Bundler-detectable factory — `createWorker: () => new SharedWorker(new URL('./worker.ts', import.meta.url), { type: 'module' })`.
+   * One of `workerUrl` / `createWorker` / `port` is required.
+   */
+  createWorker?: () => SharedWorker;
+  /** A pre-opened MessagePort — tests, or hosts that forward a port another way. */
+  port?: MessagePort;
+  sharedMemory: SharedMemory<S> & Prettify<SharedAccess<S>>;
+  /**
+   * Buffer sizing hint sent on connect — the first connected client wins;
+   * the shared worker allocates the buffer once and shares it with everyone.
+   */
+  memory?: MemoryConfig;
+  /**
+   * Named task contracts. Keys become first-class client methods:
+   * `tasks: { queryIncidents: QueryIncidents }` → `client.queryIncidents(q)`.
+   */
+  tasks?: T;
+  /** Connect handshake timeout in ms (default 10000). */
+  connectTimeoutMs?: number;
+}
+
+export interface TaskContract<Args extends any[] = any[], Result = any> {
+  taskId: string;
+  dataType?: ViewType; // element datatype of the shared-memory region this task operates on (omit for structured fields)
+  argsSchema?: Schema<Args>; // validates args as they cross the thread boundary
+  resultSchema?: Schema<Result>; // validates the handler's return value
+}
+
+/** A map of named task contracts — keys become first-class WorkerPool methods. */
+export type TaskMap = Record<string, TaskContract<any[], any>>;
+
+/**
+ * The method surface a WorkerPool exposes for a task map: each contract
+ * becomes a function with its declared args returning `Promise<Result>`.
+ */
+export type PoolTasks<T extends TaskMap> = {
+  [K in keyof T]: T[K] extends TaskContract<infer Args, infer Result>
+    ? (...args: Args) => Promise<Result>
+    : never;
+};
+
+export interface TaskMessage {
+  taskId: string;
+  args?: any[];
+  memoryOffset?: number;
+  byteLength?: number;
+}
+
+export interface TaskResult {
+  taskId: string;
+  success: boolean;
+  result?: any;
+  error?: string;
+}

package/src/index.ts

@@ -0,0 +1,114 @@
+/* ── Contract — shared memory declarations (both threads) ───────────────── */
+
+export { defineSharedMemory, field, getDefinedSharedMemoryCount, jsonCodec, registerConnectorFactory } from './contract/sharedMemory';
+export { mz } from './contract/mz';
+export type {
+  Codec,
+  Connector,
+  ConnectorContext,
+  ConnectorFactory,
+  FieldDescriptor,
+  FieldGroup,
+  FieldSchema,
+  FixedArrayDescriptor,
+  FixedObjectDescriptor,
+  FlatKey,
+  InferField,
+  PathConnector,
+  SpecPath,
+  SpecSchemas,
+  SharedAccess,
+  SharedMemory,
+  SharedMemoryOptions,
+  SharedSpec,
+  ScalarKind,
+  ListConnector,
+  ListDescriptor,
+  ListFieldSpec,
+  ListFields,
+  ListMember,
+  ListRecord,
+  ListSpec,
+} from './contract/sharedMemory';
+export { msgpackCodec } from './contract/msgpackCodec';
+export { msgpackrCodec } from './contract/msgpackrCodec';
+export { listSchema } from './contract/listSchema';
+export type { ListZodShape } from './contract/listSchema';
+export type {
+  MemoryConfig,
+  PoolTasks,
+  Prettify,
+  Schema,
+  Logger,
+  TaskContract,
+  TaskMap,
+  TaskMessage,
+  TaskResult,
+  SharedWorkerConfig,
+  ViewType,
+  WorkerPoolConfig,
+} from './contract/types';
+
+/* ── Pool — main-thread runtime ─────────────────────────────────────────── */
+
+export { MemoryManager } from './pool/memory';
+export { WorkerPool } from './pool/workerPool';
+export type { PoolStats, RunOptions } from './pool/workerPool';
+export {
+  PoolQueueFullError,
+  TaskAbortedError,
+  TaskTimeoutError,
+  WorkerCrashedError,
+} from './pool/errors';
+export { connectWorker, workerClient } from './pool/workerClient';
+export type {
+  ClientMethods,
+  ConnectWorkerConfig,
+  WorkerClient,
+  WorkerMethods,
+} from './pool/workerClient';
+
+/* ── SharedWorker — one worker (and one buffer) across tabs/iframes ──────── */
+
+export { connectSharedWorker } from './shared/sharedWorkerClient';
+export type { SharedWorkerClient } from './shared/sharedWorkerClient';
+export { sharedWorkerHost } from './shared/sharedWorkerHost';
+
+/* ── Service — framework-neutral RPC contracts shared by both threads ────── */
+
+export { createClient, defineService, implementService, rpc, serviceMethod } from './service';
+export type {
+  RpcMethodDef,
+  RpcMethodInput,
+  RpcMethodInputMap,
+  RpcMethodMap,
+  ServiceClient,
+  ServiceContract,
+  ServiceHandlers,
+  ServiceImpl,
+  ServiceMethod,
+  TaskRunner,
+} from './service';
+
+/* ── Worker — worker-side runtime ───────────────────────────────────────── */
+
+export { TaskRegistry } from './worker/registry';
+export { defineWorker, RESERVED_CLIENT_KEYS } from './worker/defineWorker';
+export type {
+  DefineWorkerConfig,
+  WorkerDefinition,
+  WorkerMethod,
+  WorkerMethodMap,
+  WorkerServiceMap,
+} from './worker/defineWorker';
+
+/* ── Shared runtime utilities ───────────────────────────────────────────── */
+
+export { reactive, shallowEqual, watch } from './reactive';
+export type { EqualityFn, ReactiveConnector, SliceOptions } from './reactive';
+export { observe } from './observable';
+export type { ObservableValue } from './observable';
+export { defineTask, isAsyncTask, toTask } from './task';
+export type { AsyncTask, TaskSnapshot } from './task';
+export { getLogLevel, log, scoped, setLogLevel, setLogSink } from './log';
+export type { LogEntry, LogLevel, LogSink } from './log';

package/src/log.ts

@@ -0,0 +1,69 @@
+/**
+ * Scoped, leveled logging for the SDK. Default level is `info` — lifecycle
+ * events (pool spawn, memory bind, seed phases) are always visible; `debug`
+ * adds per-task dispatch/completion and `trace` adds per-message detail.
+ *
+ *   import { setLogLevel } from '…/index';
+ *   setLogLevel('debug'); // or 'trace', 'warn', 'off'
+ */
+
+export type LogLevel = 'trace' | 'debug' | 'info' | 'warn' | 'error' | 'off';
+
+const LEVELS: Record<LogLevel, number> = { trace: 0, debug: 1, info: 2, warn: 3, error: 4, off: 5 };
+
+export interface LogEntry {
+  level: LogLevel;
+  scope: string;
+  message: string;
+  data?: unknown;
+  /** Which side of the boundary emitted this entry. */
+  thread: 'main' | 'worker';
+  /** performance.now() at emission — useful for sequencing across threads. */
+  at: number;
+}
+
+export type LogSink = (entry: LogEntry) => void;
+
+const THREAD: LogEntry['thread'] =
+  typeof WorkerGlobalScope !== 'undefined' && self instanceof WorkerGlobalScope ? 'worker' : 'main';
+
+const consoleSink: LogSink = ({ level, scope, message, data, thread, at }) => {
+  const prefix = `[${(at / 1000).toFixed(2)}s ${thread}:${scope}]`;
+  const fn =
+    level === 'error' ? console.error
+    : level === 'warn' ? console.warn
+    : level === 'info' ? console.info
+    : console.debug;
+  if (data !== undefined) fn(`${prefix} ${message}`, data);
+  else fn(`${prefix} ${message}`);
+};
+
+let minLevel: LogLevel = 'info';
+let sink: LogSink = consoleSink;
+
+export const setLogLevel = (level: LogLevel): void => { minLevel = level; };
+export const getLogLevel = (): LogLevel => minLevel;
+/** Replace the console sink (e.g. collect entries for a UI panel); null restores it. */
+export const setLogSink = (custom: LogSink | null): void => { sink = custom ?? consoleSink; };
+
+export function log(level: Exclude<LogLevel, 'off'>, scope: string, message: string, data?: unknown): void {
+  if (LEVELS[level] < LEVELS[minLevel]) return;
+  sink({ level, scope, message, data, thread: THREAD, at: performance.now() });
+}
+
+/** A logger bound to a subsystem scope: `poolLog.info('spawned')`. */
+export function scoped(scope: string) {
+  return {
+    trace: (message: string, data?: unknown) => log('trace', scope, message, data),
+    debug: (message: string, data?: unknown) => log('debug', scope, message, data),
+    info: (message: string, data?: unknown) => log('info', scope, message, data),
+    warn: (message: string, data?: unknown) => log('warn', scope, message, data),
+    error: (message: string, data?: unknown) => log('error', scope, message, data),
+  };
+}
+
+/** Compact byte formatting for log output. */
+export const fmtBytes = (bytes: number): string =>
+  bytes >= 1 << 20 ? `${(bytes / (1 << 20)).toFixed(1)}MB`
+  : bytes >= 1024 ? `${(bytes / 1024).toFixed(1)}KB`
+  : `${bytes}B`;

package/src/observable.ts

@@ -0,0 +1,151 @@
+import type { Connector, PathConnector, SharedMemory, SharedSpec, SpecPath } from './contract/sharedMemory';
+import { watch, type SliceOptions } from './reactive';
+import { scoped } from './log';
+
+const obsLog = scoped('observe');
+
+/**
+ * A minimal subscribable snapshot — `get()` plus `subscribe()`. This is the
+ * interface every framework binding adapts (useSyncExternalStore, refs,
+ * signals, runes). `get()` must return a stable value between notifications.
+ */
+export interface ObservableValue<T> {
+  get(): T;
+  subscribe(onChange: (value: T) => void): () => void;
+}
+
+type FieldValue<C> = C extends Connector<infer T> ? T : never;
+
+/**
+ * Observe one field of a shared memory contract as a subscribable snapshot.
+ *
+ * Safe to call before the contract is bound — `get()` returns `undefined`
+ * until binding, and a subscriber registered early activates its watch when
+ * `bind()` lands (useful for SSR, where renders happen without a buffer).
+ * The underlying `watch` starts on the first subscriber and stops when the
+ * last one leaves.
+ */
+export function observe<S extends SharedSpec, K extends SpecPath<S>>(
+  memory: SharedMemory<S>,
+  key: K
+): ObservableValue<FieldValue<PathConnector<S, K>> | undefined>;
+/**
+ * Slice form: `observe(memory, 'metrics', m => m.critical)` — the observable
+ * emits only when the selected slice changes. Slice equality defaults to
+ * `Object.is`; pass `{ equals }` to define what counts as a change —
+ * `shallowEqual` for object slices, or any `(prev, next) => boolean`.
+ */
+export function observe<S extends SharedSpec, K extends SpecPath<S>, Sel>(
+  memory: SharedMemory<S>,
+  key: K,
+  select: (value: FieldValue<PathConnector<S, K>>) => Sel,
+  options?: SliceOptions<Sel>
+): ObservableValue<Sel | undefined>;
+export function observe<S extends SharedSpec, K extends SpecPath<S>, Sel>(
+  memory: SharedMemory<S>,
+  key: K,
+  select?: (value: FieldValue<PathConnector<S, K>>) => Sel,
+  options?: SliceOptions<Sel>
+): ObservableValue<unknown> {
+  type T = FieldValue<PathConnector<S, K>>;
+  type Out = T | Sel;
+  const selectFn = (select ?? ((v: T) => v)) as (value: T) => Out;
+  const equals = (options?.equals ?? Object.is) as (a: Out, b: Out) => boolean;
+  let current: Out | undefined;
+  let lastVersion = -1;
+  let unwatch: (() => void) | null = null;
+  let pendingBind: (() => void) | null = null;
+  let rebindUn: (() => void) | null = null;
+  let watched: Connector<T> | null = null;
+  const subscribers = new Set<(value: Out | undefined) => void>();
+
+  const activate = () => {
+    if (unwatch || pendingBind) return;
+    let connector: Connector<T>;
+    try {
+      connector = memory.connector(String(key)) as Connector<T>;
+    } catch {
+      // Not bound on this thread yet — activate when it is.
+      pendingBind = memory.onBound(() => {
+        pendingBind = null;
+        activate();
+      });
+      return;
+    }
+    unwatch = watch(
+      connector,
+      selectFn as (value: T) => Sel,
+      (value) => {
+        // Same gate as get(): get() may already have stored an equal-valued
+        // snapshot — keep its reference and skip the emit.
+        if (current === undefined || !equals(current as Out, value)) {
+          current = value;
+          for (const cb of subscribers) cb(value);
+        }
+      },
+      options
+    );
+    watched = connector;
+    // bind() can REBIND this contract to a different buffer — in-process
+    // harnesses run every worker's bindSharedMemories in one module graph,
+    // so a later pool's spawn rebinds this contract too. The watch above
+    // pins the old connector (old buffer); re-watch when it swaps. onBound
+    // fires synchronously when already bound — the identity check makes
+    // that first call a no-op.
+    rebindUn ??= memory.onBound(() => {
+      let next: Connector<T>;
+      try {
+        next = memory.connector(String(key)) as Connector<T>;
+      } catch {
+        return;
+      }
+      if (next === watched) return;
+      unwatch?.();
+      unwatch = null;
+      watched = null;
+      activate();
+    });
+    obsLog.debug(`observing "${String(key)}" (offset ${connector.byteOffset})`);
+  };
+
+  const deactivate = () => {
+    unwatch?.();
+    unwatch = null;
+    watched = null;
+    pendingBind?.();
+    pendingBind = null;
+    rebindUn?.();
+    rebindUn = null;
+  };
+
+  return {
+    get() {
+      // Re-read only when the shared version counter moved — decoding a
+      // structured field on every snapshot check would be wasted work.
+      if (unwatch || memory.bound) {
+        try {
+          const connector = memory.connector(String(key)) as Connector<T>;
+          const version = connector._version ? Atomics.load(connector._version.view, connector._version.index) : 0;
+          if (version !== lastVersion) {
+            lastVersion = version;
+            // Keep the snapshot reference stable between emissions — the
+            // comparator decides what counts as a change here too.
+            const next = selectFn(connector.read());
+            if (current === undefined || !equals(current as Out, next)) current = next;
+          }
+        } catch {
+          // Racing an unbind/rebind — keep the last known value.
+        }
+      }
+      return current;
+    },
+    subscribe(onChange) {
+      subscribers.add(onChange);
+      activate();
+      return () => {
+        subscribers.delete(onChange);
+        if (subscribers.size === 0) deactivate();
+      };
+    },
+  };
+}

package/src/pool/errors.ts

@@ -0,0 +1,31 @@
+/** Task queue is at `maxQueue` — dispatch rejected without ever running. */
+export class PoolQueueFullError extends Error {
+  constructor(message = 'WorkerPool task queue is full') {
+    super(message);
+    this.name = 'PoolQueueFullError';
+  }
+}
+
+/** The task's AbortSignal fired — never dispatched, or reply discarded. */
+export class TaskAbortedError extends Error {
+  constructor(message = 'task aborted') {
+    super(message);
+    this.name = 'AbortError';
+  }
+}
+
+/** The task exceeded its `timeout` / pool `taskTimeout` budget. */
+export class TaskTimeoutError extends Error {
+  constructor(message = 'task timed out') {
+    super(message);
+    this.name = 'TaskTimeoutError';
+  }
+}
+
+/** The worker errored/exited, or the pool was terminated mid-flight. */
+export class WorkerCrashedError extends Error {
+  constructor(message = 'worker crashed') {
+    super(message);
+    this.name = 'WorkerCrashedError';
+  }
+}

package/src/pool/memory.ts

@@ -0,0 +1,68 @@
+import { MemoryConfig, ViewType } from '../contract/types';
+
+export class MemoryManager {
+  public memory: WebAssembly.Memory;
+  private maxPages: number;
+
+  constructor(config: MemoryConfig = {}) {
+    const initialPages = config.initialPages ?? 16; // 1MB default
+    this.maxPages = config.maximumPages ?? 16384; // 1GB default limit
+
+    // Initialize WebAssembly.Memory with shared: true for SharedArrayBuffer support
+    this.memory = new WebAssembly.Memory({
+      initial: initialPages,
+      maximum: this.maxPages,
+      shared: true,
+    });
+  }
+
+  /**
+   * Automatically checks if memory needs to grow to accommodate new bytes,
+   * and expands via memory.grow() if necessary. Thread-safe.
+   */
+  public ensureCapacity(requiredBytes: number): void {
+    const currentByteLength = this.memory.buffer.byteLength;
+    if (currentByteLength >= requiredBytes) return;
+
+    const bytesNeeded = requiredBytes - currentByteLength;
+    const pagesNeeded = Math.ceil(bytesNeeded / 65536);
+    const currentPages = this.memory.buffer.byteLength / 65536;
+
+    if (currentPages + pagesNeeded > this.maxPages) {
+      throw new Error(`Cannot grow memory: exceeds maximum page limit of ${this.maxPages}`);
+    }
+
+    this.memory.grow(pagesNeeded);
+  }
+
+  /**
+   * Creates a typed view over the shared memory buffer at a specific byte offset.
+   */
+  public getView<T extends Int32Array | Float64Array | BigInt64Array | Uint8Array>(
+    type: ViewType,
+    byteOffset: number = 0,
+    length?: number
+  ): T {
+    const buffer = this.memory.buffer;
+    
+    switch (type) {
+      case 'Int32':
+        return new Int32Array(buffer, byteOffset, length) as T;
+      case 'Float64':
+        return new Float64Array(buffer, byteOffset, length) as T;
+      case 'BigInt64':
+        return new BigInt64Array(buffer, byteOffset, length) as T;
+      case 'Uint8':
+        return new Uint8Array(buffer, byteOffset, length) as T;
+      default:
+        throw new Error(`Unsupported view type: ${type}`);
+    }
+  }
+
+  /**
+   * Exposes the raw SharedArrayBuffer
+   */
+  public getBuffer(): SharedArrayBuffer {
+    return this.memory.buffer as unknown as SharedArrayBuffer;
+  }
+}