@panagos/jojoscript 0.7.1 → 0.9.0

package/src/runtime/effect-log.js

@@ -0,0 +1,90 @@
+import fs from 'node:fs'
+import path from 'node:path'
+
+/*
+ * Storage interface for recorded effect calls, mirroring
+ * `checkpoint-store.js`'s `CheckpointStore`: the record/replay handler
+ * wrappers in `effects.js` only ever talk to this interface, never to a
+ * filesystem/database directly, so alternative backends can be added later
+ * without touching the effect runtime.
+ */
+export class EffectLog {
+  async record(_call) {
+    throw new Error('EffectLog.record() is not implemented')
+  }
+
+  async replayNext(_name) {
+    throw new Error('EffectLog.replayNext() is not implemented')
+  }
+}
+
+/*
+ * In-memory log. `replayNext(name)` returns recorded calls for that effect
+ * name in the order they were recorded, one per call — the same effect
+ * invoked three times during recording replays its three results in order
+ * during replay, regardless of what other effects were interleaved.
+ */
+export class MemoryEffectLog extends EffectLog {
+  constructor(entries = []) {
+    super()
+    this.entries = [...entries]
+    this.cursor = new Map()
+  }
+
+  async record(call) {
+    this.entries.push(call)
+  }
+
+  async replayNext(name) {
+    const index = this.cursor.get(name) ?? 0
+    const matches = this.entries.filter(entry => entry.name === name)
+
+    if (index >= matches.length) {
+      throw new Error(`No recorded call left to replay for effect "${name}"`)
+    }
+
+    this.cursor.set(name, index + 1)
+    return matches[index]
+  }
+
+  toJSON() {
+    return this.entries
+  }
+}
+
+/*
+ * JSON-on-disk log, one file for the whole recording. Like
+ * `FileCheckpointStore`, writes go to a temp file first and are then
+ * renamed into place so a crash mid-write never corrupts the log.
+ */
+export class FileEffectLog extends EffectLog {
+  constructor(file) {
+    super()
+    this.file = file
+    this.entries = fs.existsSync(file) ? JSON.parse(fs.readFileSync(file, 'utf8')) : []
+    this.cursor = new Map()
+  }
+
+  async record(call) {
+    this.entries.push(call)
+
+    const dir = path.dirname(this.file)
+    fs.mkdirSync(dir, { recursive: true })
+
+    const tmp = `${this.file}.tmp-${process.pid}-${Date.now()}`
+    fs.writeFileSync(tmp, JSON.stringify(this.entries, null, 2))
+    fs.renameSync(tmp, this.file)
+  }
+
+  async replayNext(name) {
+    const index = this.cursor.get(name) ?? 0
+    const matches = this.entries.filter(entry => entry.name === name)
+
+    if (index >= matches.length) {
+      throw new Error(`No recorded call left to replay for effect "${name}"`)
+    }
+
+    this.cursor.set(name, index + 1)
+    return matches[index]
+  }
+}

package/src/runtime/effects.js

@@ -0,0 +1,263 @@
+import { AsyncLocalStorage } from 'node:async_hooks'
+
+/*
+ * Algebraic effects / effect handlers runtime.
+ *
+ * The core idea: `defineEffect` turns a name into a callable that *describes
+ * an intent* ("perform fetchUser(42)") without deciding how it is carried
+ * out. `withHandlers` installs a handler map for the dynamic extent of a
+ * callback (an async-local, stack-shaped scope, exactly like
+ * `execution-context.js` already does for checkpoint/metrics state), and
+ * `performEffect` resolves an effect call against the innermost matching
+ * handler at the moment it is invoked.
+ *
+ * This is deliberately a *simplified* effect-handler model, not full
+ * algebraic effects: there is no delimited-continuation `resume`, handlers
+ * cannot resume a suspended computation more than once (or zero times) and
+ * inspect the result, and effects cannot be "re-thrown" to an outer handler
+ * after partial handling. What it *does* provide: named effects, dynamic
+ * handler dispatch, handler nesting/overriding, deterministic-vs-real
+ * handler swapping, and a clean async/JS interop story (a handler is just
+ * a normal — sync or async — JavaScript function).
+ */
+
+const handlerStorage = new AsyncLocalStorage()
+const invocationStorage = new AsyncLocalStorage()
+
+/*
+ * name -> { name, params, sideEffecting, replayable, builtin? }.
+ * Populated by every `defineEffect` call (i.e. every `effect name(...)`
+ * declaration compiles into one), plus the built-in `PARALLEL` capability.
+ * Purely descriptive metadata for inspection — never consulted by
+ * dispatch itself.
+ */
+const effectRegistry = new Map()
+
+export class UnhandledEffectError extends Error {
+  constructor(name) {
+    super(`Unhandled effect: ${name}\nNo handler registered for effect "${name}"`)
+    this.name = 'UnhandledEffectError'
+    this.effectName = name
+  }
+}
+
+export class EffectHandlerError extends Error {
+  constructor(name, cause) {
+    super(`Effect handler for "${name}" threw: ${cause?.message ?? cause}`, { cause })
+    this.name = 'EffectHandlerError'
+    this.effectName = name
+  }
+}
+
+export class RecursiveEffectError extends Error {
+  constructor(name) {
+    super(`Effect handler for "${name}" recursively invoked the same effect ("${name}")`)
+    this.name = 'RecursiveEffectError'
+    this.effectName = name
+  }
+}
+
+/*
+ * Declares a named effect and returns the callable used to *perform* it.
+ * `options.sideEffecting` (default true) and `options.replayable` (default
+ * false) are pure metadata consumed by inspection (`describeEffect`,
+ * `describeHandlers`) and by callers deciding whether it's safe to treat a
+ * result as reusable across a checkpoint/replay boundary — the runtime
+ * itself never enforces them.
+ */
+export function defineEffect(name, params = [], options = {}) {
+  effectRegistry.set(name, {
+    name,
+    params: [...params],
+    sideEffecting: options.sideEffecting ?? true,
+    replayable: options.replayable ?? false
+  })
+
+  function effectFn(...args) {
+    return performEffect(name, args)
+  }
+
+  effectFn.effectName = name
+  effectFn.isEffect = true
+  return effectFn
+}
+
+export function describeEffect(name) {
+  return effectRegistry.get(name) ?? null
+}
+
+export function listEffects() {
+  return [...effectRegistry.values()]
+}
+
+function normalizeHandlerEntry(name, value) {
+  if (typeof value === 'function') {
+    return { handle: value, deterministic: undefined }
+  }
+
+  if (value && typeof value === 'object' && typeof value.handle === 'function') {
+    return { handle: value.handle, deterministic: value.deterministic ?? undefined }
+  }
+
+  throw new TypeError(
+    `Invalid handler for effect "${name}": expected a function, e.g. \`id => fakeUser(id)\`, ` +
+    'or a `{ handle, deterministic }` descriptor'
+  )
+}
+
+function currentFrames() {
+  return handlerStorage.getStore() ?? []
+}
+
+/*
+ * Installs `handlers` (an object of `name -> handlerFn | { handle,
+ * deterministic }`) for the dynamic extent of `fn`, then restores whatever
+ * was active before. Nesting `withHandlers` calls (source: nested `handle`
+ * blocks) pushes a new innermost frame in front of the existing ones, so an
+ * inner handler for the same effect name shadows — overrides — an outer
+ * one, while effects the inner handler map does *not* mention still fall
+ * through to the outer handler.
+ *
+ * Caveat (shared with `execution-context.js`'s checkpoint/metrics context):
+ * because pipeline stages like `map`/`filter` are lazy generators, the
+ * handler frame is only active while `fn` is *synchronously/async-chain*
+ * executing. If `fn` merely constructs a lazy pipeline and returns it
+ * without consuming it (no `toArray`/`toArrayAsync`/`reduce`/...), items
+ * pulled later outside this call will see whatever handlers (if any) are
+ * active at that later point, not these ones. Always consume (or `await`)
+ * effectful pipelines inside the `handle { ... } { ... }` block itself.
+ */
+export function withHandlers(handlers, fn) {
+  if (!handlers || typeof handlers !== 'object') {
+    throw new TypeError('withHandlers requires a handler map object, e.g. { effectName: handlerFn }')
+  }
+
+  const normalized = {}
+  for (const [name, value] of Object.entries(handlers)) {
+    normalized[name] = normalizeHandlerEntry(name, value)
+  }
+
+  const frames = [{ handlers: normalized }, ...currentFrames()]
+  return handlerStorage.run(frames, fn)
+}
+
+/*
+ * Resolves and invokes the effect named `name` against the innermost
+ * handler that declares it. Always returns a Promise (even for a
+ * synchronous handler) so calling code never needs to know whether the
+ * active handler is sync or async — production (often async: HTTP/DB) and
+ * test (often sync: fake data) handlers are interchangeable.
+ */
+export async function performEffect(name, args) {
+  const inFlight = invocationStorage.getStore() ?? new Set()
+  if (inFlight.has(name)) {
+    throw new RecursiveEffectError(name)
+  }
+
+  const frame = currentFrames().find(f => name in f.handlers)
+  if (!frame) {
+    throw new UnhandledEffectError(name)
+  }
+
+  const entry = frame.handlers[name]
+  const nextInFlight = new Set(inFlight)
+  nextInFlight.add(name)
+
+  try {
+    return await invocationStorage.run(nextInFlight, () => entry.handle(...args))
+  } catch (error) {
+    if (error instanceof RecursiveEffectError || error instanceof UnhandledEffectError) throw error
+    throw new EffectHandlerError(name, error)
+  }
+}
+
+/*
+ * Introspection: which effects have an active handler *right now* (i.e.
+ * called from inside a `handle { ... } { ... }` block), and what is known
+ * about each — handled, whether the installed handler declared itself
+ * deterministic, and the declaring effect's `sideEffecting`/`replayable`
+ * metadata. Returns `[]` outside any `withHandlers` scope.
+ */
+export function describeHandlers() {
+  const seen = new Map()
+
+  for (const frame of currentFrames()) {
+    for (const [name, entry] of Object.entries(frame.handlers)) {
+      if (seen.has(name)) continue
+
+      const registryEntry = effectRegistry.get(name)
+      seen.set(name, {
+        name,
+        handled: true,
+        deterministic: entry.deterministic ?? null,
+        sideEffecting: registryEntry?.sideEffecting ?? null,
+        replayable: registryEntry?.replayable ?? null
+      })
+    }
+  }
+
+  return [...seen.values()]
+}
+
+/*
+ * `parallel(n)` as an effect-backed capability (see collections.js' `map`):
+ * if a `PARALLEL` handler is currently installed, it decides how
+ * `items |> parallel(n) |> map(fn)` actually runs (real concurrency,
+ * deterministic sequential execution, recorded scheduling, ...). With no
+ * handler installed, `map` falls back to the original bounded-concurrency
+ * implementation — existing programs that never touch `handle {}` are
+ * completely unaffected.
+ */
+effectRegistry.set('PARALLEL', {
+  name: 'PARALLEL',
+  params: ['concurrency'],
+  sideEffecting: true,
+  replayable: false,
+  builtin: true
+})
+
+export function getParallelHandler() {
+  const frame = currentFrames().find(f => 'PARALLEL' in f.handlers)
+  return frame ? frame.handlers.PARALLEL.handle : null
+}
+
+/*
+ * Record/replay extension point (see effect-log.js for the storage side).
+ * `recordHandlers` wraps a normal (typically production) handler map so
+ * every call's arguments and result are appended to `log`; `replayHandlers`
+ * builds a handler map that never runs real code at all, just replays
+ * recorded results for the given effect names, in call order, per effect.
+ */
+export function recordHandlers(handlers, log) {
+  const wrapped = {}
+
+  for (const [name, value] of Object.entries(handlers)) {
+    const entry = normalizeHandlerEntry(name, value)
+    wrapped[name] = {
+      deterministic: entry.deterministic,
+      handle: async (...args) => {
+        const result = await entry.handle(...args)
+        await log.record({ name, args, result, recordedAt: new Date().toISOString() })
+        return result
+      }
+    }
+  }
+
+  return wrapped
+}
+
+export function replayHandlers(effectNames, log) {
+  const wrapped = {}
+
+  for (const name of effectNames) {
+    wrapped[name] = {
+      deterministic: true,
+      handle: async () => {
+        const call = await log.replayNext(name)
+        return call.result
+      }
+    }
+  }
+
+  return wrapped
+}

package/src/runtime/errors.js

@@ -0,0 +1,29 @@
+/*
+ * An error thrown by user code (or attached with `.retryable = false`) that
+ * `retry()` must never retry, no matter how many attempts remain.
+ */
+export class NonRetryableError extends Error {
+  constructor(message, options) {
+    super(message, options)
+    this.name = 'NonRetryableError'
+    this.retryable = false
+  }
+}
+
+/*
+ * A structured pipeline failure. Carries enough context (which pipeline,
+ * which node, which item, how many retries were attempted) for the CLI to
+ * print a useful failure report instead of a raw stack trace.
+ */
+export class PipelineError extends Error {
+  constructor(message, details = {}) {
+    super(message, { cause: details.cause })
+    this.name = 'PipelineError'
+    this.pipelineId = details.pipelineId ?? null
+    this.nodeId = details.nodeId ?? null
+    this.operation = details.operation ?? null
+    this.itemIndex = details.itemIndex ?? null
+    this.retryCount = details.retryCount ?? 0
+    this.timestamp = details.timestamp ?? new Date().toISOString()
+  }
+}

package/src/runtime/execution-context.js

@@ -0,0 +1,86 @@
+import fs from 'node:fs'
+import { AsyncLocalStorage } from 'node:async_hooks'
+
+import { Metrics } from './metrics.js'
+import { FileCheckpointStore } from './checkpoint-store.js'
+
+const storage = new AsyncLocalStorage()
+
+/*
+ * Everything a running pipeline needs that isn't the data itself: identity,
+ * timing, metrics, where to persist checkpoints, and CLI-provided options
+ * such as which checkpoint to replay from. Threaded through explicitly via
+ * `runInContext`/`getCurrentContext` (AsyncLocalStorage) instead of bare
+ * module-level globals.
+ */
+export class ExecutionContext {
+  constructor(options = {}) {
+    this.pipelineId = options.pipelineId ?? null
+    this.executionId = options.executionId ?? randomId()
+    this.startedAt = options.startedAt ?? new Date().toISOString()
+    this.currentNode = null
+    this.metrics = options.metrics ?? new Metrics()
+    this.checkpointStore = options.checkpointStore ?? new FileCheckpointStore()
+    this.options = options.options ?? {}
+    this.replayFrom = options.replayFrom ?? null
+    this.quiet = options.quiet ?? false
+  }
+}
+
+function randomId() {
+  return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`
+}
+
+export function runInContext(context, fn) {
+  return storage.run(context, fn)
+}
+
+export function getCurrentContext() {
+  return storage.getStore()
+}
+
+/*
+ * Compiled pipelines call `checkpoint()`/`inspect()` as ordinary function
+ * calls, with no CLI wrapping required (`node compiled.js` must work on its
+ * own for the resume story to make sense). When no context has been set up
+ * via `runInContext`, a single lazily-created default context is used
+ * instead, configured from environment variables so the CLI can steer it
+ * (`jojo replay --from`, `--out`, profiling) without needing to run the
+ * compiled module in-process.
+ */
+let defaultContext = null
+
+export function getDefaultContext() {
+  const active = getCurrentContext()
+  if (active) return active
+
+  if (!defaultContext) {
+    defaultContext = new ExecutionContext({
+      checkpointStore: new FileCheckpointStore(process.env.JOJO_CHECKPOINT_DIR || '.jojo-checkpoints'),
+      replayFrom: process.env.JOJO_REPLAY_FROM || null,
+      quiet: process.env.JOJO_QUIET === '1'
+    })
+
+    if (process.env.JOJO_PROFILE_OUT) {
+      const outFile = process.env.JOJO_PROFILE_OUT
+      process.on('exit', () => {
+        try {
+          fs.writeFileSync(outFile, JSON.stringify(defaultContext.metrics.toJSON()))
+        } catch {
+          /* best-effort; a missing metrics file is handled by the CLI */
+        }
+      })
+    }
+  }
+
+  return defaultContext
+}
+
+/*
+ * Test-only escape hatch: forces the next `getDefaultContext()` call in
+ * this process to create a fresh default context instead of reusing a
+ * memoized one.
+ */
+export function resetDefaultContext() {
+  defaultContext = null
+}

package/src/runtime/index.js

@@ -1,29 +1,70 @@
-export {
-  map,
-  filter,
-  sort,
-  unique,
-  take,
-  skip,
-  flatMap,
-  reduce,
-  toArray,
-  count,
-  first,
-  find,
-  some,
-  every,
-  groupBy,
-  partition,
-  chunk,
-  window,
-  zip,
-  zipWith,
-  scan,
-  sortBy,
-  distinctBy,
-  tap,
-  mapAsync,
-  filterAsync,
-  toArrayAsync
-} from './collections.js'
+export {
+  map,
+  filter,
+  sort,
+  unique,
+  take,
+  skip,
+  flatMap,
+  reduce,
+  toArray,
+  count,
+  first,
+  find,
+  some,
+  every,
+  groupBy,
+  partition,
+  chunk,
+  window,
+  zip,
+  zipWith,
+  scan,
+  sortBy,
+  distinctBy,
+  tap,
+  mapAsync,
+  filterAsync,
+  toArrayAsync,
+  parallel,
+  batch
+} from './collections.js'
+
+export { checkpoint } from './checkpoint.js'
+export { inspect, formatInspectReport } from './inspect.js'
+export { retry } from './retry.js'
+export { traceNode } from './trace.js'
+export { PipelineError, NonRetryableError } from './errors.js'
+export { Metrics, throughputOf } from './metrics.js'
+export {
+  CheckpointStore,
+  MemoryCheckpointStore,
+  FileCheckpointStore
+} from './checkpoint-store.js'
+export {
+  ExecutionContext,
+  runInContext,
+  getCurrentContext,
+  getDefaultContext,
+  resetDefaultContext
+} from './execution-context.js'
+export {
+  defineEffect,
+  withHandlers,
+  performEffect,
+  describeEffect,
+  listEffects,
+  describeHandlers,
+  getParallelHandler,
+  recordHandlers,
+  replayHandlers,
+  UnhandledEffectError,
+  EffectHandlerError,
+  RecursiveEffectError
+} from './effects.js'
+export {
+  EffectLog,
+  MemoryEffectLog,
+  FileEffectLog
+} from './effect-log.js'
+

package/src/runtime/inspect.js

@@ -0,0 +1,62 @@
+import { getCurrentContext, getDefaultContext } from './execution-context.js'
+
+/*
+ * `inspect(items, label)` is a pure passthrough: it must never change what
+ * flows through the pipeline. It records how many items passed, how long
+ * that took, and derived throughput, then prints a small report (unless
+ * the execution context is marked quiet, e.g. under `node --test`).
+ */
+export async function* inspect(items, label, meta = {}) {
+  const ctx = getCurrentContext() ?? getDefaultContext()
+  const start = process.hrtime.bigint()
+  const startedAt = new Date().toISOString()
+  let count = 0
+  let failed = false
+
+  try {
+    for await (const item of items) {
+      count++
+      yield item
+    }
+  } catch (error) {
+    failed = true
+    throw error
+  } finally {
+    const durationMs = Number(process.hrtime.bigint() - start) / 1e6
+    const durationSeconds = durationMs / 1000
+    const throughput = durationSeconds > 0 ? count / durationSeconds : count
+
+    const report = {
+      label,
+      id: meta.id ?? null,
+      pipelineId: meta.pipelineId ?? null,
+      items: count,
+      succeeded: failed ? Math.max(count - 1, 0) : count,
+      failed: failed ? 1 : 0,
+      durationMs,
+      throughput,
+      startedAt,
+      endedAt: new Date().toISOString(),
+      memoryUsageBytes: typeof process.memoryUsage === 'function' ? process.memoryUsage().heapUsed : undefined
+    }
+
+    ctx?.metrics?.recordInspect(label, report)
+
+    if (!ctx || !ctx.quiet) {
+      console.log(formatInspectReport(report))
+    }
+  }
+}
+
+export function formatInspectReport(report) {
+  const duration = (report.durationMs / 1000).toFixed(2)
+  const throughput = Math.round(report.throughput)
+
+  return [
+    report.label,
+    '-'.repeat(Math.max(report.label.length, 20)),
+    `items:       ${report.items}`,
+    `throughput:  ${throughput}/s`,
+    `duration:    ${duration}s`
+  ].join('\n')
+}

package/src/runtime/metrics.js

@@ -0,0 +1,41 @@
+/*
+ * Aggregates per-node execution metrics for a single pipeline execution:
+ * item counts, cumulative duration, and derived throughput. Used by both
+ * `inspect()` (a single named checkpoint the user asked to observe) and the
+ * `jojo profile` compiler instrumentation (every node, automatically).
+ */
+export class Metrics {
+  constructor() {
+    this.nodes = new Map()
+    this.inspections = []
+  }
+
+  record(id, type, items, durationMs) {
+    if (!id) return
+
+    const existing = this.nodes.get(id) ?? { id, type, items: 0, durationMs: 0 }
+    existing.items += items
+    existing.durationMs += durationMs
+    existing.type = type
+    this.nodes.set(id, existing)
+  }
+
+  recordInspect(label, report) {
+    this.inspections.push(report)
+  }
+
+  toJSON() {
+    return {
+      nodes: [...this.nodes.values()].map(node => ({
+        ...node,
+        throughput: throughputOf(node.items, node.durationMs)
+      })),
+      inspections: this.inspections
+    }
+  }
+}
+
+export function throughputOf(items, durationMs) {
+  const seconds = durationMs / 1000
+  return seconds > 0 ? items / seconds : items
+}