@panagos/jojoscript 0.7.1 → 0.8.0

package/src/runtime/errors.js

@@ -0,0 +1,29 @@
+/*
+ * An error thrown by user code (or attached with `.retryable = false`) that
+ * `retry()` must never retry, no matter how many attempts remain.
+ */
+export class NonRetryableError extends Error {
+  constructor(message, options) {
+    super(message, options)
+    this.name = 'NonRetryableError'
+    this.retryable = false
+  }
+}
+
+/*
+ * A structured pipeline failure. Carries enough context (which pipeline,
+ * which node, which item, how many retries were attempted) for the CLI to
+ * print a useful failure report instead of a raw stack trace.
+ */
+export class PipelineError extends Error {
+  constructor(message, details = {}) {
+    super(message, { cause: details.cause })
+    this.name = 'PipelineError'
+    this.pipelineId = details.pipelineId ?? null
+    this.nodeId = details.nodeId ?? null
+    this.operation = details.operation ?? null
+    this.itemIndex = details.itemIndex ?? null
+    this.retryCount = details.retryCount ?? 0
+    this.timestamp = details.timestamp ?? new Date().toISOString()
+  }
+}

package/src/runtime/execution-context.js

@@ -0,0 +1,86 @@
+import fs from 'node:fs'
+import { AsyncLocalStorage } from 'node:async_hooks'
+
+import { Metrics } from './metrics.js'
+import { FileCheckpointStore } from './checkpoint-store.js'
+
+const storage = new AsyncLocalStorage()
+
+/*
+ * Everything a running pipeline needs that isn't the data itself: identity,
+ * timing, metrics, where to persist checkpoints, and CLI-provided options
+ * such as which checkpoint to replay from. Threaded through explicitly via
+ * `runInContext`/`getCurrentContext` (AsyncLocalStorage) instead of bare
+ * module-level globals.
+ */
+export class ExecutionContext {
+  constructor(options = {}) {
+    this.pipelineId = options.pipelineId ?? null
+    this.executionId = options.executionId ?? randomId()
+    this.startedAt = options.startedAt ?? new Date().toISOString()
+    this.currentNode = null
+    this.metrics = options.metrics ?? new Metrics()
+    this.checkpointStore = options.checkpointStore ?? new FileCheckpointStore()
+    this.options = options.options ?? {}
+    this.replayFrom = options.replayFrom ?? null
+    this.quiet = options.quiet ?? false
+  }
+}
+
+function randomId() {
+  return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`
+}
+
+export function runInContext(context, fn) {
+  return storage.run(context, fn)
+}
+
+export function getCurrentContext() {
+  return storage.getStore()
+}
+
+/*
+ * Compiled pipelines call `checkpoint()`/`inspect()` as ordinary function
+ * calls, with no CLI wrapping required (`node compiled.js` must work on its
+ * own for the resume story to make sense). When no context has been set up
+ * via `runInContext`, a single lazily-created default context is used
+ * instead, configured from environment variables so the CLI can steer it
+ * (`jojo replay --from`, `--out`, profiling) without needing to run the
+ * compiled module in-process.
+ */
+let defaultContext = null
+
+export function getDefaultContext() {
+  const active = getCurrentContext()
+  if (active) return active
+
+  if (!defaultContext) {
+    defaultContext = new ExecutionContext({
+      checkpointStore: new FileCheckpointStore(process.env.JOJO_CHECKPOINT_DIR || '.jojo-checkpoints'),
+      replayFrom: process.env.JOJO_REPLAY_FROM || null,
+      quiet: process.env.JOJO_QUIET === '1'
+    })
+
+    if (process.env.JOJO_PROFILE_OUT) {
+      const outFile = process.env.JOJO_PROFILE_OUT
+      process.on('exit', () => {
+        try {
+          fs.writeFileSync(outFile, JSON.stringify(defaultContext.metrics.toJSON()))
+        } catch {
+          /* best-effort; a missing metrics file is handled by the CLI */
+        }
+      })
+    }
+  }
+
+  return defaultContext
+}
+
+/*
+ * Test-only escape hatch: forces the next `getDefaultContext()` call in
+ * this process to create a fresh default context instead of reusing a
+ * memoized one.
+ */
+export function resetDefaultContext() {
+  defaultContext = null
+}

package/src/runtime/index.js

@@ -1,29 +1,51 @@
-export {
-  map,
-  filter,
-  sort,
-  unique,
-  take,
-  skip,
-  flatMap,
-  reduce,
-  toArray,
-  count,
-  first,
-  find,
-  some,
-  every,
-  groupBy,
-  partition,
-  chunk,
-  window,
-  zip,
-  zipWith,
-  scan,
-  sortBy,
-  distinctBy,
-  tap,
-  mapAsync,
-  filterAsync,
-  toArrayAsync
-} from './collections.js'
+export {
+  map,
+  filter,
+  sort,
+  unique,
+  take,
+  skip,
+  flatMap,
+  reduce,
+  toArray,
+  count,
+  first,
+  find,
+  some,
+  every,
+  groupBy,
+  partition,
+  chunk,
+  window,
+  zip,
+  zipWith,
+  scan,
+  sortBy,
+  distinctBy,
+  tap,
+  mapAsync,
+  filterAsync,
+  toArrayAsync,
+  parallel,
+  batch
+} from './collections.js'
+
+export { checkpoint } from './checkpoint.js'
+export { inspect, formatInspectReport } from './inspect.js'
+export { retry } from './retry.js'
+export { traceNode } from './trace.js'
+export { PipelineError, NonRetryableError } from './errors.js'
+export { Metrics, throughputOf } from './metrics.js'
+export {
+  CheckpointStore,
+  MemoryCheckpointStore,
+  FileCheckpointStore
+} from './checkpoint-store.js'
+export {
+  ExecutionContext,
+  runInContext,
+  getCurrentContext,
+  getDefaultContext,
+  resetDefaultContext
+} from './execution-context.js'
+

package/src/runtime/inspect.js

@@ -0,0 +1,62 @@
+import { getCurrentContext, getDefaultContext } from './execution-context.js'
+
+/*
+ * `inspect(items, label)` is a pure passthrough: it must never change what
+ * flows through the pipeline. It records how many items passed, how long
+ * that took, and derived throughput, then prints a small report (unless
+ * the execution context is marked quiet, e.g. under `node --test`).
+ */
+export async function* inspect(items, label, meta = {}) {
+  const ctx = getCurrentContext() ?? getDefaultContext()
+  const start = process.hrtime.bigint()
+  const startedAt = new Date().toISOString()
+  let count = 0
+  let failed = false
+
+  try {
+    for await (const item of items) {
+      count++
+      yield item
+    }
+  } catch (error) {
+    failed = true
+    throw error
+  } finally {
+    const durationMs = Number(process.hrtime.bigint() - start) / 1e6
+    const durationSeconds = durationMs / 1000
+    const throughput = durationSeconds > 0 ? count / durationSeconds : count
+
+    const report = {
+      label,
+      id: meta.id ?? null,
+      pipelineId: meta.pipelineId ?? null,
+      items: count,
+      succeeded: failed ? Math.max(count - 1, 0) : count,
+      failed: failed ? 1 : 0,
+      durationMs,
+      throughput,
+      startedAt,
+      endedAt: new Date().toISOString(),
+      memoryUsageBytes: typeof process.memoryUsage === 'function' ? process.memoryUsage().heapUsed : undefined
+    }
+
+    ctx?.metrics?.recordInspect(label, report)
+
+    if (!ctx || !ctx.quiet) {
+      console.log(formatInspectReport(report))
+    }
+  }
+}
+
+export function formatInspectReport(report) {
+  const duration = (report.durationMs / 1000).toFixed(2)
+  const throughput = Math.round(report.throughput)
+
+  return [
+    report.label,
+    '-'.repeat(Math.max(report.label.length, 20)),
+    `items:       ${report.items}`,
+    `throughput:  ${throughput}/s`,
+    `duration:    ${duration}s`
+  ].join('\n')
+}

package/src/runtime/metrics.js

@@ -0,0 +1,41 @@
+/*
+ * Aggregates per-node execution metrics for a single pipeline execution:
+ * item counts, cumulative duration, and derived throughput. Used by both
+ * `inspect()` (a single named checkpoint the user asked to observe) and the
+ * `jojo profile` compiler instrumentation (every node, automatically).
+ */
+export class Metrics {
+  constructor() {
+    this.nodes = new Map()
+    this.inspections = []
+  }
+
+  record(id, type, items, durationMs) {
+    if (!id) return
+
+    const existing = this.nodes.get(id) ?? { id, type, items: 0, durationMs: 0 }
+    existing.items += items
+    existing.durationMs += durationMs
+    existing.type = type
+    this.nodes.set(id, existing)
+  }
+
+  recordInspect(label, report) {
+    this.inspections.push(report)
+  }
+
+  toJSON() {
+    return {
+      nodes: [...this.nodes.values()].map(node => ({
+        ...node,
+        throughput: throughputOf(node.items, node.durationMs)
+      })),
+      inspections: this.inspections
+    }
+  }
+}
+
+export function throughputOf(items, durationMs) {
+  const seconds = durationMs / 1000
+  return seconds > 0 ? items / seconds : items
+}

package/src/runtime/retry.js

@@ -0,0 +1,67 @@
+import { NonRetryableError } from './errors.js'
+
+/*
+ * Wraps `fn` so calling the returned function retries on failure, up to
+ * `options.attempts` total tries, with an optional fixed or exponential
+ * delay between attempts. Errors marked non-retryable (`NonRetryableError`,
+ * or any error with `error.retryable === false`) are never retried and
+ * propagate immediately.
+ *
+ * Used two ways by compiled pipelines:
+ *  - fused into a preceding `map`/`mapAsync`/`filterAsync`/`tap` stage, so
+ *    each *item* is retried independently: `map(items, retry(fetch, opts))`.
+ *  - standalone, wrapping the whole pipeline expression built so far as a
+ *    zero-argument thunk: `retry(() => (expr), opts)()`.
+ */
+export function retry(fn, options = {}, meta = {}) {
+  const attempts = options.attempts ?? 1
+  const backoff = options.backoff ?? 'fixed'
+  const delayMs = options.delay ?? 0
+
+  return async function retried(...args) {
+    let attempt = 1
+
+    while (true) {
+      try {
+        return await fn(...args)
+      } catch (error) {
+        if (error instanceof NonRetryableError || error?.retryable === false) {
+          annotate(error, meta, attempt)
+          throw error
+        }
+        if (attempt >= attempts) {
+          annotate(error, meta, attempt)
+          throw error
+        }
+
+        const wait = backoff === 'exponential'
+          ? delayMs * 2 ** (attempt - 1)
+          : delayMs
+
+        if (wait > 0) await sleep(wait)
+        attempt++
+      }
+    }
+  }
+}
+
+function sleep(ms) {
+  return new Promise(resolve => setTimeout(resolve, ms))
+}
+
+/*
+ * Attaches structured failure context (§16: pipelineId, nodeId, retryCount)
+ * to the error without changing its type/instanceof behavior, so the CLI
+ * (or user code) can report exactly which pipeline node exhausted retries.
+ */
+function annotate(error, meta, retryCount) {
+  if (error && typeof error === 'object') {
+    error.jojo = {
+      pipelineId: meta.pipelineId ?? null,
+      nodeId: meta.id ?? null,
+      operation: 'retry',
+      retryCount,
+      timestamp: new Date().toISOString()
+    }
+  }
+}

package/src/runtime/trace.js

@@ -0,0 +1,66 @@
+import { getCurrentContext, getDefaultContext } from './execution-context.js'
+
+/*
+ * Generic per-node instrumentation used only by `jojo profile` (the
+ * compiler wraps every stage's output with a call to this function when
+ * compiled with `{ profile: true }`). It must be transparent to *type*,
+ * not just to values: eager results (arrays, Maps, Sets, scalars) are
+ * counted and returned completely unchanged, while lazy
+ * generator/async-generator results are wrapped in an equivalent
+ * generator that counts items as they're pulled through — never
+ * materializing anything itself.
+ */
+export function traceNode(value, meta = {}) {
+  const ctx = getCurrentContext() ?? getDefaultContext()
+  const start = process.hrtime.bigint()
+
+  function finish(count) {
+    const durationMs = Number(process.hrtime.bigint() - start) / 1e6
+    ctx.metrics.record(meta.id, meta.type, count, durationMs)
+  }
+
+  const isAsyncGenerator = value != null &&
+    typeof value.next === 'function' &&
+    typeof value[Symbol.asyncIterator] === 'function'
+
+  const isSyncGenerator = !isAsyncGenerator && value != null &&
+    typeof value.next === 'function' &&
+    typeof value[Symbol.iterator] === 'function'
+
+  if (isAsyncGenerator) {
+    return (async function* () {
+      let count = 0
+      try {
+        for await (const item of value) {
+          count++
+          yield item
+        }
+      } finally {
+        finish(count)
+      }
+    })()
+  }
+
+  if (isSyncGenerator) {
+    return (function* () {
+      let count = 0
+      try {
+        for (const item of value) {
+          count++
+          yield item
+        }
+      } finally {
+        finish(count)
+      }
+    })()
+  }
+
+  const count = Array.isArray(value)
+    ? value.length
+    : value instanceof Map || value instanceof Set
+      ? value.size
+      : 1
+
+  finish(count)
+  return value
+}