@panagos/jojoscript 0.7.0 → 0.8.0

package/src/runtime/checkpoint-store.js

@@ -0,0 +1,122 @@
+import fs from 'node:fs'
+import path from 'node:path'
+
+/*
+ * Storage interface for checkpoints. Deliberately storage-agnostic: the
+ * `checkpoint()` runtime operator only ever talks to this interface, never
+ * to a filesystem or database directly, so new backends (S3, a database,
+ * ...) can be added later without touching pipeline code.
+ */
+export class CheckpointStore {
+  async save(_checkpoint) {
+    throw new Error('CheckpointStore.save() is not implemented')
+  }
+
+  async load(_pipelineId, _checkpointId) {
+    throw new Error('CheckpointStore.load() is not implemented')
+  }
+
+  async exists(_pipelineId, _checkpointId) {
+    throw new Error('CheckpointStore.exists() is not implemented')
+  }
+
+  async delete(_pipelineId, _checkpointId) {
+    throw new Error('CheckpointStore.delete() is not implemented')
+  }
+}
+
+function key(pipelineId, checkpointId) {
+  return `${pipelineId}::${checkpointId}`
+}
+
+/*
+ * In-memory store. Useful for tests and for short-lived processes where
+ * durability across restarts is not required.
+ */
+export class MemoryCheckpointStore extends CheckpointStore {
+  constructor() {
+    super()
+    this.data = new Map()
+  }
+
+  async save(checkpoint) {
+    this.data.set(key(checkpoint.pipelineId, checkpoint.checkpointId), checkpoint)
+  }
+
+  async load(pipelineId, checkpointId) {
+    const found = this.data.get(key(pipelineId, checkpointId))
+    if (!found) throw new Error(`Checkpoint not found: ${checkpointId}`)
+    return found
+  }
+
+  async exists(pipelineId, checkpointId) {
+    return this.data.has(key(pipelineId, checkpointId))
+  }
+
+  async delete(pipelineId, checkpointId) {
+    this.data.delete(key(pipelineId, checkpointId))
+  }
+
+  list(pipelineId) {
+    return [...this.data.values()]
+      .filter(checkpoint => checkpoint.pipelineId === pipelineId)
+      .map(checkpoint => checkpoint.checkpointId)
+  }
+}
+
+/*
+ * JSON-on-disk store, one file per (pipelineId, checkpointId). Writes go to
+ * a temp file first and are then renamed into place, so a crash mid-write
+ * can never leave a corrupted checkpoint file behind that would silently
+ * poison a later resume.
+ */
+export class FileCheckpointStore extends CheckpointStore {
+  constructor(baseDir = '.jojo-checkpoints') {
+    super()
+    this.baseDir = baseDir
+  }
+
+  filePath(pipelineId, checkpointId) {
+    return path.join(this.baseDir, pipelineId, `${checkpointId}.json`)
+  }
+
+  async save(checkpoint) {
+    const file = this.filePath(checkpoint.pipelineId, checkpoint.checkpointId)
+    fs.mkdirSync(path.dirname(file), { recursive: true })
+
+    const tmp = `${file}.tmp-${process.pid}-${Date.now()}`
+    fs.writeFileSync(tmp, JSON.stringify(checkpoint), 'utf8')
+    fs.renameSync(tmp, file)
+  }
+
+  async load(pipelineId, checkpointId) {
+    const file = this.filePath(pipelineId, checkpointId)
+    if (!fs.existsSync(file)) throw new Error(`Checkpoint not found: ${checkpointId}`)
+
+    let raw
+    try {
+      raw = fs.readFileSync(file, 'utf8')
+      return JSON.parse(raw)
+    } catch (error) {
+      throw new Error(`Corrupted checkpoint "${checkpointId}": ${error.message}`)
+    }
+  }
+
+  async exists(pipelineId, checkpointId) {
+    return fs.existsSync(this.filePath(pipelineId, checkpointId))
+  }
+
+  async delete(pipelineId, checkpointId) {
+    const file = this.filePath(pipelineId, checkpointId)
+    if (fs.existsSync(file)) fs.unlinkSync(file)
+  }
+
+  list(pipelineId) {
+    const dir = path.join(this.baseDir, pipelineId)
+    if (!fs.existsSync(dir)) return []
+
+    return fs.readdirSync(dir)
+      .filter(name => name.endsWith('.json'))
+      .map(name => name.slice(0, -'.json'.length))
+  }
+}

package/src/runtime/checkpoint.js

@@ -0,0 +1,69 @@
+import { getCurrentContext, getDefaultContext } from './execution-context.js'
+
+/*
+ * A pipeline checkpoint. `items` flow through unchanged; the operator's job
+ * is purely persistence:
+ *
+ *  - if a completed checkpoint already exists for this (pipelineId, name),
+ *    the upstream `items` iterable is never touched at all (it may be an
+ *    expensive, not-yet-started generator) — the saved items are yielded
+ *    instead. This is the whole resume mechanism: re-running the same
+ *    program after a crash just naturally skips completed work.
+ *  - otherwise, items are streamed through as usual (never buffered ahead
+ *    of the consumer) and also collected so they can be durably saved once
+ *    the full checkpoint has completed. A checkpoint is only ever written
+ *    after every item has passed through, so a crash mid-checkpoint never
+ *    leaves a partial/misleading checkpoint on disk.
+ *
+ * `meta.checkpointOrder`/`meta.checkpointIndex` are injected by the
+ * compiler: the full, statically-known list of checkpoint names in this
+ * pipeline, and this call's position within it. They let `replay(--from)`
+ * decide whether a *later* checkpoint must be invalidated/recomputed
+ * without depending on the runtime order lazy generators happen to be
+ * pulled in (nested generators lazily run their upstream checks only once
+ * pulled, which can't be relied on for that decision).
+ */
+export async function* checkpoint(items, name, meta = {}) {
+  const ctx = getCurrentContext() ?? getDefaultContext()
+  const pipelineId = ctx.pipelineId ?? meta.pipelineId ?? 'default'
+  const store = ctx.checkpointStore
+
+  const replayTargetIndex = ctx.replayFrom && meta.checkpointOrder
+    ? meta.checkpointOrder.indexOf(ctx.replayFrom)
+    : -1
+
+  const isReplayTarget = ctx.replayFrom === name
+  const isAfterReplayTarget =
+    replayTargetIndex !== -1 &&
+    typeof meta.checkpointIndex === 'number' &&
+    meta.checkpointIndex > replayTargetIndex
+
+  if (isReplayTarget) {
+    if (!(await store.exists(pipelineId, name))) {
+      throw new Error(`Cannot replay from checkpoint "${name}": no such checkpoint exists`)
+    }
+    const saved = await store.load(pipelineId, name)
+    for (const item of saved.items) yield item
+    return
+  }
+
+  if (!isAfterReplayTarget && await store.exists(pipelineId, name)) {
+    const saved = await store.load(pipelineId, name)
+    for (const item of saved.items) yield item
+    return
+  }
+
+  const buffer = []
+  for await (const item of items) {
+    buffer.push(item)
+    yield item
+  }
+
+  await store.save({
+    pipelineId,
+    checkpointId: name,
+    items: buffer,
+    itemCount: buffer.length,
+    createdAt: new Date().toISOString()
+  })
+}