@rip-lang/swarm 1.0.0

package/README.md

@@ -0,0 +1,145 @@
+<img src="https://raw.githubusercontent.com/shreeve/rip-lang/main/docs/rip.svg" style="width:50px" /> <br>
+
+# Rip Swarm - @rip-lang/swarm
+
+> **Parallel job runner with worker threads — setup once, swarm many**
+
+Swarm processes large batches of tasks in parallel using Bun's worker
+threads. Define a setup function (runs once) and a perform function
+(runs per task), and swarm handles the rest — worker pool management,
+file-based task lifecycle, ANSI progress bars, crash recovery, and a
+clean summary at the end.
+
+## Quick Start
+
+```bash
+bun add -g @rip-lang/swarm
+```
+
+Create a job script:
+
+```coffee
+import { swarm, init, retry, todo } from '@rip-lang/swarm'
+
+setup = ->
+  unless retry()
+    init()
+    for i in [1..100] then todo(i)
+  { startedAt: Date.now() }
+
+perform = (task, ctx) ->
+  await Bun.sleep(Math.random() * 1000)
+  throw new Error("boom") if Math.random() < 0.03
+
+swarm { setup, perform, workers: 10 }
+```
+
+Run it:
+
+```bash
+rip jobs.rip
+rip jobs.rip -w 10       # 10 workers (default: CPU count)
+```
+
+## How It Works
+
+```
+┌──────────────────────────────────────────────┐
+│              Single Bun Process              │
+│                                              │
+│  Main Thread          Worker Threads         │
+│  ───────────          ──────────────         │
+│  setup()              perform(task, ctx)     │
+│  task dispatch        perform(task, ctx)     │
+│  progress bars        perform(task, ctx)     │
+│  file lifecycle       ...                    │
+│                                              │
+│  .swarm/todo/ ──→ .swarm/done/               │
+│                └──→ .swarm/died/             │
+└──────────────────────────────────────────────┘
+```
+
+1. **setup()** runs once in the main thread — creates tasks and returns
+   an optional context object
+2. **N worker threads** are spawned — each loads your script and gets
+   the `perform` function
+3. Tasks are dispatched from `.swarm/todo/` to workers via message passing
+4. Workers call `perform(task, ctx)` and report done or failed
+5. Main thread moves files to `.swarm/done/` or `.swarm/died/` and
+   updates the progress display
+6. When all tasks complete, a summary is printed
+
+## API
+
+### Task Queue
+
+```coffee
+import { init, retry, todo } from '@rip-lang/swarm'
+
+init()               # Remove old .swarm, create todo/done/died dirs
+retry()              # Move .swarm/died/* back to .swarm/todo/ for retry
+todo('task-1')       # Create empty task file
+todo('task-2', data) # Create task file with data (string or JSON)
+```
+
+### swarm()
+
+```coffee
+swarm { setup, perform }
+swarm { setup, perform, workers: 8, bar: 30, char: '█' }
+```
+
+Options:
+- **setup** — function, runs once in main thread, returns optional context
+- **perform** — function `(taskPath, ctx)`, runs in worker threads
+- **workers** — number of worker threads (default: CPU count)
+- **bar** — progress bar width in characters (default: 20)
+- **char** — character for progress bars (default: `•`)
+
+### CLI Flags
+
+```
+-w, --workers <n>     Number of workers (default: CPU count)
+-b, --bar <width>     Progress bar width (default: 20)
+-c, --char <ch>       Bar character (default: •)
+-r, --reset           Remove .swarm directory and quit
+```
+
+CLI flags override options passed to `swarm()`.
+
+## Task Lifecycle
+
+```
+.swarm/
+├── todo/       ← tasks waiting to be processed
+├── done/       ← successfully completed tasks
+└── died/       ← failed tasks (can be retried)
+```
+
+Tasks are plain files. The filename identifies the task. Files can be
+empty (filename is the data) or contain a payload (JSON, text, etc.).
+File moves use `renameSync` for atomic operations.
+
+## Crash Recovery
+
+| Failure | What Happens | Recovery |
+|---------|-------------|----------|
+| `perform()` throws | Worker catches it, reports failed, continues | Automatic |
+| Unhandled rejection | Worker error handler fires, continues | Automatic |
+| Worker thread dies | Main thread detects exit, respawns worker | Automatic |
+| Task timeout | (planned) AbortSignal kills task | Worker continues |
+
+Failed tasks land in `.swarm/died/`. Call `retry()` in your next
+`setup()` to move them back to `.swarm/todo/` for reprocessing.
+
+## Comparison with vete (Ruby)
+
+| Feature | vete (Ruby) | swarm (Rip/Bun) |
+|---------|------------|-----------------|
+| Parallelism | fork() per task | Worker threads (reused) |
+| Setup | Runs once (fork shares memory) | Runs once (context cloned) |
+| Per-task overhead | ~100μs (fork) | ~0 (message passing) |
+| Crash recovery | Process dies, slot freed | Exception caught, worker continues |
+| Timeout support | None | Planned (AbortSignal) |
+| Default workers | 1 | CPU count |
+| Dependencies | fileutils, optparse, thread | Zero (Bun builtins) |

package/bin/swarm

@@ -0,0 +1,15 @@
+#!/usr/bin/env bun
+
+import { execFileSync } from 'child_process';
+
+const script = process.argv[2];
+if (!script) {
+  console.error('usage: swarm [options] <script.rip>');
+  process.exit(1);
+}
+
+try {
+  execFileSync('rip', [script, ...process.argv.slice(3)], { stdio: 'inherit' });
+} catch (e) {
+  process.exit(e.status || 1);
+}

package/lib/worker.mjs

@@ -0,0 +1,43 @@
+// Worker bootstrap — loaded by each worker thread
+// The rip-loader is preloaded via Worker({ preload: [...] }), so .rip imports work.
+// Imports the user script (which calls swarm() — a no-op in worker mode),
+// then processes tasks via IPC from the main thread.
+
+import { parentPort, workerData } from 'worker_threads';
+
+const { scriptPath, context } = workerData;
+
+let perform;
+
+try {
+  // Import the user script — triggers swarm() which registers perform() in worker mode
+  await import(scriptPath);
+
+  // Get perform from the swarm module (registered by swarm() in worker mode)
+  const swarmMod = await import(new URL('../swarm.rip', import.meta.url).href);
+  perform = swarmMod._getPerform();
+
+  if (typeof perform !== 'function') {
+    throw new Error('No perform() function provided to swarm()');
+  }
+} catch (err) {
+  parentPort.postMessage({ type: 'error', error: err.message });
+  process.exit(1);
+}
+
+// Signal ready
+parentPort.postMessage({ type: 'ready' });
+
+// Process tasks as they arrive
+parentPort.on('message', async (msg) => {
+  if (msg.type === 'task') {
+    try {
+      await perform(msg.taskPath, context);
+      parentPort.postMessage({ type: 'done', taskPath: msg.taskPath });
+    } catch (err) {
+      parentPort.postMessage({ type: 'failed', taskPath: msg.taskPath, error: err.message });
+    }
+  } else if (msg.type === 'shutdown') {
+    process.exit(0);
+  }
+});

package/package.json

@@ -0,0 +1,46 @@
+{
+  "name": "@rip-lang/swarm",
+  "version": "1.0.0",
+  "description": "Parallel job runner with worker threads — setup once, swarm many",
+  "type": "module",
+  "main": "swarm.rip",
+  "bin": {
+    "swarm": "./bin/swarm"
+  },
+  "exports": {
+    ".": "./swarm.rip"
+  },
+  "scripts": {
+    "test": "rip test/example.rip -w 5"
+  },
+  "keywords": [
+    "parallel",
+    "workers",
+    "jobs",
+    "tasks",
+    "concurrency",
+    "progress",
+    "cli",
+    "rip"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/shreeve/rip-lang.git",
+    "directory": "packages/swarm"
+  },
+  "homepage": "https://github.com/shreeve/rip-lang/tree/main/packages/swarm#readme",
+  "bugs": {
+    "url": "https://github.com/shreeve/rip-lang/issues"
+  },
+  "author": "Steve Shreeve <steve.shreeve@gmail.com>",
+  "license": "MIT",
+  "dependencies": {
+    "rip-lang": "^2.9.0"
+  },
+  "files": [
+    "swarm.rip",
+    "lib/",
+    "bin/",
+    "README.md"
+  ]
+}

package/swarm.rip

@@ -0,0 +1,332 @@
+# ==============================================================================
+# @rip-lang/swarm — Parallel job runner with worker threads
+#
+# Author: Steve Shreeve (steve.shreeve@gmail.com)
+#   Date: January 18, 2026
+#
+# Usage:
+#   import { swarm, init, retry, todo } from '@rip-lang/swarm'
+#
+#   setup = ->
+#     unless retry()
+#       init()
+#       for i in [1..100] then todo(i)
+#     { startedAt: Date.now() }
+#
+#   perform = (task, ctx) ->
+#     await Bun.sleep(Math.random() * 1000)
+#
+#   swarm { setup, perform }
+# ==============================================================================
+
+import { isMainThread } from 'worker_threads'
+import { existsSync, mkdirSync, readdirSync, renameSync, writeFileSync, rmSync } from 'fs'
+import { join, resolve, dirname } from 'path'
+import { cpus } from 'os'
+
+# ==============================================================================
+# Module state
+# ==============================================================================
+
+_dir  = resolve('.swarm')
+_todo = join(_dir, 'todo')
+_done = join(_dir, 'done')
+_died = join(_dir, 'died')
+
+# ==============================================================================
+# Task queue (file-based)
+# ==============================================================================
+
+export init = ->
+  rmSync(_dir, { recursive: true, force: true })
+  mkdirSync(_todo, { recursive: true })
+  mkdirSync(_done, { recursive: true })
+  mkdirSync(_died, { recursive: true })
+
+export retry = ->
+  return false unless existsSync(_died) and existsSync(_todo)
+  died = readdirSync(_died)
+  todo = readdirSync(_todo)
+  return todo.length > 0 if died.length is 0
+  for file in died
+    renameSync(join(_died, file), join(_todo, file))
+  true
+
+export todo = (name, data) ->
+  path = join(_todo, String(name))
+  if data?
+    writeFileSync(path, if typeof data is 'string' then data else JSON.stringify(data))
+  else
+    writeFileSync(path, '')
+
+move = (path, dest) ->
+  try renameSync(path, join(dest, path.split('/').pop()))
+  catch then null
+
+# ==============================================================================
+# ANSI terminal
+# ==============================================================================
+
+STDOUT = process.stdout
+
+write  = (s) -> STDOUT.write(s)
+clear  = (line) -> if line then "\x1b[K" else "\x1b[2J"
+cursor = (show) -> write(if show then "\x1b[?25h" else "\x1b[?25l")
+go     = (r = 1, c = 1) -> "\x1b[#{r};#{c}H"
+
+_hex = {}
+
+hex = (str) ->
+  return _hex[str] if _hex[str]?
+  s = str.replace(/^#/, '')
+  m = s.match(/^([0-9a-f]{2})([0-9a-f]{2})([0-9a-f]{2})$/i) or s.match(/^([0-9a-f])([0-9a-f])([0-9a-f])$/i)
+  return '' unless m
+  result = if m[1].length is 1
+    [parseInt(m[1]+m[1], 16), parseInt(m[2]+m[2], 16), parseInt(m[3]+m[3], 16)].join(';')
+  else
+    [parseInt(m[1], 16), parseInt(m[2], 16), parseInt(m[3], 16)].join(';')
+  _hex[str] = result
+
+fg = (rgb) -> if rgb then "\x1b[38;2;#{hex(rgb)}m" else "\x1b[39m"
+bg = (rgb) -> if rgb then "\x1b[48;2;#{hex(rgb)}m" else "\x1b[49m"
+
+# ==============================================================================
+# Progress display
+# ==============================================================================
+
+_char = _wide = _len = null
+
+drawFrame = (workers) ->
+  _len = String(workers).length
+  write clear()
+  write go(2 + workers, _len + 3) + "╰" + "─".repeat(_wide + 2) + "╯"
+  write go(1,           _len + 3) + "╭" + "─".repeat(_wide + 2) + "╮"
+  for i in [0...workers]
+    write go(i + 2, 1)
+    label = String(i + 1).padStart(_len)
+    write " #{label} │ " + " ".repeat(_wide) + " │"
+  # summary bar row
+  write go(workers + 3, _len + 3) + "│ " + " ".repeat(_wide) + " │"
+
+draw = (state) ->
+  { live, done, died, jobs, workers, info } = state
+
+  ppct = (done + died) / jobs
+  most = Math.max(...Object.values(info), 1)
+
+  # worker bars
+  for slot, count of info
+    tpct = count / most
+    cols = Math.floor(ppct * tpct * _wide)
+    write go(parseInt(slot) + 1, _len + 5) + bg("5383ec") + _char.repeat(cols) + bg()
+
+  # summary bar
+  dpct = done / jobs
+  lpct = live / jobs
+  gcol = Math.floor(dpct * _wide)
+  ycol = Math.floor(lpct * _wide)
+  rcol = Math.max(0, _wide - gcol - ycol)
+  row  = workers + 3
+
+  write go(row, _len + 5)
+  write fg("fff")
+  write bg("58a65c") + _char.repeat(gcol)          # green  (done)
+  write bg("f1bf42") + _char.repeat(ycol)           # yellow (live)
+  write bg("d85140") + " ".repeat(rcol)             # red    (rest)
+  write go(row, _len + 5 + _wide + 3)
+  write bg("5383ec") + " #{(ppct * 100).toFixed(1)}% "
+  write bg() + " " + bg("58a65c") + " #{done}/#{jobs} done " if done > 0
+  write bg() + " " + bg("d85140") + " #{died} died "         if died > 0
+  write fg() + bg()
+
+# ==============================================================================
+# Worker orchestration
+# ==============================================================================
+
+# Worker mode: register perform function for the worker bootstrap
+_workerPerform = null
+export _getPerform = -> _workerPerform
+
+export swarm = (opts = {}) ->
+  unless isMainThread
+    _workerPerform = opts.perform
+    return
+
+  { Worker } = await import('worker_threads')
+
+  # parse CLI options
+  args    = process.argv.slice(2)
+  workers = parseInt(findArg(args, '-w', '--workers')) or opts.workers or cpus().length
+  barw    = parseInt(findArg(args, '-b', '--bar'))     or opts.bar    or 20
+  char    = findArg(args, '-c', '--char')              or opts.char   or '•'
+  doreset = args.includes('-r') or args.includes('--reset')
+
+  if workers < 1
+    console.error 'error: workers must be at least 1'
+    process.exit(1)
+
+  _wide = barw
+  _char = char[0]
+
+  if doreset
+    rmSync(_dir, { recursive: true, force: true })
+    console.log 'removed .swarm directory'
+    process.exit(0)
+
+  # run setup
+  unless typeof opts.perform is 'function'
+    console.error 'error: perform() function is required'
+    process.exit(1)
+
+  context = {}
+  if typeof opts.setup is 'function'
+    result = await opts.setup()
+    context = result if result? and typeof result is 'object'
+
+  # read task list
+  unless existsSync(_todo)
+    console.error 'error: no .swarm/todo directory found (did setup run?)'
+    process.exit(1)
+
+  tasks = readdirSync(_todo).sort().map (f) -> join(_todo, f)
+
+  if tasks.length is 0
+    console.log 'no tasks to process'
+    process.exit(0)
+
+  jobs = tasks.length
+
+  # resolve paths
+  workerPath = join(dirname(new URL(import.meta.url).pathname), 'lib', 'worker.mjs')
+  scriptPath = resolve(process.argv[1] or '')
+
+  # find rip-loader for workers
+  loaderPath = null
+  try
+    loaderPath = join(dirname(require.resolve('rip-lang')), '..', 'rip-loader.js')
+  catch
+    null
+
+  # state
+  live    = 0
+  done    = 0
+  died    = 0
+  info    = {}
+  taskIdx = 0
+  inflight = {}  # slot → taskPath (track in-flight tasks for crash recovery)
+
+  # signal handlers
+  process.on 'SIGINT', ->
+    cursor(true)
+    write go(workers + 5, 1) + "\n"
+    process.exit(1)
+  process.on 'SIGWINCH', ->
+    drawFrame(workers)
+    draw({ live, done, died, jobs, workers, info })
+
+  # draw initial frame
+  startTime = Date.now()
+  cursor(false)
+  drawFrame(workers)
+
+  # create workers and dispatch tasks
+  allWorkers = []
+
+  await new Promise (resolveAll) ->
+
+    finished = false
+    checkDone = ->
+      if not finished and done + died >= jobs
+        finished = true
+        for wk in allWorkers
+          try wk.postMessage { type: 'shutdown' }
+          catch then null
+        resolveAll()
+
+    dispatchNext = (worker, slot) ->
+      if taskIdx < tasks.length
+        taskPath = tasks[taskIdx++]
+        inflight[slot] = taskPath
+        live++
+        write go(slot + 1, _len + 5 + _wide + 3) + " " + taskPath.split('/').pop() + clear(true)
+        draw({ live, done, died, jobs, workers, info })
+        worker.postMessage { type: 'task', taskPath }
+      else
+        inflight[slot] = null
+        checkDone()
+
+    spawnWorker = (slot) ->
+      info[slot] ?= 0
+
+      wopts = { workerData: { scriptPath, context } }
+      wopts.preload = [loaderPath] if loaderPath
+      w = new Worker(workerPath, wopts)
+      allWorkers.push(w)
+
+      w.on 'message', (msg) ->
+        switch msg.type
+          when 'ready'
+            dispatchNext(w, slot)
+          when 'done'
+            move(msg.taskPath, _done)
+            inflight[slot] = null
+            live--
+            done++
+            info[slot]++
+            draw({ live, done, died, jobs, workers, info })
+            dispatchNext(w, slot)
+          when 'failed'
+            move(msg.taskPath, _died)
+            inflight[slot] = null
+            live--
+            died++
+            info[slot]++
+            draw({ live, done, died, jobs, workers, info })
+            dispatchNext(w, slot)
+
+      w.on 'error', (err) ->
+        console.error "\nworker #{slot} error: #{err.message}"
+
+      w.on 'exit', (code) ->
+        # if worker crashed mid-task, count the in-flight task as died
+        if inflight[slot]
+          move(inflight[slot], _died)
+          inflight[slot] = null
+          live--
+          died++
+          info[slot]++
+          draw({ live, done, died, jobs, workers, info })
+        # respawn if there's still work to do
+        if done + died < jobs
+          spawnWorker(slot)
+        else
+          checkDone()
+
+    # spawn worker pool
+    count = Math.min(workers, jobs)
+    for slot in [1..count]
+      spawnWorker(slot)
+
+  # summary
+  cursor(true)
+  secs = (Date.now() - startTime) / 1000
+  write go(workers + 5, 1)
+  write "#{secs.toFixed(2)} secs"
+  write " for #{jobs} jobs"
+  write " by #{workers} workers"
+  write " @ #{(jobs / secs).toFixed(2)} jobs/sec" if secs > 0
+  write "\n\n"
+
+# ==============================================================================
+# CLI helpers
+# ==============================================================================
+
+findArg = (args, short, long) ->
+  for arg, i in args
+    if arg is short or arg is long
+      return args[i + 1] if args[i + 1]?
+    if arg.startsWith("#{long}=")
+      return arg.split('=')[1]
+    if arg.startsWith("#{short}=")
+      return arg.split('=')[1]
+  null