@amritk/nish-x86_64-linux 0.17.0 → 0.20.0

package/runtime/runtime-parallel.c

@@ -8,10 +8,10 @@
  * and raising the syscall half's ceiling to make room would move the number a
  * reader sees for "the operating-system surface" for a reason that has nothing
  * to do with the operating-system surface. So this file carries its own
- * measured ceiling in tests/run.js beside the other two, and a program that
- * runs nothing in parallel pays nothing for it: `-ffunction-sections
- * -Wl,--gc-sections` drops every function no program calls, whichever
- * translation unit defined it.
+ * measured ceiling in tests/runtime-budgets.js beside the other two,
+ * and a program that runs nothing in parallel pays nothing for it:
+ * `-ffunction-sections -Wl,--gc-sections` drops every function no program
+ * calls, whichever translation unit defined it.
  *
  * It compiles in both configurations on purpose. With `-DNISH_THREADS` it
  * spawns; without it, and on WASI where there are no threads to spawn, the same
@@ -22,8 +22,16 @@
  * arrives with the same macro.
  *
  * docs/wp29-thread-surface.md is the surface this is the stage under: P1's
- * `parallelMapInto` and `parallelReduce` divide a range here, and P2's scopes
- * run their tasks here (`nish_scope_spawn`, `nish_scope_join`).
+ * `parallelMapInto` and `parallelReduce` divide a range here, P2's scopes
+ * run their tasks here (`nish_scope_spawn`, `nish_scope_join`), P3's lock
+ * waits here when it finds its word held (`nish_mutex_wait`), and P3's
+ * channels keep their buffers here (`nish_channel_send`, `nish_channel_count`,
+ * `nish_channel_receive`).
+ *
+ * Since the pool (docs/wp20-threads.md, section 8e) a divided region or a
+ * joined scope no longer creates a thread per chunk or task: it hands them to
+ * workers that are started once, on the first region or join that divides, and
+ * park between calls.
  */
 #include "nish.h"
 
@@ -34,37 +42,51 @@
 #if defined(NISH_THREADS) && !defined(__wasi__) && !defined(__wasm__)
 #define NISH_PAR_REAL 1
 #include <pthread.h>
-#include <unistd.h>
+#include <sched.h>
+#ifdef __GLIBC__
+#include <malloc.h> /* `mallopt` and `malloc_trim`, for the pool */
+#endif
 #endif
 
 /* The ceiling on how many threads one region may use. A fixed bound keeps the
- * job and handle arrays on the stack -- 64 jobs is 2 KB and 64 handles 512
- * bytes -- so a region never allocates to divide itself, which matters because
- * the arena it would allocate from is the very thing being made per-thread. */
+ * job array on the stack -- 64 jobs is 3.5 KB -- so a region never allocates
+ * to divide itself, which matters because the arena it would allocate from is
+ * the very thing being made per-thread. */
 #define NISH_PAR_MAX 64
 
-/* Cached, because `sysconf(_SC_NPROCESSORS_ONLN)` is not a cheap read: glibc
- * answers it from `sched_getaffinity` or by parsing
- * /sys/devices/system/cpu/online, and measured here it is 3.7 us -- which
- * `nish_parallel_range` would otherwise pay on every region, including the ones
- * too small to divide, where it was the entire cost. The race between two
- * threads filling it is benign (both compute the same number) but it is still a
- * race, so the word is read and written with relaxed atomic builtins (a plain
- * `int64_t`, since `__atomic_*` does not take an `_Atomic`-qualified pointer):
- * on x86-64 that is
- * a plain load and store and costs nothing, and it is defined behaviour.
+/* How many pieces a divided region is cut into per thread, grain permitting
+ * (`nish_parallel_range`). Eight, measured on bench/par-heavy.ts at four
+ * threads, nine interleaved rounds: 2 pieces was slower and noisier (868 ms
+ * median, 766-1121), 4 to 32 were flat within the copy band (805-822 ms), and
+ * so was bench/par-scale.ts's memory-bound map (262-270 ms). Among the flat
+ * counts, 8 bounds the tail a preempted core leaves idle at an eighth of one
+ * thread's share while keeping the claims per region at 8n. */
+#define NISH_PAR_PIECES 8
+
+/* Cached, because the answer is not a cheap read: `sysconf` alone was 3.7 us
+ * measured here (glibc parses /sys/devices/system/cpu/online), and the affinity
+ * mask and the cgroup quota `nish_cpu_online` reads are more system calls on
+ * top -- which `nish_parallel_range` would otherwise pay on every region,
+ * including the ones too small to divide, where it was the entire cost. The
+ * race between two threads filling it is benign (both compute the same number)
+ * but it is still a race, so the word is read and written with relaxed atomic
+ * builtins (a plain `int64_t`, since `__atomic_*` does not take an
+ * `_Atomic`-qualified pointer): on x86-64 that is a plain load and store and
+ * costs nothing, and it is defined behaviour.
  *
- * A machine whose online CPU count changes under a running process therefore
- * keeps the count it started with, which is the right trade for a number that
+ * A machine whose CPU count changes under a running process therefore keeps
+ * the count it started with, which is the right trade for a number that
  * decides how many threads a loop is divided into. */
 static int64_t nish_cpu_cached = 0;
 
+/* The CPUs a region may use: `nish_cpu_online` (runtime-os.c), which
+ * `availableParallelism` reads in every build, and 1 in the sequential one,
+ * which divides nothing. */
 int64_t nish_cpu_count(void) {
   int64_t n = __atomic_load_n(&nish_cpu_cached, __ATOMIC_RELAXED);
   if (n != 0) return n;
 #ifdef NISH_PAR_REAL
-  long got = sysconf(_SC_NPROCESSORS_ONLN);
-  n = got > 0 ? (int64_t)got : 1;
+  n = nish_cpu_online();
 #else
   n = 1;
 #endif
@@ -73,15 +95,6 @@ int64_t nish_cpu_count(void) {
 }
 
 #ifdef NISH_PAR_REAL
-/* One chunk of the range, and where to run it. Passed by pointer to the
- * worker, so it lives in the caller's frame until the join. */
-typedef struct {
-  nish_par_body body;
-  void *ctx;
-  int64_t lo;
-  int64_t hi;
-} nish_par_job;
-
 /* Non-zero inside a region, so a body that itself calls `nish_parallel_range`
  * runs its range on one thread instead of multiplying the thread count by the
  * nesting depth. Thread-local, so the guard is per worker rather than a lock.
@@ -89,20 +102,357 @@ typedef struct {
  * scheduler: work stealing is what a library would do here and it is not what
  * this file is for. */
 static NISH_TLS int nish_par_depth = 0;
+/* Non-zero on a pool worker or an unpooled job's thread, whose arena holds
+ * nothing of the caller's. */
+static NISH_TLS int nish_pool_thread = 0;
+
+/* ---- The worker pool (docs/wp20-threads.md, section 8e) ----
+ *
+ * One unit of work for another thread: a region's chunk or a scope's task,
+ * each of which embeds one of these as its first member. `pending` is the
+ * count its caller waits on, and lives in the caller's frame until the last
+ * job is done; the unpooled path waits on the job's own thread, `th`, instead. */
+typedef struct nish_job {
+  struct nish_job *next;
+  void (*run)(struct nish_job *);
+  union {
+    int64_t *pending; /* pooled */
+    pthread_t th;     /* unpooled */
+  };
+} nish_job;
+
+/* The workers, started once and parked between calls, and the one queue they
+ * take jobs from. It is strictly first in, first out, and the thread that
+ * hands jobs in takes them off the same front until the queue is empty, which
+ * is what keeps channels deadlock-free with any number of workers, none
+ * included: a receiving task is spawned after every task that sends to it
+ * (wp29 C6, docs/wp29-thread-surface.md section 4.4), so by the time any
+ * thread takes the receiver off the queue, every sender has already been
+ * taken by a thread that is running it. A work-stealing deque, which takes
+ * the newest job first, would break that, and is declined for that reason.
+ *
+ * `state` is 0 until the first divided region or join starts the workers, 1
+ * once it has, and -1 when `nish_pool_disable` keeps the runtime unpooled. It
+ * changes only under the lock, and is read without it once it is not 0. */
+static struct {
+  pthread_mutex_t mu;
+  pthread_cond_t work; /* a worker waits here for a job */
+  pthread_cond_t done; /* a caller waits here for its count to reach 0 */
+  nish_job *head, *tail;
+  int state;
+  int low; /* the trim threshold was lowered for a parked pool */
+} nish_pool = {PTHREAD_MUTEX_INITIALIZER, PTHREAD_COND_INITIALIZER, PTHREAD_COND_INITIALIZER, 0, 0, 0, 0};
+
+/* The front of the queue, or 0 when it is empty. The lock is held. `head` is
+ * also read without it, by a thread spinning before it parks, so it is
+ * written with an atomic store (relaxed: the lock still orders the job's
+ * contents, since nothing takes a job without it). */
+static nish_job *nish_job_take(void) {
+  nish_job *j = nish_pool.head;
+  if (j) {
+    __atomic_store_n(&nish_pool.head, j->next, __ATOMIC_RELAXED);
+    if (!j->next) nish_pool.tail = 0;
+  }
+  return j;
+}
+
+/* One job finished, with the lock held. Nothing may touch `j` after the count
+ * reaches 0, since its caller is then free to return and take its frame. The
+ * count is read without the lock by a caller spinning, as `head` is. */
+static void nish_job_done(nish_job *j) {
+  if (__atomic_sub_fetch(j->pending, 1, __ATOMIC_RELAXED) == 0) pthread_cond_broadcast(&nish_pool.done);
+}
+
+/* How long a thread that finds nothing to do looks again before it sleeps:
+ * 2,048 pauses, measured at 18 us on the four-core Xeon of docs/wp20-threads.md
+ * section 8e. A program that divides one region usually divides the next one
+ * straight after it, and a worker still spinning picks that up without the
+ * futex wake that costs more than the spin; a caller still spinning sees its
+ * last job finish without one either. The look is a relaxed load and a pause,
+ * which leaves the core to its hyperthread sibling meanwhile. */
+#define NISH_POOL_SPIN 2048
+
+static void nish_cpu_relax(void) {
+#if defined(__x86_64__) || defined(__i386__)
+  __builtin_ia32_pause();
+#elif defined(__aarch64__)
+  __asm__ __volatile__("yield");
+#endif
+}
+
+/* Takes jobs off the front of the queue and runs them until `*pending` is 0,
+ * spinning and then sleeping on `cv` while the queue is empty and the count is
+ * not. A worker passes a count that never reaches 0 and sleeps on `work`; a
+ * caller passes its own count and sleeps on `done`. The count is read under
+ * the lock before the caller returns, which orders every job's writes before
+ * it. A worker's arena is its own, not a caller's: the storage class is what
+ * makes the two disjoint (WP20 T0). Nothing a job leaves in it outlives the
+ * job -- a string or an array of scalars a chunk or a task answers, sends or
+ * stores through a guard is copied out before the job returns (runtime-cross.c),
+ * and the language refuses every other result that would point into the arena
+ * -- so it is rewound for the next one rather than freed, and kept only when it
+ * is one ordinary chunk. A larger one goes back to malloc, which under glibc
+ * keeps it for the next job (`nish_pool_ready`'s thresholds); a worker that
+ * then finds the queue empty for a second hands the memory back once before it
+ * sleeps for good, so a parked pool does not hold a burst's temporaries.
+ * `malloc_trim` alone cuts back only the main arena's top: a worker's own arena
+ * is cut back when a large free reaches its top and the top is past the trim
+ * threshold. So the worker lowers the threshold to glibc's own 128 KB, frees a
+ * 16 MB block through its arena to make that cut, and then trims. The
+ * threshold stays low until the next region starts (`nish_jobs_run`), so the
+ * blocks a join adopted from a worker's arena go back to the system too when
+ * the caller frees them later. The lowering and the restore both run under the
+ * pool lock, so a region never runs with the low threshold; the free and the
+ * trim run outside it, and a region that starts meanwhile only makes that free
+ * keep its block in the arena, to be trimmed after the next job it runs. */
+static void nish_pool_drain(int64_t *pending, pthread_cond_t *cv, int worker) {
+#ifdef __GLIBC__
+  int held = 0;
+#endif
+  pthread_mutex_lock(&nish_pool.mu);
+  for (;;) {
+    nish_job *j = nish_job_take();
+    if (j) {
+      pthread_mutex_unlock(&nish_pool.mu);
+      j->run(j);
+      if (worker) {
+        nish_reset_arena();
+        if (nish_arena.cap > 65536) nish_free_arena();
+#ifdef __GLIBC__
+        held = 1;
+#endif
+      }
+      pthread_mutex_lock(&nish_pool.mu);
+      nish_job_done(j);
+    } else if (*pending == 0) {
+      break;
+    } else {
+      pthread_mutex_unlock(&nish_pool.mu);
+      for (int i = 0; i < NISH_POOL_SPIN && __atomic_load_n(pending, __ATOMIC_RELAXED) &&
+                      !__atomic_load_n(&nish_pool.head, __ATOMIC_RELAXED);
+           i++)
+        nish_cpu_relax();
+      pthread_mutex_lock(&nish_pool.mu);
+      if (*pending != 0 && !nish_pool.head) {
+#ifdef __GLIBC__
+        if (held) {
+          struct timespec t;
+          clock_gettime(CLOCK_REALTIME, &t);
+          t.tv_sec++;
+          if (pthread_cond_timedwait(cv, &nish_pool.mu, &t) != 0) {
+            held = 0;
+            if (!nish_pool.low) mallopt(M_TRIM_THRESHOLD, 128 << 10);
+            nish_pool.low = 1;
+            pthread_mutex_unlock(&nish_pool.mu);
+            void *volatile top = malloc(16 << 20);
+            free(top);
+            malloc_trim(0);
+            pthread_mutex_lock(&nish_pool.mu);
+          }
+        } else
+#endif
+          pthread_cond_wait(cv, &nish_pool.mu);
+      }
+    }
+  }
+  pthread_mutex_unlock(&nish_pool.mu);
+}
+
+static void *nish_pool_worker(void *unused) {
+  (void)unused;
+  nish_par_depth = 1;
+  nish_pool_thread = 1;
+  int64_t never = 1;
+  nish_pool_drain(&never, &nish_pool.work, 1);
+  return 0;
+}
+
+/* A forked child has only the thread that forked, so it starts with an empty
+ * queue and no workers, and starts its own on its first divided call. It takes
+ * fresh copies of the lock and the condition variables rather than the ones
+ * the parent's workers may have held or been waiting on, so nothing has to be
+ * held across the fork. */
+static void nish_pool_child(void) {
+  nish_pool.head = nish_pool.tail = 0;
+  if (nish_pool.state > 0) nish_pool.state = 0;
+  pthread_mutex_init(&nish_pool.mu, 0);
+  pthread_cond_init(&nish_pool.work, 0);
+  pthread_cond_init(&nish_pool.done, 0);
+}
 
-static void *nish_par_worker(void *p) {
-  nish_par_job *j = (nish_par_job *)p;
+/* Whether jobs go to the pool, starting it on the first call: one worker per
+ * CPU but the caller's, up to `NISH_PAR_MAX`. A worker that cannot be created
+ * leaves fewer, and none is a pool that works, because the caller drains the
+ * queue itself. Detached, since nothing joins a worker: it parks until the
+ * process exits. */
+static int nish_pool_ready(void) {
+  int state = __atomic_load_n(&nish_pool.state, __ATOMIC_ACQUIRE);
+  if (state != 0) return state > 0;
+  pthread_mutex_lock(&nish_pool.mu);
+  if (nish_pool.state == 0) {
+    int64_t n = nish_cpu_count() - 1;
+    if (n > NISH_PAR_MAX - 1) n = NISH_PAR_MAX - 1; /* a region queues at most this many */
+    for (int64_t i = 0; i < n; i++) {
+      pthread_t th;
+      if (pthread_create(&th, 0, nish_pool_worker, 0) != 0) break;
+      pthread_detach(th);
+    }
+    pthread_atfork(0, 0, nish_pool_child);
+#ifdef __GLIBC__
+    /* glibc otherwise trims the heap top, or unmaps, every chunk of 128 KB or
+     * more an arena release frees, and a task or element that builds a 1 MB
+     * temporary then pays a page fault per 4 KB of it on every call: 200 such
+     * tasks took 314 ms at one thread, 48 ms with these. glibc's own threshold
+     * starts at 128 KB and rises only once an mmapped chunk is freed, to at
+     * most 32 MB on a 64-bit target with a trim threshold of twice that, so
+     * these are the values it reaches by itself, from the first call on; a
+     * worker's idle trim in `nish_pool_drain` lowers the trim threshold and
+     * hands the memory back once the pool parks, and the next region sets it
+     * to 64 MB again. Without that a parked pool kept each worker's arena top,
+     * 82 MB after four maps of 1.2 million strings on four threads, rather than
+     * 2.3 MB (`tests/link/par_map_idle`). Set here, once a program first runs work on other threads; a
+     * region below its grain, or at one thread, runs as the loop it would have
+     * been and keeps that loop's costs. A pool with no worker, at one CPU or
+     * `NISH_NUM_THREADS=1`, has nothing that parks to trim, and keeps what a
+     * join freed up to the 64 MB at which glibc trims the heap top itself.
+     * Trimming at the end of each join instead took that idle 18 MB to 2 MB,
+     * but took ten joins of eight 16 MB tasks from 93 ms to 121-138 ms, since
+     * every join faulted its temporaries in again. */
+    mallopt(M_MMAP_THRESHOLD, 32 << 20);
+    mallopt(M_TRIM_THRESHOLD, 64 << 20);
+#endif
+    __atomic_store_n(&nish_pool.state, 1, __ATOMIC_RELEASE);
+  }
+  state = nish_pool.state;
+  pthread_mutex_unlock(&nish_pool.mu);
+  return state > 0;
+}
+
+/* The unpooled path's thread: one job, then the arena freed, since nothing
+ * else has a pointer to it once the thread exits. */
+static void *nish_job_thread(void *p) {
+  nish_job *j = (nish_job *)p;
   nish_par_depth = 1;
-  j->body(j->lo, j->hi, j->ctx);
-  /* This thread's arena, not the parent's: the storage class is what makes the
-   * two disjoint (WP20 T0), so a worker can release every byte it bumped
-   * without the parent losing one. A worker that did not do this would leak its
-   * whole arena at thread exit, since nothing else has a pointer to it. */
+  nish_pool_thread = 1;
+  j->run(j);
   nish_free_arena();
   return 0;
 }
+
+/* Runs `lead` on this thread and the list `rest` elsewhere, oldest first, and
+ * returns once every one of them has finished. The caller runs `lead` itself,
+ * so N jobs keep N-1 other threads busy and this one is not idle, and then
+ * takes queued jobs off the front until the queue is empty: a job no worker is
+ * free for runs here, in order, so running short of threads costs speed and
+ * never an answer, and a region never waits for a worker to wake when it
+ * could be doing the work. A job taken here runs in this thread's arena, as
+ * `lead` does.
+ *
+ * The queue is shared, so a caller can take a job another caller queued. That
+ * is safe for the same reason the order is: a job only ever waits on jobs
+ * queued before it, and those have all been taken by a thread already. */
+static void nish_jobs_run(nish_job *lead, nish_job *rest) {
+  int64_t pending = 0;
+  int pooled = rest && nish_par_depth == 0 && nish_pool_ready();
+  nish_job *last = 0, *inline_from = 0;
+  if (pooled) {
+    for (nish_job *j = rest; j; j = j->next) {
+      j->pending = &pending;
+      pending++;
+      last = j;
+    }
+    pthread_mutex_lock(&nish_pool.mu);
+    if (nish_pool.tail) nish_pool.tail->next = rest;
+    else __atomic_store_n(&nish_pool.head, rest, __ATOMIC_RELAXED);
+    nish_pool.tail = last;
+#ifdef __GLIBC__
+    /* A parked pool lowered the trim threshold (`nish_pool_drain`): a burst
+       keeps its temporaries again. */
+    if (nish_pool.low) mallopt(M_TRIM_THRESHOLD, 64 << 20);
+    nish_pool.low = 0;
+#endif
+    /* One wake per job, rather than every parked worker for a job or two. */
+    for (int64_t i = 0; i < pending; i++) pthread_cond_signal(&nish_pool.work);
+    pthread_mutex_unlock(&nish_pool.mu);
+  } else {
+    /* Unpooled, a thread per job, and every job from the first thread that
+     * cannot be created on runs here. Inside a job there is no thread at all:
+     * a region or a scope in a chunk or a task runs on that one thread, in
+     * order, rather than multiplying the threads by the nesting depth. */
+    for (inline_from = rest; inline_from && nish_par_depth == 0; inline_from = inline_from->next)
+      if (pthread_create(&inline_from->th, 0, nish_job_thread, inline_from) != 0) break;
+  }
+  nish_par_depth++;
+  lead->run(lead);
+  if (pooled) {
+    nish_pool_drain(&pending, &nish_pool.done, 0);
+  } else {
+    for (nish_job *j = inline_from; j; j = j->next) j->run(j);
+    for (nish_job *j = rest; j != inline_from; j = j->next) pthread_join(j->th, 0);
+  }
+  nish_par_depth--;
+}
+
+/* A divided region: `pieces` contiguous pieces of `[0, len)`, at most one
+ * element apart in size, which the threads running it claim one at a time off
+ * `next`. It lives in the caller's frame until every piece has run. */
+typedef struct {
+  nish_par_body body;
+  void *ctx;
+  int64_t base; /* the shortest piece's length */
+  int64_t rem;  /* the first `rem` pieces are one longer */
+  int64_t pieces;
+  int64_t next; /* the next piece to claim; atomic */
+} nish_par_region;
+
+/* One thread's share of a region: the pieces it claims, however many. */
+typedef struct {
+  nish_job job;
+  nish_par_region *region;
+} nish_par_job;
+
+static void nish_par_piece(nish_par_region *r, int64_t i) {
+  int64_t lo = i * r->base + (i < r->rem ? i : r->rem);
+  r->body(lo, lo + r->base + (i < r->rem ? 1 : 0), r->ctx);
+}
+
+/* Claims pieces until none is left. A worker rewinds its arena between them,
+ * and frees it when the chunk it kept is larger than an ordinary one, so each
+ * piece it runs finds the arena a whole job finds (`nish_pool_drain`). The
+ * claim is relaxed: the pieces are fixed before the jobs are queued, under the
+ * lock, and the join orders every piece's writes before the caller goes on. */
+static void nish_par_run(nish_job *j) {
+  nish_par_region *r = ((nish_par_job *)j)->region;
+  int64_t i;
+  while ((i = __atomic_fetch_add(&r->next, 1, __ATOMIC_RELAXED)) < r->pieces) {
+    nish_par_piece(r, i);
+    if (nish_pool_thread) {
+      nish_reset_arena();
+      if (nish_arena.cap > 65536) nish_free_arena();
+    }
+  }
+}
+
+/* The caller's share: piece 0 first, which `next` starts past, then whatever
+ * is left when it is done. */
+static void nish_par_run_lead(nish_job *j) {
+  nish_par_piece(((nish_par_job *)j)->region, 0);
+  nish_par_run(j);
+}
 #endif
 
+/* Without the pool every later region and join creates its threads and joins
+ * them, as before it existed. For a library that can be unloaded, a Node addon
+ * above all: a parked worker would outlive the code it runs. nish.h calls this
+ * from a constructor in the shim, before any call can start the pool. */
+void nish_pool_disable(void) {
+#ifdef NISH_PAR_REAL
+  pthread_mutex_lock(&nish_pool.mu);
+  if (nish_pool.state == 0) __atomic_store_n(&nish_pool.state, -1, __ATOMIC_RELAXED);
+  pthread_mutex_unlock(&nish_pool.mu);
+#endif
+}
+
 void nish_parallel_range(nish_par_body body, void *ctx, int64_t len, int64_t grain) {
   if (len <= 0) return;
 #ifdef NISH_PAR_REAL
@@ -122,42 +472,34 @@ void nish_parallel_range(nish_par_body body, void *ctx, int64_t len, int64_t gra
     return;
   }
 
-  nish_par_job jobs[NISH_PAR_MAX];
-  pthread_t th[NISH_PAR_MAX];
-  /* Contiguous chunks, the remainder spread one element at a time over the
-   * first `len % n` of them, so the longest chunk is never more than one
+  /* More pieces than threads, so a thread that runs slow -- a core shared
+   * with another process, a worker that woke late -- takes fewer of them
+   * rather than holding up the join: with one piece a thread, the join on the
+   * four-core Xeon of docs/BENCHMARKS.md waited a quarter of each region for
+   * the slowest. Never finer than the grain, which is what a piece must be
+   * worth; a claim is one atomic add. */
+  int64_t pieces = n * NISH_PAR_PIECES;
+  if (chunks < pieces) pieces = chunks;
+  /* Contiguous pieces, the remainder spread one element at a time over the
+   * first `len % pieces` of them, so the longest is never more than one
    * element longer than the shortest. Contiguous rather than strided because
-   * two workers then never write the same cache line except at one boundary. */
-  int64_t base = len / n;
-  int64_t rem = len % n;
-  int64_t at = 0;
+   * two threads then never write the same cache line except at a boundary. */
+  nish_par_region region = {body, ctx, len / pieces, len % pieces, pieces, 1};
+  nish_par_job jobs[NISH_PAR_MAX];
   for (int64_t i = 0; i < n; i++) {
-    int64_t take = base + (i < rem ? 1 : 0);
-    jobs[i].body = body;
-    jobs[i].ctx = ctx;
-    jobs[i].lo = at;
-    jobs[i].hi = at + take;
-    at += take;
+    jobs[i].job.next = &jobs[i + 1].job;
+    jobs[i].job.run = i == 0 ? nish_par_run_lead : nish_par_run;
+    jobs[i].region = &region;
   }
 
-  /* The caller runs chunk 0 itself, so N-way parallelism costs N-1 spawns and
-   * the calling thread is not idle while it waits. Its chunk also runs in the
-   * caller's arena, which means chunk 0 behaves exactly as a sequential call
-   * would -- the asymmetry is deliberate and is why a body whose result is
-   * allocated is the surface's error rather than this file's. */
-  int started = 0;
-  for (int64_t i = 1; i < n; i++) {
-    if (pthread_create(&th[started], 0, nish_par_worker, &jobs[i]) != 0) break;
-    started++;
-  }
-  nish_par_depth++;
-  jobs[0].body(jobs[0].lo, jobs[0].hi, jobs[0].ctx);
-  /* A chunk whose thread could not be created runs here. Running short of
-   * threads must cost speed and never correctness, so there is no path on which
-   * a range is silently skipped. */
-  for (int64_t i = (int64_t)started + 1; i < n; i++) jobs[i].body(jobs[i].lo, jobs[i].hi, jobs[i].ctx);
-  nish_par_depth--;
-  for (int i = 0; i < started; i++) pthread_join(th[i], 0);
+  /* The caller runs piece 0 itself, in its own arena, which means piece 0
+   * behaves exactly as a sequential call would -- the asymmetry is deliberate
+   * and is why what a body's result may hold is the surface's question rather
+   * than this file's: a string or an array of scalars is copied out of the
+   * piece's arena, whichever it ran in, by the trampoline the compiler emits
+   * (runtime-cross.c), and the surface refuses every other allocated result. */
+  jobs[n - 1].job.next = 0;
+  nish_jobs_run(&jobs[0].job, jobs[0].job.next);
 #else
   (void)grain;
   body(0, len, ctx);
@@ -178,50 +520,53 @@ void nish_parallel_range(nish_par_body body, void *ctx, int64_t len, int64_t gra
  * joins it, so the list needs no lock, and scopes may nest: a join takes its
  * own scope's tasks out of the list and leaves the rest. */
 typedef struct nish_task {
+#ifdef NISH_PAR_REAL
+  nish_job job;
+#endif
   struct nish_task *next;
   void *scope;
   nish_task_fn run;
   nish_task_fn finish;
-#ifdef NISH_PAR_REAL
-  pthread_t th;
-  int started;
-#endif
   /* The copied payload, aligned for any field it holds. */
   uint64_t payload[];
 } nish_task;
 
 static NISH_TLS nish_task *nish_tasks = 0;
 
+#ifdef NISH_PAR_REAL
+static void nish_task_run(nish_job *j) {
+  nish_task *t = (nish_task *)j;
+  t->run(t->payload);
+}
+#endif
+
+/* The end of the program when an allocation of this file's fails: the scope's
+ * task list and a channel's buffer are the two, and neither has a way to go on
+ * without the memory. */
+static void nish_par_oom(void) {
+  nish_panic_claim();
+  (void)!write(2, "nish: out of memory\n", 20);
+  _exit(1);
+}
+
 void nish_scope_spawn(void *scope, nish_task_fn run, nish_task_fn finish, const void *payload, int64_t size) {
   nish_task *t = (nish_task *)malloc(sizeof(nish_task) + (size_t)size);
   /* Out of memory is the end of the program, as it is for every other
    * allocation the runtime makes: running the task here instead would store
    * its answer ahead of tasks filed before it, and a later spawn into the same
    * slot has to be the one that stays. */
-  if (!t) {
-    (void)!write(2, "nish: out of memory\n", 20);
-    _exit(1);
-  }
+  if (!t) nish_par_oom();
   t->scope = scope;
   t->run = run;
   t->finish = finish;
+#ifdef NISH_PAR_REAL
+  t->job.run = nish_task_run;
+#endif
   memcpy(t->payload, payload, (size_t)size);
   t->next = nish_tasks;
   nish_tasks = t;
 }
 
-#ifdef NISH_PAR_REAL
-static void *nish_task_worker(void *p) {
-  nish_task *t = (nish_task *)p;
-  /* A task is one thread's work, as a chunk is, so a region inside it runs on
-   * this thread rather than multiplying the threads. */
-  nish_par_depth = 1;
-  t->run(t->payload);
-  nish_free_arena(); /* as `nish_par_worker`: this thread's arena, and only it */
-  return 0;
-}
-#endif
-
 void nish_scope_join(void *scope) {
   /* This scope's tasks, oldest first: the list is newest first, and moving each
    * one to the front of `mine` reverses it. */
@@ -232,6 +577,9 @@ void nish_scope_join(void *scope) {
     if (t->scope == scope) {
       *link = t->next;
       t->next = mine;
+#ifdef NISH_PAR_REAL
+      t->job.next = (nish_job *)mine;
+#endif
       mine = t;
     } else {
       link = &t->next;
@@ -239,21 +587,11 @@ void nish_scope_join(void *scope) {
   }
   if (!mine) return;
 #ifdef NISH_PAR_REAL
-  /* Every task but the first on a thread of its own, and the first on this
-   * one, so N tasks cost N-1 spawns and this thread is not idle. A task inside
-   * a region or another task's thread runs here, as a nested region does. */
-  for (nish_task *t = mine->next; t; t = t->next) {
-    t->started = nish_par_depth == 0 && pthread_create(&t->th, 0, nish_task_worker, t) == 0;
-  }
-  nish_par_depth++;
-  mine->run(mine->payload);
-  for (nish_task *t = mine->next; t; t = t->next) {
-    if (!t->started) t->run(t->payload);
-  }
-  nish_par_depth--;
-  for (nish_task *t = mine->next; t; t = t->next) {
-    if (t->started) pthread_join(t->th, 0);
-  }
+  /* The first task on this thread and the rest on the pool, in the order they
+   * were spawned, which is the order a task no worker is free for runs in here,
+   * and the order every task runs in when the scope is joined inside another
+   * job (`nish_jobs_run`). */
+  nish_jobs_run(&mine->job, mine->job.next);
 #else
   for (nish_task *t = mine; t; t = t->next) t->run(t->payload);
 #endif
@@ -266,3 +604,148 @@ void nish_scope_join(void *scope) {
     free(t);
   }
 }
+
+/* ---- A lock's contended path (wp29 P3) ----
+ *
+ * The compiler takes a free lock inline, so this runs only when the word was
+ * held: another task is inside its critical section, which the language keeps
+ * short -- one lock at a time, nothing waited on while it is held -- so a few
+ * relaxed reads usually see it free again. Past that the holder may have been
+ * descheduled, and spinning would only keep it off the core, so each further
+ * try first yields. The swap that takes the word is the acquire the inline
+ * path would have made. */
+void nish_mutex_wait(int32_t *word) {
+#ifdef NISH_PAR_REAL
+  int spins = 0;
+#endif
+  for (;;) {
+    int32_t free_word = 0;
+    if (__atomic_load_n(word, __ATOMIC_RELAXED) == 0 &&
+        __atomic_compare_exchange_n(word, &free_word, 1, 0, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED))
+      return;
+#ifdef NISH_PAR_REAL
+    if (spins < 64) spins++;
+    else sched_yield();
+#endif
+  }
+}
+
+/* ---- A channel of scalars (wp29 P3, section 4.4) ----
+ *
+ * The `Channel` object lives in the arena of the thread that declared it, and
+ * holds one 64-bit word, `state`, that this file owns: 0 before the channel is
+ * first used, the address of its `nish_channel` after that, and
+ * `NISH_CHANNEL_DRAINED` once its receiver has drained it closed and it has
+ * been given back. Every function here takes the address of that word.
+ *
+ * The buffer is outside every arena, because the senders' arenas are freed
+ * when their scope joins and a value sent has to outlive its sender. Each slot
+ * is 8 bytes: the compiler widens a scalar into one and narrows it back
+ * (src/emit-parallel.ts). The channel is unbounded, so `send` never waits.
+ *
+ * `pending` is what keeps the channel open: 1 for the scope that has not yet
+ * started to join, plus 1 for each task filed that may send on it. The
+ * compiler adds a sender at its spawn and takes it away when the task returns,
+ * and takes the scope's 1 away just before the join (`nish_channel_count`,
+ * -1), so the channel closes when the join has started and every sender has
+ * returned -- at once, when no task may send on it. All three happen on the
+ * thread that declared the channel until the join, so the word is made there,
+ * before any task can touch it, and never by two threads at once.
+ *
+ * One receiver, the language's rule, takes values off the front. When it
+ * finds the channel closed and empty it gives the buffer back and records
+ * that, so a second loop over the channel ends at once. Nothing is left to
+ * touch it then: every sender has returned, and the parent sends only before
+ * the scope's block (docs/LANGUAGE.md, "Channels"). */
+#define NISH_CHANNEL_DRAINED 1
+
+typedef struct {
+  int64_t *slots;
+  int64_t head, tail, cap;
+  int64_t pending;
+#ifdef NISH_PAR_REAL
+  pthread_mutex_t mu;
+  pthread_cond_t cv;
+#endif
+} nish_channel;
+
+static nish_channel *nish_channel_of(int64_t *state) {
+  nish_channel *c = (nish_channel *)(intptr_t)*state;
+  if (!c) {
+    c = (nish_channel *)calloc(1, sizeof(nish_channel));
+    if (!c) nish_par_oom();
+    c->pending = 1;
+#ifdef NISH_PAR_REAL
+    pthread_mutex_init(&c->mu, 0);
+    pthread_cond_init(&c->cv, 0);
+#endif
+    *state = (int64_t)(intptr_t)c;
+  }
+#ifdef NISH_PAR_REAL
+  pthread_mutex_lock(&c->mu);
+#endif
+  return c;
+}
+
+/* Every path out of `nish_channel_of` holds the lock; this gives it back,
+ * waking the receiver first, since whatever changed may be what it waits on. */
+static void nish_channel_done(nish_channel *c) {
+#ifdef NISH_PAR_REAL
+  pthread_cond_broadcast(&c->cv);
+  pthread_mutex_unlock(&c->mu);
+#else
+  (void)c;
+#endif
+}
+
+void nish_channel_send(int64_t *state, int64_t value) {
+  /* The checker keeps a send off a drained channel (every sender has returned
+   * before it closes, and the parent sends only before its scope); this keeps
+   * the sentinel from being read as an address if that ever slips. */
+  if (*state == NISH_CHANNEL_DRAINED) return;
+  nish_channel *c = nish_channel_of(state);
+  if (c->tail == c->cap) {
+    c->cap = c->cap ? c->cap * 2 : 64;
+    c->slots = (int64_t *)realloc(c->slots, (size_t)c->cap * sizeof(int64_t));
+    if (!c->slots) nish_par_oom();
+  }
+  c->slots[c->tail++] = value;
+  nish_channel_done(c);
+}
+
+void nish_channel_count(int64_t *state, int64_t delta) {
+  if (*state == NISH_CHANNEL_DRAINED) return; /* as in `nish_channel_send` */
+  nish_channel *c = nish_channel_of(state);
+  c->pending += delta;
+  nish_channel_done(c);
+}
+
+int32_t nish_channel_receive(int64_t *state, int64_t *out) {
+  if (*state == NISH_CHANNEL_DRAINED) return 0;
+  nish_channel *c = nish_channel_of(state);
+#ifdef NISH_PAR_REAL
+  while (c->head == c->tail && c->pending != 0) pthread_cond_wait(&c->cv, &c->mu);
+#endif
+  if (c->head != c->tail) {
+    *out = c->slots[c->head++];
+    /* Emptied: the next value goes back to the front, so a buffer only grows
+     * while the senders are ahead of the receiver. */
+    if (c->head == c->tail) c->head = c->tail = 0;
+    nish_channel_done(c);
+    return 1;
+  }
+  /* Without threads every sender has returned by the time a receiver runs:
+   * the tasks run one after another in spawn order, and every sender is
+   * spawned before the one receiver (C6). An empty channel that is still open
+   * here is a broken invariant, not a wait. */
+  if (c->pending != 0) __builtin_trap();
+#ifdef NISH_PAR_REAL
+  pthread_mutex_unlock(&c->mu);
+  pthread_mutex_destroy(&c->mu);
+  pthread_cond_destroy(&c->cv);
+#endif
+  free(c->slots);
+  free(c);
+  *state = NISH_CHANNEL_DRAINED;
+  return 0;
+}