@amritk/nish-x86_64-linux 0.17.0 → 0.20.0

package/runtime/runtime-cross.c

@@ -0,0 +1,366 @@
+/* Nish runtime, the copy-at-join half: how a string or an array of scalars
+ * crosses from one thread's arena to another's, and how a `Mutex` keeps the
+ * strings stored through its guard.
+ *
+ * A translation unit of its own rather than more of runtime-parallel.c, for
+ * the reason that file's header gives for being one: each file carries its own
+ * measured `.text*` ceiling in tests/runtime-budgets.js, and this one would not
+ * fit in what runtime-parallel.c had left (RUNTIME_CROSS_TEXT_BUDGET says how
+ * much it needs). Apart, a program that divides a range of numbers links none
+ * of this, and section GC drops what a program that crosses no string calls.
+ *
+ * A task's or a chunk's arena is rewound once its job is done
+ * (`nish_pool_drain`, and the release the compiler emits after a task or a
+ * chunk returns), so a string or an array it answers, sends or stores through
+ * a guard cannot stay where it was made. `nish_cross_out` runs on the thread
+ * that made the value: one that was bumped in its arena since `mark` (0: at
+ * any time) is copied into a block of its own, outside every arena, and
+ * answered with its low bit set. Any other value -- a literal, or one the
+ * parent made, which outlives the join -- is answered as it is, so a task that
+ * answers its parent's array answers that array, as it does under Node.
+ * `nish_cross_send` is a channel's: it copies every string, which costs the
+ * string's length and nothing that grows with the sender's arena.
+ * `nish_cross_in` runs on the thread that takes the value: a tagged one is
+ * copied into its arena and its block freed, or, when it is large, its block
+ * becomes a chunk of that arena, so every block is either freed by the one
+ * `cross_in` its value meets or freed with the arena that adopted it. A map's
+ * elements do not go one block each: a chunk packs them into blocks of its
+ * own (`nish_cross_keep`), which the caller adopts whole, or copies in and
+ * frees when they hold little (`nish_cross_adopt`). Arena blocks,
+ * literals and the copies here are all 8-aligned, which is what keeps the low
+ * bit free for the tag.
+ *
+ * `esize` is 0 for a string and the element's size for an array of scalars,
+ * which holds no pointer, so one flat copy of its header and its elements is
+ * the whole value. A panic ends the process, and every block with it.
+ *
+ * It compiles in both configurations, as runtime-parallel.c does: a task run
+ * on the calling thread at the join is rewound all the same, so the copies are
+ * needed whether or not there are threads; only the guard set's lock is
+ * `-DNISH_THREADS` alone. */
+#include "nish.h"
+
+#include <stddef.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#if defined(NISH_THREADS) && !defined(__wasi__) && !defined(__wasm__)
+#define NISH_CROSS_LOCKED 1
+#include <pthread.h>
+#endif
+
+/* runtime.c's `nish_chunk`, which nish.h leaves opaque (`nish_arena.chunks` is
+ * a `void *` there). runtime.c asserts the same three facts of its own
+ * definition, so a change to either side fails to compile until the other
+ * follows. */
+typedef struct nish_xchunk { struct nish_xchunk *next; uint64_t cap; } nish_xchunk;
+_Static_assert(sizeof(nish_xchunk) == 16, "runtime.c's nish_chunk: { next, cap }, the bytes after it");
+_Static_assert(offsetof(nish_xchunk, next) == 0, "runtime.c's nish_chunk: next first");
+_Static_assert(offsetof(nish_xchunk, cap) == 8, "runtime.c's nish_chunk: cap second");
+
+/* The end of the program when a copy cannot be allocated: nothing that
+ * crosses has a way to go on without the memory. */
+static void nish_cross_oom(void) {
+  nish_panic_claim();
+  (void)!write(2, "nish: out of memory\n", 20);
+  _exit(1);
+}
+
+static void *nish_cross_block(uint64_t size) {
+  void *b = size <= (uint64_t)SIZE_MAX ? malloc((size_t)size) : 0;
+  if (!b) nish_cross_oom();
+  return b;
+}
+
+/* Whether `p` was bumped since `mark`: one walk of the chunks, for a single
+ * value. */
+static int nish_since(uint64_t mark, uintptr_t p) {
+  for (const nish_xchunk *c = (const nish_xchunk *)nish_arena.chunks; c; c = c->next) {
+    uintptr_t base = (uintptr_t)(c + 1), end = base + c->cap;
+    int marked = mark >= base && mark <= end;
+    if (p >= (marked ? (uintptr_t)mark : base) && p < end) return 1;
+    if (marked) return 0;
+  }
+  return 0;
+}
+
+/* The bytes `v` takes, rounded to 8: a string's header, bytes and NUL, or an
+ * array's header and elements. */
+static uint64_t nish_cross_size(const void *v, uint64_t esize) {
+  uint64_t n = esize ? sizeof(nish_array) + ((const nish_array *)v)->len * esize
+                     : 8 + ((const nish_str *)v)->len + 1;
+  return (n + 7) & ~(uint64_t)7;
+}
+
+/* `v` copied to `at`, flat (an array's elements follow its header). */
+static void nish_cross_put(char *at, const void *v, uint64_t esize) {
+  if (esize == 0) {
+    memcpy(at, v, (size_t)(8 + ((const nish_str *)v)->len + 1));
+  } else {
+    const nish_array *a = (const nish_array *)v;
+    nish_array *c = (nish_array *)at;
+    c->len = c->cap = a->len;
+    c->data = (char *)(c + 1);
+    if (a->len) memcpy(c->data, a->data, (size_t)(a->len * esize));
+  }
+}
+
+/* `n` bytes after a chunk header, so the block can become a chunk of the arena
+ * that takes it: runtime.c's `nish_free_until` frees it as it frees one
+ * `nish_arena_grow` made. */
+static nish_xchunk *nish_cross_chunk(uint64_t n) {
+  nish_xchunk *c = (nish_xchunk *)nish_cross_block(sizeof *c + n);
+  c->cap = n;
+  return c;
+}
+
+/* Links the block `c` into this thread's arena as its newest chunk, and bumps
+ * from it, full to its `cap`. A chunk grown here would
+ * sit in the same place, so a release, a keep and a reset treat these as they
+ * treat it: whatever rewinds the arena past this point frees them. */
+static void nish_cross_link(nish_xchunk *c) {
+  c->next = (nish_xchunk *)nish_arena.chunks;
+  nish_arena = (struct nish_arena){(char *)(c + 1), c->cap, c->cap, c};
+}
+
+/* `v` in a block of its own, tagged. */
+static void *nish_cross_copy(const void *v, uint64_t esize) {
+  nish_xchunk *c = nish_cross_chunk(nish_cross_size(v, esize));
+  nish_cross_put((char *)(c + 1), v, esize);
+  return (void *)((uintptr_t)(c + 1) | 1);
+}
+
+void *nish_cross_out(void *v, uint64_t esize, uint64_t mark) {
+  return v && nish_since(mark, (uintptr_t)v) ? nish_cross_copy(v, esize) : v;
+}
+
+void *nish_cross_send(void *s) { return nish_cross_copy(s, 0); }
+
+/* The transfer state of the chunk this thread is running
+ * (`nish_cross_begin`): the list its blocks go on, the block being filled and
+ * how much of it is, and the state it hides, which `nish_cross_end` restores
+ * -- a map nested in a chunk's body runs its own chunks inside it. A map's
+ * sequential path begins with no list, which hides the outer state without
+ * pushing one, so its elements are not packed into the outer chunk's block. */
+typedef struct nish_xfer {
+  void **list;
+  nish_xchunk *block;
+  uint64_t off;
+  struct nish_xfer *prev;
+} nish_xfer;
+_Static_assert(sizeof(nish_xfer) == (sizeof(void *) == 8 ? 32 : 24),
+               "src/emit-parallel.ts allocas the state as { i8*, i8*, i64, i8* }");
+static NISH_TLS nish_xfer *nish_xfer_top;
+
+void nish_cross_begin(void *state, void **list) {
+  nish_xfer *x = (nish_xfer *)state;
+  *x = (nish_xfer){list, 0, 0, nish_xfer_top};
+  nish_xfer_top = list ? x : 0;
+}
+
+/* The block being filled records how much of it is, as its `cap`, so the
+ * join reads what each chunk kept and an adopted block is full where it ends. */
+void nish_cross_end(void *state) {
+  nish_xfer *x = (nish_xfer *)state;
+  if (x->block) x->block->cap = x->off;
+  nish_xfer_top = x->prev;
+}
+
+/* `nish_arena_release(mark)` for an element's rewind, but keeping the newest
+ * chunk when the body grew it past `mark`'s: a mark at the end of a full chunk
+ * would otherwise have every element's body malloc a fresh chunk and its
+ * release free it again. Only a 64 KB one is kept, emptied, in front of
+ * `mark`'s chunk, with those between freed; an older mark still finds its own
+ * chunk below it, and what `mark`'s chunk had left is given up once. A mark
+ * in no chunk below the newest leaves the arena as it was, as the release
+ * does. */
+static void nish_rewind(uint64_t mark) {
+  struct nish_arena *a = &nish_arena;
+  nish_xchunk *top = a->chunks;
+  char *buf = (char *)(top + 1);
+  if (mark && top && top->cap == 65536 && mark - (uintptr_t)buf > 65536) {
+    a->chunks = top->next;
+    nish_arena_release(mark);
+    if (a->buf != buf) {
+      top->next = a->chunks;
+      *a = (struct nish_arena){ buf, 0, 65536, top };
+    } else
+      a->chunks = top;
+    return;
+  }
+  nish_arena_release(mark);
+}
+
+/* A map's or a `parallelFrom`'s element, kept as its chunk loop rewinds the
+ * arena to `mark`, the bump position before the body was called: everything
+ * else the body bumped is dead once it has answered, so a chunk holds one
+ * element's temporaries at a time rather than all of its elements'. A value
+ * made since `mark` is copied, in a chunk of a divided region, to the end of
+ * the chunk's transfer block -- a new one, pushed on the region's list when it
+ * does not fit, of 4 KB doubling to 64 KB or of the value's size, so what a
+ * chunk leaves unused is bounded by what it keeps -- and is answered there;
+ * the caller adopts the blocks at the join (`nish_cross_adopt`). On a map's
+ * sequential path, where no state is pushed, it goes out to a block of its own
+ * and back in below `mark`. One made before `mark` is answered as it is. */
+void *nish_cross_keep(void *v, uint64_t esize, uint64_t mark) {
+  nish_xfer *x = nish_xfer_top;
+  if (v && nish_since(mark, (uintptr_t)v)) {
+    if (!x) {
+      v = nish_cross_copy(v, esize);
+    } else {
+      uint64_t n = nish_cross_size(v, esize);
+      if (!x->block || x->block->cap - x->off < n) {
+        uint64_t want = x->block && x->block->cap < 32768 ? x->block->cap * 2 : x->block ? 65536 : 4096;
+        x->block = nish_cross_chunk(n > want ? n : want);
+        x->block->next = __atomic_exchange_n((nish_xchunk **)x->list, x->block, __ATOMIC_RELAXED);
+        x->off = 0;
+      }
+      char *at = (char *)(x->block + 1) + x->off;
+      x->off += n;
+      nish_cross_put(at, v, esize);
+      v = at;
+    }
+  }
+  nish_rewind(mark);
+  return nish_cross_in(v, esize);
+}
+
+/* A value of 64 KB or more that does not fit what is left of the current
+ * chunk is adopted rather than copied: `nish_arena_grow` would have made a
+ * chunk of exactly its size for the copy, and the block is that chunk
+ * already. A smaller one is copied, so a run of small values never makes a
+ * chunk each, which a release would walk. */
+void *nish_cross_in(void *p, uint64_t esize) {
+  if (!((uintptr_t)p & 1)) return p;
+  void *v = (void *)((uintptr_t)p & ~(uintptr_t)1);
+  nish_xchunk *c = (nish_xchunk *)v - 1;
+  uint64_t n = c->cap;
+  if (n >= 65536 && n > nish_arena.cap - nish_arena.off) {
+    nish_cross_link(c);
+    return v;
+  }
+  char *r = (char *)nish_alloc_struct(n);
+  nish_cross_put(r, v, esize);
+  free(c);
+  return r;
+}
+
+/* Once every chunk is done, on the thread that called the region, with `dst`
+ * the array the region wrote. When the blocks the chunks filled hold 64 KB or
+ * more, they become the newest chunks of its arena (`nish_cross_link`), so
+ * the elements in them stay where they are, and the cost is one link per
+ * block rather than a copy per element. Less is copied, element by element,
+ * into the arena and the blocks freed, as `nish_cross_in` copies a small
+ * value: an adopted block leaves the arena's current chunk behind, and a
+ * region run in a loop would leave one per run. */
+void nish_cross_adopt(void *list, void *dst, uint64_t esize) {
+  nish_xchunk *first = (nish_xchunk *)list, *c, *next;
+  uint64_t kept = 0;
+  for (c = first; c; c = c->next) kept += c->cap;
+  if (kept >= 65536) {
+    for (c = first; c; c = next) {
+      next = c->next;
+      nish_cross_link(c);
+    }
+    return;
+  }
+  if (!first) return;
+  nish_array *a = (nish_array *)dst;
+  void **e = (void **)a->data;
+  for (uint64_t i = 0; i < a->len; i++) {
+    for (c = first; c; c = c->next) {
+      if ((uintptr_t)e[i] - (uintptr_t)(c + 1) < c->cap) {
+        char *r = (char *)nish_alloc_struct(nish_cross_size(e[i], esize));
+        nish_cross_put(r, e[i], esize);
+        e[i] = r;
+        break;
+      }
+    }
+  }
+  for (c = first; c; c = next) {
+    next = c->next;
+    free(c);
+  }
+}
+
+/* ---- A string a guard stores ----
+ *
+ * `g.value = s` keeps `s` in memory the `Mutex` owns, since the task that
+ * stores it may have made it in its arena: `nish_guard_keep` copies it into a
+ * block of its own and answers the copy, and frees `old`, the string the slot
+ * held, when it is a copy made here; `g.value = null` into a `string | null`
+ * slot frees that copy and keeps nothing. A read that keeps the string past the
+ * lock's block copies it into the reader's arena (the compiler calls
+ * `nish_str_new`), so nothing outside the lock points at a block, and the
+ * next store frees it. The copies are recorded in one set, `nish_kept`, which
+ * is how `old` is told apart from the string `new Mutex` was given or a
+ * literal: those are never freed. A block still held when its `Mutex` is
+ * dropped stays allocated until the process ends -- one per string slot of
+ * the `Mutex`, not one per store. The set is one for the process, behind one
+ * lock of its own, since two `Mutex`es are locked by two tasks at once; it is
+ * taken only by a store, so stores through two different guards wait on each
+ * other for the moment it is held. The set holds each block's header, the
+ * pointer `malloc` answered, so a leak checker counts a block a dropped `Mutex`
+ * left as reachable rather than as possibly lost. */
+static struct {
+#ifdef NISH_CROSS_LOCKED
+  pthread_mutex_t mu;
+#endif
+  uintptr_t *slots; /* open addressing; 0 empty, 1 a removed entry */
+  uint64_t cap, used;
+} nish_kept = {
+#ifdef NISH_CROSS_LOCKED
+  PTHREAD_MUTEX_INITIALIZER,
+#endif
+  0, 0, 0};
+
+static uint64_t nish_kept_at(uintptr_t p) { return (uint64_t)((p >> 3) * 0x9E3779B97F4A7C15ull); }
+
+static void nish_kept_put(uintptr_t p) {
+  if ((nish_kept.used + 1) * 2 > nish_kept.cap) {
+    uintptr_t *old = nish_kept.slots;
+    uint64_t n = nish_kept.cap;
+    nish_kept.cap = n ? n * 2 : 64;
+    nish_kept.slots = (uintptr_t *)calloc((size_t)nish_kept.cap, sizeof(uintptr_t));
+    if (!nish_kept.slots) nish_cross_oom();
+    nish_kept.used = 0;
+    for (uint64_t i = 0; i < n; i++)
+      if (old[i] > 1) nish_kept_put(old[i]);
+    free(old);
+  }
+  uint64_t i = nish_kept_at(p) & (nish_kept.cap - 1);
+  while (nish_kept.slots[i] > 1) i = (i + 1) & (nish_kept.cap - 1);
+  if (nish_kept.slots[i] == 0) nish_kept.used++;
+  nish_kept.slots[i] = p;
+}
+
+/* Whether `p` is in the set, taking it out when it is. */
+static int nish_kept_take(uintptr_t p) {
+  if (!nish_kept.cap) return 0;
+  uint64_t i = nish_kept_at(p) & (nish_kept.cap - 1);
+  for (; nish_kept.slots[i] != 0; i = (i + 1) & (nish_kept.cap - 1)) {
+    if (nish_kept.slots[i] == p) {
+      nish_kept.slots[i] = 1;
+      return 1;
+    }
+  }
+  return 0;
+}
+
+/* `old` is null when the slot is a `string | null` that held none, and `s`
+ * is null for a `null` stored into one: the slot's copy is freed and nothing
+ * is kept in its place, so the store answers null. */
+void *nish_guard_keep(void *old, const void *s) {
+  void *b = s ? (void *)((uintptr_t)nish_cross_copy(s, 0) & ~(uintptr_t)1) : 0;
+#ifdef NISH_CROSS_LOCKED
+  pthread_mutex_lock(&nish_kept.mu);
+#endif
+  int mine = old && nish_kept_take((uintptr_t)((nish_xchunk *)old - 1));
+  if (b) nish_kept_put((uintptr_t)((nish_xchunk *)b - 1));
+#ifdef NISH_CROSS_LOCKED
+  pthread_mutex_unlock(&nish_kept.mu);
+#endif
+  if (mine) free((nish_xchunk *)old - 1);
+  return b;
+}

package/runtime/runtime-host.c

@@ -5,12 +5,12 @@
  * before it trusts one.
  *
  * A translation unit of its own for the reason runtime-os.c is one: each file
- * carries its own measured `.text*` ceiling in tests/run.js. runtime-os.c had
- * 143 bytes of its ceiling left when these arrived, the four measure more than
- * that, and a ceiling is not raised to make room (docs/MASTER_PLAN.md §2 has
- * the numbers). Section GC still means a program that calls none of them pays
- * for none of them, and scripts/build.sh pairs this file with runtime.c like
- * the other two halves, so a link line still names one runtime.
+ * carries its own measured `.text*` ceiling in tests/runtime-budgets.js.
+ * runtime-os.c had 143 bytes of its ceiling left when these arrived, the
+ * four measure more than that, and a ceiling is not raised to make room
+ * (docs/MASTER_PLAN.md §2 has the numbers). Section GC still means a program that
+ * calls none of them pays for none of them, and scripts/build.sh pairs this file
+ * with runtime.c like the other two halves, so a link line still names one runtime.
  *
  * Every function here is platform code, and the two platforms differ in three
  * places: `getrandom` on Linux and `getentropy` elsewhere, `st_mtim`
@@ -57,6 +57,7 @@ double nish_date_now(void) {
 }
 
 static NISH_COLD void nish_entropy_fail(uint64_t asked) {
+  nish_panic_claim();
   if (asked > 65536) {
     dprintf(2, "crypto.getRandomValues: %llu bytes asked for, and one call fills at most 65536\n",
             (unsigned long long)asked);

package/runtime/runtime-net.c

@@ -5,9 +5,9 @@
  * `spawnSync` starts inherits none of them.
  *
  * A translation unit of its own for the reason runtime-host.c is one: each
- * file carries its own measured `.text*` ceiling in tests/run.js, and sockets
- * are a surface that grows, which would push another unit past its ceiling
- * rather than into a new one. Section GC
+ * file carries its own measured `.text*` ceiling in tests/runtime-budgets.js,
+ * and sockets are a surface that grows, which would push another unit past its
+ * ceiling rather than into a new one. Section GC
  * means a program that calls none of these pays for none of them, and
  * scripts/build.sh pairs this file with runtime.c like the other halves, so a
  * link line still names one runtime.
@@ -39,7 +39,8 @@
  * receive leaves `meta` at 0, 0, and a `meta[1]` of 0 is also what a real
  * Not-ECT datagram reads as, so a program learns that Darwin has no ECN from
  * the send, never from the receive. The Darwin branch is compiled by CI's Darwin bootstrap
- * rows and run by nothing. A WASI build has none of this (the checker refuses
+ * rows and run by its `runtime-net (macos-latest)` job: tests/runtime-test.c and
+ * the `net_` sections of tests/run.js, on a mac. A WASI build has none of this (the checker refuses
  * every `nish:net` call under a wasm target), so there the file is empty.
  */
 #if defined(__wasi__) || defined(__wasm__)
@@ -54,6 +55,7 @@ typedef int nish_net_unused;
 #include <errno.h>
 #include <fcntl.h>
 #include <netinet/in.h>
+#include <netinet/tcp.h>
 #include <stdint.h>
 #include <string.h>
 #include <sys/socket.h>
@@ -188,10 +190,11 @@ int32_t nish_net_local_port(int32_t fd) {
    IPv4 host is an `AF_INET` socket; any other is `AF_INET6` with
    `IPV6_V6ONLY` off, so `::` hears both families. A stream gets
    `SO_REUSEADDR` and `arg` its backlog; a datagram socket's `arg` is
-   `udpBind`'s flags. Every option is set before the bind, which is where
-   `SO_REUSEPORT` has to be, and where `UDP_GRO` has to be for the first
-   datagram to be coalesced. */
-static int32_t nish_net_bound(const nish_str *host, int32_t port, int type, int32_t arg) {
+   `udpBind`'s flags. `opts` is a stream's `tcpListenFlags` word: bit 1 asks
+   for `SO_REUSEPORT`, bit 2 for `TCP_NODELAY`. Every option is set before the
+   bind, which is where `SO_REUSEPORT` has to be, and where `UDP_GRO` has to be
+   for the first datagram to be coalesced. */
+static int32_t nish_net_bound(const nish_str *host, int32_t port, int type, int32_t arg, int32_t opts) {
   unsigned char a[16];
   if ((uint32_t)port > 65535 || !nish_net_parse(host, a)) return -22;
   int v4 = memcmp(a, nish_mapped_prefix, 12) == 0;
@@ -209,11 +212,11 @@ static int32_t nish_net_bound(const nish_str *host, int32_t port, int type, int3
   /* Both families at once on an IPv6 socket; on an IPv4 one this fails with
      `ENOPROTOOPT` and changes nothing, which is cheaper than asking. */
   setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &zero, sizeof zero);
-  int bad = 0;
+  int bad = (opts & 1) ? setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, &one, sizeof one) : 0;
   if (type == SOCK_STREAM) {
     setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &one, sizeof one);
+    if (opts & 2) bad |= setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &one, sizeof one);
   } else {
-    if (arg & 1) bad = setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, &one, sizeof one);
 #if defined(__linux__)
     if (arg & 2) bad |= setsockopt(fd, IPPROTO_UDP, UDP_GRO, &one, sizeof one);
     /* ECN is always read. An IPv6 socket needs both: a datagram from an IPv4
@@ -236,7 +239,17 @@ static int32_t nish_net_bound(const nish_str *host, int32_t port, int type, int3
    so a restarted server rebinds a port whose old connections are still in
    TIME_WAIT. */
 int32_t nish_tcp_listen(const nish_str *host, int32_t port, int32_t backlog) {
-  return nish_net_bound(host, port, SOCK_STREAM, backlog);
+  return nish_net_bound(host, port, SOCK_STREAM, backlog, 0);
+}
+
+/* `tcpListenFlags(host, port, backlog, flags)`: `tcpListen`'s socket, with
+   `SO_REUSEPORT` for flag 1, so several listeners of one uid share the port
+   (Linux spreads new connections across them; Darwin hands them all to one),
+   and `TCP_NODELAY` for flag 2, which the listener carries so that
+   `tcpAccept` sets it on each connection it takes. Any other bit is -22. */
+int32_t nish_tcp_listen_flags(const nish_str *host, int32_t port, int32_t backlog, int32_t flags) {
+  if ((uint32_t)flags > 3) return -22;
+  return nish_net_bound(host, port, SOCK_STREAM, backlog, flags);
 }
 
 /* A socket address of `n` bytes into the caller's 18-byte form. An IPv4 peer
@@ -264,7 +277,18 @@ static void nish_net_put(nish_array *out, const nish_sockaddr *s, socklen_t n) {
 /* `tcpAccept(fd, peer)`: the next connection as a descriptor of its own,
    non-blocking and close-on-exec, with the peer's address written into
    `peer`; -11 when none is waiting. A `peer` shorter than 18 bytes is -22
-   before anything is accepted, so no connection is taken and lost. */
+   before anything is accepted, so no connection is taken and lost.
+
+   A listener opened with `tcpListenFlags` flag 2 holds `TCP_NODELAY`, and the
+   connection gets it here. Linux copies the option to an accepted socket,
+   but that is not a rule other systems keep, so it is read off the listener
+   and set on the connection everywhere. The listener is the record of the
+   flag, rather than a table beside the descriptors, so it holds for a
+   descriptor however the program came by it, and a closed listener's number
+   reused leaves nothing stale behind. A listener that cannot answer (not
+   TCP) has no Nagle to turn off; a connection on which the set fails is
+   closed and its failure answered, so a flag-2 listener never hands out a
+   socket without it. */
 int32_t nish_tcp_accept(int32_t fd, nish_array *peer) {
   if (peer->len < NISH_ADDR_BYTES) return -22;
   nish_sockaddr s;
@@ -275,6 +299,14 @@ int32_t nish_tcp_accept(int32_t fd, nish_array *peer) {
   int c = nish_net_setup(accept(fd, &s.sa, &n));
 #endif
   if (c < 0) return nish_net_fail();
+  int nodelay = 0;
+  socklen_t size = sizeof nodelay;
+  if (getsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &nodelay, &size) == 0 && nodelay != 0 &&
+      setsockopt(c, IPPROTO_TCP, TCP_NODELAY, &nodelay, sizeof nodelay) != 0) {
+    int e = errno;
+    close(c);
+    return nish_net_err(e);
+  }
   nish_net_put(peer, &s, n);
   return c;
 }
@@ -348,7 +380,7 @@ int32_t nish_udp_bind(const nish_str *host, int32_t port, int32_t flags) {
 #if !defined(__linux__)
   if (flags & 2) return -95;
 #endif
-  return nish_net_bound(host, port, SOCK_DGRAM, flags);
+  return nish_net_bound(host, port, SOCK_DGRAM, flags, flags & 1);
 }
 
 /* Room for the two control messages either direction carries: a segment size
@@ -503,11 +535,15 @@ static int nish_poll_change(int32_t loop, int32_t fd, int filter, int flags, int
 
 /* Add, modify or remove `fd` in `loop`. epoll does each in one call, with
    the token in `data.u32`. kqueue has a filter per direction rather than a
-   set of events, so a change is the filters deleted and then the ones asked
-   for added, each carrying the token in `udata`. Deleting a filter that was
-   never added is not an error, except to a removal that finds neither, which
-   answers epoll's ENOENT; a modification adds what it asks for either way, so
-   a descriptor added watching nothing, which has no filter, can be modified. */
+   set of events, so the answers epoll gives are built from filters: a
+   descriptor in the loop has at least one, a disabled read filter standing for
+   one added watching nothing; an add first touches both filters with no flags,
+   which answers ENOENT for a filter not there, and is EEXIST when either is;
+   a modification or a removal deletes both, and is ENOENT when neither was
+   there; a modification then adds the ones asked for, each carrying the token
+   in `udata`. A touch rewrites the filter's `udata`, and the add touches both
+   filters, so the add that answers -17 leaves the descriptor watched for its
+   old events, both ways, under the token it gave rather than the old one. */
 static int32_t nish_poll_ctl(int32_t loop, int op, int32_t fd, int32_t events, int32_t token) {
   if ((uint32_t)events > 3) return -22;
 #if defined(__linux__)
@@ -517,14 +553,23 @@ static int32_t nish_poll_ctl(int32_t loop, int op, int32_t fd, int32_t events, i
   e.data.u32 = (uint32_t)token;
   return epoll_ctl(loop, op, fd, &e) != 0 ? nish_net_fail() : 0;
 #else
-  int e = 0;
-  if (op != NISH_POLL_ADD) {
-    int r = nish_poll_change(loop, fd, EVFILT_READ, EV_DELETE, 0);
-    int w = nish_poll_change(loop, fd, EVFILT_WRITE, EV_DELETE, 0);
-    if (op == NISH_POLL_REMOVE) return r != 0 && w != 0 ? nish_net_err(r) : 0;
+  int r, w;
+  if (op == NISH_POLL_ADD) {
+    r = nish_poll_change(loop, fd, EVFILT_READ, 0, token);
+    w = nish_poll_change(loop, fd, EVFILT_WRITE, 0, token);
+    if (r == 0 || w == 0) return -17;
+    if (r != ENOENT) return nish_net_err(r);
+    if (w != ENOENT) return nish_net_err(w);
+  } else {
+    r = nish_poll_change(loop, fd, EVFILT_READ, EV_DELETE, 0);
+    w = nish_poll_change(loop, fd, EVFILT_WRITE, EV_DELETE, 0);
+    if (r != 0 && w != 0) return nish_net_err(r);
+    if (op == NISH_POLL_REMOVE) return 0;
   }
+  int e = 0;
   if (events & 1) e = nish_poll_change(loop, fd, EVFILT_READ, EV_ADD | EV_ENABLE, token);
   if (e == 0 && (events & 2)) e = nish_poll_change(loop, fd, EVFILT_WRITE, EV_ADD | EV_ENABLE, token);
+  if (events == 0) e = nish_poll_change(loop, fd, EVFILT_READ, EV_ADD | EV_DISABLE, token);
   return e != 0 ? nish_net_err(e) : 0;
 #endif
 }
@@ -549,8 +594,7 @@ int32_t nish_poll_remove(int32_t loop, int32_t fd) { return nish_poll_ctl(loop,
    `NISH_POLL_CHUNK`. A signal that interrupts the wait answers 0, as a
    timeout does, rather than waiting again past the deadline; the program's
    loop comes round and, if it watches `signalFd()`, finds the signal ready.
-   On Darwin a descriptor both readable and writable is two pairs with the
-   same token, one per filter. A `ready` shorter than 2 is -22. */
+   A `ready` shorter than 2 is -22. */
 int32_t nish_poll_wait(int32_t loop, nish_array *ready, int32_t timeout_ms) {
   uint64_t room = ready->len / 2;
   if (room == 0) return -22;
@@ -560,22 +604,37 @@ int32_t nish_poll_wait(int32_t loop, nish_array *ready, int32_t timeout_ms) {
   struct epoll_event e[NISH_POLL_CHUNK];
   /* epoll waits forever for any negative timeout, as the language does. */
   int n = epoll_wait(loop, e, max, timeout_ms);
-#else
-  struct kevent e[NISH_POLL_CHUNK];
-  struct timespec t = {timeout_ms / 1000, (long)(timeout_ms % 1000) * 1000000};
-  int n = kevent(loop, NULL, 0, e, max, timeout_ms < 0 ? NULL : &t);
-#endif
   if (n < 0) return errno == EINTR ? 0 : nish_net_fail();
   for (int i = 0; i < n; i++) {
-#if defined(__linux__)
     uint32_t got = e[i].events;
     out[2 * i] = (int32_t)e[i].data.u32;
     out[2 * i + 1] = ((got & EPOLLIN) ? 1 : 0) | ((got & EPOLLOUT) ? 2 : 0) | ((got & (EPOLLHUP | EPOLLERR)) ? 4 : 0);
+  }
+  return n;
 #else
-    out[2 * i] = (int32_t)(intptr_t)e[i].udata;
-    out[2 * i + 1] = (e[i].filter == EVFILT_READ ? 1 : 2) | ((e[i].flags & (EV_EOF | EV_ERROR)) ? 4 : 0);
-#endif
+  /* A record per filter, so a descriptor ready both ways is two, folded here
+     into one pair as epoll's one event is. Twice the room is asked for, so a
+     full `ready` is not cut short by the folding; a descriptor that finds no
+     pair free is still ready at the next wait. */
+  struct kevent e[NISH_POLL_CHUNK];
+  struct timespec t = {timeout_ms / 1000, (long)(timeout_ms % 1000) * 1000000};
+  int got = kevent(loop, NULL, 0, e, 2 * max < NISH_POLL_CHUNK ? 2 * max : NISH_POLL_CHUNK, timeout_ms < 0 ? NULL : &t);
+  if (got < 0) return errno == EINTR ? 0 : nish_net_fail();
+  int n = 0;
+  for (int i = 0; i < got; i++) {
+    int32_t bits = (e[i].filter == EVFILT_READ ? 1 : 2) | ((e[i].flags & (EV_EOF | EV_ERROR)) ? 4 : 0);
+    int k = 0;
+    while (k < n && e[k].ident != e[i].ident) k++;
+    if (k == n) {
+      if (n == max) continue;
+      e[n] = e[i];
+      out[2 * n] = (int32_t)(intptr_t)e[i].udata;
+      out[2 * n + 1] = 0;
+      n++;
+    }
+    out[2 * k + 1] |= bits;
   }
   return n;
+#endif
 }
 #endif