@amritk/nish-x86_64-linux 0.23.1 → 0.25.0

package/runtime/nish.mjs

@@ -84,12 +84,16 @@ globalThis.console = {
 // `Array#sort`'s UTF-16 order — the two agree on ASCII names only.
 provide("write", shim.write);
 provide("writeError", shim.writeError);
+provide("writeBytes", shim.writeBytes);
 provide("panic", shim.panic);
 provide("readFileSync", shim.readFileSync);
 provide("readFileSyncOrNull", shim.readFileSyncOrNull);
 provide("readFileBytesSync", shim.readFileBytesSync);
 provide("writeFileSync", shim.writeFileSync);
 provide("appendFileSync", shim.appendFileSync);
+provide("writeFileBytesSync", shim.writeFileBytesSync);
+provide("renameSync", shim.renameSync);
+provide("unlinkSync", shim.unlinkSync);
 provide("mkdirSync", shim.mkdirSync);
 provide("isDirectorySync", shim.isDirectorySync);
 provide("readdirSync", shim.readdirSync);
@@ -103,6 +107,14 @@ provide("kill", shim.kill);
 provide("waitPid", shim.waitPid);
 provide("availableParallelism", shim.availableParallelism);
 provide("totalMemory", shim.totalMemory);
+// `setNice` and `setIdlePriority` throw, naming themselves: Node has a reading
+// of the nice value (`os.setPriority`) but none of `SCHED_IDLE`, and a reading
+// of one without the other would differ from the native build on the same host.
+for (const name of ["setNice", "setIdlePriority"]) {
+  provide(name, () => {
+    throw new Error(`\`${name}\` has no reading under Node: it changes how the kernel schedules the native program`);
+  });
+}
 provide("getenv", shim.getenv);
 provide("statMtimeSync", shim.statMtimeSync);
 provide("signalFd", shim.signalFd);
@@ -130,9 +142,17 @@ for (const name of [
   "pollModify",
   "pollRemove",
   "pollWait",
+  "netSetOption",
 ]) {
   provide(name, () => shim.noNetReading(name));
 }
+// `netSetOption`'s option ids are plain numbers, so they read the same here
+// as natively: only the call that takes one throws.
+provide("NET_TCP_NODELAY", 1);
+provide("NET_TCP_QUICKACK", 2);
+provide("NET_SO_KEEPALIVE", 3);
+provide("NET_SO_SNDBUF", 4);
+provide("NET_SO_RCVBUF", 5);
 
 // `nish:term`. Whether a descriptor is a terminal and how big stdout's is have
 // Node's answers; reading stdin, waiting for it and raw mode throw, each naming
@@ -180,6 +200,7 @@ if (!("set" in Array.prototype)) {
 // describes. A printed reading can never match a native run: the two origins are
 // both arbitrary, and only the difference between two reads means anything.
 provide("monotonicNanos", shim.monotonicNanos);
+provide("wallClockNanos", shim.wallClockNanos);
 
 // Conversions. In f64 mode `toF64` is the identity and `toI32` is the one that
 // matters (it saturates, where a JavaScript cast would not).
@@ -265,15 +286,12 @@ globalThis.parseFloat = shim.parseFloat;
 // script under Node, which is one index later than Node's own `argv`.
 process.argv = shim.argv();
 
-// There is no arena. `mark`/`release`/`reset` are no-ops rather than errors so
-// that a program which manages memory explicitly still runs; `used()` answers
-// zero, which is the honest number for a host that is not bump-allocating.
+// There is no arena. `mark()` answers zero rather than failing so that a
+// program which reads it still runs; `used()` answers zero too, which is the honest number for a host that is not bump-allocating.
 // Delegated like everything else here rather than written inline, so that the
 // answers cannot drift from the ones the rewritten runner gives.
 provide("Arena", {
   mark: shim.arenaMark,
-  release: shim.arenaRelease,
-  reset: shim.arenaReset,
   used: shim.arenaUsed,
 });
 // `using a = arena()` releases nothing here either: what the block allocated

package/runtime/node.mjs

@@ -72,6 +72,48 @@ function appendFileSync(path, data) {
   }
 }
 
+/**
+ * The negative errno a failed `fs` call names, as `runtime-os.c` answers it.
+ * Node names the errno by its code; one it gives no number for, such as the
+ * `ERR_INVALID_ARG_VALUE` a path holding a NUL throws, is -22 (`EINVAL`), the
+ * runtime's answer to that path.
+ */
+function negErrno(e) {
+  return -(os.constants.errno[e.code] ?? 22);
+}
+
+/** `ok` when `f` returns, or the negative errno of what it threw. */
+function errnoOr(ok, f) {
+  try {
+    f();
+    return ok;
+  } catch (e) {
+    return negErrno(e);
+  }
+}
+
+/** `renameSync(from, to)`: 0, or a negative errno (-18 across file systems). */
+const renameSync = (from, to) => errnoOr(0, () => fs.renameSync(from, to));
+
+/** `unlinkSync(path)`: 0, or a negative errno; a directory is refused, not removed. */
+const unlinkSync = (path) => errnoOr(0, () => fs.unlinkSync(path));
+
+/** `writeFileBytesSync(path, bytes)`: the byte count, or a negative errno; mode 0644 and `WRITE_FLAGS`. */
+const writeFileBytesSync = (path, bytes) =>
+  errnoOr(bytes.length, () => fs.writeFileSync(path, Uint8Array.from(bytes), { flag: WRITE_FLAGS, mode: 0o644 }));
+
+/**
+ * `writeBytes`'s write, after the core checked the range `[from, to)`: every
+ * byte to stdout, then the count, or a negative errno. `fs.writeSync` may
+ * write less than it was given, so it is asked again for the rest.
+ */
+function writeBytes(bytes, from, to) {
+  const buf = Uint8Array.from(bytes.slice(from, to));
+  return errnoOr(buf.length, () => {
+    for (let at = 0; at < buf.length; ) at += fs.writeSync(1, buf, at, buf.length - at);
+  });
+}
+
 /**
  * `mkdirSync(path)` (WP14 D4): one directory, not recursive, answering whether
  * a directory is there afterwards. Node throws where the runtime answers
@@ -234,12 +276,7 @@ function spawnSyncTo(argv, out, err) {
  */
 function kill(pid, sig) {
   if (pid <= 0 || !(sig === 0 || sig === 2 || sig === 9 || sig === 15)) return -22;
-  try {
-    process.kill(pid, sig);
-    return 0;
-  } catch (e) {
-    return -(os.constants.errno[e.code] ?? 22);
-  }
+  return errnoOr(0, () => process.kill(pid, sig));
 }
 
 /**
@@ -275,6 +312,15 @@ function monotonicNanos() {
   return process.hrtime.bigint();
 }
 
+/**
+ * `wallClockNanos()`: the wall clock in nanoseconds since the epoch, a BigInt
+ * like `monotonicNanos`'s. Node's wall clock is `Date.now`, whole
+ * milliseconds, so the last six digits are zeros where a native run has them.
+ */
+function wallClockNanos() {
+  return BigInt(Date.now()) * 1000000n;
+}
+
 /**
  * `statMtimeSync(path)`: Node's `mtimeMs` for the path, or NaN when it cannot
  * be stat'd, which is the native answer too. `mtimeMs` is the same arithmetic
@@ -344,6 +390,10 @@ const CALLS = {
   readFileBytesSync,
   writeFileSync,
   appendFileSync,
+  writeFileBytesSync,
+  renameSync,
+  unlinkSync,
+  writeBytes,
   mkdirSync,
   isDirectorySync,
   readdirSync,
@@ -357,6 +407,7 @@ const CALLS = {
   availableParallelism,
   totalMemory,
   monotonicNanos,
+  wallClockNanos,
   statMtimeSync,
   lstatOwnerModeSync,
   geteuid,

package/runtime/runtime-net.c

@@ -17,7 +17,8 @@
  * errno to it: where `signalFd` and `readSignal` answer a plain -1, a call
  * here answers the failure's errno, negated. The codes a loop branches on are Linux's numbers on every platform (-11
  * would block, -95 unsupported, -32 the peer is gone, -104 reset, -98 the
- * address is in use, -111 refused, -110 timed out, -22 a bad argument), so
+ * address is in use, -111 refused, -110 timed out, -22 a bad argument, -88
+ * not a socket), so
  * `nish_net_err` translates Darwin's; any other failure is the host's own
  * `-errno`. Nothing here allocates: the addresses a call reads or writes are
  * the caller's `u8[]`.
@@ -29,13 +30,14 @@
  * (`EAFNOSUPPORT`, as in many containers) `::` falls back to IPv4's
  * `0.0.0.0`.
  *
- * Linux and Darwin differ in five places: `SOCK_NONBLOCK | SOCK_CLOEXEC` and
+ * Linux and Darwin differ in six places: `SOCK_NONBLOCK | SOCK_CLOEXEC` and
  * `accept4` against a `socket` or `accept` followed by `fcntl`, `MSG_NOSIGNAL`
  * against the `SO_NOSIGPIPE` socket option (a write to a gone peer is -32 on
  * both, never SIGPIPE), the errno numbers, UDP's segmentation offload
  * (`UDP_SEGMENT`, `UDP_GRO`) and ECN marks, which Linux has and for which
  * Darwin answers -95, and the readiness loop, which is epoll on Linux and
- * kqueue on Darwin. Only the sending side and the GRO bind say -95 there: a
+ * kqueue on Darwin, and `TCP_QUICKACK`, which only Linux has (-95 on
+ * Darwin). Only the sending side and the GRO bind say -95 there: a
  * receive leaves `meta` at 0, 0, and a `meta[1]` of 0 is also what a real
  * Not-ECT datagram reads as, so a program learns that Darwin has no ECN from
  * the send, never from the receive. The Darwin branch is compiled by CI's Darwin bootstrap
@@ -100,6 +102,7 @@ static int32_t nish_net_err(int e) {
   if (e == EADDRINUSE) return -98;
   if (e == ECONNREFUSED) return -111;
   if (e == ETIMEDOUT) return -110;
+  if (e == ENOTSOCK) return -88;
 #endif
   return -e;
 }
@@ -359,7 +362,7 @@ int32_t nish_connect_result(int32_t fd) {
 
 /* `netRead(fd, buf, off, len)`: at most `len` bytes into `buf` from `off`.
    The compiled call has already checked `0 <= off <= off + len <=
-   buf.length`, unless the program was built with `--unchecked-indexing`. */
+   buf.length`. */
 int32_t nish_net_read(int32_t fd, nish_array *buf, int64_t off, int64_t len) {
   ssize_t n = recv(fd, buf->data + off, (size_t)len, 0);
   return n < 0 ? nish_net_fail() : (int32_t)n;
@@ -383,6 +386,44 @@ int32_t nish_net_shutdown(int32_t fd, int32_t how) {
    promises on Linux, so a failure is information rather than a retry. */
 int32_t nish_net_close(int32_t fd) { return close(fd) != 0 ? nish_net_fail() : 0; }
 
+/* `netSetOption(fd, option, value)`: one option from a whitelist, named by
+   the runtime's own id rather than a level and an option number, so a
+   program can set these five and nothing else. `value` is 0 or 1 for the
+   flags and a byte count for the buffers, passed to the kernel as it is
+   (Linux stores twice a buffer size, to cover its bookkeeping). Linux does
+   not keep quick-ACK mode: the kernel leaves it of its own accord, so a
+   server that wants it on or off sets it again after each response. */
+int32_t nish_net_set_option(int32_t fd, int32_t option, int32_t value) {
+  int level = SOL_SOCKET;
+  int name;
+  switch (option) {
+  case 1:
+    level = IPPROTO_TCP;
+    name = TCP_NODELAY;
+    break;
+  case 2:
+#if defined(__linux__)
+    level = IPPROTO_TCP;
+    name = TCP_QUICKACK;
+    break;
+#else
+    return -95;
+#endif
+  case 3:
+    name = SO_KEEPALIVE;
+    break;
+  case 4:
+    name = SO_SNDBUF;
+    break;
+  case 5:
+    name = SO_RCVBUF;
+    break;
+  default:
+    return -22;
+  }
+  return setsockopt(fd, level, name, &value, sizeof value) != 0 ? nish_net_fail() : 0;
+}
+
 /* `udpBind(host, port, flags)`: a datagram socket, non-blocking and
    close-on-exec, bound like `tcpListen`'s, with `SO_REUSEPORT` for flag 1
    and `UDP_GRO` for flag 2; any other bit is -22. */

package/runtime/runtime-os.c

@@ -51,6 +51,7 @@
 /* WASI has no processes, so `spawn.h` and `wait.h` do not exist there. */
 #include <signal.h>
 #include <spawn.h>
+#include <sys/resource.h>
 #include <sys/wait.h>
 extern char **environ;
 #endif
@@ -178,23 +179,89 @@ nish_str *nish_read_file(const nish_str *path) {
 
    `O_CLOEXEC` for the reason the read has it (RT-12), and `EINTR` is retried
    as the read retries it (RT-13): a write cut short is finished, and one that
-   fails is `cannot write`, never a silently short file. */
+   fails is `cannot write`, never a silently short file.
+
+   The loop is `nish_put_all`, which `writeFileBytesSync` and `writeBytes`
+   share: all `n` bytes of `p` to `fd`, answering 0 or the negative errno of
+   the write that failed (-EIO for one that wrote nothing). */
+static int32_t nish_put_all(int fd, const char *p, uint64_t n) {
+  for (uint64_t done = 0; done < n;) {
+    ssize_t k = write(fd, p + done, n - done);
+    if (k > 0) done += k;
+    else if (k == 0) return -EIO;
+    else if (errno != EINTR) return -errno;
+  }
+  return 0;
+}
 static void nish_put_file(const nish_str *path, const nish_str *data, int flags) {
   int fd = open(nish_cpath(path), O_WRONLY | O_CREAT | O_NOFOLLOW | O_CLOEXEC | flags, 0644);
-  if (fd < 0) nish_io_fail("write ", path);
-  for (uint64_t done = 0; done < data->len;) {
-    ssize_t n = write(fd, data->data + done, data->len - done);
-    if (n > 0) {
-      done += n;
-    } else if (n == 0 || errno != EINTR) {
-      nish_io_fail("write ", path);
-    }
-  }
+  if (fd < 0 || nish_put_all(fd, data->data, data->len) < 0) nish_io_fail("write ", path);
   close(fd);
 }
 void nish_write_file(const nish_str *path, const nish_str *data) { nish_put_file(path, data, O_TRUNC); }
 void nish_append_file(const nish_str *path, const nish_str *data) { nish_put_file(path, data, O_APPEND); }
 
+/* ---- Replacing and removing files: `renameSync`, `unlinkSync` and
+   `writeFileBytesSync`. Unlike the writers above, none exits: each answers 0
+   (or the bytes written) on success and a negative errno otherwise, so a
+   program that writes a file beside its target and renames it over the
+   target owns its own diagnostic. A path holding a NUL answers -EINVAL before
+   any system call, rather than reaching the file its prefix names
+   (docs/security/runtime.md, RT-3). */
+
+/* `renameSync(from, to)`: rename(2). On one file system it replaces `to`
+   atomically, so a reader sees the old file or the new one and never a part;
+   across two it answers -EXDEV and moves nothing, since a copy and a delete
+   would not be atomic. A link at `from` is moved as a link, not followed. */
+int32_t nish_rename(const nish_str *from, const nish_str *to) {
+  const char *f = nish_cpath(from), *t = nish_cpath(to);
+  if (f != from->data || t != to->data) return -EINVAL;
+  return rename(f, t) == 0 ? 0 : -errno;
+}
+
+/* `unlinkSync(path)`: unlink(2). A directory is refused with whatever the
+   kernel answers (-EISDIR on Linux, -EPERM elsewhere) and is never handed to
+   rmdir. A link is removed, not the file it names. */
+int32_t nish_unlink(const nish_str *path) {
+  const char *p = nish_cpath(path);
+  if (p != path->data) return -EINVAL;
+  return unlink(p) == 0 ? 0 : -errno;
+}
+
+/* `writeFileBytesSync(path, bytes)`: every element of a `u8[]` to the file,
+   opened as `writeFileSync` opens it (`O_NOFOLLOW`, `O_CLOEXEC`, `O_TRUNC`,
+   0644 before the umask) and written through `nish_put_all`, a short write
+   finished and `EINTR` retried. It answers the length written or the negative
+   errno of the `open` or `write` that failed (-EIO for a write that wrote
+   nothing); a failed write leaves the file truncated to what got out. */
+int32_t nish_write_file_bytes(const nish_str *path, const nish_array *bytes) {
+  const char *p = nish_cpath(path);
+  if (p != path->data) return -EINVAL;
+  if (bytes->len > NISH_LENGTH_MAX) return -EFBIG;
+  int fd = open(p, O_WRONLY | O_CREAT | O_TRUNC | O_NOFOLLOW | O_CLOEXEC, 0644);
+  if (fd < 0) return -errno;
+  int32_t r = nish_put_all(fd, bytes->data, bytes->len);
+  /* A write NFS or a quota refuses can surface only at `close`, and the file
+     is then short: an answer of the full length would let the caller rename
+     it over the one it replaces. */
+  if (close(fd) != 0 && r == 0 && errno != EINTR) r = -errno;
+  return r < 0 ? r : (int32_t)bytes->len;
+}
+
+/* `writeBytes(bytes, off, len)` (nish:io): the bytes `[off, off + len)` of a
+   `u8[]` on stdout, through `nish_put_all`, so a short write is finished and `EINTR` retried. The
+   emitter has range-checked `off` and `len` against `bytes->len`
+   (`emitNetCall`). It answers `len`, every byte out, or the negative errno of
+   the write that failed (`-EIO` for one that wrote nothing), and never exits:
+   how much of a failed call reached the descriptor is not said. A pipe whose
+   reader has gone raises SIGPIPE, as it does for `write`. */
+int32_t nish_write_bytes(const nish_array *bytes, int64_t off, int64_t len) {
+  /* An empty array's data may be NULL, and NULL + 0 is undefined. */
+  if (len == 0) return 0;
+  int32_t r = nish_put_all(1, bytes->data + off, (uint64_t)len);
+  return r < 0 ? r : (int32_t)len;
+}
+
 /* ---- Directories and subprocesses (WP14 D4): what a driver needs to create
    its own output directory and shell out for `--link`. Both answer a value
    instead of exiting, as `nish_read_file_or_null` does — there are no
@@ -477,6 +544,36 @@ int64_t nish_total_memory(void) {
 #endif
 }
 
+/* `setNice(n)`: this process's nice value set to `n`, 0 or a negative errno.
+   The kernel clamps `n` to its range (-20 to 19), so only lowering the value
+   below what the process has can fail: -EACCES (Linux, Darwin) or -EPERM
+   without the privilege to raise a priority. Linux keeps the value per thread,
+   so it is the calling thread's and every thread started after it. */
+int32_t nish_set_nice(int32_t n) {
+#ifdef __wasi__
+  (void)n;
+  return -ENOSYS;
+#else
+  return setpriority(PRIO_PROCESS, 0, n) == 0 ? 0 : -errno;
+#endif
+}
+
+/* `setIdlePriority()`: this process moved to `SCHED_IDLE`, which runs it only
+   when nothing else wants the CPU. Linux alone has the policy; elsewhere the
+   answer is -EOPNOTSUPP's Linux number, -95, as runtime-net.c answers it. An
+   unprivileged process cannot leave the policy again. Like the nice value, the
+   policy is the calling thread's. */
+int32_t nish_set_idle_priority(void) {
+#if defined(__linux__)
+  struct sched_param param = {0};
+  return sched_setscheduler(0, SCHED_IDLE, &param) == 0 ? 0 : -errno;
+#elif defined(__wasi__)
+  return -ENOSYS;
+#else
+  return -95;
+#endif
+}
+
 /* `readdirSync(path)`: the listing a driver needs to discover its own inputs.
 
    **Sorted by bytes**, which `readdir(3)` and Node are not, for two reasons
@@ -545,11 +642,17 @@ nish_array *nish_readdir(const nish_str *path) {
    backwards and make an elapsed time negative. Only the difference between two
    reads means anything — the origin is arbitrary and is not comparable across
    processes or machines — and an i64 of nanoseconds is 292 years of it. */
-int64_t nish_monotonic_nanos(void) {
+static int64_t nish_clock_nanos(clockid_t clock) {
   struct timespec ts;
-  clock_gettime(CLOCK_MONOTONIC, &ts);
+  clock_gettime(clock, &ts);
   return (int64_t)ts.tv_sec * 1000000000 + ts.tv_nsec;
 }
+int64_t nish_monotonic_nanos(void) { return nish_clock_nanos(CLOCK_MONOTONIC); }
+
+/* `wallClockNanos()`: `CLOCK_REALTIME`, nanoseconds since the Unix epoch, for
+   dating an event. It steps when the system time is set, so a duration is
+   `monotonicNanos`'s to measure. */
+int64_t nish_wall_clock_nanos(void) { return nish_clock_nanos(CLOCK_REALTIME); }
 
 /* A C string copied into the arena, or NULL for NULL: the shape of every
    answer libc hands back by pointer, `getenv`'s here and `realpath`'s below. */

package/runtime/runtime-term.c

@@ -145,8 +145,8 @@ static int nish_stdin_ready(int32_t timeout_ms) {
 
 /* `readStdin(buf, off, len)`: at most `len` bytes into `buf` from `off`, the
    count, 0 at the end of the stream and -11 when nothing is waiting. The
-   compiled call has already checked `0 <= off <= off + len <= buf.length`,
-   unless the program was built with `--unchecked-indexing`. A `read` that a
+   compiled call has already checked `0 <= off <= off + len <= buf.length`.
+   A `read` that a
    signal interrupts is asked again; one that would block (a descriptor some
    other program made non-blocking) is -11 too, and so is a readiness check a
    signal interrupts, which would otherwise fall through to a `read` that

package/runtime/runtime.c

@@ -1354,7 +1354,7 @@ nish_array *nish_alloc_array(uint64_t elem_size, uint64_t len) {
   return a;
 }
 
-/* process.argv: malloc, not the arena, so Arena.reset() cannot free it. Built once by @main.
+/* process.argv: malloc, not the arena, so no arena scope can free it. Built once by @main.
    Deliberately *not* NISH_TLS (WP20 T0 §3.2): `nish_argv_init` writes it once
    before the program's `main` body runs and nothing writes it afterwards, so
    every thread may read the one copy. That holds only while no thread is

package/scripts/build.sh

@@ -62,6 +62,10 @@
 # put in the .ll (line table, variables) reaches the binary. `nish
 # --link -g` passes it through automatically.
 #
+# Build id: every native ELF binary carries an NT_GNU_BUILD_ID note
+# (--build-id=sha1). A -g link hashes other bytes than a plain one, so strip
+# the -g build to pair a binary with its debug file (docs/guide/using-nish/cli.md).
+#
 # CPU level (WP38 S2): `--cpu x86-64-v3` compiles and links every input --
 # the module, the runtime and any other C -- with -march=x86-64-v3
 # -ffp-contract=off, so the code generator may use AVX2, BMI2 and the other
@@ -281,8 +285,8 @@ case "$(uname -s)" in
     # followed by SIGABRT, so the stage1 that linked went on to abort the
     # moment the bootstrap ran it. Measured too, on the run after the one that
     # named the UUID. An unloadable compiler is worse than any reproducibility,
-    # so the UUID stays. The ELF branch's --build-id=none below is the flag
-    # this would have been; ELF has no loader that insists on one.
+    # so the UUID stays. ELF's twin is the build-id note below, which ELF's
+    # loader never asks for and which is a hash of the output's bytes anyway.
     #
     # It costs nothing, because the UUID was measured stable across two links
     # of one input to one output path, and differing on a link to another. That
@@ -294,8 +298,14 @@ case "$(uname -s)" in
     strip_flag=(-Wl,-x) ;;
   *)
     # shellcheck disable=SC2054 # -Wl,<flag> is one argument: the comma is the linker's
-    gc=(-Wl,--gc-sections -Wl,--as-needed -Wl,-O2 -Wl,--build-id=none); strip_flag=(-s)
+    gc=(-Wl,--gc-sections -Wl,--as-needed -Wl,-O2); strip_flag=(-s)
     elf=(-fno-plt)
+    # A GNU build-id note on every native link, debug included, so perf, gdb
+    # and debuginfod can match a stripped binary to its debug file. sha1 is a
+    # hash of the output's own bytes in lld, GNU ld and gold, so one input
+    # links to one id and `stage3 == stage2` holds; uuid would be random.
+    # shellcheck disable=SC2054
+    common+=(-Wl,--build-id=sha1)
     # GNU ld needs the gold plugin for LTO; prefer lld when clang can find it.
     if command -v ld.lld >/dev/null 2>&1; then common+=(-fuse-ld=lld); lld=1; fi ;;
 esac