@amritk/nish-x86_64-linux 0.20.0 → 0.22.1

package/scripts/build.sh

@@ -3,20 +3,23 @@
 #
 #   scripts/build.sh <module.ll> [more .ll/.c files...] -o <out> [--profile debug|speed|size|wasm|wasi|napi]
 #
-# The C runtime is eight translation units and is named as one: an input
+# The C runtime is ten translation units and is named as one: an input
 # <dir>/runtime.c also compiles <dir>/runtime-os.c, the half that wraps the
 # system calls (files, directories, subprocesses, the environment, the clock),
 # <dir>/runtime-parallel.c, the half that divides a range of work across
 # threads, <dir>/runtime-cross.c, the copies that carry a string or an array
 # from one thread's arena to another's, <dir>/runtime-host.c, the wall clock, entropy, file times and
 # signals, <dir>/runtime-net.c, the sockets of `nish:net`,
-# <dir>/runtime-simd.c, the byte searches that read a vector at a time, and
-# <dir>/runtime-aes.c, AES-GCM on the CPU's AES instructions. Each of those
+# <dir>/runtime-simd.c, the byte searches that read a vector at a time,
+# <dir>/runtime-aes.c, AES-GCM on the CPU's AES instructions,
+# <dir>/runtime-term.c, stdin and the terminal of `nish:term`, and
+# <dir>/runtime-bytes.c, the byte-class runs of `nish/bytes`. Each of those
 # files says why they are compiled and measured apart.
 #
 # Profiles:
 #   debug  clang defaults: no optimisation, symbols kept. The "before" number.
-#   speed  -O3 + LTO + section GC + strip. Rust `--release` equivalent.
+#   speed  -O3 + LTO + section GC + strip. Rust `--release` equivalent. With
+#          lld, LTO code generation runs on every core.
 #   size   -Oz + LTO + section GC + strip + no unwind tables. Rust
 #          `opt-level="z"`, `panic="abort"`, `strip=true` equivalent.
 #   wasm   wasm32 freestanding module exporting every non-internal function;
@@ -68,6 +71,20 @@
 # stops with SIGILL on a CPU without AVX2. `nish --link --cpu x86-64-v3`
 # passes it through.
 #
+# SQLite: an input .ll that declares an `sqlite3_*` or `nishsqlite_*` function
+# also compiles and links every runtime/sqlite/*.c, the vendored amalgamation
+# (runtime/sqlite/README.md). Any other program compiles none of it. sqlite3.c
+# is compiled to native code at the profile's level, even where the profile
+# compiles to LTO bitcode. The options are most of SQLite's recommended set
+# (see the defaults below), with THREADSAFE=0 (=2 under --threads).
+# NISH_SQLITE_CFLAGS is appended to them: its first line, split on whitespace
+# with no quoting, and a relative -I in it names a directory below
+# runtime/sqlite/, where each unit is compiled. A -D or -U in it for one of
+# their names replaces it. The napi profile compiles SQLite -fPIC and
+# -fvisibility=hidden, so an addon does not export its symbols. Every profile
+# but wasm links it; wasm has no libc to build it on and is refused with exit 2.
+# On wasi, SQLite locks with dot files and has no WAL shared memory.
+#
 # The runtime compiled once (the debug, speed and size profiles only; `nish
 # --link` and `nish run` use these through their runtime cache, src/compile.ts):
 #   --runtime-objects <dir>  compile the runtime.c input and its seven siblings,
@@ -80,7 +97,13 @@
 #                            runtime.c input, after compiling every other input
 #                            with its own $CC -c, all at once. The link line is
 #                            the one the profile always uses.
-# Both refuse the wasm, wasi and napi profiles and --pgo-*, with exit 2.
+#   --sqlite-objects <dir>   compile runtime/sqlite/*.c alone, into
+#                            <dir>/<unit>.o, as a link of a program that calls
+#                            SQLite would, and link nothing. Takes no inputs and
+#                            no -o, and prints nothing on success.
+#   --sqlite-from <dir>      link <dir>/<unit>.o in place of compiling
+#                            runtime/sqlite/, when the IR calls SQLite.
+# All four refuse the wasm, wasi and napi profiles and --pgo-*, with exit 2.
 #
 # Works on Linux (clang + lld preferred, GNU ld tolerated) and macOS (Apple ld64
 # or Homebrew llvm). Set CC to pick a compiler (default: clang on PATH).
@@ -95,6 +118,8 @@ threads=0                              # --threads: -DNISH_THREADS, the thread-l
 objects_to=""                          # --runtime-objects <dir>: compile the runtime alone, into <dir>
 objects_from=""                        # --runtime-from <dir>: link against the runtime compiled there
 cpu=""                                 # --cpu x86-64-v3: the one CPU level above the baseline (WP38 S2)
+sqlite_to=""                           # --sqlite-objects <dir>: compile runtime/sqlite/ alone, into <dir>
+sqlite_from=""                         # --sqlite-from <dir>: link runtime/sqlite/ compiled there
 while [ $# -gt 0 ]; do
   case "$1" in
     -o) out="$2"; shift 2 ;;
@@ -106,15 +131,17 @@ while [ $# -gt 0 ]; do
       cpu="$2"; shift 2 ;;
     --runtime-objects) objects_to="$2"; shift 2 ;;
     --runtime-from) objects_from="$2"; shift 2 ;;
+    --sqlite-objects) sqlite_to="$2"; shift 2 ;;
+    --sqlite-from) sqlite_from="$2"; shift 2 ;;
     --pgo-generate) pgo=(-fprofile-generate); shift ;;
     --pgo-use)
       [ -f "$2" ] || { echo "error: --pgo-use: profile '$2' not found (run the instrumented binary, then llvm-profdata merge)" >&2; exit 2; }
       pgo=("-fprofile-use=$2"); shift 2 ;;
-    -h|--help) sed -n '2,85p' "$0"; exit 0 ;;
+    -h|--help) sed -n '2,/^set -euo pipefail$/{/^set -euo pipefail$/!p;}' "$0"; exit 0 ;;   # the header, to the first command
     *) inputs+=("$1"); shift ;;
   esac
 done
-[ ${#inputs[@]} -gt 0 ] || { echo "error: no input files" >&2; exit 2; }
+[ ${#inputs[@]} -gt 0 ] || [ -n "$sqlite_to" ] || { echo "error: no input files" >&2; exit 2; }
 if [ -n "$cpu" ]; then
   [ "$cpu" = x86-64-v3 ] || { echo "error: --cpu takes x86-64-v3, not '$cpu'" >&2; exit 2; }
   case "$profile" in
@@ -151,15 +178,27 @@ if [ -n "$objects_to" ] || [ -n "$objects_from" ]; then
   [ ${#pgo[@]} -eq 0 ] || { echo "error: --runtime-objects and --runtime-from do not take --pgo-*" >&2; exit 2; }
   [ -z "$objects_to" ] || [ -z "$objects_from" ] || { echo "error: --runtime-objects and --runtime-from are two different builds" >&2; exit 2; }
 fi
-[ -n "$out" ] || [ -n "$objects_to" ] || { echo "error: -o <out> is required" >&2; exit 2; }
+if [ -n "$sqlite_to" ] || [ -n "$sqlite_from" ]; then
+  # The cached SQLite objects sit beside the cached runtime and take what it takes.
+  case "$profile" in
+    debug|speed|size) ;;
+    *) echo "error: --sqlite-objects and --sqlite-from take the debug, speed and size profiles, not '$profile'" >&2; exit 2 ;;
+  esac
+  [ ${#pgo[@]} -eq 0 ] || { echo "error: --sqlite-objects and --sqlite-from do not take --pgo-*" >&2; exit 2; }
+  [ -z "$sqlite_to" ] || [ -z "$sqlite_from" ] || { echo "error: --sqlite-objects and --sqlite-from are two different builds" >&2; exit 2; }
+  [ -z "$sqlite_to" ] || [ -z "$objects_to$objects_from" ] || { echo "error: --sqlite-objects compiles runtime/sqlite/ alone, without --runtime-objects or --runtime-from" >&2; exit 2; }
+fi
+[ -n "$out" ] || [ -n "$objects_to" ] || [ -n "$sqlite_to" ] || { echo "error: -o <out> is required" >&2; exit 2; }
 
-# The runtime is eight translation units, and a caller names one: whoever passes
+# The runtime is ten translation units, and a caller names one: whoever passes
 # <dir>/runtime.c gets <dir>/runtime-os.c, <dir>/runtime-parallel.c,
-# <dir>/runtime-cross.c, <dir>/runtime-host.c, <dir>/runtime-net.c, <dir>/runtime-simd.c and
-# <dir>/runtime-aes.c compiled beside it, on every profile, wasi included:
-# runtime-simd.c takes its scalar path there unless the build adds -msimd128,
-# and runtime-aes.c compiles to an answer of 0 and stubs that nothing calls,
-# because the compiler emits no call to its kernels for wasm32. They were one file until the
+# <dir>/runtime-cross.c, <dir>/runtime-host.c, <dir>/runtime-net.c, <dir>/runtime-simd.c,
+# <dir>/runtime-aes.c, <dir>/runtime-term.c and <dir>/runtime-bytes.c compiled beside it, on every profile,
+# wasi included: runtime-simd.c and runtime-bytes.c take their scalar paths there
+# unless the build adds -msimd128,
+# runtime-aes.c compiles to an answer of 0 and stubs that nothing calls,
+# because the compiler emits no call to its kernels for wasm32, and
+# runtime-term.c reads stdin and answers "not a terminal". They were one file until the
 # operating-system half was split out for its own size budget, and the parallel
 # half followed for the same reason (each file's header comment says why), and
 # a link line is where those splits
@@ -169,7 +208,7 @@ fi
 # here keeps every one of those correct, and keeps "the runtime" one thing to
 # name from the outside. A caller that names one itself is left alone, because
 # naming one object twice is a duplicate-symbol error.
-halves="runtime-os.c runtime-parallel.c runtime-cross.c runtime-host.c runtime-net.c runtime-simd.c runtime-aes.c"
+halves="runtime-os.c runtime-parallel.c runtime-cross.c runtime-host.c runtime-net.c runtime-simd.c runtime-aes.c runtime-term.c runtime-bytes.c"
 for i in ${inputs[@]+"${inputs[@]}"}; do
   case "$i" in
     */runtime.c|runtime.c)
@@ -184,9 +223,49 @@ for i in ${inputs[@]+"${inputs[@]}"}; do
   esac
 done
 
+# SQLite (runtime/sqlite/) is an optional unit, and the IR selects it. A module
+# that declares a function of SQLite's C API (`sqlite3_*`) or of the adapter
+# beside it (`nishsqlite_*`) gets every runtime/sqlite/*.c compiled and
+# linked. Any other program compiles none of it, so its link line and binary
+# stay as they were. The test reads `declare` lines, and only before the first
+# `"` on one, rather than any `@sqlite3_` in the file, because a string
+# constant or a quoted name may hold one. It is one rule with `callsSqlite` in
+# src/run-cache.ts, which decides whether a link asks the runtime cache for
+# SQLite's objects; change both or neither. A caller
+# that names a runtime/sqlite/*.c input itself is left alone, as `halves`
+# leaves a runtime half: naming one object twice is a duplicate-symbol error.
+# A tree without the directory still links every program that does not call it.
+sqlite_dir="$(cd "$(dirname "$0")/../runtime/sqlite" 2>/dev/null && pwd -P)" || sqlite_dir=""
+sqlite_wanted=0
+if [ -n "$sqlite_to" ]; then
+  if [ ${#inputs[@]} -gt 0 ]; then
+    echo "error: --sqlite-objects compiles runtime/sqlite/ alone, not ${inputs[0]}" >&2; exit 2
+  fi
+  sqlite_wanted=1
+fi
+for i in ${inputs[@]+"${inputs[@]}"}; do
+  case "$i" in
+    *.ll) if grep -Eq '^declare [^"]*@(sqlite3|nishsqlite)_' "$i"; then sqlite_wanted=1; fi ;;
+  esac
+done
+for i in ${inputs[@]+"${inputs[@]}"}; do
+  case "$i" in
+    *.c) if [ -n "$sqlite_dir" ] && [ "$(cd "$(dirname -- "$i")" && pwd -P)" = "$sqlite_dir" ]; then sqlite_wanted=0; break; fi ;;
+  esac
+done
+if [ "$sqlite_wanted" = 1 ] && [ -z "$sqlite_dir" ]; then
+  echo "error: the module calls SQLite, and $(dirname "$0")/../runtime/sqlite/ is not there" >&2
+  exit 2
+fi
+if [ "$sqlite_wanted" = 1 ] && [ "$profile" = wasm ]; then
+  echo "error: the module calls SQLite, which the wasm profile cannot link: it has no libc for runtime/sqlite/sqlite3.c to build on; use --profile wasi" >&2
+  exit 2
+fi
+
 CC=${CC:-clang}
 common=(-Wno-override-module)          # our IR is target-neutral; clang fills the triple in
 elf=()                                 # flags that only make sense for ELF targets
+lld=0                                  # 1 when the link goes through ld.lld
 
 # Platform-specific dead-stripping, symbol stripping and LTO linker selection.
 case "$(uname -s)" in
@@ -218,7 +297,7 @@ case "$(uname -s)" in
     gc=(-Wl,--gc-sections -Wl,--as-needed -Wl,-O2 -Wl,--build-id=none); strip_flag=(-s)
     elf=(-fno-plt)
     # GNU ld needs the gold plugin for LTO; prefer lld when clang can find it.
-    if command -v ld.lld >/dev/null 2>&1; then common+=(-fuse-ld=lld); fi ;;
+    if command -v ld.lld >/dev/null 2>&1; then common+=(-fuse-ld=lld); lld=1; fi ;;
 esac
 
 # -g: compile everything with debug info and never strip, whatever the profile,
@@ -255,6 +334,39 @@ if [ "$debug" = 1 ]; then common+=(-g); strip_flag=(); fi
 # float than the baseline build (docs/LANGUAGE.md, "The CPU level").
 if [ -n "$cpu" ]; then common+=(-march=x86-64-v3 -ffp-contract=off); fi
 
+# Hardening, on every native profile (the wasm and wasi ones build their own
+# command line and have none of these to give). docs/security/runtime.md
+# records what each one closes and what it costs.
+#
+# The control-flow flag names the target's own scheme: -fcf-protection is an
+# error on arm64 and -mbranch-protection one on x86, so the triple decides.
+# Nish's IR carries both targets' module flags, and each backend reads its own;
+# these make clang's C units agree, because the linker marks the binary only
+# when every object asks for it. -fstack-clash-protection is ELF's alone: on
+# Mach-O clang ignores it with a warning, and the kernel's guard is the stack
+# itself. -z relro -z now leaves the GOT read-only after relocation, and
+# -z noexecstack says so even when an object without the note slips in.
+#
+# -fstack-protector-strong and _FORTIFY_SOURCE reach the C runtime alone: on a
+# .ll input clang adds no ssp attribute and the macro has no source to change,
+# so one flag list serves the one-command link too. FORTIFY wants an optimiser
+# and is left out of debug, where it would only warn. -U first, because a
+# distribution clang may already define it at another level.
+case "$("$CC" -dumpmachine 2>/dev/null)" in
+  x86_64*|amd64*) common+=(-fcf-protection=full); landing=1 ;;
+  aarch64*|arm64*) common+=(-mbranch-protection=standard); landing=1 ;;
+  *) landing=0 ;;
+esac
+common+=(-fstack-protector-strong)
+if [ "$profile" != debug ]; then common+=(-U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=3); fi
+case "$(uname -s)" in
+  Darwin) ;;
+  *)
+    common+=(-fstack-clash-protection)
+    # shellcheck disable=SC2054 # -Wl,<flag> is one argument: the comma is the linker's
+    common+=(-Wl,-z,relro,-z,now -Wl,-z,noexecstack) ;;
+esac
+
 tls=()
 if [ "$threads" = 1 ]; then
   common+=(-DNISH_THREADS=1 -ftls-model=initial-exec -pthread)
@@ -274,47 +386,169 @@ case "$profile" in
     optimise=(-Oz -flto -DNDEBUG ${pgo[@]+"${pgo[@]}"}
       -ffunction-sections -fdata-sections -fomit-frame-pointer
       -fno-asynchronous-unwind-tables -fno-unwind-tables ${elf[@]+"${elf[@]}"}
-      -fno-stack-protector -fvisibility=hidden) ;;
+      -fvisibility=hidden) ;;
+  napi)
+    optimise=(-O3 -flto -DNDEBUG ${pgo[@]+"${pgo[@]}"}
+      -ffunction-sections -fdata-sections -fomit-frame-pointer
+      -fno-asynchronous-unwind-tables -fno-unwind-tables ${elf[@]+"${elf[@]}"}) ;;
 esac
 
+# The speed link splits full LTO's code generation into one partition per core,
+# run in parallel: lld's --lto-partitions, after the whole program is optimised
+# as one module, so inlining does not change. But a partition boundary promotes
+# the internal functions called across it to external ones, and under the
+# control-flow hardening above every external function opens with a landing pad
+# (endbr64, or bti c), which the hot calls then run: fib runs 6.5% more
+# instructions with two partitions than with one. So a target with landing pads
+# links in one partition, and the split is left to the rest. The bytes depend
+# on the count, so one machine links one input to one output, as
+# scripts/bootstrap.sh needs. ld64 and GNU ld have no such flag and link as
+# before, and so does every other profile. The count comes from nproc (Linux),
+# else getconf (both), else sysctl (macOS).
+lto_parts=()
+if [ "$profile" = speed ] && [ "$lld" = 1 ] && [ "$landing" = 0 ]; then
+  cores=$(nproc 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null ||
+    sysctl -n hw.ncpu 2>/dev/null || echo 1)
+  # shellcheck disable=SC2054 # -Wl,<flag> is one argument: the comma is the linker's
+  lto_parts=(-Wl,--lto-partitions="$cores")
+fi
+
 # --runtime-objects and --runtime-from: the one-command link split in two, with
 # every compile it held run at once. A `$CC -c` takes `common` and `optimise`,
 # which is what the one command compiles each input with -- less
-# -fuse-ld=lld, which only names the linker -- so a runtime unit compiles to the
-# same object either way: native code for debug, -flto bitcode for speed and
-# size, which the link still optimises together with the program. The link
-# takes the same line the profile always uses, with objects for sources.
-if [ -n "$objects_to" ] || [ -n "$objects_from" ]; then
-  compile=()
-  for flag in "${common[@]}"; do
-    case "$flag" in -fuse-ld=*) ;; *) compile+=("$flag") ;; esac
+# -fuse-ld=lld and the -Wl, flags, which only speak to the linker -- so a
+# runtime unit compiles to the same object either way: native code for debug,
+# -flto bitcode for speed and size, which the link still optimises together
+# with the program. The link takes the same line the profile always uses, with
+# objects for sources.
+compile=()
+for flag in "${common[@]}"; do
+  case "$flag" in -fuse-ld=*|-Wl,*) ;; *) compile+=("$flag") ;; esac
+done
+compile+=(${optimise[@]+"${optimise[@]}"})
+# A program's modules are .ll alone, and clang warns that a -U it cannot
+# apply to one went unused, so their compile leaves the FORTIFY pair out.
+module_compile=()
+for flag in "${compile[@]}"; do
+  case "$flag" in -U_FORTIFY_SOURCE|-D_FORTIFY_SOURCE=*) ;; *) module_compile+=("$flag") ;; esac
+done
+# Waits for every job, not just the last, and fails when any of them did.
+# bash 3.2 has no `wait -n`, so the jobs are waited for in order.
+pids=()
+join_jobs() {
+  local failed=0 pid
+  for pid in ${pids[@]+"${pids[@]}"}; do
+    wait "$pid" || failed=1
   done
-  compile+=(${optimise[@]+"${optimise[@]}"})
-  # Waits for every job, not just the last, and fails when any of them did.
-  # bash 3.2 has no `wait -n`, so the jobs are waited for in order.
   pids=()
-  join_jobs() {
-    local failed=0 pid
-    for pid in ${pids[@]+"${pids[@]}"}; do
-      wait "$pid" || failed=1
-    done
+  return "$failed"
+}
+# The compiles are this script's children, so a signal that stops it stops
+# them too, before anything they write into is removed. A finished script
+# has waited for every one of them, so there is nothing left to stop.
+stop_jobs() {
+  if [ ${#pids[@]} -gt 0 ]; then
+    kill "${pids[@]}" 2>/dev/null || true
+    wait "${pids[@]}" 2>/dev/null || true
     pids=()
-    return "$failed"
-  }
-  # The compiles are this script's children, so a signal that stops it stops
-  # them too, before anything they write into is removed. A finished script
-  # has waited for every one of them, so there is nothing left to stop.
-  stop_jobs() {
-    if [ ${#pids[@]} -gt 0 ]; then
-      kill "${pids[@]}" 2>/dev/null || true
-      wait "${pids[@]}" 2>/dev/null || true
-      pids=()
-    fi
-  }
-  work=""
-  trap 'stop_jobs; [ -z "$work" ] || rm -rf -- "$work"' EXIT
-  trap 'exit 130' INT
-  trap 'exit 143' TERM
+  fi
+}
+work=""
+trap 'stop_jobs; [ -z "$work" ] || rm -rf -- "$work"' EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+# A link's own objects go in a private directory of its own and go with it.
+make_work() {
+  [ -n "$work" ] || work=$(mktemp -d "${TMPDIR:-/tmp}/nish-link.XXXXXX")
+}
+# A CC named by a relative path is named from here, not from a unit's directory.
+case "$CC" in /*) ;; */*) CC="$(pwd)/$CC" ;; esac
+
+# SQLite's compile-time options (https://sqlite.org/compile.html): the set the
+# SQLite project recommends for a program that needs none of the legacy
+# behaviours, less two of it. SQLITE_OMIT_DECLTYPE is left out because it
+# removes sqlite3_column_decltype, which a binding reads to tell a column's
+# declared type. SQLITE_OMIT_AUTOINIT is left out because it makes a call
+# before sqlite3_initialize() undefined behaviour, and the check it saves is
+# a branch in a few entry points. Double-quoted strings are identifiers only. There are no memory
+# statistics, no progress callback, no shared cache and no loadable extensions.
+# Expression depth is bounded by the stack, not a counter. A program has one
+# thread unless it was built with --threads. SQLite takes no mutex for one
+# thread (THREADSAFE=0). Under --threads it is multi-thread mode (=2): no mutex
+# guards a connection, so a program must not share one across threads. wasi
+# has no threads, so it gets 0 whatever the module says. NISH_SQLITE_CFLAGS is
+# appended: `read` takes its first line and splits it on whitespace, so a
+# word cannot hold a space and a quote is an ordinary character. A relative
+# path in it (-Iinclude) resolves against runtime/sqlite/, the directory every
+# unit is compiled from. A -D or -U in it for one of these names replaces the
+# default, so the compile does not warn about a macro defined twice.
+if [ "$threads" = 1 ] && [ "$profile" != wasi ]; then sqlite_threadsafe=2; else sqlite_threadsafe=0; fi
+sqlite_extra=()
+read -r -a sqlite_extra <<< "${NISH_SQLITE_CFLAGS:-}"
+sqlite_defs=()
+for d in -DSQLITE_DQS=0 -DSQLITE_DEFAULT_MEMSTATUS=0 -DSQLITE_DEFAULT_WAL_SYNCHRONOUS=1 \
+         -DSQLITE_LIKE_DOESNT_MATCH_BLOBS -DSQLITE_MAX_EXPR_DEPTH=0 -DSQLITE_OMIT_DEPRECATED \
+         -DSQLITE_OMIT_PROGRESS_CALLBACK -DSQLITE_OMIT_SHARED_CACHE -DSQLITE_USE_ALLOCA \
+         -DSQLITE_OMIT_LOAD_EXTENSION "-DSQLITE_THREADSAFE=$sqlite_threadsafe"; do
+  name=${d#-D}; name=${name%%=*}
+  keep=1
+  for e in ${sqlite_extra[@]+"${sqlite_extra[@]}"}; do
+    case "$e" in "-D$name"|"-D$name="*|"-U$name") keep=0 ;; esac
+  done
+  if [ "$keep" = 1 ]; then sqlite_defs+=("$d"); fi
+done
+sqlite_defs+=(${sqlite_extra[@]+"${sqlite_extra[@]}"})
+
+# Compiles every runtime/sqlite/*.c with the flags given, all at once, into
+# $1/<unit>.o. Each unit is compiled from inside runtime/sqlite/ by its bare
+# name, as the runtime is, so -g's DWARF names no directory the link was
+# started in. sqlite3.c, the 9.6 MB amalgamation, is compiled to native code
+# even where the profile compiles to LTO bitcode. As bitcode, every link of a
+# program that uses it would generate its code again: measured on a program
+# that opens a database and runs a query, that is 10.6 s a link at -O3,
+# against 0.04 s for the native object. The adapter beside it stays LTO, and
+# it is what inlines into the program. So `-flto` comes off for that one unit
+# alone.
+sqlite_units() {
+  local to=$1 unit flag
+  local -a flags
+  shift
+  for unit in "$sqlite_dir"/*.c; do
+    unit=${unit##*/}
+    flags=()
+    for flag in "$@"; do
+      case "$unit:$flag" in sqlite3.c:-flto) ;; *) flags+=("$flag") ;; esac
+    done
+    (cd -- "$sqlite_dir" && exec "$CC" "${flags[@]}" "${sqlite_defs[@]}" -I. -c "$unit" -o "$to/${unit%.c}.o") &
+    pids+=("$!")
+  done
+  join_jobs || { echo "error: runtime/sqlite/ did not compile" >&2; exit 1; }
+}
+# Adds the objects of runtime/sqlite/ to this link, in the order of the units.
+# They are compiled into the link's own directory with the flags given, or
+# taken from --sqlite-from. Nothing is added for a module that does not call
+# SQLite.
+sqlite_objects() {
+  local from=$sqlite_from unit
+  [ "$sqlite_wanted" = 1 ] || return 0
+  if [ -z "$from" ]; then
+    make_work
+    from="$work/sqlite"
+    mkdir -p -- "$from"
+    sqlite_units "$from" "$@"
+  fi
+  for unit in "$sqlite_dir"/*.c; do
+    unit=${unit##*/}
+    [ -f "$from/${unit%.c}.o" ] || { echo "error: --sqlite-from: $from/${unit%.c}.o is missing" >&2; exit 2; }
+    inputs+=("$from/${unit%.c}.o")
+  done
+}
+
+if [ -n "$sqlite_to" ]; then
+  mkdir -p -- "$sqlite_to"
+  sqlite_to=$(cd -- "$sqlite_to" && pwd)
+  sqlite_units "$sqlite_to" "${compile[@]}"
+  exit 0
 fi
 
 if [ -n "$objects_to" ]; then
@@ -334,8 +568,6 @@ if [ -n "$objects_to" ]; then
   # the file `runtime.c` in the runtime's own directory, whichever directory
   # the first link that compiled it was run in.
   runtime_dir=$(dirname -- "$runtime")
-  # A CC named by a relative path is named from here, not from there.
-  case "$CC" in /*) ;; */*) CC="$(pwd)/$CC" ;; esac
   for unit in runtime.c $halves; do
     (cd -- "$runtime_dir" && exec "$CC" "${compile[@]}" -c "$unit" -o "$objects_to/${unit%.c}.o") &
     pids+=("$!")
@@ -348,15 +580,13 @@ if [ -n "$objects_from" ]; then
   for unit in runtime.c $halves; do
     [ -f "$objects_from/${unit%.c}.o" ] || { echo "error: --runtime-from: $objects_from/${unit%.c}.o is missing" >&2; exit 2; }
   done
-  # The program's own objects are this link's alone, so they go in a private
-  # directory of its own and go with it.
-  work=$(mktemp -d "${TMPDIR:-/tmp}/nish-link.XXXXXX")
+  make_work
   objects=()
   n=0
   for i in "${inputs[@]}"; do
     case "$i" in */runtime.c|runtime.c) echo "error: --runtime-from links the runtime in $objects_from, so $i is not an input" >&2; exit 2 ;; esac
     n=$((n + 1))
-    "$CC" "${compile[@]}" -c "$i" -o "$work/$n.o" &
+    "$CC" "${module_compile[@]}" -c "$i" -o "$work/$n.o" &
     pids+=("$!")
     objects+=("$work/$n.o")
   done
@@ -378,11 +608,12 @@ fi
 # under `set -u` is a fatal "unbound variable" -- bash 4.4 made it legal, which is
 # why Linux never noticed. `pgo`, `elf`, `strip_flag` and `libs` are all empty on
 # ordinary builds, so please do not simplify these back.
+case "$profile" in debug|speed|size) sqlite_objects "${compile[@]}" ;; esac
 case "$profile" in
   debug)
     "$CC" "${common[@]}" "${inputs[@]}" -lm -o "$out" ;;
   speed|size)
-    "$CC" "${common[@]}" "${optimise[@]}" \
+    "$CC" "${common[@]}" "${optimise[@]}" ${lto_parts[@]+"${lto_parts[@]}"} \
       "${gc[@]}" ${strip_flag[@]+"${strip_flag[@]}"} "${inputs[@]}" -lm -o "$out" ;;
   wasm)
     # clang resolves wasm-ld next to its own binary first, then on PATH; ask it
@@ -409,6 +640,21 @@ case "$profile" in
     if [ "$aes_wanted" = 1 ] && [ "$aes_named" = 0 ]; then
       inputs+=("$(cd "$(dirname "$0")/../runtime" && pwd)/runtime-aes.c")
     fi
+    # IR of either kind that imports std/json or calls nish/bytes's byte-class runs
+    # calls runtime-bytes.c's kernels (src/kernels.ts), which compile freestanding:
+    # the scalar path, or the 128-bit one under -msimd128. It is added the same way,
+    # only for such IR and only when the caller has not named it.
+    bytes_named=0
+    bytes_wanted=0
+    for i in ${inputs[@]+"${inputs[@]}"}; do
+      case "$i" in
+        */runtime-bytes.c|runtime-bytes.c) bytes_named=1 ;;
+        *.ll) if grep -q '@nish_bytes_skip_' "$i"; then bytes_wanted=1; fi ;;
+      esac
+    done
+    if [ "$bytes_wanted" = 1 ] && [ "$bytes_named" = 0 ]; then
+      inputs+=("$(cd "$(dirname "$0")/../runtime" && pwd)/runtime-bytes.c")
+    fi
     # -mbulk-memory lowers llvm.memset/memcpy (`new Array<T>(n)`, `push` growth) to the
     # memory.fill/memory.copy instructions instead of libc calls the freestanding link lacks.
     "$CC" -Wno-override-module --target=wasm32-unknown-unknown -Oz -nostdlib -mbulk-memory \
@@ -450,6 +696,10 @@ case "$profile" in
         exit 2
       fi
     fi
+    # SQLite builds on wasi-libc as it is: the amalgamation sees __wasi__ and
+    # takes dot-file locking, no mmap and no WAL shared memory.
+    sqlite_objects --target=wasm32-wasi --sysroot="$sysroot" -Oz -DNDEBUG \
+      ${tls[@]+"${tls[@]}"} -ffunction-sections -fdata-sections
     "$CC" -Wno-override-module --target=wasm32-wasi --sysroot="$sysroot" -Oz -DNDEBUG \
       ${tls[@]+"${tls[@]}"} \
       -ffunction-sections -fdata-sections -Wl,--gc-sections -Wl,--strip-all \
@@ -476,9 +726,11 @@ case "$profile" in
     case "$(uname -s)" in Darwin) shared+=(-Wl,-undefined,dynamic_lookup) ;; esac
     # runtime/nish.h is the public ABI header the generated shim includes.
     runtime_inc="$(cd "$(dirname "$0")/../runtime" && pwd)"
-    "$CC" "${common[@]}" -O3 -flto -DNDEBUG ${pgo[@]+"${pgo[@]}"} -I"$node_inc" -I"$runtime_inc" \
-      -ffunction-sections -fdata-sections -fomit-frame-pointer \
-      -fno-asynchronous-unwind-tables -fno-unwind-tables ${elf[@]+"${elf[@]}"} \
+    # SQLite's API stays inside the addon: a Node linked against a system
+    # libsqlite3 exports the same names, and a call bound to that copy would
+    # run another version with other options.
+    sqlite_objects "${compile[@]}" -fPIC -fvisibility=hidden
+    "$CC" "${common[@]}" "${optimise[@]}" -I"$node_inc" -I"$runtime_inc" \
       "${shared[@]}" "${gc[@]}" ${strip_flag[@]+"${strip_flag[@]}"} "${inputs[@]}" -lm -o "$out" ;;
   *) echo "error: unknown profile '$profile'" >&2; exit 2 ;;
 esac

package/std/LICENSE-fast-float

@@ -0,0 +1,23 @@
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.

package/std/LICENSE-fdlibm

@@ -0,0 +1,8 @@
+====================================================
+Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved.
+
+Developed at SunPro, a Sun Microsystems, Inc. business.
+Permission to use, copy, modify, and distribute this
+software is freely granted, provided that this notice
+is preserved.
+====================================================