@elixir-playground/git 0.0.0-stage → 1.0.0

package/source/git/tools/wasi-compat/autoconf.py

@@ -0,0 +1,158 @@
+"""A GNU tool (autoconf and gnulib) built to WebAssembly (wasm32-wasip1)
+over tools/wasi-compat/: what tools/sed/build.py and the others like it share
+(docs/tools.md). Each names its release and what to keep:
+
+    build(HERE, "sed", ("4.10", URL, SHA256), {"sed": "sed/sed"}, ["--disable-acl"])
+    # → target/sed/sed.wasm
+
+The release tarball by URL and SHA-256, the tool's patches/ applied, its
+own configure --host=wasm32-wasip1 and make in wasi-sdk 33's image (as
+tools/git/build.py's, pinned by digest), -Oz. What WASI lacks is stubs.c's
+(built as git's is, wasi-compat.h forced in), declared by wasi-posix.h,
+which goes at the end of the tool's config.h: gnulib's replacement
+headers want config.h first, and bring their own signals. What configure
+cannot learn when cross-compiling, config.site answers: this directory's
+for stubs.c's functions, then the tool's own.
+
+Loaded by path (python3 -I leaves the script's directory off sys.path).
+"""
+
+import hashlib
+import importlib.util
+import os
+import shutil
+import subprocess
+import tarfile
+import urllib.request
+
+# The repository (this file is tools/wasi-compat/autoconf.py).
+ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+COMPAT = os.path.dirname(os.path.abspath(__file__))
+DOCKER = os.environ.get("DOCKER", "sudo -n docker").split()
+# wasi-sdk 33 (clang 23), as the backend's NIF toolchain and tools/git/build.py.
+IMAGE = "ghcr.io/webassembly/wasi-sdk@sha256:52595085a0ffbaf574d8d678a9aa81305f0b3af81c639291f15bc7713e42848e"
+CC = "/opt/wasi-sdk/bin/clang --target=wasm32-wasip1 --sysroot=/opt/wasi-sdk/share/wasi-sysroot"
+# wasi-libc's emulations (signal, raise, clock, getpid, mmap), as git's.
+EMULATED = ("signal", "process-clocks", "getpid", "mman")
+CFLAGS = "-Oz -I/wasi/include " + " ".join(f"-D_WASI_EMULATED_{e.upper().replace('-', '_')}" for e in EMULATED)
+# wrap.o wraps wasi-libc's fcntl (F_DUPFD by dup) and stat's kin (modes).
+WRAPPED = ("fcntl", "stat", "lstat", "fstat", "fstatat")
+LIBS = ("/wasi/stubs.o /wasi/wrap.o " + " ".join(f"-Wl,--wrap={name}" for name in WRAPPED) + " "
+        + " ".join(f"-lwasi-emulated-{e}" for e in EMULATED))
+
+
+def fetch(source, sha256, path):
+    if not os.path.exists(path):
+        urllib.request.urlretrieve(source, path + ".part")
+        with open(path + ".part", "rb") as f:
+            got = hashlib.sha256(f.read()).hexdigest()
+        if got != sha256:
+            raise SystemExit(f"{source}: sha256 {got}, not {sha256}")
+        os.rename(path + ".part", path)
+
+
+def stamp(here, release, configure):
+    """What the build is made from: the image, the release, the options,
+    the tool's files and this directory's (not .py files)."""
+    hashed = hashlib.sha256("\0".join([IMAGE, release[2], *configure, ""]).encode())
+    for top in (here, COMPAT):
+        for directory, _, names in sorted(os.walk(top)):
+            for name in sorted(names):
+                if name.endswith(".py") or name == "__pycache__":
+                    continue
+                path = os.path.join(directory, name)
+                with open(path, "rb") as f:
+                    hashed.update(os.path.relpath(path, ROOT).encode() + b"\0" + f.read())
+    return hashed.hexdigest()
+
+
+def licensed(here, name, release, made):
+    """target/NAME/licenses (tools/licenses.py) and, the tool being GPL,
+    target/NAME/source: the release, its patches and the build's files."""
+    spec = importlib.util.spec_from_file_location("licenses", os.path.join(ROOT, "tools", "licenses.py"))
+    licenses = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(licenses)
+    out = os.path.join(ROOT, "target", name)
+    if licenses.current(out, made):
+        return
+    version, source, sha256 = release
+    archive = os.path.join(out, os.path.basename(source))
+    fetch(source, sha256, archive)
+    tool = os.path.relpath(here, ROOT)
+    sources = {os.path.basename(source): archive, f"{tool}/build.py": os.path.join(here, "build.py"),
+               "tools/wasi-compat": COMPAT}
+    for extra in ("patches", "config.site"):
+        if os.path.exists(os.path.join(here, extra)):
+            sources[f"{tool}/{extra}"] = os.path.join(here, extra)
+    inventory = licenses.c_tool(out, name, version, "GPL-3.0-or-later", f"https://www.gnu.org/software/{name}/",
+                                archive, ["COPYING"], sources, f"{tool}/build.py")
+    licenses.finish(inventory, out, made)
+
+
+def build(here, name, release, programs, configure=()):
+    """`name`'s `release` (version, URL, SHA-256) built; `programs` maps
+    each wasm to keep (target/NAME/KEY.wasm) to what make builds (a path
+    in the source tree, its make target too)."""
+    out = os.path.join(ROOT, "target", name)
+    os.makedirs(out, exist_ok=True)
+    built = os.path.join(out, "stamp")
+    wasm = {key: os.path.join(out, f"{key}.wasm") for key in programs}
+    made = stamp(here, release, configure)
+    if all(map(os.path.exists, wasm.values())) and os.path.exists(built):
+        with open(built) as f:
+            if f.read() == made:
+                return licensed(here, name, release, made)
+    version, source, sha256 = release
+    archive = os.path.join(out, os.path.basename(source))
+    fetch(source, sha256, archive)
+    src = os.path.join(out, "src")
+    shutil.rmtree(src, ignore_errors=True)
+    with tarfile.open(archive) as tar:
+        tar.extractall(out, filter="data")
+    os.rename(os.path.join(out, f"{name}-{version}"), src)
+    patches = os.path.join(here, "patches")
+    for patch in sorted(os.listdir(patches)) if os.path.isdir(patches) else ():
+        subprocess.run(["patch", "-p1", "-s", "-d", src, "-i", os.path.join(patches, patch)], check=True)
+    wasi = os.path.join(out, "wasi")
+    shutil.rmtree(wasi, ignore_errors=True)
+    shutil.copytree(COMPAT, wasi, ignore=shutil.ignore_patterns("*.py", "__pycache__"))
+    sites = "/wasi/config.site" + (" /tool/config.site" if os.path.exists(os.path.join(here, "config.site")) else "")
+    script = (
+        f"{CC} {CFLAGS} -include /wasi/include/wasi-compat.h -c /wasi/stubs.c -o /wasi/stubs.o\n"
+        f"{CC} {CFLAGS} -c /wasi/wrap.c -o /wasi/wrap.o\n"
+        "cd /src\n"
+        f"CONFIG_SITE='{sites}' ./configure --host=wasm32-wasip1 --disable-nls {' '.join(configure)} "
+        f"CC='{CC}' CFLAGS='{CFLAGS}' LIBS='{LIBS}' >configure.out 2>&1 "
+        "|| { tail -20 configure.out; tail -40 config.log; exit 1; }\n"
+        # After config.h, whose #defines gnulib's headers want first.
+        # (config.h, or lib/config.h: diffutils'.)
+        "header=$(ls config.h lib/config.h 2>/dev/null | head -n 1)\n"
+        "echo '#include <wasi-posix.h>' >> ${header:?configure made no config.h}\n"
+        # The programs alone (gnulib-tests/ does not build for WASI), after
+        # what automake makes first for `all` (gnulib's generated headers).
+        "printf 'built-sources: $(BUILT_SOURCES)\\n' > built-sources.mk\n"
+        "make -s -j$(nproc) -f Makefile -f built-sources.mk built-sources >/dev/null\n"
+        # A recursive tree (grep's, diffutils': lib/, src/) makes gnulib's
+        # lib/ (gawk's support/; findutils' gl/lib/, then its own lib/)
+        # first, then each program in its directory; a flat one (sed's)
+        # each from the top.
+        "for lib in gl/lib lib support; do\n"
+        "  if [ -f $lib/Makefile ]; then make -s -j$(nproc) -C $lib >/dev/null; fi\n"
+        "done\n"
+        f"for program in {' '.join(programs.values())}; do\n"
+        "  dir=${program%/*}\n"
+        "  if [ -f $dir/Makefile ]; then make -s -j$(nproc) -C $dir ${program##*/} >/dev/null\n"
+        "  else make -s -j$(nproc) $program >/dev/null; fi\n"
+        "done\n"
+    )
+    subprocess.run(DOCKER + [
+        "run", "--rm", "--user", f"{os.getuid()}:{os.getgid()}", "-v", f"{src}:/src", "-v", f"{wasi}:/wasi",
+        "-v", f"{here}:/tool:ro", IMAGE, "sh", "-ec", script,
+    ], check=True)
+    for key, path in programs.items():
+        shutil.copy(os.path.join(src, path), wasm[key])
+    with open(built, "w") as f:
+        f.write(made)
+    licensed(here, name, release, made)
+    for path in wasm.values():
+        print(f"{path}: {os.path.getsize(path) / 1e3:.0f} KB")

package/source/git/tools/wasi-compat/config.site

@@ -0,0 +1,31 @@
+# Answers for a tool's configure (CONFIG_SITE, before the tool's own):
+# what wasi-posix.h declares and stubs.c defines is there and declared.
+# configure's checks see neither header (the build appends it to
+# config.h), so they would take each for missing and have gnulib replace
+# it, or not use it at all. Not fork and exec, which stubs.c has fail:
+# without them a tool starts programs by popen and system (diff3's diff,
+# sdiff's editor, diff --paginate's pr).
+ac_cv_func_fork=no
+ac_cv_func_fork_works=no
+ac_cv_func_vfork=no
+ac_cv_func_vfork_works=no
+for ac_func in getuid geteuid getgid getegid getgroups getppid getpgid getpgrp \
+    setsid tcgetpgrp umask chmod fchmod chown fchown dup dup2 pipe kill alarm getdtablesize getpass \
+    mkdtemp mkstemp popen pclose system sigaction sigemptyset sigfillset sigaddset sigdelset sigismember \
+    sigprocmask tzset setmntent getmntent endmntent hasmntopt; do
+  eval "ac_cv_func_$ac_func=yes"
+  eval "ac_cv_have_decl_$ac_func=yes"
+done
+# The mount table: mntent.h's (empty), as Linux's getmntent.
+ac_cv_header_mntent_h=yes
+fu_cv_sys_mounted_getmntent1=yes
+# getgroups: no supplementary groups (stubs.c), as gid_t.
+gl_cv_func_getgroups_works=yes
+ac_cv_type_getgroups=gid_t
+# Its sigset_t and struct sigaction (wasi-posix.h's), not wasi-libc's.
+gl_cv_type_sigset_t=yes
+ac_cv_type_sigset_t=yes
+ac_cv_member_struct_sigaction_sa_sigaction=no
+# sys/wait.h's macros are tools/wasi-compat/include's; configure's check also
+# calls wait(), which is not (the host waits for a pid).
+ac_cv_header_sys_wait_h=yes

package/source/git/tools/wasi-compat/include/grp.h

@@ -0,0 +1,6 @@
+#pragma once
+#include <sys/types.h>
+struct group { char *gr_name; char *gr_passwd; gid_t gr_gid; char **gr_mem; };
+struct group *getgrgid(gid_t);
+struct group *getgrnam(const char *);
+int initgroups(const char *, gid_t);

package/source/git/tools/wasi-compat/include/mntent.h

@@ -0,0 +1,10 @@
+/* The mount table, for gnulib's mountlist (find -fstype): WASI has none
+   to list, so it reads as empty (stubs.c: setmntent opens /dev/null). */
+#pragma once
+#include <stdio.h>
+#define MOUNTED "/etc/mtab"
+struct mntent { char *mnt_fsname; char *mnt_dir; char *mnt_type; char *mnt_opts; int mnt_freq; int mnt_passno; };
+FILE *setmntent(const char *, const char *);
+struct mntent *getmntent(FILE *);
+int endmntent(FILE *);
+char *hasmntopt(const struct mntent *, const char *);

package/source/git/tools/wasi-compat/include/netdb.h

@@ -0,0 +1,7 @@
+#pragma once
+#include <sys/socket.h>
+struct hostent { char *h_name; char **h_aliases; int h_addrtype; int h_length; char **h_addr_list; };
+#define h_addr h_addr_list[0]
+struct hostent *gethostbyname(const char *);
+const char *hstrerror(int);
+extern int h_errno;

package/source/git/tools/wasi-compat/include/paths.h

@@ -0,0 +1,2 @@
+#pragma once
+#define _PATH_DEFPATH "/usr/bin:/bin"

package/source/git/tools/wasi-compat/include/pwd.h

@@ -0,0 +1,5 @@
+#pragma once
+#include <sys/types.h>
+struct passwd { char *pw_name; char *pw_passwd; uid_t pw_uid; gid_t pw_gid; char *pw_gecos; char *pw_dir; char *pw_shell; };
+struct passwd *getpwuid(uid_t);
+struct passwd *getpwnam(const char *);

package/source/git/tools/wasi-compat/include/setjmp.h

@@ -0,0 +1,17 @@
+/* setjmp without the jump: WASI has none without wasm's exception
+   handling (wasi-libc's setjmp.h refuses). setjmp returns 0; a longjmp,
+   which the tools here take only where WASI never leads them (gawk's
+   debugger, diff -r's failing strcoll), ends the program saying so. */
+#pragma once
+#include <stdio.h>
+#include <stdlib.h>
+typedef int jmp_buf[1];
+typedef int sigjmp_buf[1];
+#define setjmp(env) 0
+#define sigsetjmp(env, save) 0
+#define longjmp(env, val) wasi_longjmp()
+#define siglongjmp(env, val) wasi_longjmp()
+static inline _Noreturn void wasi_longjmp(void) {
+  fputs("longjmp: not on WASI (no exception handling)\n", stderr);
+  exit(70);
+}

package/source/git/tools/wasi-compat/include/sys/wait.h

@@ -0,0 +1,8 @@
+#pragma once
+#include <sys/types.h>
+#define WNOHANG 1
+#define WIFEXITED(s) (((s) & 0x7f) == 0)
+#define WEXITSTATUS(s) (((s) >> 8) & 0xff)
+#define WIFSIGNALED(s) (((s) & 0x7f) != 0 && ((s) & 0x7f) != 0x7f)
+#define WTERMSIG(s) ((s) & 0x7f)
+pid_t waitpid(pid_t, int *, int);

package/source/git/tools/wasi-compat/include/syslog.h

@@ -0,0 +1,7 @@
+#pragma once
+#define LOG_ERR 3
+#define LOG_INFO 6
+#define LOG_PID 1
+#define LOG_DAEMON (3<<3)
+static inline void openlog(const char *i, int o, int f) { (void)i; (void)o; (void)f; }
+static inline void syslog(int p, const char *f, ...) { (void)p; (void)f; }

package/source/git/tools/wasi-compat/include/termios.h

@@ -0,0 +1,10 @@
+#pragma once
+typedef unsigned tcflag_t; typedef unsigned char cc_t;
+struct termios { tcflag_t c_iflag, c_oflag, c_cflag, c_lflag; cc_t c_cc[32]; };
+#define ECHO 8
+#define ICANON 2
+#define VMIN 6
+#define VTIME 5
+#define TCSAFLUSH 2
+int tcgetattr(int, struct termios *);
+int tcsetattr(int, int, const struct termios *);

package/source/git/tools/wasi-compat/include/wasi-compat.h

@@ -0,0 +1,38 @@
+/* What git asks of POSIX that WASI preview 1 lacks, forced into each of
+   its files (config.mak's -include): wasi-posix.h's declarations, and
+   what git wants of stat, timers and sockets beyond them. A tool built
+   with autoconf and gnulib takes wasi-posix.h alone, after its config.h
+   (gnulib's replacement headers want config.h first). */
+#pragma once
+#include "wasi-posix.h"
+/* stat, lstat and fstat with a mode's permission bits (stubs.c): WASI's
+   have the type alone, and git takes a mode of S_IFDIR alone for a sparse
+   index's directory, not a directory (a submodule's gitlink). Function-like,
+   so that `struct stat` stays itself. */
+struct stat;
+int wasi_stat(const char *, struct stat *);
+int wasi_lstat(const char *, struct stat *);
+int wasi_fstat(int, struct stat *);
+#define stat(path, st) wasi_stat(path, st)
+#define lstat(path, st) wasi_lstat(path, st)
+#define fstat(fd, st) wasi_fstat(fd, st)
+/* The host's coroutines for start_async (run-command.c's WASI branch):
+   one runs wasi_run_async(async) on a stack of its own (*stack, for the
+   caller to free once it waited for the pid); the async that runs now
+   (NULL: git's own); and ending it. */
+pid_t wasi_async_start(void *async, void **stack);
+void *wasi_async_current(void);
+_Noreturn void wasi_async_exit(int code);
+
+/* Signals beyond wasi-posix.h's (whose sigset_t signal.h then takes). */
+#include <signal.h>
+#define ITIMER_REAL 0
+
+/* Sockets: none (git://, ssh go through the host or not at all). */
+#include <sys/socket.h>
+#define SO_KEEPALIVE 9
+struct servent { char *s_name; char **s_aliases; int s_port; char *s_proto; };
+struct servent *getservbyname(const char *, const char *);
+int socket(int, int, int);
+int connect(int, const struct sockaddr *, socklen_t);
+int setsockopt(int, int, int, const void *, socklen_t);

package/source/git/tools/wasi-compat/include/wasi-posix.h

@@ -0,0 +1,79 @@
+/* What a C tool asks of POSIX that WASI preview 1 lacks: declarations of
+   what stubs.c defines (wasi-libc declares some of them, defines none).
+   Processes are the host's (wasi_spawn and the `processes` imports);
+   users, modes and terminals answer as one user's would. Declarations
+   only, so that a tool's own portability layer (gnulib) keeps its
+   types: an autoconf build appends `#include <wasi-posix.h>` to its
+   config.h, git's wasi-compat.h includes it. */
+#pragma once
+#include <stdio.h>
+#include <sys/types.h>
+pid_t fork(void);
+int execv(const char *, char *const []);
+int execvp(const char *, char *const []);
+int execve(const char *, char *const [], char *const []);
+int execl(const char *, const char *, ...);
+int execlp(const char *, const char *, ...);
+uid_t getuid(void); uid_t geteuid(void); gid_t getgid(void); gid_t getegid(void);
+int getgroups(int, gid_t []);
+pid_t getppid(void);
+pid_t getpgid(pid_t);
+pid_t getpgrp(void);
+pid_t setsid(void);
+pid_t tcgetpgrp(int);
+mode_t umask(mode_t);
+int chmod(const char *, mode_t);
+int fchmod(int, mode_t);
+int chown(const char *, uid_t, gid_t);
+int fchown(int, uid_t, gid_t);
+int dup(int); int dup2(int, int);
+int pipe(int [2]);
+int kill(pid_t, int);
+unsigned alarm(unsigned);
+int getdtablesize(void);
+char *getpass(const char *);
+char *mkdtemp(char *);
+int mkstemp(char *);
+/* The host's spawn: argv[0] found by name as the shell finds it, in `dir`
+   (NULL: here), its standard streams our descriptors in, out and err; its
+   pid, or -1 (errno ENOENT: no such program). */
+pid_t wasi_spawn(const char **argv, char **env, const char *dir, int in, int out, int err);
+/* sh -c COMMAND (brush, a process of the host's) over wasi_spawn. */
+FILE *popen(const char *, const char *);
+int pclose(FILE *);
+int system(const char *);
+
+/* Signals: wasi-libc emulates signal() and raise() (-D_WASI_EMULATED_SIGNAL),
+   and its sigset_t is a placeholder of one byte. Sets of 64 signals,
+   masks kept and never applied (no signal arrives but raise()'s and the
+   host's kill, which ends a process), sigaction over signal() (stubs.c).
+   As POSIX's, gnulib has no blocking of its own to emulate (its emulation
+   wants NSIG <= 32; wasi-libc's is 65). */
+#include <__typedef_sigset_t.h>
+typedef unsigned long long wasi_sigset_t;
+#define sigset_t wasi_sigset_t
+#define SIG_BLOCK 0
+#define SIG_UNBLOCK 1
+#define SIG_SETMASK 2
+#define SA_RESTART 0x10000000
+struct sigaction { void (*sa_handler)(int); sigset_t sa_mask; int sa_flags; };
+int sigaction(int, const struct sigaction *, struct sigaction *);
+int sigemptyset(sigset_t *);
+int sigfillset(sigset_t *);
+int sigaddset(sigset_t *, int);
+int sigdelset(sigset_t *, int);
+int sigismember(const sigset_t *, int);
+int sigprocmask(int, const sigset_t *, sigset_t *);
+
+/* No time zones (WASI has no tables): UTC, and tzset does nothing. */
+void tzset(void);
+
+/* wasi-libc has an opendirat(fd, path) of its own; gnulib's (fts, for
+   grep -r and find) takes four arguments, so is gnulib's by another name
+   from here on. */
+#include <dirent.h>
+#define opendirat gnulib_opendirat
+
+/* wasi-libc has qsort_r (GNU's signature) and declares it only within its
+   `#ifdef` for wait, which WASI lacks. */
+void qsort_r(void *, size_t, size_t, int (*)(const void *, const void *, void *), void *);