@lakutata/ffi 0.0.0-stage → 3.1.5

package/dist/esm/lib/library/Resolve.js

@@ -0,0 +1,122 @@
+import { existsSync, readdirSync } from 'node:fs';
+import path from 'node:path';
+import koffi from 'koffi';
+import { NativeLibraryNotFoundException } from '../../exceptions/NativeLibraryNotFoundException.js';
+/**
+ * The file of the dynamic loader of musl, which holds its C library (Alpine has no libc.so.6)
+ */
+function muslLoader() {
+    try {
+        const loader = readdirSync('/lib').find((name) => /^ld-musl-.*\.so\.1$/.test(name));
+        return loader ? `/lib/${loader}` : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+/**
+ * The file of the C library (c) or of its mathematical functions (m) on a system
+ * @param name
+ * @param os
+ */
+function systemLibrary(name, os) {
+    switch (os) {
+        case 'darwin':
+            return '/usr/lib/libSystem.B.dylib';
+        case 'win32':
+            return 'ucrtbase.dll';
+        case 'linux':
+            return muslLoader() ?? (name === 'c' ? 'libc.so.6' : 'libm.so.6');
+        case 'freebsd':
+            return name === 'c' ? 'libc.so.7' : 'libm.so.5';
+        default:
+            return name === 'c' ? 'libc.so' : 'libm.so';
+    }
+}
+/**
+ * The file names a name gives on a system: the name itself when it has a directory or an extension, else the names the
+ * system gives libraries
+ * @param name
+ * @param os
+ */
+export function fileNames(name, os = process.platform) {
+    if (name === 'c' || name === 'm')
+        return [systemLibrary(name, os)];
+    if (/[/\\]/.test(name) || /\.(dylib|dll|so)(\.\d+)*$/i.test(name))
+        return [name];
+    switch (os) {
+        case 'darwin':
+            return [`lib${name}.dylib`, `${name}.dylib`];
+        case 'win32':
+            return [`${name}.dll`, `lib${name}.dll`];
+        default:
+            return [`lib${name}.so`, `${name}.so`];
+    }
+}
+/**
+ * The names of a library on a system
+ * @param names
+ * @param os
+ */
+export function libraryNames(names, os = process.platform) {
+    if (typeof names === 'string')
+        return fileNames(names, os);
+    const given = names[os] ?? names.default;
+    if (given === undefined)
+        throw new NativeLibraryNotFoundException(`The library has no name for ${os} (give one for ${os}, or a default)`);
+    return (typeof given === 'string' ? [given] : given).flatMap((name) => fileNames(name, os));
+}
+/**
+ * Load a library: the file given; else each name in the search directories, in the directory of a packaged
+ * application, then where the system looks for libraries (dlopen, LoadLibrary)
+ * @param names
+ * @param options
+ * @returns The library and the file it was loaded from.
+ */
+export function loadLibrary(names, options = {}) {
+    const attempts = [];
+    const attempt = (file) => {
+        try {
+            return koffi.load(file);
+        }
+        catch (error) {
+            attempts.push(`${file}: ${error.message}`);
+            return undefined;
+        }
+    };
+    if (options.path) {
+        const file = path.resolve(options.path);
+        const library = attempt(file);
+        if (library)
+            return { library: library, file: file };
+        throw new NativeLibraryNotFoundException(`The library cannot be loaded:\n${attempts.join('\n')}`);
+    }
+    const directories = [...options.search ?? [], ...options.packaged ? [options.packaged] : []].map((directory) => path.resolve(directory));
+    const candidates = libraryNames(names);
+    for (const candidate of candidates) {
+        //A name with a directory: that file only
+        if (/[/\\]/.test(candidate)) {
+            const library = attempt(candidate);
+            if (library)
+                return { library: library, file: candidate };
+            continue;
+        }
+        for (const directory of directories) {
+            const file = path.join(directory, candidate);
+            if (!existsSync(file)) {
+                attempts.push(`${file}: not found`);
+                continue;
+            }
+            const library = attempt(file);
+            if (library)
+                return { library: library, file: file };
+        }
+    }
+    for (const candidate of candidates.filter((name) => !/[/\\]/.test(name))) {
+        const library = attempt(candidate);
+        if (library)
+            return { library: library, file: candidate };
+    }
+    const versioned = process.platform === 'linux' && candidates.some((name) => name.endsWith('.so')) ? '\nOn Linux, the libraries of the system are often installed with their version only (libsqlite3.so.0): give that name.' : '';
+    throw new NativeLibraryNotFoundException(`The library ${candidates.join(' or ')} cannot be loaded:\n${attempts.join('\n')}${versioned}`);
+}

package/dist/esm/lib/thread/NativeThreads.js

@@ -0,0 +1,377 @@
+import { Worker } from 'node:worker_threads';
+import koffi from 'koffi';
+import { NativeDeclarationException } from '../../exceptions/NativeDeclarationException.js';
+import { NativeLibraryClosedException } from '../../exceptions/NativeLibraryClosedException.js';
+import { NativeLibraryNotFoundException } from '../../exceptions/NativeLibraryNotFoundException.js';
+import { NativeMemoryException } from '../../exceptions/NativeMemoryException.js';
+import { convertedCallback } from '../backend/Functions.js';
+import { TYPES } from '../Runtime.js';
+import { nativeWorkerCode } from './NativeWorkerCode.js';
+import { packageRequire } from './PackageRequire.js';
+/**
+ * How long the threads have to close once asked, in milliseconds
+ */
+const CLOSE_TIMEOUT = 5000;
+/**
+ * An error of a thread, made again with its class
+ * @param error
+ */
+function errorOf(error) {
+    const classes = {
+        NativeDeclarationException: NativeDeclarationException,
+        NativeLibraryClosedException: NativeLibraryClosedException,
+        NativeLibraryNotFoundException: NativeLibraryNotFoundException,
+        NativeMemoryException: NativeMemoryException,
+        TypeError: TypeError,
+        RangeError: RangeError
+    };
+    const made = new (classes[error.name] ?? Error)(error.message);
+    if (!classes[error.name])
+        made.name = error.name;
+    return made;
+}
+/**
+ * The value a callback returns to the library when it throws: zero, false, a null pointer
+ * @param type
+ */
+function zeroOf(type) {
+    if (type.kind === 'primitive')
+        return type.name === 'void' ? undefined : type.name === 'bool' ? false : 0;
+    if (type.kind === 'enum')
+        return 0;
+    if (type.kind === 'pointer')
+        return null;
+    return {};
+}
+/**
+ * Whether a parameter points to memory the function may write: not const, or marked _Out_ or _Inout_
+ * @param parameter
+ */
+function written(parameter) {
+    if (!parameter)
+        return true;
+    if (parameter.direction !== 'in')
+        return true;
+    return parameter.type.kind === 'pointer' && !parameter.type.constTarget;
+}
+/**
+ * The worker threads of a library: they load it, and run its asynchronous calls (synchronously, each in its thread);
+ * with one thread, in the order they are made. The arguments are copied to the thread and the ones the function writes
+ * copied back; a JS function given as a callback is registered for the call in the thread of the library, which runs it
+ * when the library calls it
+ */
+export class NativeThreads {
+    #data;
+    #count;
+    #slots;
+    #declarations = [];
+    #nextFunction = 1;
+    #nextCall = 1;
+    #closed = false;
+    /**
+     * Start the threads of a library.
+     * @param file The file of the library, as loaded by this thread.
+     * @param declarations Its C declarations.
+     * @param count The number of threads.
+     * @param crt On Windows, the C library whose errno is read after each call.
+     */
+    constructor(file, declarations, count, crt) {
+        if (!nativeWorkerCode || !packageRequire)
+            throw new Error('The code of the worker threads of the libraries is not built (scripts/worker.mjs)');
+        this.#data = { koffi: packageRequire.resolve('koffi'), file: file, declarations: declarations, ...crt ? { crt: crt } : {} };
+        this.#count = count;
+        this.#slots = Array.from({ length: count }, () => this.#start());
+    }
+    /**
+     * Settled once every thread has loaded the library: rejected with the error of a thread that could not.
+     */
+    get ready() {
+        return Promise.all(this.#slots.map((slot) => slot?.ready ?? Promise.resolve())).then(() => undefined);
+    }
+    /**
+     * The number of threads.
+     */
+    get count() {
+        return this.#count;
+    }
+    /**
+     * Add declarations to the ones of the threads (and of the threads started later).
+     * @param declarations The declarations.
+     */
+    declare(declarations) {
+        this.#declarations.push(declarations);
+        for (const slot of this.#slots)
+            if (slot && !slot.exited)
+                this.#post(slot, { type: 'declare', declarations: declarations });
+    }
+    /**
+     * Bind a function in the threads now rather than at its first call: when the library is made, while the threads
+     * start (a thread started later binds it at its first call in it).
+     * @param fn The function.
+     */
+    bind(fn) {
+        if (this.#closed)
+            return;
+        for (const slot of this.#slots)
+            if (slot && !slot.exited)
+                this.#bound(slot, fn);
+    }
+    /**
+     * Call a function in the least busy thread (with one thread, after the calls made before).
+     * @param fn The function.
+     * @param args Its arguments.
+     * @returns Its value and the errno of the thread after it.
+     */
+    call(fn, args) {
+        if (this.#closed)
+            return Promise.reject(new NativeLibraryClosedException(`The library ${this.#data.file} is closed`));
+        const slot = this.#leastBusy();
+        const id = this.#bound(slot, fn);
+        const type = fn.type.type;
+        const passed = args.slice();
+        const back = [];
+        const views = new Map();
+        const transfer = [];
+        const callbacks = [];
+        let callbackError;
+        for (let index = 0; index < passed.length; index++) {
+            const value = passed[index];
+            const parameter = type.parameters[index];
+            //A JS function given as a callback: registered for the call, run in this thread when the library calls it
+            if (typeof value === 'function') {
+                const target = parameter?.type.kind === 'pointer' ? parameter.type.to : undefined;
+                if (!target || target.kind !== 'function')
+                    return this.#refuse(callbacks, new TypeError(`The argument ${index} of ${fn.type.name} is a function, but its parameter is not a pointer to a function`));
+                const prototype = target;
+                const zero = zeroOf(prototype.result);
+                const safe = (...callbackArgs) => {
+                    try {
+                        return value(...callbackArgs);
+                    }
+                    catch (error) {
+                        callbackError ??= { error: error };
+                        return zero;
+                    }
+                };
+                const pointer = koffi.register(convertedCallback(prototype, TYPES, safe), koffi.pointer(TYPES.prototype(prototype)));
+                callbacks.push(pointer);
+                passed[index] = pointer;
+                continue;
+            }
+            //A buffer: its own bytes only, copied and transferred (a Buffer of the pool would copy all its pool); copied
+            //back after the call when the function may write it
+            if (ArrayBuffer.isView(value)) {
+                const bytes = new Uint8Array(value.byteLength);
+                bytes.set(new Uint8Array(value.buffer, value.byteOffset, value.byteLength));
+                //A Buffer as its bytes (koffi takes them as it takes the Buffer)
+                const copy = Buffer.isBuffer(value) || value instanceof Uint8Array ? bytes : value instanceof DataView ? new DataView(bytes.buffer) : new value.constructor(bytes.buffer);
+                passed[index] = copy;
+                transfer.push(bytes.buffer);
+                if (written(parameter)) {
+                    back.push(index);
+                    views.set(index, value);
+                }
+                continue;
+            }
+            //An output parameter: an array of one element, an object filled by the function
+            if (parameter && parameter.direction !== 'in' && value && typeof value === 'object')
+                back.push(index);
+        }
+        const callId = this.#nextCall++;
+        return new Promise((resolve, reject) => {
+            const release = () => {
+                for (const pointer of callbacks)
+                    koffi.unregister(pointer);
+            };
+            slot.calls.set(callId, {
+                resolve: resolve,
+                reject: (error) => {
+                    release();
+                    reject(error);
+                },
+                abandon: reject,
+                settle: (message) => {
+                    release();
+                    //The arguments the function wrote, given back to the caller
+                    back.forEach((index, at) => {
+                        const given = args[index];
+                        const returned = message.back[at];
+                        const view = views.get(index);
+                        if (view && ArrayBuffer.isView(returned))
+                            new Uint8Array(view.buffer, view.byteOffset, view.byteLength).set(new Uint8Array(returned.buffer, returned.byteOffset, returned.byteLength));
+                        else if (Array.isArray(given) && Array.isArray(returned))
+                            given.splice(0, given.length, ...returned);
+                        else if (given && typeof given === 'object' && returned && typeof returned === 'object')
+                            Object.assign(given, returned);
+                    });
+                    if (callbackError)
+                        return reject(callbackError.error);
+                    if (message.error)
+                        return reject(errorOf(message.error));
+                    resolve({ value: message.value, errno: message.errno, ...message.windowsError !== undefined ? { windowsError: message.windowsError } : {} });
+                }
+            });
+            try {
+                this.#post(slot, { type: 'call', id: callId, function: id, args: passed, back: back }, transfer);
+            }
+            catch (error) {
+                slot.calls.delete(callId);
+                release();
+                reject(error);
+            }
+        });
+    }
+    /**
+     * Stop a thread as if it had ended on its own (for the tests): its calls fail, the next call starts a new one.
+     * @param index The index of the thread.
+     */
+    async stopThread(index) {
+        await this.#slots[index]?.worker.terminate();
+    }
+    /**
+     * Close the threads: each unloads the library and ends; the ones still running after a while are stopped. A thread
+     * still running a call (its native code did not return) is stopped at once, the call rejected: the thread ends
+     * once the native code returns.
+     */
+    async close() {
+        if (this.#closed)
+            return;
+        this.#closed = true;
+        await Promise.all(this.#slots.map(async (slot) => {
+            if (!slot || slot.exited)
+                return;
+            if (slot.calls.size) {
+                const error = new NativeLibraryClosedException(`The library ${this.#data.file} is closed while a call runs in its thread`);
+                for (const call of slot.calls.values())
+                    call.abandon(error);
+                slot.calls.clear();
+                this.#stop(slot);
+                return;
+            }
+            const exited = new Promise((resolve) => {
+                slot.worker.once('exit', () => resolve());
+            });
+            //A thread still loading the library gets it once loaded: its port keeps the messages until it listens
+            this.#post(slot, { type: 'close' });
+            let timer;
+            const timeout = new Promise((resolve) => {
+                timer = setTimeout(() => resolve(true), CLOSE_TIMEOUT);
+            });
+            const late = await Promise.race([exited, timeout]);
+            clearTimeout(timer);
+            if (late)
+                this.#stop(slot);
+        }));
+    }
+    /**
+     * A thread stopped, not awaited: a thread running native code ends only once it returns
+     * @param slot
+     */
+    #stop(slot) {
+        slot.worker.terminate().catch(() => undefined);
+    }
+    /**
+     * A call refused before it is sent: its callbacks released
+     * @param callbacks
+     * @param error
+     */
+    #refuse(callbacks, error) {
+        for (const pointer of callbacks)
+            koffi.unregister(pointer);
+        return Promise.reject(error);
+    }
+    /**
+     * The identifier of a function in a thread, which binds it once
+     * @param slot
+     * @param fn
+     */
+    #bound(slot, fn) {
+        let id = slot.bound.get(fn);
+        if (id === undefined) {
+            id = this.#nextFunction++;
+            slot.bound.set(fn, id);
+            this.#post(slot, { type: 'bind', id: id, declaration: fn.declaration, named: fn.named, options: fn.options });
+        }
+        return id;
+    }
+    /**
+     * The thread with the fewest calls waiting, started again if it ended
+     */
+    #leastBusy() {
+        for (let index = 0; index < this.#slots.length; index++) {
+            if (!this.#slots[index] || this.#slots[index].exited)
+                this.#slots[index] = this.#start();
+        }
+        let chosen = this.#slots[0];
+        for (const slot of this.#slots)
+            if (slot.calls.size < chosen.calls.size)
+                chosen = slot;
+        return chosen;
+    }
+    /**
+     * A message to a thread
+     * @param slot
+     * @param message
+     * @param transfer
+     */
+    #post(slot, message, transfer = []) {
+        slot.worker.postMessage(message, transfer);
+        //Kept alive while calls wait in it
+        if (message.type === 'call')
+            slot.worker.ref();
+    }
+    /**
+     * A thread started: it loads the library, then serves its calls
+     */
+    #start() {
+        const worker = new Worker(nativeWorkerCode, { eval: true, workerData: { ...this.#data, declarations: [this.#data.declarations, ...this.#declarations].join('\n') } });
+        let readied = () => undefined;
+        let failed = () => undefined;
+        const ready = new Promise((resolve, reject) => {
+            readied = resolve;
+            failed = reject;
+        });
+        //Not unhandled when nothing awaits it: the calls report the failure
+        ready.catch(() => undefined);
+        const slot = { worker: worker, ready: ready, calls: new Map(), bound: new Map(), exited: false };
+        //A thread keeps the process alive while it starts and while calls wait in it, not once idle
+        const refresh = () => {
+            if (!slot.calls.size)
+                worker.unref();
+        };
+        worker.on('message', (message) => {
+            switch (message.type) {
+                case 'ready':
+                    readied();
+                    refresh();
+                    return;
+                case 'failed':
+                    slot.error = errorOf(message.error);
+                    failed(slot.error);
+                    refresh();
+                    return;
+                case 'result': {
+                    const call = slot.calls.get(message.id);
+                    slot.calls.delete(message.id);
+                    refresh();
+                    call?.settle(message);
+                    return;
+                }
+                case 'closed':
+                    return;
+            }
+        });
+        const end = (error) => {
+            if (slot.exited)
+                return;
+            slot.exited = true;
+            failed(error);
+            for (const call of slot.calls.values())
+                call.reject(slot.error ?? error);
+            slot.calls.clear();
+        };
+        worker.on('error', (error) => end(error));
+        worker.on('exit', (code) => end(new Error(`The worker thread of the library ${this.#data.file} ended (exit code ${code})`)));
+        return slot;
+    }
+}

package/dist/esm/lib/thread/NativeWorker.js

@@ -0,0 +1,93 @@
+import { parentPort, workerData } from 'node:worker_threads';
+import { NativeLibraryHandle } from '../library/NativeLibraryHandle.js';
+import { nativeWindowsError } from '../Runtime.js';
+/**
+ * An error as a thread sends it
+ * @param error
+ */
+function threadError(error) {
+    return error instanceof Error ? { name: error.name, message: error.message } : { name: 'Error', message: String(error) };
+}
+/**
+ * The buffers of values sent back, transferred rather than copied (they are the copies of the thread)
+ * @param values
+ */
+function transferables(values) {
+    const buffers = new Set();
+    for (const value of values)
+        if (ArrayBuffer.isView(value) && value.buffer instanceof ArrayBuffer)
+            buffers.add(value.buffer);
+    return [...buffers];
+}
+/**
+ * Serve the calls of a library in this worker thread: it loads the library, binds the functions it is asked to bind
+ * (synchronous: the thread is the one waiting), calls them in the order of the messages, and sends back their results,
+ * the arguments asked back (the output parameters, the buffers written) and its errno after each call
+ */
+export function runNativeWorker() {
+    const port = parentPort;
+    const data = workerData;
+    const post = (message, transfer = []) => port.postMessage(message, transfer);
+    let library;
+    try {
+        library = new NativeLibraryHandle(data.file, data.declarations, { crt: data.crt });
+    }
+    catch (error) {
+        post({ type: 'failed', error: threadError(error) });
+        port.close();
+        return;
+    }
+    //The functions bound, or why they could not be
+    const functions = new Map();
+    port.on('message', (message) => {
+        switch (message.type) {
+            case 'bind':
+                try {
+                    functions.set(message.id, message.named ? library.function(message.declaration, message.options) : library.func(message.declaration, message.options));
+                }
+                catch (error) {
+                    functions.set(message.id, threadError(error));
+                }
+                return;
+            case 'declare':
+                try {
+                    library.declare(message.declarations);
+                }
+                catch {
+                    //The thread of the library read them first: it reported their error
+                }
+                return;
+            case 'call': {
+                const bound = functions.get(message.function);
+                let value;
+                let error;
+                if (typeof bound !== 'function') {
+                    error = bound ?? { name: 'Error', message: 'The function is not bound in the thread' };
+                }
+                else {
+                    try {
+                        value = bound(...message.args);
+                    }
+                    catch (thrown) {
+                        error = threadError(thrown);
+                    }
+                }
+                //GetLastError first: reading the errno of the C library of Windows is a call too
+                const windowsError = nativeWindowsError();
+                const errno = library.errno();
+                const back = message.back.map((index) => message.args[index]);
+                post({ type: 'result', id: message.id, ...error ? { error: error } : { value: value }, back: back, errno: errno, ...windowsError !== undefined ? { windowsError: windowsError } : {} }, transferables(back));
+                return;
+            }
+            case 'close': {
+                const end = () => {
+                    post({ type: 'closed' });
+                    port.close();
+                };
+                library.close().then(end, end);
+                return;
+            }
+        }
+    });
+    post({ type: 'ready' });
+}