koffi 2.5.10 → 2.5.12

package/doc/migration.md

@@ -10,7 +10,7 @@ You may need to change your code if you use:
 - Opaque types
 - `koffi.introspect()`
 
-### Callback types
+### Callback type changes
 
 In Koffi 1.x, callbacks were defined in a way that made them usable directly as parameter and return types, obscuring the underlying pointer. Now, you must use them through a pointer: `void CallIt(CallbackType func)` in Koffi 1.x becomes `void CallIt(CallbackType *func)` in version 2.0 and newer.
 
@@ -61,13 +61,13 @@ let ret = TransferToJS('Niels', 27, (str, age) => {
 console.log(ret);
 ```
 
-Koffi 1.x only supported [transient callbacks](callbacks.md#callbacks), you must use Koffi 2.x for registered callbacks.
+Koffi 1.x only supported [transient callbacks](callbacks.md#javascript-callbacks), you must use Koffi 2.x for registered callbacks.
 
 ```{note}
 The function `koffi.proto()` was introduced in Koffi 2.4, it was called `koffi.callback()` in earlier versions.
 ```
 
-### Opaque types
+### Opaque type changes
 
 In Koffi 1.x, opaque handles were defined in a way that made them usable directly as parameter and return types, obscuring the underlying pointer. Now, in Koffi 2.0, you must use them through a pointer, and use an array for output parameters.
 
@@ -145,7 +145,7 @@ db = ptr[0];
 sqlite3_close_v2(db);
 ```
 
-### koffi.introspect()
+### New koffi.introspect()
 
 In Koffi 1.x, `koffi.introspect()` would only work with struct types, and return the object passed to `koffi.struct()` to initialize the type. Now this function works with all types.
 

package/doc/output.md

@@ -0,0 +1,169 @@
+# Output parameters
+
+## Output and input/output
+
+For simplicity, and because Javascript only has value semantics for primitive types, Koffi can marshal out (or in/out) multiple types of parameters:
+
+- [Structs](input.md#struct-types) (to/from JS objects)
+- [Unions](unions.md)
+- [Opaque types](input.md#opaque-types)
+- String buffers
+
+In order to change an argument from input-only to output or input/output, use the following functions:
+
+- `koffi.out()` on a pointer, e.g. `koffi.out(koffi.pointer(timeval))` (where timeval is a struct type)
+- `koffi.inout()` for dual input/output parameters
+
+The same can be done when declaring a function with a C-like prototype string, with the MSDN-like type qualifiers:
+
+- `_Out_` for output parameters
+- `_Inout_` for dual input/output parameters
+
+### Primitive value
+
+This Windows example enumerate all Chrome windows along with their PID and their title. The `GetWindowThreadProcessId()` function illustrates how to get a primitive value from an output argument.
+
+```js
+// ES6 syntax: import koffi from 'koffi';
+const koffi = require('koffi');
+
+const user32 = koffi.load('user32.dll');
+
+const DWORD = koffi.alias('DWORD', 'uint32_t');
+const HANDLE = koffi.pointer(koffi.opaque('HANDLE'));
+const HWND = koffi.alias('HWND', HANDLE);
+
+const FindWindowEx = user32.func('HWND __stdcall FindWindowExW(HWND hWndParent, HWND hWndChildAfter, const char16_t *lpszClass, const char16_t *lpszWindow)');
+const GetWindowThreadProcessId = user32.func('DWORD __stdcall GetWindowThreadProcessId(HWND hWnd, _Out_ DWORD *lpdwProcessId)');
+const GetWindowText = user32.func('int __stdcall GetWindowTextA(HWND hWnd, _Out_ uint8_t *lpString, int nMaxCount)');
+
+for (let hwnd = null;;) {
+    hwnd = FindWindowEx(0, hwnd, 'Chrome_WidgetWin_1', null);
+
+    if (!hwnd)
+        break;
+
+    // Get PID
+    let pid;
+    {
+        let ptr = [null];
+        let tid = GetWindowThreadProcessId(hwnd, ptr);
+
+        if (!tid) {
+            // Maybe the process ended in-between?
+            continue;
+        }
+
+        pid = ptr[0];
+    }
+
+    // Get window title
+    let title;
+    {
+        let buf = Buffer.allocUnsafe(1024);
+        let length = GetWindowText(hwnd, buf, buf.length);
+
+        if (!length) {
+            // Maybe the process ended in-between?
+            continue;
+        }
+
+        title = koffi.decode(buf, 'char', length);
+    }
+
+    console.log({ PID: pid, Title: title });
+}
+```
+
+### Struct example
+
+This example calls the POSIX function `gettimeofday()`, and uses the prototype-like syntax.
+
+```js
+// ES6 syntax: import koffi from 'koffi';
+const koffi = require('koffi');
+
+const lib = koffi.load('libc.so.6');
+
+const timeval = koffi.struct('timeval', {
+    tv_sec: 'unsigned int',
+    tv_usec: 'unsigned int'
+});
+const timezone = koffi.struct('timezone', {
+    tz_minuteswest: 'int',
+    tz_dsttime: 'int'
+});
+
+// The _Out_ qualifiers instruct Koffi to marshal out the values
+const gettimeofday = lib.func('int gettimeofday(_Out_ timeval *tv, _Out_ timezone *tz)');
+
+let tv = {};
+gettimeofday(tv, null);
+
+console.log(tv);
+```
+
+### Opaque type example
+
+This example opens an in-memory SQLite database, and uses the node-ffi-style function declaration syntax.
+
+```js
+// ES6 syntax: import koffi from 'koffi';
+const koffi = require('koffi');
+
+const lib = koffi.load('sqlite3.so');
+
+const sqlite3 = koffi.opaque('sqlite3');
+
+// Use koffi.out() on a double pointer to copy out (from C to JS) after the call
+const sqlite3_open_v2 = lib.func('sqlite3_open_v2', 'int', ['str', koffi.out(koffi.pointer(sqlite3, 2)), 'int', 'str']);
+const sqlite3_close_v2 = lib.func('sqlite3_close_v2', 'int', [koffi.pointer(sqlite3)]);
+
+const SQLITE_OPEN_READWRITE = 0x2;
+const SQLITE_OPEN_CREATE = 0x4;
+
+let out = [null];
+if (sqlite3_open_v2(':memory:', out, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, null) != 0)
+    throw new Error('Failed to open database');
+let db = out[0];
+
+sqlite3_close_v2(db);
+```
+
+### String buffer example
+
+*New in Koffi 2.2*
+
+This example calls a C function to concatenate two strings to a pre-allocated string buffer. Since JS strings are immutable, you must pass an array with a single string instead.
+
+```c
+void ConcatToBuffer(const char *str1, const char *str2, char *out)
+{
+    size_t len = 0;
+
+    for (size_t i = 0; str1[i]; i++) {
+        out[len++] = str1[i];
+    }
+    for (size_t i = 0; str2[i]; i++) {
+        out[len++] = str2[i];
+    }
+
+    out[len] = 0;
+}
+```
+
+```js
+const ConcatToBuffer = lib.func('void ConcatToBuffer(const char *str1, const char *str2, _Out_ char *out)');
+
+let str1 = 'Hello ';
+let str2 = 'Friends!';
+
+// We need to reserve space for the output buffer! Including the NUL terminator
+// because ConcatToBuffer() expects so, but Koffi can convert back to a JS string
+// without it (if we reserve the right size).
+let out = ['\0'.repeat(str1.length + str2.length + 1)];
+
+ConcatToBuffer(str1, str2, out);
+
+console.log(out[0]);
+```

package/doc/packaging.md

@@ -1,4 +1,4 @@
-# Packaging
+# Bundlers and Koffi
 
 ## Bundling and native modules
 

package/doc/pointers.md

@@ -130,13 +130,63 @@ let total = ComputeTotalLength(strings);
 console.log(total); // Prints 14
 ```
 
-By default, just like for objects, array arguments are copied from JS to C but not vice-versa. You can however change the direction as documented in the section on [output parameters](parameters.md#output-parameters).
+By default, just like for objects, array arguments are copied from JS to C but not vice-versa. You can however change the direction as documented in the section on [output parameters](output.md).
 
 ## Disposable types
 
+*New in Koffi 2.0*
+
 Disposable types allow you to register a function that will automatically called after each C to JS conversion performed by Koffi. This can be used to avoid leaking heap-allocated strings, for example.
 
-Read the documentation for [disposable types](parameters.md#heap-allocated-values) on the page about special parameters.
+Some C functions return heap-allocated values directly or through output parameters. While Koffi automatically converts values from C to JS (to a string or an object), it does not know when something needs to be freed, or how.
+
+For opaque types, such as FILE, this does not matter because you will explicitly call `fclose()` on them. But some values (such as strings) get implicitly converted by Koffi, and you lose access to the original pointer. This creates a leak if the string is heap-allocated.
+
+To avoid this, you can instruct Koffi to call a function on the original pointer once the conversion is done, by creating a **disposable type** with `koffi.dispose(name, type, func)`. This creates a type derived from another type, the only difference being that *func* gets called with the original pointer once the value has been converted and is not needed anymore.
+
+The *name* can be omitted to create an anonymous disposable type. If *func* is omitted or is null, Koffi will use `koffi.free(ptr)` (which calls the standard C library *free* function under the hood).
+
+```js
+const AnonHeapStr = koffi.disposable('str'); // Anonymous type (cannot be used in function prototypes)
+const NamedHeapStr = koffi.disposable('HeapStr', 'str'); // Same thing, but named so usable in function prototypes
+const ExplicitFree = koffi.disposable('HeapStr16', 'str16', koffi.free); // You can specify any other JS function
+```
+
+The following example illustrates the use of a disposable type derived from *str*.
+
+```js
+// ES6 syntax: import koffi from 'koffi';
+const koffi = require('koffi');
+
+const lib = koffi.load('libc.so.6');
+
+const HeapStr = koffi.disposable('str');
+const strdup = lib.cdecl('strdup', HeapStr, ['str']);
+
+let copy = strdup('Hello!');
+console.log(copy); // Prints Hello!
+```
+
+When you declare functions with the [prototype-like syntax](functions.md#definition-syntax), you can either use named disposable types or use the '!' shortcut qualifier with compatibles types, as shown in the example below. This qualifier creates an anonymous disposable type that calls `koffi.free(ptr)`.
+
+```js
+// ES6 syntax: import koffi from 'koffi';
+const koffi = require('koffi');
+
+const lib = koffi.load('libc.so.6');
+
+// You can also use: const strdup = lib.func('const char *! strdup(const char *str)')
+const strdup = lib.func('str! strdup(const char *str)');
+
+let copy = strdup('World!');
+console.log(copy); // Prints World!
+```
+
+Disposable types can only be created from pointer or string types.
+
+```{warning}
+Be careful on Windows: if your shared library uses a different CRT (such as msvcrt), the memory could have been allocated by a different malloc/free implementation or heap, resulting in undefined behavior if you use `koffi.free()`.
+```
 
 ## Unwrap pointers
 

package/doc/polymorphism.md

@@ -0,0 +1,108 @@
+# Polymorphic arguments
+
+## Input polymorphism
+
+*New in Koffi 2.1*
+
+Many C functions use `void *` parameters in order to pass polymorphic objects and arrays, meaning that the data format changes can change depending on one other argument, or on some kind of struct tag member.
+
+Koffi provides two features to deal with this:
+
+- Buffers and typed JS arrays can be used as values in place everywhere a pointer is expected. See [dynamic arrays](pointers.md#array-pointers-dynamic-arrays) for more information, for input or output.
+- You can use `koffi.as(value, type)` to tell Koffi what kind of type is actually expected.
+
+The example below shows the use of `koffi.as()` to read the header of a PNG file with `fread()`.
+
+```js
+// ES6 syntax: import koffi from 'koffi';
+const koffi = require('koffi');
+
+const lib = koffi.load('libc.so.6');
+
+const FILE = koffi.opaque('FILE');
+
+const PngHeader = koffi.pack('PngHeader', {
+    signature: koffi.array('uint8_t', 8),
+    ihdr: koffi.pack({
+        length: 'uint32_be_t',
+        chunk: koffi.array('char', 4),
+        width: 'uint32_be_t',
+        height: 'uint32_be_t',
+        depth: 'uint8_t',
+        color: 'uint8_t',
+        compression: 'uint8_t',
+        filter: 'uint8_t',
+        interlace: 'uint8_t',
+        crc: 'uint32_be_t'
+    })
+});
+
+const fopen = lib.func('FILE *fopen(const char *path, const char *mode)');
+const fclose = lib.func('int fclose(FILE *fp)');
+const fread = lib.func('size_t fread(_Out_ void *ptr, size_t size, size_t nmemb, FILE *fp)');
+
+let filename = process.argv[2];
+if (filename == null)
+    throw new Error('Usage: node png.js <image.png>');
+
+let hdr = {};
+{
+    let fp = fopen(filename, 'rb');
+    if (!fp)
+        throw new Error(`Failed to open '${filename}'`);
+
+    try {
+        let len = fread(koffi.as(hdr, 'PngHeader *'), 1, koffi.sizeof(PngHeader), fp);
+        if (len < koffi.sizeof(PngHeader))
+            throw new Error('Failed to read PNG header');
+    } finally {
+        fclose(fp);
+    }
+}
+
+console.log('PNG header:', hdr);
+```
+
+## Output buffers
+
+*New in Koffi 2.3*
+
+You can use buffers and typed arrays for output (and input/output) pointer parameters. Simply pass the buffer as an argument and the native function will receive a pointer to its contents.
+
+Once the native function returns, you can decode the content with `koffi.decode(value, type)` as in the following example:
+
+```js
+// ES6 syntax: import koffi from 'koffi';
+const koffi = require('koffi');
+
+const lib = koffi.load('libc.so.6');
+
+const Vec3 = koffi.struct('Vec3', {
+    x: 'float32',
+    y: 'float32',
+    z: 'float32'
+})
+
+const memcpy = lib.func('void *memcpy(_Out_ void *dest, const void *src, size_t size)');
+
+let vec1 = { x: 1, y: 2, z: 3 };
+let vec2 = null;
+
+// Copy the vector in a convoluted way through memcpy
+{
+    let src = koffi.as(vec1, 'Vec3 *');
+    let dest = Buffer.allocUnsafe(koffi.sizeof(Vec3));
+
+    memcpy(dest, src, koffi.sizeof(Vec3));
+
+    vec2 = koffi.decode(dest, Vec3);
+}
+
+// CHange vector1, leaving copy alone
+[vec1.x, vec1.y, vec1.z] = [vec1.z, vec1.y, vec1.x];
+
+console.log(vec1); // { x: 3, y: 2, z: 1 }
+console.log(vec2); // { x: 1, y: 2, z: 3 }
+```
+
+See [pointer arguments](callbacks.md#decoding-pointer-arguments) for another example with the decode function.

package/package.json

@@ -1,7 +1,7 @@
 {
     "name": "koffi",
-    "version": "2.5.10",
-    "stable": "2.5.10",
+    "version": "2.5.12",
+    "stable": "2.5.12",
     "description": "Fast and simple C FFI (foreign function interface) for Node.js",
     "keywords": [
         "foreign",
@@ -17,7 +17,11 @@
         "url": "https://github.com/Koromix/koffi"
     },
     "homepage": "https://koffi.dev/",
-    "author": "Niels Martignène <niels.martignene@protonmail.com>",
+    "author": {
+        "name": "Niels Martignène",
+        "email": "niels.martignene@protonmail.com",
+        "url": "https://koromix.dev/"
+    },
     "main": "src/index.js",
     "types": "src/index.d.ts",
     "scripts": {

package/src/cnoke/package.json

@@ -13,7 +13,11 @@
     "type": "git",
     "url": "https://github.com/Koromix/rygel.git"
   },
-  "author": "Niels Martignène <niels.martignene@protonmail.com>",
+  "author": {
+    "name": "Niels Martignène",
+    "email": "niels.martignene@protonmail.com",
+    "url": "https://koromix.dev/"
+  },
   "index": "src/index.js",
   "bin": "cnoke.js",
   "license": "MIT"

package/src/core/libcc/libcc.cc

@@ -801,6 +801,108 @@ bool ParseBool(Span<const char> str, bool *out_value, unsigned int flags,
     return false;
 }
 
+bool ParseSize(Span<const char> str, int64_t *out_size, unsigned int flags, Span<const char> *out_remaining)
+{
+    uint64_t size = 0;
+
+    if (!ParseInt(str, &size, flags & ~(int)ParseFlag::End, &str))
+        return false;
+    if (size > INT64_MAX) [[unlikely]]
+        goto overflow;
+
+    if (str.len) {
+        uint64_t multiplier = 1;
+        int next = 1;
+
+        switch (str[0]) {
+            case 'B': { multiplier = 1; } break;
+            case 'k': { multiplier = 1000; } break;
+            case 'M': { multiplier = 1000000; } break;
+            case 'G': { multiplier = 1000000000; } break;
+            case 'T': { multiplier = 1000000000000; } break;
+            default: { next = 0; } break;
+        }
+
+        if ((flags & (int)ParseFlag::End) && str.len > next) [[unlikely]] {
+            if (flags & (int)ParseFlag::Log) {
+                LogError("Unknown size unit '%1'", str[0]);
+            }
+            return false;
+        }
+        str = str.Take(next, str.len - next);
+
+        uint64_t total = size * multiplier;
+        if ((size && total / size != multiplier) || total > INT64_MAX) [[unlikely]]
+            goto overflow;
+        size = total;
+    }
+
+    *out_size = (int64_t)size;
+    if (out_remaining) {
+        *out_remaining = str;
+    }
+    return true;
+
+overflow:
+    if (flags & (int)ParseFlag::Log) {
+        LogError("Size value is too high");
+    }
+    return false;
+}
+
+bool ParseDuration(Span<const char> str, int64_t *out_duration, unsigned int flags, Span<const char> *out_remaining)
+{
+    uint64_t duration = 0;
+
+    if (!ParseInt(str, &duration, flags & ~(int)ParseFlag::End, &str))
+        return false;
+    if (duration > INT64_MAX) [[unlikely]]
+        goto overflow;
+
+    if (str.len) {
+        uint64_t multiplier = 1;
+        int next = 1;
+
+        switch (str[0]) {
+            case 's': { multiplier = 1000; } break;
+            case 'm': { multiplier = 60000; } break;
+            case 'h': { multiplier = 3600000; } break;
+            case 'd': { multiplier = 86400000; } break;
+            default: { next = 0; } break;
+        }
+
+        if ((flags & (int)ParseFlag::End) && str.len > next) [[unlikely]] {
+            if (flags & (int)ParseFlag::Log) {
+                LogError("Unknown duration unit '%1'", str[0]);
+            }
+            return false;
+        }
+        str = str.Take(next, str.len - next);
+
+        uint64_t total = duration * multiplier;
+        if ((duration && total / duration != multiplier) || total > INT64_MAX) [[unlikely]]
+            goto overflow;
+        duration = total;
+    } else {
+        uint64_t total = duration * 1000;
+        if ((duration && total / duration != 1000) || total > INT64_MAX) [[unlikely]]
+            goto overflow;
+        duration = total;
+    }
+
+    *out_duration = (int64_t)duration;
+    if (out_remaining) {
+        *out_remaining = str;
+    }
+    return true;
+
+overflow:
+    if (flags & (int)ParseFlag::Log) {
+        LogError("Duration value is too high");
+    }
+    return false;
+}
+
 // ------------------------------------------------------------------------
 // Format
 // ------------------------------------------------------------------------
@@ -2576,14 +2678,12 @@ bool TestFile(const char *filename)
 
 bool TestFile(const char *filename, FileType type)
 {
-    RG_ASSERT(type != FileType::Link);
-
     FileInfo file_info;
     if (StatFile(filename, (int)StatFlag::IgnoreMissing, &file_info) != StatResult::Success)
         return false;
 
     // Don't follow, but don't warn if we just wanted a file
-    if (file_info.type == FileType::Link) {
+    if (type != FileType::Link && file_info.type == FileType::Link) {
         file_info.type = FileType::File;
     }
 
@@ -4392,7 +4492,7 @@ Size ReadCommandOutput(const char *cmd_line, Span<char> out_output)
 bool ReadCommandOutput(const char *cmd_line, HeapArray<char> *out_output)
 {
     int exit_code;
-    if (!ExecuteCommandLine(cmd_line, nullptr, {}, Kilobytes(4), out_output, &exit_code))
+    if (!ExecuteCommandLine(cmd_line, nullptr, {}, Mebibytes(1), out_output, &exit_code))
         return false;
     if (exit_code) {
         LogDebug("Command '%1 failed (exit code: %2)", cmd_line, exit_code);
@@ -6760,6 +6860,16 @@ bool SpliceStream(StreamReader *reader, int64_t max_len, StreamWriter *writer)
     return true;
 }
 
+bool IsCompressorAvailable(CompressionType compression_type)
+{
+    return CompressorFunctions[(int)compression_type];
+}
+
+bool IsDecompressorAvailable(CompressionType compression_type)
+{
+    return DecompressorFunctions[(int)compression_type];
+}
+
 // ------------------------------------------------------------------------
 // INI
 // ------------------------------------------------------------------------