flatsql 3.11.1 → 3.12.1

package/bin/flatsql.mjs

@@ -0,0 +1,76 @@
+#!/usr/bin/env node
+// Binary streams only. Schema/text conversion is a separate program.
+import fs from 'node:fs/promises';
+import { once } from 'node:events';
+import { FlatSQL } from '../wasm/index.js';
+
+const help = `Usage:
+  flatsql ingest --root DIR --schema FILE.bfbs --type NAME --id '$FID' < records.fb > receipts.fb
+  flatsql query --root DIR [--records] [--request FILE.fb] > results.fb
+
+Ingest reads concatenated size-prefixed FlatBuffers from stdin.
+Query reads one SDS QRP FlatBuffer request from stdin or --request.
+Results are size-prefixed SDS QRP pages, or unchanged record frames with --records.
+Optional ingest metadata: --provider NAME --source NAME --producer NAME --batch NAME.
+Use a separate converter such as flatc-wasm for text input and output.
+`;
+async function main() {
+  const [command,...args]=process.argv.slice(2), options={};
+  if (!command || command==='--help' || command==='-h') { process.stdout.write(help); return; }
+  const allowed=new Set(['root','schema','type','id','request','provider','source','producer','batch']);
+  for (let i=0;i<args.length;i++) {
+    const key=args[i].slice(2);
+    if (args[i]==='--records') { options.records=true; continue; }
+    if (!args[i].startsWith('--') || !allowed.has(key) || !args[i+1] || args[i+1].startsWith('--'))
+      throw new Error(`Unknown or incomplete option: ${args[i]}`);
+    options[key]=args[++i];
+  }
+  if (!options.root || !['ingest','query'].includes(command)) throw new Error(help);
+  if (command==='ingest' && (!options.schema || !options.type || !options.id)) throw new Error(help);
+  const signal=new AbortController();
+  const interrupt=()=>signal.abort();
+  process.once('SIGINT',interrupt); process.once('SIGTERM',interrupt);
+  const write=async data=>{ if (!process.stdout.write(data)) await once(process.stdout,'drain'); };
+  const db=await FlatSQL.open({root:options.root});
+  try {
+    if (command==='query') {
+      let request;
+      if (options.request) {
+        if ((await fs.stat(options.request)).size>8*1024*1024) throw new Error('QRP request exceeds 8 MiB');
+        request=await fs.readFile(options.request);
+      } else {
+        const parts=[];let size=0;
+        for await (const chunk of process.stdin) {
+          size+=chunk.length;if(size>8*1024*1024)throw new Error('QRP request exceeds 8 MiB');parts.push(chunk);
+        }
+        request=Buffer.concat(parts);
+      }
+      for await (const chunk of db.query(request,{records:options.records,signal:signal.signal})) await write(chunk);
+    } else {
+      await db.registerSchema({name:options.type,fileIdentifier:options.id,bfbs:await fs.readFile(options.schema)});
+      const limit=4*1024*1024;
+      let pending=Buffer.alloc(0),batch=[],size=0;
+      const flush=async()=>{
+        if(!size)return;
+        await write(await db.ingest(options.type,Buffer.concat(batch,size),{...options,signal:signal.signal}));
+        batch=[];size=0;
+      };
+      for await(const chunk of process.stdin) {
+        pending=Buffer.concat([pending,chunk]);let at=0;
+        while(pending.length-at>=4) {
+          const n=pending.readUInt32LE(at)+4;
+          if(n<12 || n>limit)throw new Error('Record frame must be between 12 bytes and 4 MiB');
+          if(pending.length-at<n)break;
+          if(size+n>limit)await flush();
+          batch.push(Buffer.from(pending.subarray(at,at+n)));size+=n;at+=n;
+        }
+        pending=Buffer.from(pending.subarray(at));
+      }
+      if(pending.length)throw new Error('Truncated FlatBuffer record frame');
+      await flush();
+    }
+  } finally {
+    await db.close();process.removeListener('SIGINT',interrupt);process.removeListener('SIGTERM',interrupt);
+  }
+}
+main().catch(error=>{process.stderr.write(`flatsql: ${error.message}\n`);process.exitCode=1;});

package/cpp/README.md

@@ -0,0 +1,18 @@
+# FlatSQL native engine
+
+FlatSQL 3.12 has one public C ABI: `include/flatsql/flatsql.h`.
+Build the `flatsql` static library with CMake and the pinned DigitalArsenal
+FlatBuffers checkout at `8af3053eb8400de4a0a654805b8257e93fae3a8e`.
+
+```sh
+cmake -S cpp -B cpp/build -DFLATBUFFERS_DIR=/path/to/flatbuffers -DCMAKE_BUILD_TYPE=Release
+cmake --build cpp/build --target flatsql
+```
+
+The same engine builds as `wasm/flatsql.wasm` using wasi-sdk 30.
+Native hosts provide the seven filesystem imports described in the ABI header.
+Public SQL requests/results use SDS QRP; records are size-prefixed FlatBuffers.
+See `docs/API-3.12.md` for the binary and geospatial contracts.
+
+The private source directory names `p4` and `p4sql` identify the storage
+implementation. They are not alternate engines or public entry points.

package/cpp/include/flatsql/flatsql.h

@@ -0,0 +1,162 @@
+/* FlatSQL: one engine and one ABI for binary records and SQL.
+ * SQL input is SDS QRP; all projected results are size-prefixed SDS QRP.
+ * Record streams contain unchanged size-prefixed FlatBuffers. Configuration,
+ * registration and storage metadata use the documented packed control layout;
+ * they never parse or render record text. Mailboxes keep queries off the host
+ * executor thread and retain independent cancellation and resource budgets.
+ */
+#ifndef FLATSQL_H
+#define FLATSQL_H
+
+#include <stdint.h>
+
+#if defined(__wasm__)
+#  define FLATSQL_EXPORT(name) __attribute__((export_name(name)))
+#else
+#  define FLATSQL_EXPORT(name)
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* ---- §3.2 Status and reject codes ------------------------------------- */
+
+#define FLATSQL_OK               0
+#define FLATSQL_E_ARG           -1   /* malformed request or config */
+#define FLATSQL_E_NOMEM         -2
+#define FLATSQL_E_IO            -3
+#define FLATSQL_E_CORRUPT       -4   /* the file is quarantined, never rewritten; reads elsewhere continue */
+#define FLATSQL_E_FORMAT        -5   /* unsupported store/file format or layout; marker/state mismatch; C-5 rule change */
+#define FLATSQL_E_BUSY          -6   /* a queue or partition credit is full: nothing was done, retry; reader paths never wait */
+#define FLATSQL_E_CANCELLED     -7
+#define FLATSQL_E_BUDGET        -8   /* rows-examined, bytes-read, result rows/bytes, work or heap cap */
+#define FLATSQL_E_UNSUPPORTED   -9
+#define FLATSQL_E_NOTYPE       -10   /* type not registered */
+#define FLATSQL_E_STOPPED      -11   /* stop word set, or the instance is stopping */
+#define FLATSQL_E_NOSPACE      -12   /* quota or device full: writes refuse, reads continue */
+#define FLATSQL_E_SQL          -13   /* SQL error; the text is in the slot's err */
+#define FLATSQL_E_INTERNAL     -14
+
+/* Per-record rejects (PUT rows, `reject` column). -100..-109 are
+ * ps::RejectCode from flatsql/typecfg/extract.h, unchanged. */
+#define FLATSQL_REJ_BAD_ENTRY   -100
+#define FLATSQL_REJ_FRAME_SIZE  -101
+#define FLATSQL_REJ_FID         -102
+#define FLATSQL_REJ_VERIFY      -103
+#define FLATSQL_REJ_CID         -104
+#define FLATSQL_REJ_SEALED      -105
+#define FLATSQL_REJ_ATTR        -106
+#define FLATSQL_REJ_QUARANTINED -107
+#define FLATSQL_REJ_NO_TYPE     -108
+#define FLATSQL_REJ_TXN_TOO_LARGE -109
+#define FLATSQL_REJ_CID_FORM    -110 /* the CID is not a CIDv1 raw sha2-256 (bafkrei...) */
+#define FLATSQL_REJ_SEQ         -111 /* a migrate-mode seq is missing, <= 0, or held by a different CID */
+#define FLATSQL_REJ_TOO_LARGE   -112 /* above the type's max frame, or above the write pool's request area (C-6) */
+#define FLATSQL_REJ_TAG         -113 /* a tag without provider or source, a tag index out of range, or a record
+                                   PEER whose token differs from the call peer's */
+
+/* ---- §3.5 Ops, classes, slot states -------------------------------------- */
+
+enum {
+    FLATSQL_OPC_PUT = 1, FLATSQL_OPC_SUPERSEDE = 2, FLATSQL_OPC_DELETE = 3, FLATSQL_OPC_QUOTA_GC = 4, FLATSQL_OPC_REBUILD = 5,
+    FLATSQL_OPC_GET = 10, FLATSQL_OPC_TAGS = 11, FLATSQL_OPC_SCAN = 12, FLATSQL_OPC_HEAD = 13, FLATSQL_OPC_WINDOW = 14,
+    FLATSQL_OPC_INDEX_PAGE = 15, FLATSQL_OPC_EPOCH = 16, FLATSQL_OPC_SUMMARY = 17,
+    FLATSQL_OPC_RANGES = 18, FLATSQL_OPC_FEED_ROOT = 19, FLATSQL_OPC_DROPS = 20,
+    FLATSQL_OPC_EXPOSURE = 21, FLATSQL_OPC_RELEASE = 22,
+    FLATSQL_OPC_SQL = 30, FLATSQL_OPC_SURFACE = 31
+};
+/* IPFS ops: tag 1 type; tags 3/4 provider/source (both omitted = every feed,
+ * both empty = local). Tag 80 u64: inclusive from-commit for RANGES/DROPS,
+ * through-commit for RELEASE. Tag 81 u8: EXPOSURE (0 default, 1 on).
+ * Tag 82 u32: RELEASE generation (default current); offsets are generation-local.
+ * RELEASE of a generation older than the current one releases that whole
+ * (immutable) generation; tag 80 is ignored. RELEASE of the current generation
+ * releases every older one once tag 80 reaches its compaction checkpoint (the
+ * first FCK after genesis), and is a no-op before that.
+ * RANGES: path,offset,length,cid,kind,unit,block,provider,source,generation.
+ * Paths are relative to config root; inline block bytes only for dag-pb nodes.
+ * LINK unit is its publication/change commit, including rewrites of old units.
+ * FEED_ROOT: provider,source,cid,unit,generation,blocked_generations.
+ * DROPS: cid,unit,provider,source,generation. Includes removed records, large
+ * roots/leaves and obsolete raw metadata. Derived LINK blocks are reclaimed
+ * by pin-update/GC. Unit names the removal or replacement checkpoint.
+ * RANGES is the live closure in the current generation, filtered by unit.
+ * Reset the cursor when FEED_ROOT reports a changed generation.
+ * Exposure is runtime policy: the node must reapply it after opening a store
+ * before the 30-second retired-reader window expires.
+ * EXPOSURE/RELEASE return no rows. These run on a read lane; exposure/release
+ * change only the node's generation-release bookkeeping, never stream bytes.
+ */
+enum { FLATSQL_RANGE_RECORD=1, FLATSQL_RANGE_LARGE_LEAF=2, FLATSQL_RANGE_LARGE_ROOT=3,
+       FLATSQL_RANGE_FMB=4, FLATSQL_RANGE_FCK=5, FLATSQL_RANGE_FSG=6, FLATSQL_RANGE_LINK=7 };
+enum { FLATSQL_CLASS_WRITE = 1, FLATSQL_CLASS_INTERACTIVE = 2, FLATSQL_CLASS_BULK = 3, FLATSQL_CLASS_SANDBOX = 4,
+       FLATSQL_CLASS_MAINTENANCE = 5 };
+enum { FLATSQL_SLOT_FREE = 0, FLATSQL_SLOT_CLAIMED = 1, FLATSQL_SLOT_QUEUED = 2, FLATSQL_SLOT_RUNNING = 3, FLATSQL_SLOT_DONE = 5 };
+enum { FLATSQL_ACT_REJECTED = 0, FLATSQL_ACT_NEW = 1, FLATSQL_ACT_COPY = 2, FLATSQL_ACT_RETAG = 3, FLATSQL_ACT_DUP = 4,
+       FLATSQL_ACT_IDENT_DUP = 5, FLATSQL_ACT_MIGRATED = 6 };
+
+/* ---- §3.4 The mailbox ------------------------------------------------------ */
+
+#define FLATSQL_LAYOUT_VERSION 1
+#define FLATSQL_SLOT_HEADER 448
+typedef struct FlatSQLLayout {
+    uint32_t version;          /*   0: 1 */
+    uint32_t size;             /*   4: 640 */
+    uint32_t headerSize;       /*   8: 448 */
+    uint32_t stopWord;         /*  12: address of the u32 stop word */
+    uint32_t nSlots[2];        /*  16: [0] write pool, [1] read pool */
+    uint32_t slotBase[2];      /*  24 */
+    uint32_t slotStride[2];    /*  32: headerSize + reqBytes + ringBytes, 64-aligned */
+    uint32_t reqBytes[2];      /*  40 */
+    uint32_t ringBytes[2];     /*  48: powers of two */
+    uint32_t queueCells[4];    /*  56: per class, index = class-1 */
+    uint32_t queueMask[4];     /*  72 */
+    uint32_t queueEnq[4];      /*  88: address of the u64 enqueue counter */
+    uint32_t queueDeq[4];      /* 104: address of the u64 dequeue counter */
+    uint32_t nThreads;         /* 120: service threads (writers, lanes, maintenance), <= 64 */
+    uint32_t rsv;              /* 124: 0 */
+    uint32_t threadClass[64];  /* 128: 1 write, 2 interactive, 3 bulk, 4 sandbox, 5 maintenance */
+    uint32_t doorbell[64];     /* 384: address of the thread's doorbell u32; its state u32 is at doorbell+4
+                                        (0 idle, 1 busy, 2 stopped) */
+} FlatSQLLayout;
+
+/* Slot header byte offsets (version 1). */
+enum {
+    FLATSQL_SLOT_OFF_STATE = 0, FLATSQL_SLOT_OFF_CANCEL = 4, FLATSQL_SLOT_OFF_OUTSEQ = 8, FLATSQL_SLOT_OFF_SPACESEQ = 12,
+    FLATSQL_SLOT_OFF_OP = 16, FLATSQL_SLOT_OFF_CLASS = 20, FLATSQL_SLOT_OFF_FLAGS = 24, FLATSQL_SLOT_OFF_REQLEN = 28,
+    FLATSQL_SLOT_OFF_REQID = 32, FLATSQL_SLOT_OFF_MAX_ROWS_EXAMINED = 40, FLATSQL_SLOT_OFF_MAX_BYTES_READ = 48,
+    FLATSQL_SLOT_OFF_MAX_RESULT_ROWS = 56, FLATSQL_SLOT_OFF_MAX_RESULT_BYTES = 64, FLATSQL_SLOT_OFF_RING_HEAD = 72,
+    FLATSQL_SLOT_OFF_RING_TAIL = 80, FLATSQL_SLOT_OFF_STATUS = 88, FLATSQL_SLOT_OFF_ERRLEN = 92, FLATSQL_SLOT_OFF_ROWS_OUT = 96,
+    FLATSQL_SLOT_OFF_ROWS_EXAMINED = 104, FLATSQL_SLOT_OFF_BYTES_READ = 112, FLATSQL_SLOT_OFF_SUBMIT_NS = 120,
+    FLATSQL_SLOT_OFF_START_NS = 128, FLATSQL_SLOT_OFF_END_NS = 136, FLATSQL_SLOT_OFF_ERR = 144, FLATSQL_SLOT_ERR_CAP = 256,
+    FLATSQL_SLOT_OFF_THREAD = 400
+};
+
+/* ---- §3.3 Exports (exact) ------------------------------------------------ */
+
+FLATSQL_EXPORT("flatsql_init")          int32_t flatsql_init(const uint8_t* cfg, int32_t cfgLen);
+FLATSQL_EXPORT("flatsql_start")         int32_t flatsql_start(void);      /* >= 0: threads started */
+FLATSQL_EXPORT("flatsql_layout")        int32_t flatsql_layout(FlatSQLLayout* out); /* bytes written = 640 */
+FLATSQL_EXPORT("flatsql_wake")          int32_t flatsql_wake(uint32_t* addr, int32_t n); /* memory.atomic.notify; woken count */
+FLATSQL_EXPORT("flatsql_register_type") int32_t flatsql_register_type(const uint8_t* spec, int32_t specLen);
+FLATSQL_EXPORT("flatsql_set_quota")     int32_t flatsql_set_quota(double bytes);   /* 0 = none */
+FLATSQL_EXPORT("flatsql_activate")      int32_t flatsql_activate(void);            /* create mode 2 only */
+FLATSQL_EXPORT("flatsql_stats")         int32_t flatsql_stats(uint8_t* out, int32_t cap); /* bytes written; u64 LE array §3.10 */
+FLATSQL_EXPORT("flatsql_stop")          int32_t flatsql_stop(double deadlineMs);   /* FLATSQL_OK, or FLATSQL_E_BUSY at the deadline */
+FLATSQL_EXPORT("flatsql_alloc")         void*   flatsql_alloc(int32_t n);          /* wasm only */
+FLATSQL_EXPORT("flatsql_free")          void    flatsql_free(void* p);             /* wasm only */
+
+/* Prepare a PUT or DELETE mailbox request from an unchanged stream of size-prefixed
+ * records. Metadata is the control TLV (type, producer, source tags, timestamp;
+ * tag 55: u32 signature count, then u16 length + bytes per PUT record).
+ * CID computation stays in the guest. Returns status and an allocated buffer.
+ * Caller frees *output with flatsql_free. */
+FLATSQL_EXPORT("flatsql_record_request") int32_t flatsql_record_request(
+    int32_t opcode, const uint8_t* frames, int32_t size, const uint8_t* metadata, int32_t metadataSize, uint8_t** output, uint32_t* outLen);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

package/cpp/include/flatsql/flatsql_io.h

@@ -0,0 +1,159 @@
+#ifndef FLATSQL_IO_H
+#define FLATSQL_IO_H
+
+/*
+ * FlatSQL host I/O contract — SEVEN imports, one contract, every host.
+ * =====================================================================
+ *
+ * FlatSQL does not use emscripten's filesystem and does not use WASI file
+ * descriptors. It registers its OWN sqlite3_vfs ("flatsql_io") whose methods
+ * call out through the seven functions declared below. Every host — the
+ * browser JS shim, the standalone WASI shim, WasmEdge/wazero in the Go node —
+ * satisfies the SAME seven names with the SAME signatures. There is no runtime
+ * detection anywhere in module code; the difference between a browser
+ * key->bytes store and a real POSIX file lives entirely in the host shim.
+ * See docs/STORAGE-DURABILITY.md §3.5.
+ *
+ * WHY NOT WASI path_open: MEASURED (§3.5). `-s WASMFS=1` routes file I/O to an
+ * in-memory backend that looks durable and is not; `-s FORCE_FILESYSTEM=1`
+ * emits ten emscripten `__syscall_*` imports that no host satisfies and still
+ * never emits path_open. Both were reverted.
+ *
+ * SIGNATURE LAW — i32 and f64 ONLY.
+ * Offsets and sizes are `double`, never `int64_t`. Emscripten legalizes i64 at
+ * the JS boundary for the browser target (splitting one i64 parameter into two
+ * i32s) but NOT for STANDALONE_WASM. An i64 in this contract would therefore
+ * give the two lanes genuinely different wasm signatures for the same function
+ * name — the exact divergence class this design exists to prevent. f64 is
+ * exact to 2^53 (9 PB), passes through JS unmodified, and is identical in
+ * every lane.
+ *
+ * ERROR LAW: errors are return values, never traps. The -fignore-exceptions
+ * WASI artifact turns any throw into a guest-poisoning `unreachable`
+ * (commit b26ed45), so nothing below may throw.
+ */
+
+#include <stdint.h>
+
+/* ---- open flags ------------------------------------------------------- */
+#define FLATSQL_IO_READ       0x0001
+#define FLATSQL_IO_WRITE      0x0002
+#define FLATSQL_IO_CREATE     0x0004
+#define FLATSQL_IO_EXCL       0x0008
+#define FLATSQL_IO_TRUNC      0x0010
+/* Advisory: the host may drop the file when the handle is closed. */
+#define FLATSQL_IO_DELETE_ON_CLOSE 0x0020
+
+/* Namespace operations. These are carried on `open` rather than spent as two
+ * more imports: a VFS needs xAccess and xDelete, and both are pure
+ * path->status questions that allocate no handle. With either bit set,
+ * flatsql_io_open returns a STATUS (0 = success/exists, negative = error or
+ * absent) and no handle is created. Hosts must not allocate for them. */
+#define FLATSQL_IO_PROBE      0x0040  /* xAccess: does the path exist?      */
+#define FLATSQL_IO_UNLINK     0x0080  /* xDelete: remove the path           */
+
+/* Partition store flags (docs/PARTITION-STORE.md §5.4, design A12/A38).
+ * They are flags, not imports: the import set stays seven. The values are the
+ * ones every space-data-module-sdk host implements (module-sdk 0.8.22,
+ * docs/isomorphic-pthreads.md "Flags and statuses"); they must not diverge.
+ *
+ * CREATE_PARENTS: create every missing parent directory (fsyncing the parent
+ *   of each one created) and, when the file itself is newly created, fsync its
+ *   parent directory before returning. Best effort on OPFS.
+ * UNLINK_IF_UNUSED: unlink, or FLATSQL_IO_ERR_BUSY while any handle names the
+ *   path. After a successful unlink a POSIX host fsyncs the parent directory.
+ *   (A plain UNLINK of an open path is BUSY on OPFS and allowed on POSIX: the
+ *   engine never unlinks a path it holds open.)
+ * OPEN_DEFERRED: return a handle before the open finishes; the first use
+ *   waits, or fails with the open's status. Synchronous hosts treat it as a
+ *   plain open. */
+#define FLATSQL_IO_CREATE_PARENTS   0x0100
+#define FLATSQL_IO_UNLINK_IF_UNUSED 0x0200
+#define FLATSQL_IO_OPEN_DEFERRED    0x0400
+
+/* ---- status codes (all negative; every call may return any of them) ---- */
+#define FLATSQL_IO_ERR_GENERIC    (-1)
+#define FLATSQL_IO_ERR_NOENT      (-2)
+#define FLATSQL_IO_ERR_ACCESS     (-3)
+#define FLATSQL_IO_ERR_IO         (-4)
+#define FLATSQL_IO_ERR_NOSPACE    (-5)
+#define FLATSQL_IO_ERR_BADHANDLE  (-6)
+#define FLATSQL_IO_ERR_BUSY       (-7)  /* path in use, or its lock held elsewhere */
+/* EEXIST (CREATE|EXCL on an existing path) maps to GENERIC on every host. */
+
+#if defined(__wasm__)
+#  define FLATSQL_IO_IMPORT(name) \
+      __attribute__((import_module("env"), import_name(name)))
+#else
+#  define FLATSQL_IO_IMPORT(name)
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Open (or probe/unlink) `path`. Returns a non-negative opaque handle, or a
+ * negative status. `pathLen` is explicit: hosts never scan for NUL. */
+FLATSQL_IO_IMPORT("flatsql_io_open")
+int32_t flatsql_io_open(const char* path, int32_t pathLen, int32_t flags);
+
+/* Offset-addressed read. Returns bytes actually read (may be < len at EOF), or
+ * a negative status. Never moves an implicit cursor — there isn't one. */
+FLATSQL_IO_IMPORT("flatsql_io_read")
+int32_t flatsql_io_read(int32_t handle, void* dst, int32_t len, double offset);
+
+/* Offset-addressed write. Writing past EOF extends the file (the gap reads as
+ * zeroes). Returns bytes written or a negative status. */
+FLATSQL_IO_IMPORT("flatsql_io_write")
+int32_t flatsql_io_write(int32_t handle, const void* src, int32_t len,
+                         double offset);
+
+/* Set the file length exactly (grow or shrink). */
+FLATSQL_IO_IMPORT("flatsql_io_truncate")
+int32_t flatsql_io_truncate(int32_t handle, double size);
+
+/* Durability barrier. Returns 0 only when prior writes are durable. */
+FLATSQL_IO_IMPORT("flatsql_io_sync")
+int32_t flatsql_io_sync(int32_t handle);
+
+/* Current length in bytes, or a negative status. */
+FLATSQL_IO_IMPORT("flatsql_io_size")
+double flatsql_io_size(int32_t handle);
+
+/* Release the handle. */
+FLATSQL_IO_IMPORT("flatsql_io_close")
+int32_t flatsql_io_close(int32_t handle);
+
+#ifdef __cplusplus
+}  /* extern "C" */
+
+namespace flatsql {
+
+/* Name of the VFS registered over the seven imports above. */
+extern const char* const kFlatSqlVfsName;  /* "flatsql_io" */
+
+/* Idempotent. Registers the VFS with SQLite. `makeDefault` decides whether an
+ * open with a NULL vfs name lands here. Returns SQLITE_OK on success. Never
+ * throws. */
+int registerFlatSqlIoVfs(bool makeDefault);
+
+/* Connections opened with the URI parameter ra=1 read ahead sequentially:
+ * up to `streams` streams per connection (1..8, default 2), `bytes` per
+ * read-ahead (default 1 MiB). Applies to reads after the call. */
+void setFlatSqlIoReadahead(int streams, int bytes);
+
+/* Process-wide VFS counters. */
+struct FlatSqlIoVfsStats {
+    int64_t walIndexBytes;      /* heap WAL-index regions mapped now */
+    int64_t walIndexBytesPeak;
+    int64_t readaheadReads;     /* host reads issued by readahead */
+    int64_t readaheadHits;      /* SQLite reads served from readahead buffers */
+    int64_t readaheadBytes;
+    int64_t nodes;              /* paths with an open database file */
+};
+FlatSqlIoVfsStats flatSqlIoVfsStats();
+
+}  /* namespace flatsql */
+#endif /* __cplusplus */
+
+#endif /* FLATSQL_IO_H */

package/docs/API-3.12.md

@@ -0,0 +1,126 @@
+# FlatSQL 3.12 binary API
+
+FlatSQL has one public JavaScript entry point, `flatsql`, exporting `FlatSQL`
+and `FlatSQLError`, and one native ABI, `cpp/include/flatsql/flatsql.h`.
+`flatsql/flatsql.wasm` identifies the same WASI threads artifact on every host.
+There are no PS/P4 selectors, old engine aliases, or compatibility modes.
+
+## Schemas, records, and queries
+
+- `FlatSQL.open(options)` opens persistent host storage. Node `root` is a
+  filesystem directory; browser `root` is an OPFS directory. Node without a
+  root creates a temporary store. `close()` drains/cancels operations and stops
+  the engine. Close an instance before opening the same directory again.
+- `registerSchema({name, fileIdentifier, bfbs, ...})` registers a binary
+  reflection schema. Type names are SQL identifiers, up to 64 characters; the
+  file identifier is four bytes. Indexed extraction rules and full-text
+  indexing are schema configuration, executed inside the guest.
+- `ingest(type, records, metadata)` accepts concatenated frames:
+  `[u32 little-endian payload length][FlatBuffer payload]`. The guest validates
+  records, computes identities, persists streams and updates indexes. Provider,
+  source, producer, batch and timestamp metadata are optional. `signatures`
+  supplies one opaque byte array per record, retained in `_signature`.
+- `remove(type, records)` deletes those original binary payloads and returns a
+  QRP receipt. Record hashing and deletion stay inside the engine.
+- `query(request, options)` returns an async stream of byte chunks; `execute`
+  collects it within a byte budget. Requests are canonical SDS QRP buffers,
+  optionally size-prefixed, with `KIND=Request` and `REPRESENTATION=FlatBuffer`
+  (or their schema defaults). `SQL` and `PARAMETERS` drive the query. Express
+  filters, sort, selection, and paging in SQL; unrelated QRP selection fields
+  are rejected instead of silently ignored.
+- Projection output is a stream of size-prefixed SDS QRP pages: column names,
+  typed rows, and a final summary containing counts, status and truncation.
+  Chunk boundaries need not coincide with frame boundaries. QRP null, integer,
+  float, text and bytes preserve SQLite values; boolean parameters bind as
+  SQLite integers 0/1. Integers remain signed 64-bit values.
+- `SELECT _data` with `{records:true}` streams original record frames unchanged.
+  IDs, properties, altitude, null geometry and bounding boxes are preserved
+  byte for byte. Derived geometry has the separate semantics below.
+- With `registerSchema({..., fullText:true})`, SQL `_search MATCH ?1` uses the
+  engine's FTS5 index over the record's text fields, including nested text.
+  The engine returns `BUSY` while its background index catches up; retry the
+  query, and never treat that condition as zero matches.
+- Hidden metadata columns `_cid`, `_seq`, `_ts`, `_epoch`, `_producer`, `_peer`
+  and `_signature` are available alongside `_source`, `_rowid` and `_data`.
+- `flatsql_provenance(type, cid)` is a SQL table-valued function returning every
+  persisted copy and source tag for that content record. Both arguments are
+  required; `cid` is the `_cid` text from the record relation. Join it to the
+  record relation to preserve provider, source, batch, producer, original peer,
+  timestamp and signature when copying records. Its columns are `cid`, `seq`,
+  `producer`, `peer`, `ts`, `sig`, `provider`, `source`, `source_url`, `batch`,
+  `content_key_id`, `producer_peer`, `producer_pubkey` and `at`. Untagged copies
+  have NULL tag columns. Use distinct CIDs when joining the ordinary record
+  relation: a CID can occur in several feeds, and `_source` alone does not
+  enumerate every membership. Results use the same
+  typed QRP pages and query budgets as other SQL queries.
+  The function name is reserved and cannot be registered as a schema name.
+- Root scalar/string/byte-vector fields are exposed by BFBS reflection.
+  Complex/nested fields remain in `_data`. Unsigned integers above the SQLite
+  signed 64-bit range produce an error; retrieve `_data` to preserve them.
+- `surface()` returns the SQL catalog in QRP pages. `setQuota(bytes)` changes the
+  disk budget; zero removes that budget. Low-level mailbox maintenance and
+  storage controls are documented in the C header, behind the same engine.
+
+`QueryOptions` provides cancellation (`AbortSignal`), elapsed timeout,
+rows/result bytes, rows examined, bytes read, and bulk/sandbox lane selection.
+QRP `MAX_ROWS`, `LIMIT`, `MAX_BYTES`, and `MAX_ELAPSED_MS` are enforced by the
+engine. Breaking a stream cancels and drains its lane before its slot is reused.
+`execute` defaults to a 16 MiB collected result budget, with bounded room for
+terminal framing. A budget failure throws `FlatSQLError`; already streamed
+chunks are partial, never a successful complete answer.
+
+The data boundary accepts `Uint8Array`, not JavaScript rows or SQL strings.
+The package does not parse or render JSON, GeoJSON or WKT, compile schema text,
+or provide the SQLite JSON extension. Manifest JSON is tooling metadata.
+Convert separately with flatc-wasm or generated SDS bindings. Private packed
+control messages configure memory and files; they do not carry a second public
+record or SQL result format.
+
+## Geometry
+
+Geometry arguments are canonical SDS GJN (`$GJN`, root `GJN`) FlatBuffers,
+provided as a payload or one size-prefixed frame. Results are GJN payloads in
+QRP `Bytes` values. Every result verifies against the same SDS schema. No
+custom geometry blob or JSON parser exists inside the engine.
+
+All seven GJN geometry kinds are decoded, including multi-geometries and
+collections. Multiple features are evaluated as a collection; null feature
+geometries contribute no points. SQL NULL propagates. Malformed buffers,
+non-finite positions, open polygon rings, and unsupported combinations produce
+SQL errors. Properties remain opaque, including SDS `JSON_VALUE` strings.
+
+| Function | Behavior |
+| --- | --- |
+| `geo_distance`, `geo_within_radius`, `geo_bearing` | Spherical calculations; distances/radii in kilometers, coordinates in latitude/longitude order. |
+| `geo_bbox_contains`, `geo_area_bbox` | Scalar bounding-box predicates and spherical area. |
+| `geo_contains(g,lat,lon)` | Point/polygon membership, including polygon holes and collection members. |
+| `geo_area_geom(g)` | Sum of planar polygon areas minus holes, in square degrees. Non-area members contribute zero. |
+| `geo_length_geom(g)` | Haversine line length in kilometers; collection members contribute their lengths. |
+| `geo_envelope(g)`, `geo_convex_hull(g)` | Two-dimensional envelope/hull over every member's coordinates. |
+| `geo_centroid(g)` | Planar centroid, weighted by polygon area minus holes, then line length, then point count. Highest-dimensional members contribute; empty/degenerate input returns NULL. |
+| `geo_intersection`, `geo_union`, `geo_difference`, `geo_sym_difference` | Convex, hole-free polygon operands, either winding. Exact clipped pieces; concave or holed operands are rejected. Results may contain multiple polygon pieces with touching boundaries. |
+| `geo_buffer(g,d)` | Point buffer with positive distance in degrees, approximated by a 16-segment circle. Other geometry kinds are rejected. |
+| `geo_voronoi`, `geo_delaunay` | Point/multipoint sites in the longitude/latitude plane. Voronoi bounds must be a convex, hole-free polygon. |
+| `geo_destination`, `geo_midpoint` | GJN point results from spherical calculations. |
+| `geo_to_ecef_x/y/z(lat,lon,alt)` | Separate numeric Cartesian components in meters; altitude input is meters. |
+| `geo_from_ecef(x,y,z)` | GJN point with WGS84 longitude, latitude and altitude in meters. |
+
+Derived map geometry is explicitly 2D: it contains newly computed coordinates
+and no feature IDs, properties, source bbox or altitude. `geo_from_ecef` computes
+altitude. Scalar map measurements ignore altitude. Original record queries
+preserve the complete original 3D record. Geometry computations are planar
+unless explicitly described as spherical; they do not unwrap antimeridian
+crossings or act as a global spherical topology engine.
+
+## Integrity and hosts
+
+Default loads verify the adjacent `integrity.json`: browser Fetch SRI and Node
+SHA-256. Supplying `wasmBytes` requires a verification callback, allowing a host
+to enforce its signed artifact policy before instantiation. Generic adapters
+provide threads, memory, filesystem I/O and transport; query, geometry, indexing,
+and record identity logic stay inside C++/WASM.
+
+Browser hosting requires shared memory and cross-origin isolation. Worker URLs
+can be supplied explicitly for bundlers. The distributed private runtime
+adapters bundle only the necessary published SDK host code and its license;
+they do not bring the SDK's legacy FlatSQL dependency into the package.