lithos 0.1.0

data/ext/lithos/lithos.cpp

@@ -0,0 +1,1026 @@
+/*
+ * lithos — a small embedded, ordered, crash-safe key-value store for Ruby.
+ *
+ * Engine: a log-structured merge (LSM) tree.
+ *   - Every put/delete is appended to a write-ahead log (WAL) with a CRC32 and,
+ *     in sync mode, fsync'd before the call returns (durable-on-return).
+ *   - Live data sits in an in-memory sorted memtable (std::map, memcmp order).
+ *   - When the memtable grows past a threshold it is flushed to an immutable,
+ *     sorted SSTable file (sparse index + bloom filter + footer/magic), and a
+ *     fresh WAL is started.
+ *   - Reads check the memtable, then SSTables newest->oldest (bloom-filtered).
+ *   - compact() merges all SSTables into one, dropping shadowed keys/tombstones.
+ *   - The live set (SSTables + active WAL + file counter) is recorded in a
+ *     MANIFEST that is rewritten atomically (temp + FlushFileBuffers + rename).
+ *   - On open, the MANIFEST is read and the active WAL replayed; a torn/garbage
+ *     WAL tail is detected by CRC and discarded.
+ *
+ * Keys and values are arbitrary BINARY strings (embedded NULs allowed); ordering
+ * is lexicographic over unsigned bytes (memcmp). Single-writer (exclusive dir
+ * lock); a DB is not safe to share across threads.
+ *
+ * Build (mswin/MSVC): <ruby.h> before <windows.h>; $CXXFLAGS << " -EHsc".
+ */
+
+#include <ruby.h>
+#include <ruby/encoding.h>
+
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h>
+#include <io.h>
+
+#include <stdint.h>
+#include <string>
+#include <map>
+#include <vector>
+#include <queue>
+#include <algorithm>
+#include <cstring>
+#include <cstdio>
+
+/* ===================== little-endian + varint + crc + hash ============== */
+
+static void put_u32(std::string &b, uint32_t v) { for (int i = 0; i < 4; i++) b.push_back((char)((v >> (8 * i)) & 0xFF)); }
+static void put_u64(std::string &b, uint64_t v) { for (int i = 0; i < 8; i++) b.push_back((char)((v >> (8 * i)) & 0xFF)); }
+static uint32_t get_u32(const uint8_t *p) { uint32_t v = 0; for (int i = 0; i < 4; i++) v |= (uint32_t)p[i] << (8 * i); return v; }
+static uint64_t get_u64(const uint8_t *p) { uint64_t v = 0; for (int i = 0; i < 8; i++) v |= (uint64_t)p[i] << (8 * i); return v; }
+
+static void put_varint(std::string &b, uint64_t v) {
+    do { uint8_t x = (uint8_t)(v & 0x7F); v >>= 7; if (v) x |= 0x80; b.push_back((char)x); } while (v);
+}
+/* Advances p; returns false on malformed/overflow/out-of-bounds. */
+static bool get_varint(const uint8_t *&p, const uint8_t *end, uint64_t *out) {
+    uint64_t v = 0; int sh = 0;
+    while (p < end) {
+        uint8_t b = *p++;
+        v |= (uint64_t)(b & 0x7F) << sh;
+        if (!(b & 0x80)) { *out = v; return true; }
+        sh += 7; if (sh > 63) return false;
+    }
+    return false;
+}
+
+static uint32_t crc_table[256];
+static bool crc_ready = false;
+static void crc32_init(void) {
+    for (uint32_t i = 0; i < 256; i++) {
+        uint32_t c = i;
+        for (int k = 0; k < 8; k++) c = (c & 1) ? (0xEDB88320u ^ (c >> 1)) : (c >> 1);
+        crc_table[i] = c;
+    }
+    crc_ready = true;
+}
+static uint32_t crc32_buf(const uint8_t *b, size_t n) {
+    if (!crc_ready) crc32_init();
+    uint32_t c = 0xFFFFFFFFu;
+    for (size_t i = 0; i < n; i++) c = crc_table[(c ^ b[i]) & 0xFF] ^ (c >> 8);
+    return c ^ 0xFFFFFFFFu;
+}
+static uint64_t fnv1a64(const char *p, size_t n) {
+    uint64_t h = 1469598103934665603ull;
+    for (size_t i = 0; i < n; i++) { h ^= (uint8_t)p[i]; h *= 1099511628211ull; }
+    return h;
+}
+
+/* unsigned-byte (memcmp) comparator with length tiebreak */
+struct ByteCmp {
+    bool operator()(const std::string &a, const std::string &b) const {
+        size_t n = a.size() < b.size() ? a.size() : b.size();
+        int c = n ? memcmp(a.data(), b.data(), n) : 0;
+        if (c != 0) return c < 0;
+        return a.size() < b.size();
+    }
+};
+static int bytecmp(const std::string &a, const std::string &b) {
+    size_t n = a.size() < b.size() ? a.size() : b.size();
+    int c = n ? memcmp(a.data(), b.data(), n) : 0;
+    if (c != 0) return c < 0 ? -1 : 1;
+    if (a.size() == b.size()) return 0;
+    return a.size() < b.size() ? -1 : 1;
+}
+
+/* ===================== records / memtable =============================== */
+
+enum { KIND_VALUE = 0, KIND_TOMB = 1 };
+struct Record { uint8_t kind; std::string value; };
+typedef std::map<std::string, Record, ByteCmp> Memtable;
+
+static VALUE mLithos, cDB, eError;
+
+/* ===================== path / file helpers ============================== */
+
+static std::wstring utf8_to_wide(const char *s, size_t n) {
+    if (n == 0) return std::wstring();
+    int need = MultiByteToWideChar(CP_UTF8, 0, s, (int)n, NULL, 0);
+    std::wstring w((size_t)(need > 0 ? need : 0), L'\0');
+    if (need > 0) MultiByteToWideChar(CP_UTF8, 0, s, (int)n, &w[0], need);
+    return w;
+}
+static std::wstring child_path(const std::wstring &dir, const char *name) {
+    std::wstring w = dir;
+    if (!w.empty() && w.back() != L'\\' && w.back() != L'/') w.push_back(L'\\');
+    for (const char *p = name; *p; p++) w.push_back((wchar_t)(unsigned char)*p);
+    return w;
+}
+static std::wstring numbered(const std::wstring &dir, uint64_t num, const char *ext) {
+    char buf[64];
+    snprintf(buf, sizeof(buf), "%06llu%s", (unsigned long long)num, ext);
+    return child_path(dir, buf);
+}
+
+static bool write_all(HANDLE h, const void *buf, size_t len) {
+    const char *p = (const char *)buf;
+    while (len > 0) {
+        DWORD chunk = (len > 0x40000000u) ? 0x40000000u : (DWORD)len;
+        DWORD wrote = 0;
+        if (!WriteFile(h, p, chunk, &wrote, NULL) || wrote == 0) return false;
+        p += wrote; len -= wrote;
+    }
+    return true;
+}
+static bool fsync_handle(HANDLE h) { return FlushFileBuffers(h) != 0; }
+
+/* Transient locks (antivirus / the search indexer briefly holding a freshly
+   touched file) surface as ERROR_ACCESS_DENIED / ERROR_SHARING_VIOLATION on
+   rename/delete/create. Retry those a few times before giving up. */
+static bool transient_err(DWORD e) {
+    return e == ERROR_ACCESS_DENIED || e == ERROR_SHARING_VIOLATION;
+}
+static HANDLE create_retry(const std::wstring &path, DWORD access, DWORD share,
+                           DWORD disp, DWORD flags) {
+    for (int i = 0; i < 60; i++) {
+        HANDLE h = CreateFileW(path.c_str(), access, share, NULL, disp, flags, NULL);
+        if (h != INVALID_HANDLE_VALUE) return h;
+        if (!transient_err(GetLastError())) return INVALID_HANDLE_VALUE;
+        Sleep(10);
+    }
+    return INVALID_HANDLE_VALUE;
+}
+static bool move_replace_retry(const std::wstring &from, const std::wstring &to) {
+    for (int i = 0; i < 60; i++) {
+        if (MoveFileExW(from.c_str(), to.c_str(), MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH))
+            return true;
+        if (!transient_err(GetLastError())) return false;
+        Sleep(10);
+    }
+    return false;
+}
+static void delete_retry(const std::wstring &path) {
+    for (int i = 0; i < 60; i++) {
+        if (DeleteFileW(path.c_str())) return;
+        DWORD e = GetLastError();
+        if (e == ERROR_FILE_NOT_FOUND || e == ERROR_PATH_NOT_FOUND) return;
+        if (!transient_err(e)) return;
+        Sleep(10);
+    }
+}
+
+/* Write `data` to a temp file, fsync, then atomically rename over `final`. */
+static bool publish_file(const std::wstring &final_path, const std::string &data) {
+    std::wstring tmp = final_path + L".tmp";
+    HANDLE h = create_retry(tmp, GENERIC_WRITE, 0, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL);
+    if (h == INVALID_HANDLE_VALUE) return false;
+    bool ok = write_all(h, data.data(), data.size()) && fsync_handle(h);
+    CloseHandle(h);
+    if (!ok) { delete_retry(tmp); return false; }
+    if (!move_replace_retry(tmp, final_path)) { delete_retry(tmp); return false; }
+    return true;
+}
+/* Read an entire file into `out`. Returns false if it doesn't exist / fails. */
+static bool read_whole(const std::wstring &path, std::string &out) {
+    HANDLE h = CreateFileW(path.c_str(), GENERIC_READ, FILE_SHARE_READ, NULL,
+                           OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
+    if (h == INVALID_HANDLE_VALUE) return false;
+    LARGE_INTEGER sz;
+    if (!GetFileSizeEx(h, &sz)) { CloseHandle(h); return false; }
+    out.resize((size_t)sz.QuadPart);
+    size_t off = 0;
+    bool ok = true;
+    while (off < out.size()) {
+        DWORD want = (DWORD)((out.size() - off > 0x40000000u) ? 0x40000000u : (out.size() - off));
+        DWORD got = 0;
+        if (!ReadFile(h, &out[off], want, &got, NULL) || got == 0) { ok = false; break; }
+        off += got;
+    }
+    CloseHandle(h);
+    return ok;
+}
+
+/* ===================== SSTable format ===================================
+ * data:   [ entry ]*  (ascending key)
+ *   entry = varint klen | key | u8 kind | varint vlen | value
+ * bloom:  u32 k | u32 m | bitmap[ceil(m/8)]
+ * index:  [ varint klen | key | u64 dataoffset ]*  (sparse, ascending)
+ * footer (40 bytes, at EOF):
+ *   u64 index_off | u64 index_len | u64 bloom_off | u64 bloom_len | u64 MAGIC
+ * ====================================================================== */
+
+static const uint64_t SST_MAGIC = 0x4C495448534F5331ull; /* "LITHSOS1" */
+static const size_t   SST_FOOTER = 40;
+static const int      SST_INDEX_STRIDE = 16; /* one sparse index entry per N data entries */
+
+/* Build an SSTable image (as a byte string) from sorted entries. */
+struct OutEntry { std::string key; uint8_t kind; std::string value; };
+
+static std::string build_sstable(const std::vector<OutEntry> &entries) {
+    std::string data, index;
+    size_t n = entries.size();
+
+    /* bloom params: ~10 bits/key, k≈7 */
+    uint32_t m = (uint32_t)(n * 10 + 64);
+    uint32_t kfun = 7;
+    std::string bloom_bits((m + 7) / 8, '\0');
+
+    for (size_t i = 0; i < n; i++) {
+        const OutEntry &e = entries[i];
+        if ((i % SST_INDEX_STRIDE) == 0) {
+            put_varint(index, e.key.size());
+            index.append(e.key);
+            put_u64(index, (uint64_t)data.size());
+        }
+        put_varint(data, e.key.size());
+        data.append(e.key);
+        data.push_back((char)e.kind);
+        put_varint(data, e.value.size());
+        data.append(e.value);
+
+        uint64_t hv = fnv1a64(e.key.data(), e.key.size());
+        uint32_t h1 = (uint32_t)hv, h2 = (uint32_t)(hv >> 32);
+        for (uint32_t j = 0; j < kfun; j++) {
+            uint32_t bit = (uint32_t)((uint64_t)h1 + (uint64_t)j * h2) % m;
+            bloom_bits[bit >> 3] |= (char)(1 << (bit & 7));
+        }
+    }
+
+    std::string out;
+    out.append(data);
+    uint64_t bloom_off = out.size();
+    put_u32(out, kfun);
+    put_u32(out, m);
+    out.append(bloom_bits);
+    uint64_t bloom_len = out.size() - bloom_off;
+    uint64_t index_off = out.size();
+    out.append(index);
+    uint64_t index_len = out.size() - index_off;
+    put_u64(out, index_off);
+    put_u64(out, index_len);
+    put_u64(out, bloom_off);
+    put_u64(out, bloom_len);
+    put_u64(out, SST_MAGIC);
+    return out;
+}
+
+/* A memory-mapped, immutable SSTable opened for reading. */
+struct SSTable {
+    uint64_t number;
+    HANDLE hFile, hMap;
+    const uint8_t *base;
+    size_t size;
+    uint64_t data_end;          /* data spans [0, data_end) */
+    const uint8_t *bloom_bits;  /* points into the map */
+    uint32_t bloom_k, bloom_m;
+    std::vector<std::pair<std::string, uint64_t> > index; /* sparse, ascending */
+
+    SSTable() : number(0), hFile(INVALID_HANDLE_VALUE), hMap(NULL), base(NULL),
+                size(0), data_end(0), bloom_bits(NULL), bloom_k(0), bloom_m(0) {}
+    ~SSTable() {
+        if (base) UnmapViewOfFile(base);
+        if (hMap) CloseHandle(hMap);
+        if (hFile != INVALID_HANDLE_VALUE) CloseHandle(hFile);
+    }
+
+    /* Open + parse footer/index/bloom. Returns false on any malformation. */
+    bool open(const std::wstring &path, uint64_t num) {
+        number = num;
+        hFile = CreateFileW(path.c_str(), GENERIC_READ, FILE_SHARE_READ, NULL,
+                            OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
+        if (hFile == INVALID_HANDLE_VALUE) return false;
+        LARGE_INTEGER sz;
+        if (!GetFileSizeEx(hFile, &sz) || (uint64_t)sz.QuadPart < SST_FOOTER) return false;
+        size = (size_t)sz.QuadPart;
+        hMap = CreateFileMappingW(hFile, NULL, PAGE_READONLY, 0, 0, NULL);
+        if (!hMap) return false;
+        base = (const uint8_t *)MapViewOfFile(hMap, FILE_MAP_READ, 0, 0, 0);
+        if (!base) return false;
+
+        const uint8_t *foot = base + size - SST_FOOTER;
+        uint64_t index_off = get_u64(foot);
+        uint64_t index_len = get_u64(foot + 8);
+        uint64_t bloom_off = get_u64(foot + 16);
+        uint64_t bloom_len = get_u64(foot + 24);
+        uint64_t magic = get_u64(foot + 32);
+        if (magic != SST_MAGIC) return false;
+        /* Subtractive bounds checks: the additive form `off + len > size` can
+           wrap mod 2^64 for an attacker/corruption-controlled 64-bit len and
+           bypass the check. Establish `off <= size` first, then `len <= size-off`. */
+        if (bloom_off > size || bloom_len > size - bloom_off || bloom_len < 8) return false;
+        if (index_off > size || index_len > size - index_off) return false;
+        if (bloom_off > index_off) return false;
+
+        data_end = bloom_off;
+        bloom_k = get_u32(base + bloom_off);
+        bloom_m = get_u32(base + bloom_off + 4);
+        bloom_bits = base + bloom_off + 8;
+        /* (bloom_m + 7) in 32-bit would wrap for bloom_m near 2^32; widen first. */
+        uint64_t bloom_need = 8 + ((uint64_t)bloom_m + 7) / 8;
+        if (bloom_m == 0 || bloom_need > bloom_len) return false;
+
+        const uint8_t *p = base + index_off;
+        const uint8_t *iend = p + index_len;
+        while (p < iend) {
+            uint64_t klen;
+            if (!get_varint(p, iend, &klen) || klen > (uint64_t)(iend - p)) return false;
+            std::string key((const char *)p, (size_t)klen); p += klen;
+            if ((uint64_t)(iend - p) < 8) return false;
+            uint64_t off = get_u64(p); p += 8;
+            if (off > data_end) return false;
+            index.push_back(std::make_pair(key, off));
+        }
+        return true;
+    }
+
+    bool bloom_maybe(const std::string &key) const {
+        uint64_t hv = fnv1a64(key.data(), key.size());
+        uint32_t h1 = (uint32_t)hv, h2 = (uint32_t)(hv >> 32);
+        for (uint32_t j = 0; j < bloom_k; j++) {
+            uint32_t bit = (uint32_t)((uint64_t)h1 + (uint64_t)j * h2) % bloom_m;
+            if (!(bloom_bits[bit >> 3] & (1 << (bit & 7)))) return false;
+        }
+        return true;
+    }
+
+    /* Offset to start scanning for `key` (largest sparse-index key <= key). */
+    uint64_t start_offset(const std::string &key) const {
+        size_t lo = 0, hi = index.size();
+        while (lo < hi) { /* first index entry with key > target */
+            size_t mid = (lo + hi) / 2;
+            if (bytecmp(index[mid].first, key) <= 0) lo = mid + 1; else hi = mid;
+        }
+        return lo == 0 ? 0 : index[lo - 1].second;
+    }
+
+    /* Parse the entry at p (within [base,data_end)); advance p. false if malformed. */
+    bool parse_entry(const uint8_t *&p, std::string &key, uint8_t &kind, std::string &value) const {
+        const uint8_t *end = base + data_end;
+        uint64_t klen;
+        if (!get_varint(p, end, &klen) || klen > (uint64_t)(end - p)) return false;
+        key.assign((const char *)p, (size_t)klen); p += klen;
+        if (p >= end) return false; /* need the kind byte */
+        kind = *p++;
+        uint64_t vlen;
+        if (!get_varint(p, end, &vlen) || vlen > (uint64_t)(end - p)) return false;
+        value.assign((const char *)p, (size_t)vlen); p += vlen;
+        return true;
+    }
+
+    /* 0 = absent, 1 = value (sets *out), 2 = tombstone. */
+    int get(const std::string &key, std::string *out) const {
+        if (!bloom_maybe(key)) return 0;
+        const uint8_t *p = base + start_offset(key);
+        const uint8_t *end = base + data_end;
+        std::string k, v; uint8_t kind;
+        while (p < end) {
+            if (!parse_entry(p, k, kind, v)) return 0;
+            int c = bytecmp(k, key);
+            if (c == 0) { if (kind == KIND_VALUE) { *out = v; return 1; } return 2; }
+            if (c > 0) return 0;
+        }
+        return 0;
+    }
+};
+
+/* ===================== merge iterator =================================== */
+
+struct Source {
+    bool is_mem;
+    uint64_t recency;            /* memtable highest; SSTables by number */
+    /* memtable */
+    Memtable::const_iterator mit, mend;
+    /* sstable */
+    const SSTable *sst;
+    const uint8_t *p, *pend;
+    /* current head */
+    std::string key, value;
+    uint8_t kind;
+    bool valid;
+    /* upper bound */
+    const std::string *upper; bool upper_incl;
+
+    bool within_upper(const std::string &k) const {
+        if (!upper) return true;
+        int c = bytecmp(k, *upper);
+        return upper_incl ? (c <= 0) : (c < 0);
+    }
+    void load_current() {
+        if (is_mem) {
+            if (mit == mend) { valid = false; return; }
+            key = mit->first; kind = mit->second.kind; value = mit->second.value;
+        } /* sstable head already parsed into key/kind/value by advance */
+        valid = within_upper(key);
+    }
+    void advance() {
+        if (is_mem) {
+            ++mit;
+            if (mit == mend) { valid = false; return; }
+            key = mit->first; kind = mit->second.kind; value = mit->second.value;
+            valid = within_upper(key);
+        } else {
+            if (p >= pend) { valid = false; return; }
+            if (!sst->parse_entry(p, key, kind, value)) { valid = false; return; }
+            valid = within_upper(key);
+        }
+    }
+};
+
+/* Pick the valid source with the smallest key; ties go to the newest (highest
+   recency). Returns its index, or -1 when all sources are exhausted. */
+static int merge_pick(std::vector<Source> &src) {
+    int best = -1;
+    for (int i = 0; i < (int)src.size(); i++) {
+        if (!src[i].valid) continue;
+        if (best < 0) { best = i; continue; }
+        int c = bytecmp(src[i].key, src[best].key);
+        if (c < 0 || (c == 0 && src[i].recency > src[best].recency)) best = i;
+    }
+    return best;
+}
+
+/* Collecting merge for compaction (no Ruby calls — C++ temporaries are fine). */
+static void merge_collect(std::vector<Source> &src, std::vector<OutEntry> &out, bool drop_tombstones) {
+    int best;
+    while ((best = merge_pick(src)) >= 0) {
+        std::string key = src[best].key, value = src[best].value;
+        uint8_t kind = src[best].kind;
+        src[best].advance();
+        for (int j = 0; j < (int)src.size(); j++) {     /* drain older duplicates */
+            if (j == best) continue;
+            while (src[j].valid && bytecmp(src[j].key, key) == 0) src[j].advance();
+        }
+        if (kind == KIND_TOMB) {
+            if (!drop_tombstones) { OutEntry e; e.key = key; e.kind = KIND_TOMB; e.value.clear(); out.push_back(e); }
+            continue;
+        }
+        OutEntry e; e.key = key; e.kind = KIND_VALUE; e.value = value; out.push_back(e);
+    }
+}
+
+/* Build the [key,value] strings and yield — runs under rb_protect so any
+   exception/break from the user block (or a NoMemError from rb_str_new) is
+   captured locally instead of longjmp-ing through C++ frames. */
+struct YieldArg { const std::string *k; const std::string *v; };
+static VALUE merge_yield_one(VALUE arg) {
+    YieldArg *y = (YieldArg *)arg;
+    VALUE pair[2];
+    pair[0] = rb_str_new(y->k->data(), (long)y->k->size());
+    pair[1] = rb_str_new(y->v->data(), (long)y->v->size());
+    rb_enc_associate(pair[0], rb_ascii8bit_encoding());
+    rb_enc_associate(pair[1], rb_ascii8bit_encoding());
+    return rb_yield_values2(2, pair);
+}
+
+/* Yielding merge for #each/#scan. ek/ev are caller-owned (heap) buffers. The
+   only Ruby calls that can raise happen inside rb_protect, so NO longjmp ever
+   crosses this C++ frame. Returns 0 normally, or a non-zero Ruby tag (break /
+   exception) for the caller to re-raise with rb_jump_tag from a clean frame. */
+static int merge_yield(std::vector<Source> &src, std::string &ek, std::string &ev) {
+    int best;
+    while ((best = merge_pick(src)) >= 0) {
+        ek = src[best].key; ev = src[best].value;
+        uint8_t kind = src[best].kind;
+        src[best].advance();
+        for (int j = 0; j < (int)src.size(); j++) {
+            if (j == best) continue;
+            while (src[j].valid && bytecmp(src[j].key, ek) == 0) src[j].advance();
+        }
+        if (kind == KIND_TOMB) continue;
+        YieldArg y; y.k = &ek; y.v = &ev;
+        int state = 0;
+        rb_protect(merge_yield_one, (VALUE)&y, &state);
+        if (state) return state;
+    }
+    return 0;
+}
+
+/* ===================== the database ===================================== */
+
+struct DB {
+    std::wstring dir;
+    std::string  dir_utf8;
+    HANDLE lock;
+    HANDLE wal;
+    uint64_t wal_number;
+    uint64_t next_number;
+    Memtable mem;
+    size_t mem_bytes;
+    std::vector<SSTable *> ssts; /* ascending by number; newest at back */
+    bool sync_writes;
+    size_t memtable_limit;
+    size_t compact_trigger;
+    bool closed;
+    char errbuf[256]; /* last engine error; raised by the thin Ruby wrappers */
+
+    DB() : lock(INVALID_HANDLE_VALUE), wal(INVALID_HANDLE_VALUE), wal_number(0),
+           next_number(1), mem_bytes(0), sync_writes(true),
+           memtable_limit(4u * 1024 * 1024), compact_trigger(8), closed(true) {
+        errbuf[0] = '\0';
+    }
+
+    /* Engine methods NEVER call rb_raise (a longjmp through these /EHsc C++
+       frames is UB on MSVC). They record the error here and return false; the
+       Ruby-facing wrapper raises from a frame holding no live C++ object. */
+    bool fail(const char *what) {
+        snprintf(errbuf, sizeof(errbuf), "%s (GetLastError=%lu)", what,
+                 (unsigned long)GetLastError());
+        return false;
+    }
+    ~DB() { release(); }
+
+    void release() {
+        if (wal != INVALID_HANDLE_VALUE) { CloseHandle(wal); wal = INVALID_HANDLE_VALUE; }
+        for (size_t i = 0; i < ssts.size(); i++) delete ssts[i];
+        ssts.clear();
+        if (lock != INVALID_HANDLE_VALUE) { CloseHandle(lock); lock = INVALID_HANDLE_VALUE; }
+        closed = true;
+    }
+
+    /* ---- MANIFEST: framed [u32 len][u32 crc][payload]; payload =
+       u32 MAGIC | u32 ver | u64 next | u64 wal | u32 sstcount | u64 sst* ---- */
+    static const uint32_t MAN_MAGIC = 0x4C54484Du; /* "LTHM" */
+    std::wstring manifest_path() { return child_path(dir, "MANIFEST"); }
+
+    bool write_manifest() {
+        std::string pay;
+        put_u32(pay, MAN_MAGIC);
+        put_u32(pay, 1);
+        put_u64(pay, next_number);
+        put_u64(pay, wal_number);
+        put_u32(pay, (uint32_t)ssts.size());
+        for (size_t i = 0; i < ssts.size(); i++) put_u64(pay, ssts[i]->number);
+        std::string framed;
+        put_u32(framed, (uint32_t)pay.size());
+        put_u32(framed, crc32_buf((const uint8_t *)pay.data(), pay.size()));
+        framed.append(pay);
+        return publish_file(manifest_path(), framed);
+    }
+
+    /* Parse MANIFEST -> sst numbers + counters. false if missing/corrupt. */
+    bool read_manifest(std::vector<uint64_t> &sst_numbers) {
+        std::string buf;
+        if (!read_whole(manifest_path(), buf)) return false;
+        if (buf.size() < 8) return false;
+        const uint8_t *b = (const uint8_t *)buf.data();
+        uint32_t len = get_u32(b), crc = get_u32(b + 4);
+        if ((uint64_t)8 + len > buf.size()) return false;
+        const uint8_t *pay = b + 8;
+        if (crc32_buf(pay, len) != crc) return false;
+        const uint8_t *p = pay, *end = pay + len;
+        if (end - p < 4 + 4 + 8 + 8 + 4) return false;
+        if (get_u32(p) != MAN_MAGIC) return false; p += 4;
+        p += 4; /* version */
+        next_number = get_u64(p); p += 8;
+        wal_number = get_u64(p); p += 8;
+        uint32_t cnt = get_u32(p); p += 4;
+        for (uint32_t i = 0; i < cnt; i++) {
+            if (end - p < 8) return false;
+            sst_numbers.push_back(get_u64(p)); p += 8;
+        }
+        return true;
+    }
+
+    /* ---- WAL ---- */
+    bool open_wal(uint64_t num, bool create) {
+        std::wstring path = numbered(dir, num, ".wal");
+        wal = CreateFileW(path.c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ, NULL,
+                          create ? OPEN_ALWAYS : OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
+        if (wal == INVALID_HANDLE_VALUE) return false;
+        wal_number = num;
+        return true;
+    }
+    bool wal_seek_end() {
+        LARGE_INTEGER zero; zero.QuadPart = 0; LARGE_INTEGER np;
+        return SetFilePointerEx(wal, zero, &np, FILE_END) != 0;
+    }
+    bool wal_append(uint8_t op, const std::string &key, const std::string &value) {
+        std::string pay;
+        pay.push_back((char)op);
+        put_varint(pay, key.size()); pay.append(key);
+        if (op == KIND_VALUE) { put_varint(pay, value.size()); pay.append(value); }
+        std::string rec;
+        put_u32(rec, (uint32_t)pay.size());
+        put_u32(rec, crc32_buf((const uint8_t *)pay.data(), pay.size()));
+        rec.append(pay);
+        if (!write_all(wal, rec.data(), rec.size())) return false;
+        if (sync_writes && !fsync_handle(wal)) return false;
+        return true;
+    }
+    /* Replay a WAL file (read by path, BEFORE the writable handle is opened, so
+       there is no share conflict) into the memtable. Returns the durable byte
+       length: everything before a torn/garbage tail. */
+    size_t wal_replay_file(const std::wstring &path) {
+        std::string buf;
+        if (!read_whole(path, buf)) return 0;
+        const uint8_t *b = (const uint8_t *)buf.data();
+        size_t fsize = buf.size(), pos = 0, good = 0;
+        while (pos + 8 <= fsize) {
+            uint32_t len = get_u32(b + pos), crc = get_u32(b + pos + 4);
+            if ((uint64_t)pos + 8 + len > fsize) break;            /* torn tail */
+            const uint8_t *pay = b + pos + 8;
+            if (crc32_buf(pay, len) != crc) break;                 /* bad crc */
+            const uint8_t *p = pay, *end = pay + len;
+            if (p >= end) break;
+            uint8_t op = *p++;
+            uint64_t klen;
+            if (!get_varint(p, end, &klen) || (uint64_t)(end - p) < klen) break;
+            std::string key((const char *)p, (size_t)klen); p += klen;
+            if (op == KIND_VALUE) {
+                uint64_t vlen;
+                if (!get_varint(p, end, &vlen) || (uint64_t)(end - p) < vlen) break;
+                std::string val((const char *)p, (size_t)vlen); p += vlen;
+                Record r; r.kind = KIND_VALUE; r.value = val;
+                mem[key] = r; mem_bytes += key.size() + val.size();
+            } else {
+                Record r; r.kind = KIND_TOMB;
+                mem[key] = r; mem_bytes += key.size();
+            }
+            pos += 8 + len; good = pos;
+        }
+        return good;
+    }
+
+    /* ---- flush memtable -> new SSTable, rotate WAL, rewrite manifest.
+       Returns false (with errbuf set) on failure; never raises. ---- */
+    bool flush() {
+        if (mem.empty()) return true;
+        uint64_t sst_num = next_number++;
+        {
+            std::vector<OutEntry> entries;
+            entries.reserve(mem.size());
+            for (Memtable::const_iterator it = mem.begin(); it != mem.end(); ++it) {
+                OutEntry e; e.key = it->first; e.kind = it->second.kind; e.value = it->second.value;
+                entries.push_back(e);
+            }
+            std::string image = build_sstable(entries);
+            if (!publish_file(numbered(dir, sst_num, ".sst"), image)) return fail("write SSTable");
+        } /* entries + image freed before any later return */
+
+        SSTable *t = new SSTable();
+        if (!t->open(numbered(dir, sst_num, ".sst"), sst_num)) { delete t; return fail("re-open SSTable"); }
+        ssts.push_back(t);
+
+        uint64_t old_wal = wal_number;        /* rotate to a fresh WAL */
+        uint64_t new_wal = next_number++;
+        HANDLE old = wal;
+        if (!open_wal(new_wal, true)) { wal = old; return fail("open new WAL"); }
+        if (old != INVALID_HANDLE_VALUE) CloseHandle(old);
+
+        if (!write_manifest()) return fail("write MANIFEST");
+        delete_retry(numbered(dir, old_wal, ".wal"));
+
+        mem.clear();
+        mem_bytes = 0;
+        return true;
+    }
+
+    bool maybe_flush() {
+        if (mem_bytes >= memtable_limit && !flush()) return false;
+        if (ssts.size() > compact_trigger && !compact()) return false;
+        return true;
+    }
+
+    /* ---- compaction: merge all SSTables into one (drops tombstones).
+       Returns false (with errbuf set) on failure; never raises. ---- */
+    bool compact() {
+        if (!flush()) return false;       /* fold the memtable in first */
+        if (ssts.size() <= 1) return true;
+
+        uint64_t sst_num = next_number++;
+        {
+            std::vector<OutEntry> merged = merge_all(/*drop_tombstones=*/true);
+            std::string image = build_sstable(merged);
+            if (!publish_file(numbered(dir, sst_num, ".sst"), image)) return fail("write compacted SSTable");
+        } /* merged + image freed */
+
+        SSTable *t = new SSTable();
+        if (!t->open(numbered(dir, sst_num, ".sst"), sst_num)) { delete t; return fail("re-open compacted SSTable"); }
+
+        std::vector<SSTable *> old = ssts;
+        ssts.clear();
+        ssts.push_back(t);
+        if (!write_manifest()) { ssts = old; ssts.push_back(t); return fail("write MANIFEST (compact)"); }
+
+        for (size_t i = 0; i < old.size(); i++) {
+            uint64_t num = old[i]->number;
+            delete old[i]; /* unmap+close so the file can be deleted */
+            delete_retry(numbered(dir, num, ".sst"));
+        }
+        return true;
+    }
+
+    /* Acquire the lock, load SSTables from the MANIFEST, replay the WAL (or
+       create a fresh DB). Returns false (errbuf set) on failure; never raises. */
+    bool bootstrap() {
+        CreateDirectoryW(dir.c_str(), NULL); /* ok if it already exists */
+        {
+            std::wstring lockp = child_path(dir, "LOCK");
+            lock = CreateFileW(lockp.c_str(), GENERIC_READ | GENERIC_WRITE, 0 /* exclusive */,
+                               NULL, OPEN_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
+        }
+        if (lock == INVALID_HANDLE_VALUE) return fail("could not lock directory (already open elsewhere?)");
+
+        std::vector<uint64_t> sst_numbers;
+        if (read_manifest(sst_numbers)) {
+            for (size_t i = 0; i < sst_numbers.size(); i++) {
+                SSTable *t = new SSTable();
+                if (!t->open(numbered(dir, sst_numbers[i], ".sst"), sst_numbers[i])) {
+                    delete t;
+                    snprintf(errbuf, sizeof(errbuf), "SSTable %06llu missing or corrupt",
+                             (unsigned long long)sst_numbers[i]);
+                    return false;
+                }
+                ssts.push_back(t);
+            }
+            /* Replay the WAL by path FIRST (no writable handle held -> no share
+               conflict), then open it for appending and truncate any torn tail. */
+            size_t good = wal_replay_file(numbered(dir, wal_number, ".wal"));
+            if (!open_wal(wal_number, true)) return fail("open WAL");
+            LARGE_INTEGER fsz;
+            if (GetFileSizeEx(wal, &fsz) && good < (size_t)fsz.QuadPart) {
+                LARGE_INTEGER gp; gp.QuadPart = (LONGLONG)good;
+                if (SetFilePointerEx(wal, gp, NULL, FILE_BEGIN)) SetEndOfFile(wal);
+            }
+            if (!wal_seek_end()) return fail("seek WAL");
+        } else {
+            next_number = 1;
+            uint64_t wal_num = next_number++;
+            if (!open_wal(wal_num, true)) return fail("create WAL");
+            if (!write_manifest()) return fail("write MANIFEST");
+        }
+        closed = false;
+        return true;
+    }
+
+    /* Full ordered merge into a vector (used by compaction). */
+    std::vector<OutEntry> merge_all(bool drop_tombstones) {
+        std::vector<OutEntry> out;
+        std::vector<Source> src;
+        build_sources(src, NULL, false, NULL, false);
+        merge_collect(src, out, drop_tombstones);
+        return out;
+    }
+
+    /* Build merge sources seeked to the lower bound; upper stored per source. */
+    void build_sources(std::vector<Source> &src,
+                        const std::string *lower, bool lower_incl,
+                        const std::string *upper, bool upper_incl) {
+        /* memtable source (recency = max) */
+        {
+            Source s; s.is_mem = true; s.recency = (uint64_t)-1; s.sst = NULL;
+            s.upper = upper; s.upper_incl = upper_incl;
+            if (lower) {
+                s.mit = lower_incl ? mem.lower_bound(*lower) : mem.upper_bound(*lower);
+            } else s.mit = mem.begin();
+            s.mend = mem.end();
+            s.valid = false;
+            if (s.mit != s.mend) { s.key = s.mit->first; s.kind = s.mit->second.kind; s.value = s.mit->second.value; s.valid = s.within_upper(s.key); }
+            src.push_back(s);
+        }
+        for (size_t i = 0; i < ssts.size(); i++) {
+            SSTable *t = ssts[i];
+            Source s; s.is_mem = false; s.recency = t->number; s.sst = t;
+            s.upper = upper; s.upper_incl = upper_incl;
+            uint64_t off = lower ? t->start_offset(*lower) : 0;
+            s.p = t->base + off; s.pend = t->base + t->data_end;
+            s.valid = false;
+            /* parse forward to the first in-range key */
+            while (s.p < s.pend) {
+                if (!t->parse_entry(s.p, s.key, s.kind, s.value)) { s.valid = false; break; }
+                if (lower) {
+                    int c = bytecmp(s.key, *lower);
+                    if (c < 0 || (c == 0 && !lower_incl)) continue;
+                }
+                s.valid = s.within_upper(s.key);
+                break;
+            }
+            src.push_back(s);
+        }
+    }
+
+    /* (Ordered merge lives in the free functions merge_collect / merge_yield
+       above; merge_yield keeps no C++ destructor on the stack across rb_yield.) */
+};
+
+/* ===================== Ruby bindings ==================================== */
+
+typedef struct { DB *db; } DBWrap;
+
+static void db_free(void *p) {
+    DBWrap *w = (DBWrap *)p;
+    if (w) { delete w->db; w->db = NULL; xfree(w); }
+}
+static size_t db_memsize(const void *p) {
+    const DBWrap *w = (const DBWrap *)p;
+    size_t n = sizeof(DBWrap);
+    if (w && w->db) { n += sizeof(DB) + w->db->mem_bytes; }
+    return n;
+}
+static const rb_data_type_t db_type = { "Lithos::DB", { 0, db_free, db_memsize }, 0, 0, RUBY_TYPED_FREE_IMMEDIATELY };
+
+static VALUE db_alloc(VALUE klass) {
+    DBWrap *w;
+    VALUE obj = TypedData_Make_Struct(klass, DBWrap, &db_type, w);
+    w->db = new DB();
+    return obj;
+}
+static DB *get_db(VALUE self) {
+    DBWrap *w;
+    TypedData_Get_Struct(self, DBWrap, &db_type, w);
+    if (!w || !w->db) rb_raise(eError, "lithos: store is not initialized");
+    if (w->db->closed) rb_raise(eError, "lithos: store is closed");
+    return w->db;
+}
+static std::string to_bytes(VALUE v) {
+    StringValue(v);
+    return std::string(RSTRING_PTR(v), (size_t)RSTRING_LEN(v));
+}
+static VALUE from_bytes(const std::string &s) {
+    VALUE r = rb_str_new(s.data(), (long)s.size());
+    rb_enc_associate(r, rb_ascii8bit_encoding());
+    return r;
+}
+
+static VALUE db_initialize(int argc, VALUE *argv, VALUE self) {
+    VALUE vpath, vopts;
+    rb_scan_args(argc, argv, "1:", &vpath, &vopts);
+    StringValue(vpath);
+    DBWrap *w; TypedData_Get_Struct(self, DBWrap, &db_type, w);
+    DB *db = w->db;
+    if (!db->closed) rb_raise(eError, "lithos: store is already open"); /* no double-init */
+    db->dir_utf8.assign(RSTRING_PTR(vpath), (size_t)RSTRING_LEN(vpath));
+    db->dir = utf8_to_wide(db->dir_utf8.data(), db->dir_utf8.size());
+
+    if (!NIL_P(vopts)) {
+        VALUE s = rb_hash_aref(vopts, ID2SYM(rb_intern("sync")));
+        if (!NIL_P(s)) db->sync_writes = RTEST(s);
+        VALUE ml = rb_hash_aref(vopts, ID2SYM(rb_intern("memtable_bytes")));
+        if (!NIL_P(ml)) db->memtable_limit = (size_t)NUM2ULL(ml);
+    }
+
+    /* All filesystem work (lock, load SSTables, WAL replay) happens in
+       bootstrap(), which never raises; we raise here from a clean frame. */
+    if (!db->bootstrap()) {
+        db->release();
+        rb_raise(eError, "lithos: %s", db->errbuf);
+    }
+    return self;
+}
+
+static VALUE db_put(VALUE self, VALUE key, VALUE val) {
+    DB *db = get_db(self);
+    StringValue(key); StringValue(val); /* TypeError here: no C++ object live yet */
+    bool ok;
+    {
+        std::string k(RSTRING_PTR(key), (size_t)RSTRING_LEN(key));
+        std::string v(RSTRING_PTR(val), (size_t)RSTRING_LEN(val));
+        ok = db->wal_append(KIND_VALUE, k, v);
+        if (!ok) db->fail("WAL append");
+        else {
+            Record r; r.kind = KIND_VALUE; r.value = v;
+            db->mem[k] = r;
+            db->mem_bytes += k.size() + v.size();
+            ok = db->maybe_flush();
+        }
+    } /* k, v freed before any raise */
+    if (!ok) rb_raise(eError, "lithos: %s", db->errbuf);
+    return self; /* chainable; `db[k]=v` still evaluates to v per Ruby semantics */
+}
+
+static VALUE db_get(VALUE self, VALUE key) {
+    DB *db = get_db(self);
+    std::string k = to_bytes(key);
+    Memtable::const_iterator it = db->mem.find(k);
+    if (it != db->mem.end()) return it->second.kind == KIND_VALUE ? from_bytes(it->second.value) : Qnil;
+    std::string out;
+    for (size_t i = db->ssts.size(); i-- > 0;) {
+        int r = db->ssts[i]->get(k, &out);
+        if (r == 1) return from_bytes(out);
+        if (r == 2) return Qnil; /* tombstone shadows older */
+    }
+    return Qnil;
+}
+
+static VALUE db_key_p(VALUE self, VALUE key) {
+    DB *db = get_db(self);
+    std::string k = to_bytes(key);
+    Memtable::const_iterator it = db->mem.find(k);
+    if (it != db->mem.end()) return it->second.kind == KIND_VALUE ? Qtrue : Qfalse;
+    std::string out;
+    for (size_t i = db->ssts.size(); i-- > 0;) {
+        int r = db->ssts[i]->get(k, &out);
+        if (r == 1) return Qtrue;
+        if (r == 2) return Qfalse;
+    }
+    return Qfalse;
+}
+
+static VALUE db_delete(VALUE self, VALUE key) {
+    DB *db = get_db(self);
+    bool existed = RTEST(db_key_p(self, key));
+    bool ok;
+    {
+        std::string k = to_bytes(key);
+        ok = db->wal_append(KIND_TOMB, k, std::string());
+        if (!ok) db->fail("WAL append");
+        else {
+            Record r; r.kind = KIND_TOMB;
+            db->mem[k] = r;
+            db->mem_bytes += k.size();
+            ok = db->maybe_flush();
+        }
+    } /* k freed before any raise */
+    if (!ok) rb_raise(eError, "lithos: %s", db->errbuf);
+    return existed ? Qtrue : Qfalse;
+}
+
+/* _each_range(lower|nil, lower_incl_bool, upper|nil, upper_incl_bool) { |k,v| } */
+struct MergeRun { DB *db; std::vector<Source> src; std::string lo, hi; std::string ek, ev; };
+
+static VALUE db_each_range(VALUE self, VALUE vlo, VALUE vloi, VALUE vhi, VALUE vhii) {
+    DB *db = get_db(self);
+    if (!rb_block_given_p())
+        rb_raise(rb_eLocalJumpError, "lithos: no block given");
+
+    MergeRun *m = new MergeRun();
+    m->db = db;
+    {
+        /* Convert the bounds FIRST (StringValue can raise TypeError); copy into
+           the heap MergeRun so these stack locals are gone before any later jump. */
+        bool have_lo = !NIL_P(vlo), have_hi = !NIL_P(vhi);
+        const std::string *lo = NULL, *hi = NULL;
+        try {
+            if (have_lo) { m->lo = to_bytes(vlo); lo = &m->lo; }
+            if (have_hi) { m->hi = to_bytes(vhi); hi = &m->hi; }
+            db->build_sources(m->src, lo, RTEST(vloi), hi, RTEST(vhii));
+        } catch (...) { delete m; rb_raise(eError, "lithos: out of memory building scan"); }
+    }
+
+    /* Run the merge. merge_yield catches block exceptions/breaks via rb_protect
+       and returns a Ruby tag; NO longjmp crosses a C++ frame. We free our state,
+       then re-raise the tag from this clean C frame. (Mutation-during-iteration
+       is forbidden in the Ruby layer, which keeps that guard a pure-Ruby raise.) */
+    int state = merge_yield(m->src, m->ek, m->ev);
+    delete m;
+    if (state) rb_jump_tag(state); /* propagate the block's break/exception cleanly */
+    return self;
+}
+
+static VALUE db_flush(VALUE self) {
+    DB *db = get_db(self);
+    if (!db->flush()) rb_raise(eError, "lithos: %s", db->errbuf);
+    return self;
+}
+static VALUE db_compact(VALUE self) {
+    DB *db = get_db(self);
+    if (!db->compact()) rb_raise(eError, "lithos: %s", db->errbuf);
+    return self;
+}
+
+static VALUE db_close(VALUE self) {
+    DBWrap *w; TypedData_Get_Struct(self, DBWrap, &db_type, w);
+    if (w && w->db && !w->db->closed) {
+        DB *db = w->db;
+        db->flush();   /* best-effort clean shutdown; data is already WAL-durable */
+        db->release();
+    }
+    return Qnil;
+}
+static VALUE db_closed_p(VALUE self) {
+    DBWrap *w; TypedData_Get_Struct(self, DBWrap, &db_type, w);
+    return (w && w->db && !w->db->closed) ? Qfalse : Qtrue;
+}
+static VALUE db_path(VALUE self) {
+    DBWrap *w; TypedData_Get_Struct(self, DBWrap, &db_type, w);
+    /* the path is conceptually UTF-8 text (not binary like keys/values) */
+    return rb_utf8_str_new(w->db->dir_utf8.data(), (long)w->db->dir_utf8.size());
+}
+
+extern "C" void Init_lithos(void) {
+    mLithos = rb_define_module("Lithos");
+    cDB = rb_define_class_under(mLithos, "DB", rb_cObject);
+    eError = rb_define_class_under(mLithos, "Error", rb_eStandardError);
+
+    rb_define_alloc_func(cDB, db_alloc);
+    rb_define_method(cDB, "initialize", RUBY_METHOD_FUNC(db_initialize), -1);
+    /* Read-only methods are safe during iteration -> exposed directly. */
+    rb_define_method(cDB, "get", RUBY_METHOD_FUNC(db_get), 1);
+    rb_define_method(cDB, "key?", RUBY_METHOD_FUNC(db_key_p), 1);
+    rb_define_method(cDB, "_each_range", RUBY_METHOD_FUNC(db_each_range), 4);
+    rb_define_method(cDB, "closed?", RUBY_METHOD_FUNC(db_closed_p), 0);
+    rb_define_method(cDB, "path", RUBY_METHOD_FUNC(db_path), 0);
+    /* Mutators are primitives wrapped by the Ruby layer, which forbids calling
+       them during iteration (a pure-Ruby raise — never a longjmp through C++). */
+    rb_define_method(cDB, "_put", RUBY_METHOD_FUNC(db_put), 2);
+    rb_define_method(cDB, "_delete", RUBY_METHOD_FUNC(db_delete), 1);
+    rb_define_method(cDB, "_flush", RUBY_METHOD_FUNC(db_flush), 0);
+    rb_define_method(cDB, "_compact", RUBY_METHOD_FUNC(db_compact), 0);
+    rb_define_method(cDB, "_close", RUBY_METHOD_FUNC(db_close), 0);
+}