keri-ts 1.0.0 → 2.0.0

package/esm/keri/npm/src/db/core/lmdber.js

@@ -3,7 +3,7 @@
  */
 import * as dntShim from "../../_dnt.shims.js";
 import { b, bytesEqual, bytesHex, t, toBytes } from "cesr-ts";
-import { action } from "effection";
+import { call, ensure } from "effection";
 import { open } from "lmdb";
 import { startsWith } from "../../core/bytes.js";
 import { DatabaseKeyError, DatabaseNotOpenError, DatabaseOperationError } from "../../core/errors.js";
@@ -209,6 +209,10 @@ const DEFAULT_DB_VERSION = "1.0.0";
  *   `PathManager` lifecycle instead of inheriting KERIpy's exact structure
  */
 export class LMDBer {
+    /** Expose only a usable environment, never one whose shutdown has started or failed. */
+    get env() {
+        return this.closing ? null : this.root;
+    }
     constructor(options = {}, defaults) {
         Object.defineProperty(this, "pathManager", {
             enumerable: true,
@@ -216,12 +220,19 @@ export class LMDBer {
             writable: true,
             value: void 0
         });
-        Object.defineProperty(this, "env", {
+        Object.defineProperty(this, "root", {
             enumerable: true,
             configurable: true,
             writable: true,
             value: void 0
         });
+        /** A rejected close remains authoritative: the binding may have disabled only part of the handle. */
+        Object.defineProperty(this, "closing", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
         Object.defineProperty(this, "readonly", {
             enumerable: true,
             configurable: true,
@@ -247,6 +258,13 @@ export class LMDBer {
             writable: true,
             value: void 0
         });
+        /** Existing-store maintenance may neither create tables nor clear its target. */
+        Object.defineProperty(this, "existingOnly", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: false
+        });
         this.defaults = { ...LMDBER_DEFAULTS, ...defaults };
         this.logger = options.logger ?? consoleLogger;
         this.configuredMapSize = options.mapSize;
@@ -264,7 +282,7 @@ export class LMDBer {
             perm: this.defaults.perm,
         };
         this.pathManager = new PathManager(options, pathDefaults);
-        this.env = null;
+        this.root = null;
         this.readonly = options.readonly || false;
     }
     /** Expose the resolved logical database name used for path derivation and temp dirs. */
@@ -307,23 +325,32 @@ export class LMDBer {
         const message = error instanceof Error ? error.message : String(error);
         return new DatabaseKeyError(`Key is empty, too big, or wrong size: ${message}`, { cause: error });
     }
-    /** Bridge async LMDB close into an Effection operation. */
-    *closeEnv(env) {
-        yield* action((resolve, reject) => {
-            env.close().then(() => resolve(undefined)).catch(reject);
-            return () => { };
-        });
-    }
-    /**
-     * Allow LMDB-js internal read-reset timers to flush before test/resource
-     * teardown completes. LMDB uses setTimeout(0/1) internally around read txn
-     * renewal/reset and deferred close paths.
-     */
-    *quiesceEnvTimers() {
-        yield* action((resolve) => {
-            const timeout = setTimeout(() => resolve(undefined), 1);
-            return () => clearTimeout(timeout);
-        });
+    /** Join native shutdown even when the caller is cancelled; failed handles cannot be reused. */
+    *closeEnv() {
+        if (!this.closing && this.root) {
+            const root = this.root;
+            // lmdb-js disables reads/writes before completion. A rejection cannot prove
+            // whether the environment is usable or fully closed; retain it as failed.
+            this.closing = new Promise((resolve, reject) => {
+                // In 3.5.3 the callback runs after doClose, including outstanding reads.
+                // The returned Promise alone can resolve before that deferred branch.
+                const native = root;
+                native.close(resolve).catch(reject);
+            }).then(() => {
+                this.root = null;
+                this.pathManager.opened = false;
+            });
+        }
+        const pending = this.closing;
+        if (pending) {
+            yield* call(function* () {
+                // Register joining before the async leaf. Effection reports a failing
+                // ensure as destruction failure, including when its task is halted.
+                yield* ensure(() => call(() => pending));
+                yield* call(() => pending);
+            });
+            this.closing = null;
+        }
     }
     /**
      * Reopen the LMDB environment at the resolved path.
@@ -333,19 +360,9 @@ export class LMDBer {
      * and stamps the default version marker on newly-created writable envs.
      */
     *reopen(options = {}) {
+        yield* this.closeEnv();
         const readonly = options.readonly ?? this.readonly;
         this.readonly = readonly;
-        // Close existing database if open (prevents double-free when reopening)
-        if (this.env) {
-            try {
-                yield* this.closeEnv(this.env);
-            }
-            catch (error) {
-                // Ignore close errors (database might already be closed)
-                this.logger.warn(`Warning: Error closing existing LMDB environment: ${error}`);
-            }
-            this.env = null;
-        }
         // Reopen path manager (now an Effection operation)
         yield* this.pathManager.reopen(options);
         if (!this.pathManager.path) {
@@ -358,16 +375,42 @@ export class LMDBer {
         // For readonly mode, database files must exist
         if (readonly && !dbExists) {
             this.logger.error(`Cannot open readonly database: database files do not exist at ${dbPath}`);
-            this.env = null;
+            this.root = null;
             return false;
         }
-        // Prefer reopen-time option, then constructor option, then env, then default.
+        if (options.mapSize !== undefined)
+            this.configuredMapSize = options.mapSize;
+        this.existingOnly = false;
+        this.openEnvironment(dbPath, dbExists);
+        // KERIpy parity: stamp version metadata on newly-created DBs and temp DBs.
+        if (this.opened && !readonly && (!dbExists || this.temp)) {
+            this.setVer(DEFAULT_DB_VERSION);
+        }
+        return this.opened;
+    }
+    /**
+     * Open a resolved existing environment without path creation or version writes.
+     * All subsequent named-table bindings refuse creation, including writable ones.
+     */
+    *openExisting(path) {
+        if (this.root || this.closing || this.temp) {
+            throw new DatabaseOperationError("Existing-store opening requires a closed, persistent LMDBer");
+        }
+        const data = yield* this.pathManager.statFileOp(`${path}/data.mdb`);
+        if (!data.isFile) {
+            throw new DatabaseOperationError(`Existing database not found at ${path}`);
+        }
+        this.existingOnly = true;
+        this.openEnvironment(path, true);
+        this.pathManager.path = path;
+        this.pathManager.opened = true;
+    }
+    /** Open the native environment with the shared binary and map-size contract. */
+    openEnvironment(dbPath, dbExists) {
+        // Prefer the configured option, then env, then default.
         // When data.mdb already exists, raise to at least the stored map size so
         // KERIpy databases opened with KERI_BASER_MAP_SIZE=10GiB (etc.) do not abort
         // under an undersized lmdb-js virtual mapping.
-        if (options.mapSize !== undefined) {
-            this.configuredMapSize = options.mapSize;
-        }
         const dataMdbPath = dbExists ? `${dbPath}/data.mdb` : null;
         const effectiveMapSize = resolveLmdbMapSize({
             optionMapSize: this.configuredMapSize,
@@ -379,29 +422,25 @@ export class LMDBer {
         });
         const dbConfig = {
             path: dbPath, // Use directory path (Node.js lmdb should handle this)
+            noSubdir: false, // A dotted directory name must not select LMDB's single-file layout.
             maxDbs: this.defaults.maxNamedDBs,
             mapSize: effectiveMapSize,
-            readOnly: readonly,
+            readOnly: this.readonly,
             compression: false, // Disable compression for compatibility
             encoding: "binary", // to mimic KERIpy behavior
             keyEncoding: "binary", // to mimic KERIpy behavior
         };
-        this.logger.debug(`Opening LMDB at: ${dbPath} (readonly: ${readonly}, mapSize: ${effectiveMapSize})`);
+        this.logger.debug(`Opening LMDB at: ${dbPath} (readonly: ${this.readonly}, mapSize: ${effectiveMapSize})`);
         // Open LMDB environment
         // LMDB's open() will create data.mdb and lock.mdb if they don't exist
         try {
             // do sync because wrapping synchronous native operations in action() can cause
             // memory management issues with native bindings (double-free errors)
-            this.env = open(dbConfig);
+            this.root = open(dbConfig);
             this.logger.debug(`LMDB environment opened successfully`);
-            // KERIpy parity: stamp version metadata on newly-created DBs and temp DBs.
-            if (this.opened && !readonly && (!dbExists || this.temp)) {
-                this.setVer(DEFAULT_DB_VERSION);
-            }
-            return this.opened;
         }
         catch (error) {
-            this.env = null;
+            this.root = null;
             const message = error instanceof Error ? error.message : String(error);
             throw new DatabaseOperationError(`Failed to open LMDB environment: ${message}`, { cause: error });
         }
@@ -417,25 +456,15 @@ export class LMDBer {
         return pathStat.isFile ?? false;
     }
     /**
-     * Close the root LMDB env and path manager.
-     *
-     * This waits for LMDB-js async close completion and its internal read-reset
-     * timers before optional path cleanup, so test/resource teardown does not
-     * leak across later work.
+     * Close the environment before path cleanup. Observable binding failures
+     * propagate and permanently prevent this wrapper from reusing or deleting it.
+     * Repeated closure of a known-closed environment is harmless.
      */
     *close(clear = false) {
-        if (this.env) {
-            try {
-                yield* this.closeEnv(this.env);
-                yield* this.quiesceEnvTimers(); // lmdb-js uses internal teardown timers to manage post-close; this lets them finish
-            }
-            catch (error) {
-                // Intentional recovery: close errors are non-fatal during shutdown paths.
-                this.logger.warn(`Error closing LMDB: ${error}`);
-            }
-            this.env = null;
+        if (clear && this.existingOnly) {
+            throw new DatabaseOperationError("Existing-store maintenance cannot clear its environment");
         }
-        // Close path manager (now an Effection operation)
+        yield* this.closeEnv();
         yield* this.pathManager.close(clear);
         return true;
     }
@@ -477,11 +506,21 @@ export class LMDBer {
      */
     openDB(name, dupsort = false) {
         const env = this.requireEnv();
-        return env.openDB(name, {
+        // lmdb-js 3.5.3 supports create:false in open.js even though its public
+        // DatabaseOptions declaration omits it. Keep that native detail here.
+        const options = {
             keyEncoding: "binary",
             encoding: "binary", // Use binary encoding for values (raw bytes) to match KERIpy
             dupSort: dupsort,
-        });
+            create: !this.existingOnly,
+        };
+        const table = env.openDB(name, options);
+        // The native API returns undefined for an absent non-creating binding,
+        // despite declaring a non-null result. Maintenance must reject it; ordinary
+        // readonly Baser bindings already tolerate tables absent from Python layouts.
+        if (this.existingOnly && !table)
+            throw new DatabaseOperationError(`Database not found: ${name}`);
+        return table;
     }
     /*
      * Plain key/value family (`dupsort=false` parity surface)

package/esm/keri/npm/src/db/escrowing.js

@@ -1,6 +1,7 @@
 import { Cigar, Dater, Diger, Prefixer, Seqner, Siger, Verfer } from "cesr-ts";
 import { TransIdxSigGroup } from "../core/dispatch.js";
-import { dropEscrow } from "../core/kever-decisions.js";
+import { dropEscrow } from "../core/kel/kever-decisions.js";
+import { consoleLogger } from "../core/logger.js";
 import { makeNowIso8601, timestampMicros } from "../time/mod.js";
 import { CatCesrIoSetSuber, CesrIoSetSuber, CesrSuber, SerderSuber } from "./subing.js";
 function hexToFixedBytes(hex, size) {
@@ -137,6 +138,8 @@ export class Broker {
      * propagate and leave that row eligible for retry. Kept rows and insertions behind
      * the checkpoint return next pass; new rows beyond the horizon cannot extend it.
      * Removing an index retains artifacts that another dependency family may own.
+     * Terminal drops are reported only after the index removal succeeds; dependency
+     * migration must return accept for the relinquished family, rather than drop.
      */
     retryStateNotices(typ, processReply, { maxRows = 64, now = makeNowIso8601() } = {}) {
         if (!/^[a-z-]+$/.test(typ) || !Number.isSafeInteger(maxRows) || maxRows < 1 || maxRows > 1024) {
@@ -147,58 +150,78 @@ export class Broker {
             throw new Error("Invalid Broker retry time.");
         const encode = (value) => new TextEncoder().encode(value);
         const decode = (value) => new TextDecoder("utf-8", { fatal: true }).decode(value);
-        let scan = this.retryScans.get(typ);
-        if (!scan) {
-            const start = encode(`${typ}.`), ceiling = encode(`${typ}/`);
-            let tail;
-            for (const key of this.escrowdb.sdb.getKeys({ start: ceiling, end: start, reverse: true, limit: 1 })) {
-                tail = key.slice();
+        try {
+            let scan = this.retryScans.get(typ);
+            if (!scan) {
+                const start = encode(`${typ}.`), ceiling = encode(`${typ}/`);
+                let tail;
+                for (const key of this.escrowdb.sdb.getKeys({ start: ceiling, end: start, reverse: true, limit: 1 })) {
+                    tail = key.slice();
+                }
+                if (!tail)
+                    return { processed: 0, done: true };
+                scan = { start, end: encode(`${decode(tail)}\0`) };
+                this.retryScans.set(typ, scan);
             }
-            if (!tail)
-                return { processed: 0, done: true };
-            scan = { start, end: encode(`${decode(tail)}\0`) };
-            this.retryScans.set(typ, scan);
-        }
-        const rows = [];
-        for (const key of this.escrowdb.sdb.getKeys({ start: scan.start, end: scan.end, limit: maxRows })) {
-            rows.push(key.slice());
-        }
-        for (const key of rows) {
-            const parts = decode(key).split(".");
-            if (parts.length !== 4 || parts[0] !== typ || !parts[1] || !parts[2]
-                || parts.some((part) => part.includes("\0")) || !/^[0-9a-f]{32}$/.test(parts[3])) {
-                throw new Error("Corrupt Broker retry key.");
+            const rows = [];
+            for (const key of this.escrowdb.sdb.getKeys({ start: scan.start, end: scan.end, limit: maxRows })) {
+                rows.push(key.slice());
             }
-            const raw = this.db.getVal(this.escrowdb.sdb, key);
-            if (!raw || raw.length > 128)
-                throw new Error("Corrupt Broker retry digest row.");
-            const diger = new Diger({ qb64b: raw });
-            const dater = this.daterdb.get([diger.qb64]), serder = this.serderdb.get([diger.qb64]);
-            const tsgs = this.fetchTsgs(diger), couples = this.cigardb.get([diger.qb64]);
-            if (!dater || !serder || (!tsgs.length && !couples.length) || !serder.route) {
-                throw new Error("Corrupt Broker retry artifacts.");
+            for (const key of rows) {
+                const parts = decode(key).split(".");
+                if (parts.length !== 4 || parts[0] !== typ || !parts[1] || !parts[2]
+                    || parts.some((part) => part.includes("\0")) || !/^[0-9a-f]{32}$/.test(parts[3])) {
+                    throw new Error("Corrupt Broker retry key.");
+                }
+                const raw = this.db.getVal(this.escrowdb.sdb, key);
+                if (!raw || raw.length > 128)
+                    throw new Error("Corrupt Broker retry digest row.");
+                const diger = new Diger({ qb64b: raw });
+                const dater = this.daterdb.get([diger.qb64]), serder = this.serderdb.get([diger.qb64]);
+                const tsgs = this.fetchTsgs(diger), couples = this.cigardb.get([diger.qb64]);
+                if (!dater || !serder || (!tsgs.length && !couples.length) || !serder.route) {
+                    throw new Error("Corrupt Broker retry artifacts.");
+                }
+                const stamp = timestampMicros(dater.dts);
+                if (stamp === null)
+                    throw new Error("Corrupt Broker retry timestamp.");
+                const decision = when - stamp > BigInt(ttl) ? dropEscrow("stale") : processReply({
+                    aid: parts[2],
+                    diger,
+                    serder,
+                    route: serder.route,
+                    tsgs,
+                    cigars: couples.map(([verfer, cigar]) => new Cigar(cigar, verfer)),
+                });
+                if (decision.kind === "accept" || decision.kind === "drop") {
+                    const removed = this.escrowdb.rem(parts.slice(0, 3), diger);
+                    if (removed && decision.kind === "drop") {
+                        consoleLogger.warn("Broker ejected transaction-state notice from escrow", {
+                            family: typ,
+                            said: diger.qb64,
+                            pre: parts[1],
+                            aid: parts[2],
+                            route: serder.route,
+                            reason: decision.reason,
+                            detail: decision.message,
+                            ...(decision.cause !== undefined ? { cause: decision.cause } : {}),
+                        });
+                    }
+                }
+                else if (decision.kind !== "keep")
+                    throw new Error("Invalid Broker retry decision.");
+                scan.start = encode(`${decode(key)}\0`);
             }
-            const stamp = timestampMicros(dater.dts);
-            if (stamp === null)
-                throw new Error("Corrupt Broker retry timestamp.");
-            const decision = when - stamp > BigInt(ttl) ? dropEscrow("stale") : processReply({
-                aid: parts[2],
-                diger,
-                serder,
-                route: serder.route,
-                tsgs,
-                cigars: couples.map(([verfer, cigar]) => new Cigar(cigar, verfer)),
-            });
-            if (decision.kind === "accept" || decision.kind === "drop")
-                this.escrowdb.rem(parts.slice(0, 3), diger);
-            else if (decision.kind !== "keep")
-                throw new Error("Invalid Broker retry decision.");
-            scan.start = encode(`${decode(key)}\0`);
+            const done = rows.length < maxRows;
+            if (done)
+                this.retryScans.delete(typ);
+            return { processed: rows.length, done };
         }
-        const done = rows.length < maxRows;
-        if (done)
+        catch (error) {
+            // Restart after failure so earlier kept rows can use evidence recovered before the fault.
             this.retryScans.delete(typ);
-        return { processed: rows.length, done };
+            throw error;
+        }
     }
     /**
      * Persist one escrowed transaction-state notice and its attachments.
@@ -218,7 +241,8 @@ export class Broker {
         this.daterdb.put(keys, dater);
         this.serderdb.put(keys, serder);
         for (const group of tsgs) {
-            this.tigerdb.put([diger.qb64, group.pre, group.snh, group.said], group.sigers);
+            // Both Python references key equivalent ordinal encodings by the same numeric value.
+            this.tigerdb.put([diger.qb64, group.pre, group.sn.toString(16).padStart(32, "0"), group.said], group.sigers);
         }
         for (const cigar of cigars) {
             if (!cigar.verfer) {

package/esm/keri/npm/src/db/exchange/exchange-nests.js

@@ -0,0 +1,54 @@
+/** KERIpy enst. wire storage and finite nested evidence hydration. */
+import { b, collectPayload, concatBytes, Counter, nestedPayload, parseBytes, t, Vrsn_2_0, } from "cesr-ts";
+import { suffix, unsuffix } from "../core/keys.js";
+/** Read one physical row and close the LMDB key cursor before any caller suspension. */
+function nextRow(db, table, said, ordinal) {
+    for (const key of table.sdb.getKeys({ start: suffix(b(said), ordinal) })) {
+        const [owner, index] = unsuffix(key);
+        if (t(owner) !== said)
+            return null;
+        const wire = db.getVal(table.sdb, key);
+        if (!wire)
+            throw new Error("Stored nested row disappeared");
+        return { wire, next: index + 1 };
+    }
+    return null;
+}
+/** Test row presence without hydrating a potentially large wire value. */
+export function hasNests(db, said, ordinal = 0) {
+    for (const key of db.enst.sdb.getKeys({ start: suffix(b(said), ordinal) }))
+        return t(unsuffix(key)[0]) === said;
+    return false;
+}
+/** Serialize resident children before the caller enters its synchronous evidence writer. */
+export async function prepareNests(nests, signal) {
+    const wires = [];
+    for (const child of nests)
+        wires.push(await collectPayload(nestedPayload(child, Vrsn_2_0), signal));
+    return wires;
+}
+/** Publish ordinary KERIpy nested wire rows inside the caller's exchange transaction. */
+export function publishNests(db, said, wires) {
+    if (!wires.length) {
+        db.enst.rem([said]);
+        return;
+    }
+    if (!db.enst.pin([said], wires.map((wire) => t(wire))))
+        throw new Error("Nested evidence write failed");
+}
+/** Hydrate each stored wrapper through the finite decoder without parser-owned files. */
+export function loadNests(db, said) {
+    const nests = [];
+    let ordinal = 0;
+    for (;;) {
+        const row = nextRow(db, db.enst, said, ordinal);
+        if (!row)
+            break;
+        ordinal = row.next;
+        const frames = parseBytes(concatBytes(Counter.makeGVC(Vrsn_2_0), row.wire));
+        if (frames.length !== 1)
+            throw new Error("Stored nested row must contain exactly one child substream");
+        nests.push(frames[0]);
+    }
+    return nests;
+}

package/esm/keri/npm/src/db/exchange/exchange-records.js

@@ -0,0 +1,7 @@
+import { RawRecord } from "../../core/records.js";
+/** Private metadata; the ordinary KERIpy exchange tables retain their wire schema. */
+export class ExchangeRecord extends RawRecord {
+}
+/** Python exrs.; keys are (Grant SAID, node SAID), never a credential-validity index. */
+export class ExchangeRegistryStateRecord extends RawRecord {
+}

package/esm/keri/npm/src/db/index.js

@@ -2,15 +2,16 @@
  * Database module exports
  */
 export * from "../acdc/regbasing.js";
-export { openRegistry, RegistryRecord as AcdcRegistryRecord, RegistryStore, } from "../acdc/registering.js";
+export { openRegistry, publicOpeningAttachment, RegistryRecord as AcdcRegistryRecord, RegistryStore, } from "../acdc/registering.js";
 export * from "./basing.js";
 export * from "./core/index.js";
 export * from "./escrowing.js";
+export { loadNests } from "./exchange/exchange-nests.js";
+export { ExchangeRegistryStateRecord } from "./exchange/exchange-records.js";
 export * from "./keeping.js";
 export * from "./koming.js";
-export * from "./mailboxing.js";
+export * from "./mailbox/mailboxing.js";
 export * from "./noting.js";
-export { PayloadArchive } from "./payload-archive.js";
-export { PayloadStore } from "./payload-store.js";
 export * from "./reger.js";
+export { UnsupportedTelReplayError } from "./replay/tel-replay.js";
 export * from "./subing.js";