nedb-engine 3.0.0 → 3.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +104 -57
- package/index.d.ts +59 -0
- package/index.js +1 -2
- package/nedb.darwin-arm64.node +0 -0
- package/nedb.darwin-x64.node +0 -0
- package/nedb.linux-arm64-gnu.node +0 -0
- package/nedb.linux-arm64-musl.node +0 -0
- package/nedb.linux-x64-gnu.node +0 -0
- package/nedb.linux-x64-musl.node +0 -0
- package/nedb.win32-x64-msvc.node +0 -0
- package/nedbd-v2-darwin-arm64 +0 -0
- package/nedbd-v2-darwin-x64 +0 -0
- package/nedbd-v2-linux-arm64 +0 -0
- package/nedbd-v2-linux-arm64-musl +0 -0
- package/nedbd-v2-linux-x64 +0 -0
- package/nedbd-v2-linux-x64-musl +0 -0
- package/nedbd-v2-win-x64.exe +0 -0
- package/package.json +14 -3
- package/wrap/index.js +24 -0
- package/wrap/mongo.js +91 -0
- package/wrap/redis.js +134 -0
- package/wrap/sql.js +151 -0
- package/wrap/surface.js +330 -0
package/README.md
CHANGED
|
@@ -26,86 +26,114 @@ One Rust core → ships to **PyPI** and **npm** from a single source.
|
|
|
26
26
|
|
|
27
27
|
---
|
|
28
28
|
|
|
29
|
-
##
|
|
29
|
+
## New in 3.2.0 — wrap the databases you already run
|
|
30
30
|
|
|
31
|
-
|
|
32
|
-
|
|
31
|
+
NEDB adds **tamper-evident causal provenance to a database you already have**, in one line, without
|
|
32
|
+
rip-and-replace. Five adapters, one surface:
|
|
33
33
|
|
|
34
|
-
|
|
34
|
+
```python
|
|
35
|
+
from nedb import wrap_redis, wrap_sqlite, wrap_mysql, wrap_mongo, wrap_postgresql
|
|
35
36
|
|
|
36
|
-
|
|
37
|
-
|
|
37
|
+
r = wrap_redis(redis.Redis()) # or wrap_sqlite(sqlite3.connect("app.db")), ...
|
|
38
|
+
r.nedb.register("driver:*", "driver") # teach NEDB the host's shape
|
|
39
|
+
r.nedb.backfill() # import what is already there
|
|
40
|
+
r.nedb.shadow_writes = True # every future write is chained
|
|
38
41
|
|
|
39
|
-
|
|
40
|
-
returned 0 after reopen — while `verify()` reported all 30 objects healthy.** The content-addressed
|
|
41
|
-
objects were durable; the id-index entries that make them findable were gone.
|
|
42
|
+
r.set("driver:d1", json.dumps({"name": "Bob", "status": "active"}))
|
|
42
43
|
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
...free space, retry -> Ok reopen -> 30 rows
|
|
44
|
+
r.nedb.query('FROM driver WHERE status = "active"') # NQL over your Redis data
|
|
45
|
+
r.nedb.query('FROM driver AS OF 41') # what it looked like at seq 41
|
|
46
|
+
r.nedb.verify() # True — BLAKE2b chain intact
|
|
47
47
|
```
|
|
48
48
|
|
|
49
|
-
|
|
50
|
-
|
|
49
|
+
| wrapper | host | shadowing |
|
|
50
|
+
|---|---|---|
|
|
51
|
+
| `wrap_redis` | `redis.Redis` / compatible | automatic — every write command intercepted |
|
|
52
|
+
| `wrap_sqlite` | `sqlite3.Connection` | automatic — `execute()` intercepted post-write |
|
|
53
|
+
| `wrap_mysql` | DB-API 2.0 (mysql-connector, PyMySQL) | explicit `shadow_row()` |
|
|
54
|
+
| `wrap_mongo` | `pymongo.MongoClient` | explicit `shadow_row()` |
|
|
55
|
+
| `wrap_postgresql` | DB-API 2.0 (psycopg2, psycopg 3) | explicit `shadow_row()` |
|
|
56
|
+
|
|
57
|
+
**NEDB never writes into the host database's namespace.** Shadow data lives only in the NEDB engine.
|
|
58
|
+
|
|
59
|
+
Three backends behind the same surface, selected by `backend="auto"`: **nedbd over HTTP** (`nedbd_url=`),
|
|
60
|
+
**embedded v2/v3 DAG** (the Rust core, in-process, no server — `dag_path=` for a durable store,
|
|
61
|
+
`dag_tmk=` for AES-256-GCM at rest), or the **v1 in-process AOF** engine as a universal fallback. On the
|
|
62
|
+
DAG backend you also get `tip()`, `tip_collection()`, `since()` (changefeed) and `scan_status()`.
|
|
63
|
+
|
|
64
|
+
### 🟢 MIT licensed since 3.0.0
|
|
65
|
+
|
|
66
|
+
No production restriction, no copyleft, no Change Date. Use it in production, embed it commercially,
|
|
67
|
+
ship it closed-source, fork it, sell it. License review is a wall, not a speed bump — that wall is gone.
|
|
68
|
+
|
|
69
|
+
### Also in 3.2.0
|
|
70
|
+
|
|
71
|
+
- **A durability defect that pinned every embedded database.** The background flush ticker held a
|
|
72
|
+
strong `Arc<Db>` in an unconditional loop, so the handle was never dropped: the exclusive data-dir
|
|
73
|
+
`LOCK` was never released (reopening the same path *in the same process* failed with "locked by
|
|
74
|
+
another process" naming your own pid), every `open()` leaked a thread and the whole `Db`, and
|
|
75
|
+
flush-on-close could never fire. The ticker now holds a `Weak<Db>` and exits when its owner does.
|
|
76
|
+
**Live in 2.8.5 through 3.1.0 — upgrade if you embed the engine.**
|
|
77
|
+
- **`wrap_redis` crashed on any install without the native wheel** — the pure-Python fallback path
|
|
78
|
+
raised `AttributeError` from inside `wrap_redis()`. Fixed.
|
|
79
|
+
- **Prebuilt binaries for `linux-arm64` and musl/Alpine**, on npm and PyPI. Graviton, Ampere, Linux
|
|
80
|
+
containers on Apple Silicon, and Alpine images previously installed cleanly and then failed at
|
|
81
|
+
import.
|
|
82
|
+
- **CI actually runs the test suites.** Until now the only workflows fired on a version tag, so the
|
|
83
|
+
first automated opinion about a change arrived *after* it was published to three registries. All
|
|
84
|
+
26 suites now run on every push and pull request. It found four real defects in its first hour,
|
|
85
|
+
including two of the ones listed above.
|
|
51
86
|
|
|
52
|
-
|
|
87
|
+
---
|
|
53
88
|
|
|
54
|
-
|
|
55
|
-
flush from a failed one. Anything that takes a destructive or externally-visible action on the
|
|
56
|
-
strength of a persisted record needs to know.
|
|
89
|
+
## Earlier — 2.8.6 durability & recovery
|
|
57
90
|
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
db.try_flush_all()?; // Result<()> — id-index WAL + segment sync + MANIFEST
|
|
91
|
+
Three defects found by killing a real engine at every persistence boundary and by filling a real
|
|
92
|
+
filesystem to zero free blocks. **If you are on 2.8.5 or earlier, upgrade.**
|
|
61
93
|
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
94
|
+
**1. A failed flush silently discarded acknowledged writes.** `IdIndex::flush_write_buf` cleared every
|
|
95
|
+
buffered entry regardless of whether its disk write succeeded, so a flush that hit `ENOSPC` threw the
|
|
96
|
+
entry away and no later flush retried it. Reproduced on a full 22 MiB filesystem: 30 rows acknowledged
|
|
97
|
+
by `put() -> Ok`, then `list()` returned 0 after reopen — while `verify()` reported all 30 objects
|
|
98
|
+
healthy. The content-addressed objects were durable; the id-index entries that make them findable were
|
|
99
|
+
gone. An entry now leaves the WAL only when its write actually landed.
|
|
100
|
+
|
|
101
|
+
**2. Flush errors were unobservable.** `flush_all()` returns `()`, so a caller could not tell a durable
|
|
102
|
+
flush from a failed one.
|
|
65
103
|
|
|
66
|
-
|
|
104
|
+
```rust
|
|
105
|
+
db.try_flush_all()?; // Result<()> — use this when the outcome matters
|
|
106
|
+
db.flush_all(); // still logs; for ticker / Drop, nowhere to propagate
|
|
107
|
+
```
|
|
67
108
|
|
|
68
|
-
|
|
109
|
+
Also added: `Db::try_flush_manifest()` and `IdIndex::try_flush_write_buf()`.
|
|
69
110
|
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
`
|
|
73
|
-
|
|
111
|
+
**3. `repair` could not repair, and `since()` claimed "caught up" while behind.** The cold scan rebuilt
|
|
112
|
+
`seq_index`, per-collection tips, the Merkle head and `MANIFEST` — but never the id index, and
|
|
113
|
+
`start_cold_scan()` is a deliberate no-op on a warm store, so `nedb-cli repair` printed success on
|
|
114
|
+
exactly the database it exists to fix.
|
|
74
115
|
|
|
75
116
|
```bash
|
|
76
117
|
nedb-cli repair ./data
|
|
77
118
|
# repaired: 203 id-index entr(ies) rebuilt, 203 node(s) verified, flushed
|
|
78
119
|
```
|
|
79
120
|
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
database reopens **warm** instead of coming back up cold with an empty head.
|
|
87
|
-
|
|
88
|
-
Separately, `since()` set `has_more = hit_limit` alone. On a warm boot the seq index is empty **by
|
|
89
|
-
design** (that is why warm start is O(1)), so every lookup missed and `since()` returned zero nodes
|
|
90
|
-
with `has_more = false` — indistinguishable from genuinely up to date. A consumer following the
|
|
91
|
-
documented drain loop stopped one call in, with every record unread.
|
|
92
|
-
|
|
93
|
-
**Fixed:** `has_more` is true whenever the cursor is behind the log head. `ScanStatus` gains
|
|
94
|
-
**`seq_index_ready`** — replication consumers should gate on that, not on `scan_complete`, which is
|
|
95
|
-
true on a warm boot precisely because the scan was skipped.
|
|
121
|
+
Every object carries its own `coll`, `id` and `seq`, so the index is fully derivable — nothing is
|
|
122
|
+
invented. Separately, `since()` set `has_more = hit_limit` alone; on a warm boot the seq index is empty
|
|
123
|
+
*by design*, so `since()` returned zero nodes with `has_more = false` — indistinguishable from
|
|
124
|
+
genuinely up to date, and a consumer following the documented drain loop stopped one call in with every
|
|
125
|
+
record unread. `has_more` is now true whenever the cursor is behind head, and `ScanStatus` gains
|
|
126
|
+
**`seq_index_ready`** — gate replication on that, not on `scan_complete`.
|
|
96
127
|
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
very first write in a database (seq 0) is unreachable through any cursor value. Ten writes drain as
|
|
101
|
-
nine records. Changing the convention would break existing consumers; a replica seeded from
|
|
102
|
-
`since()` alone starts one record short.
|
|
128
|
+
**Known sharp edge (documented, not changed):** `since()`'s cursor is **exclusive** and seqs start at 0,
|
|
129
|
+
so `since(0, _)` returns `(0, head]` and the very first write (seq 0) is unreachable through any cursor
|
|
130
|
+
value. Ten writes drain as nine records. Changing the convention would break existing consumers.
|
|
103
131
|
|
|
104
132
|
---
|
|
105
133
|
|
|
106
|
-
## NEDB
|
|
134
|
+
## NEDB v3.2.0 — Production Stable
|
|
107
135
|
|
|
108
|
-
**Current stable: 2.
|
|
136
|
+
**Current stable: 3.2.0** — NEDB ships as **three version-aligned distributions** on one tag — `nedb-engine` (flagship), `crypto-database` (verifiable v2/v3 DAG), and `aof-db` (fast append-only) — across npm / PyPI / crates.io with native addons for **macOS (arm64 + x86_64), Linux (x86_64 + aarch64, glibc + musl) and Windows x86_64** (see [**Releasing**](#releasing) below). All native wheels (Linux + Windows on GitHub Actions; macOS on Codemagic M2 Mac Minis) **plus** the universal pure-Python wheel ship from a single `v*` tag, with the `nedbd-v2` binary bundled inside `pip install nedb-engine`.
|
|
109
137
|
|
|
110
138
|
### New in 2.8.0 — Cast: the database understands English
|
|
111
139
|
|
|
@@ -150,7 +178,7 @@ nedbd --dag --data ./data
|
|
|
150
178
|
NEDBD_DAG=1 NEDB_TMK=<32-byte-hex> nedbd --data ./data
|
|
151
179
|
|
|
152
180
|
curl http://127.0.0.1:7070/health
|
|
153
|
-
# {"ok":true,"version":"
|
|
181
|
+
# {"ok":true,"version":"3.2.0","service":"nedbd","engine":"dag","startup_ready":true,"encrypted":true}
|
|
154
182
|
|
|
155
183
|
# Tail the live event stream (new in v2.2.31)
|
|
156
184
|
curl http://127.0.0.1:7070/events
|
|
@@ -204,6 +232,24 @@ pip install nedb-engine # Python ≥ 3.8 — pure-Python + optional Rust na
|
|
|
204
232
|
npm install nedb-engine # Node ≥ 16 — napi-rs prebuilt binaries
|
|
205
233
|
```
|
|
206
234
|
|
|
235
|
+
### Prebuilt platforms
|
|
236
|
+
|
|
237
|
+
Both registries ship prebuilt binaries for:
|
|
238
|
+
|
|
239
|
+
| Platform | libc | Python wheel | Node addon |
|
|
240
|
+
|---|---|---|---|
|
|
241
|
+
| Linux x86_64 | glibc | ✅ manylinux | ✅ |
|
|
242
|
+
| Linux x86_64 | musl (Alpine) | ✅ musllinux | ✅ |
|
|
243
|
+
| Linux aarch64 (Graviton, Ampere, Apple-Silicon containers) | glibc | ✅ manylinux | ✅ |
|
|
244
|
+
| Linux aarch64 | musl (Alpine) | ✅ musllinux | ✅ |
|
|
245
|
+
| macOS arm64 + x86_64 | — | ✅ | ✅ |
|
|
246
|
+
| Windows x86_64 | MSVC | ✅ | ✅ |
|
|
247
|
+
|
|
248
|
+
On Python, any platform without a prebuilt wheel still installs: pip falls back
|
|
249
|
+
to the universal `py3-none-any` wheel and you get the pure-Python v1 AOF engine
|
|
250
|
+
(correct, slower, no embedded DAG). On Node there is no such fallback — an
|
|
251
|
+
unlisted platform has no addon.
|
|
252
|
+
|
|
207
253
|
---
|
|
208
254
|
|
|
209
255
|
## Python — 5-minute tour
|
|
@@ -913,7 +959,8 @@ Requires `GITHUB_TOKEN` (`repo` + `workflow` scope) in the environment. It never
|
|
|
913
959
|
## Authors
|
|
914
960
|
|
|
915
961
|
Built by **[Mark Allen Evans Jr.](https://interchained.org)** (INTERCHAINED, LLC)
|
|
916
|
-
with **
|
|
962
|
+
with the **Interchained AI fleet** on [Hyperagent](https://hyperagent.com/refer/J2G6TCD7) —
|
|
963
|
+
Vex (GLM · Claude Sonnet · Opus · Fable · GPT-6 Astra/Sol), across hundreds of sessions.
|
|
917
964
|
|
|
918
965
|
> *"Take one idea, turn it into an LP, then an app, then a system, then a platform, then infrastructure that is irreplaceable."*
|
|
919
966
|
|
package/index.d.ts
CHANGED
|
@@ -3,3 +3,62 @@
|
|
|
3
3
|
// Runtime behavior (durable-mode auto-flush-on-exit) is added by the wrapper in
|
|
4
4
|
// index.js; the type surface is exactly the generated native binding's.
|
|
5
5
|
export * from './native';
|
|
6
|
+
|
|
7
|
+
// ── wrap adapter family (wrap/*.js) ─────────────────────────────────────────
|
|
8
|
+
|
|
9
|
+
/** Options accepted by every wrap_* constructor. */
|
|
10
|
+
export interface WrapOptions {
|
|
11
|
+
/** Logical database name (default "default"). */
|
|
12
|
+
dbName?: string;
|
|
13
|
+
/** HTTP nedbd server (v1 AOF, `--dag` v2, `--dag-v3` v3). Overrides embedded DAG. */
|
|
14
|
+
nedbdUrl?: string;
|
|
15
|
+
/** Bearer token for nedbd (NEDBD_TOKEN on the server). */
|
|
16
|
+
nedbdToken?: string;
|
|
17
|
+
/** Durable DAG store directory (embedded mode). */
|
|
18
|
+
dagPath?: string;
|
|
19
|
+
/** 64-hex TMK → AES-256-GCM at-rest encryption (embedded DAG mode). */
|
|
20
|
+
dagTmk?: string;
|
|
21
|
+
/** Explicit NedbCore class (testing / custom builds). */
|
|
22
|
+
native?: unknown;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** The `.nedb` attribute — full NEDB layer-2 API. */
|
|
26
|
+
export interface NedbSurface {
|
|
27
|
+
register(pattern: string, collection: string, opts?: {
|
|
28
|
+
idExtractor?: (key: string) => string;
|
|
29
|
+
valueParser?: (raw: unknown) => Record<string, unknown>;
|
|
30
|
+
valueType?: 'string' | 'hash' | 'json';
|
|
31
|
+
}): NedbSurface;
|
|
32
|
+
backfill(opts?: { pattern?: string; collection?: string; batchSize?: number }): number;
|
|
33
|
+
shadowWrites: boolean;
|
|
34
|
+
readonly engineKind: 'dag-embedded' | 'nedbd-http' | 'aof-embedded';
|
|
35
|
+
|
|
36
|
+
put(coll: string, id: string, doc: Record<string, unknown>): Record<string, unknown>;
|
|
37
|
+
get(coll: string, id: string, asOf?: number): Record<string, unknown> | null;
|
|
38
|
+
query(nql: string): Array<Record<string, unknown>>;
|
|
39
|
+
createIndex(coll: string, field: string, kind?: string): void;
|
|
40
|
+
delete(coll: string, id: string): void;
|
|
41
|
+
link(frm: string, rel: string, to: string): void;
|
|
42
|
+
unlink(frm: string, rel: string, to: string): void;
|
|
43
|
+
neighbors(frm: string, rel: string, asOf?: number): string[];
|
|
44
|
+
inbound(to: string, rel: string, asOf?: number): string[];
|
|
45
|
+
verify(): boolean;
|
|
46
|
+
readonly head: string;
|
|
47
|
+
readonly seq: number;
|
|
48
|
+
checkpoint(): string;
|
|
49
|
+
/** DAG-native: latest node (null on non-DAG backends). */
|
|
50
|
+
tip(): Record<string, unknown> | null;
|
|
51
|
+
/** DAG-native: changefeed page, after_seq exclusive. */
|
|
52
|
+
since(afterSeq: number | bigint, limit?: number): {
|
|
53
|
+
nodes: Array<Record<string, unknown>>; from_seq: number; to_seq: number;
|
|
54
|
+
head_seq: number; has_more: boolean;
|
|
55
|
+
};
|
|
56
|
+
/** DAG-native: replication readiness. */
|
|
57
|
+
scanStatus(): { scan_complete: boolean; tip_seq: number; indexed_count: number; [k: string]: unknown };
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export declare function wrapRedis<T extends object = object>(client: T, opts?: WrapOptions): T & { nedb: NedbSurface };
|
|
61
|
+
export declare function wrapSqlite<T extends object = object>(conn: T, opts?: WrapOptions): T & { nedb: NedbSurface };
|
|
62
|
+
export declare function wrapMysql<T extends object = object>(conn: T, opts?: WrapOptions): T & { nedb: NedbSurface };
|
|
63
|
+
export declare function wrapPg<T extends object = object>(conn: T, opts?: WrapOptions): T & { nedb: NedbSurface };
|
|
64
|
+
export declare function wrapMongo<T extends object = object>(client: T, opts?: WrapOptions): T & { nedb: NedbSurface };
|
package/index.js
CHANGED
|
@@ -15,8 +15,7 @@
|
|
|
15
15
|
// Escape hatch: set NEDB_NO_EXIT_FLUSH=1 to leave signal handling entirely to
|
|
16
16
|
// the host app (it can still call `db.flush()` itself).
|
|
17
17
|
//
|
|
18
|
-
// © INTERCHAINED LLC × Claude Opus
|
|
19
|
-
|
|
18
|
+
// © INTERCHAINED LLC × Vex (Interchained AI fleet: GLM · Claude · Opus · Fable · GPT-6)
|
|
20
19
|
const native = require('./native.js');
|
|
21
20
|
|
|
22
21
|
const Native = native.NedbCore;
|
package/nedb.darwin-arm64.node
CHANGED
|
Binary file
|
package/nedb.darwin-x64.node
CHANGED
|
Binary file
|
|
Binary file
|
|
Binary file
|
package/nedb.linux-x64-gnu.node
CHANGED
|
Binary file
|
|
Binary file
|
package/nedb.win32-x64-msvc.node
CHANGED
|
Binary file
|
package/nedbd-v2-darwin-arm64
CHANGED
|
Binary file
|
package/nedbd-v2-darwin-x64
CHANGED
|
Binary file
|
|
Binary file
|
|
Binary file
|
package/nedbd-v2-linux-x64
CHANGED
|
Binary file
|
|
Binary file
|
package/nedbd-v2-win-x64.exe
CHANGED
|
Binary file
|
package/package.json
CHANGED
|
@@ -1,8 +1,15 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "nedb-engine",
|
|
3
|
-
"version": "3.
|
|
4
|
-
"description": "NEDB
|
|
3
|
+
"version": "3.2.1",
|
|
4
|
+
"description": "NEDB \u2014 hash-chained, time-traveling, bi-temporal embedded database with Rust native core. SQL, Redis, MongoDB adapters. Causal Write Provenance. RESP2 wire protocol.",
|
|
5
5
|
"main": "index.js",
|
|
6
|
+
"exports": {
|
|
7
|
+
".": {
|
|
8
|
+
"import": "./index.js",
|
|
9
|
+
"require": "./index.js"
|
|
10
|
+
},
|
|
11
|
+
"./wrap": "./wrap/index.js"
|
|
12
|
+
},
|
|
6
13
|
"types": "index.d.ts",
|
|
7
14
|
"bin": {
|
|
8
15
|
"nedbd-v2": "./nedbd-v2.js",
|
|
@@ -14,6 +21,7 @@
|
|
|
14
21
|
"index.d.ts",
|
|
15
22
|
"native.js",
|
|
16
23
|
"native.d.ts",
|
|
24
|
+
"wrap/",
|
|
17
25
|
"nedbd-v2.js",
|
|
18
26
|
"nedb-inspector.mjs",
|
|
19
27
|
"*.node",
|
|
@@ -53,7 +61,10 @@
|
|
|
53
61
|
"additional": [
|
|
54
62
|
"aarch64-apple-darwin",
|
|
55
63
|
"x86_64-unknown-linux-gnu",
|
|
56
|
-
"x86_64-pc-windows-msvc"
|
|
64
|
+
"x86_64-pc-windows-msvc",
|
|
65
|
+
"aarch64-unknown-linux-gnu",
|
|
66
|
+
"x86_64-unknown-linux-musl",
|
|
67
|
+
"aarch64-unknown-linux-musl"
|
|
57
68
|
]
|
|
58
69
|
}
|
|
59
70
|
},
|
package/wrap/index.js
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
// nedb/wrap/index.js — the JS wrap adapter family.
|
|
3
|
+
//
|
|
4
|
+
// const { wrapRedis, wrapSqlite, wrapMysql, wrapPg, wrapMongo } =
|
|
5
|
+
// require('nedb-engine/wrap');
|
|
6
|
+
//
|
|
7
|
+
// Every wrapper: register → backfill → shadowWrites=true → full NEDB API on
|
|
8
|
+
// `.nedb`, with the embedded v2/v3 DAG (Rust napi core) as the default engine.
|
|
9
|
+
//
|
|
10
|
+
// © INTERCHAINED LLC × Claude Sonnet 4.6
|
|
11
|
+
'use strict';
|
|
12
|
+
|
|
13
|
+
const { WrapSurface, openEngine, CollectionMapping, NedbdProxy, NativeEngine } = require('./surface');
|
|
14
|
+
const { wrapRedis, WRITE_CMDS } = require('./redis');
|
|
15
|
+
const { wrapSqlite, wrapMysql, wrapPg, SqlSurface } = require('./sql');
|
|
16
|
+
const { wrapMongo, MongoSurface } = require('./mongo');
|
|
17
|
+
|
|
18
|
+
module.exports = {
|
|
19
|
+
// adapters
|
|
20
|
+
wrapRedis, wrapSqlite, wrapMysql, wrapPg, wrapMongo,
|
|
21
|
+
// surface primitives (for custom adapters)
|
|
22
|
+
WrapSurface, openEngine, CollectionMapping, NedbdProxy, NativeEngine,
|
|
23
|
+
SqlSurface, MongoSurface, WRITE_CMDS,
|
|
24
|
+
};
|
package/wrap/mongo.js
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
// nedb/wrap/mongo.js — wrapMongo: causal provenance for MongoDB (JS).
|
|
3
|
+
//
|
|
4
|
+
// const { wrapMongo } = require('nedb-engine/wrap');
|
|
5
|
+
// const client = wrapMongo(mongoClient, { dbName: 'app' });
|
|
6
|
+
//
|
|
7
|
+
// client.nedb.register('app.drivers', 'driver'); // "db.collection" ns
|
|
8
|
+
// client.nedb.backfill();
|
|
9
|
+
// client.nedb.shadowWrites = true;
|
|
10
|
+
//
|
|
11
|
+
// // your code unchanged; after each write, chain it:
|
|
12
|
+
// await client.db().collection('drivers').insertOne({ name: 'Bob' });
|
|
13
|
+
// client.nedb.shadowRow('app.drivers', { _id: r.insertedId, name: 'Bob' });
|
|
14
|
+
//
|
|
15
|
+
// © INTERCHAINED LLC × Claude Sonnet 4.6
|
|
16
|
+
'use strict';
|
|
17
|
+
|
|
18
|
+
const { WrapSurface, openEngine } = require('./surface');
|
|
19
|
+
|
|
20
|
+
class MongoSurface extends WrapSurface {
|
|
21
|
+
constructor(client, dbName, engine) {
|
|
22
|
+
super(dbName, engine);
|
|
23
|
+
this.client = client;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/** hostScan: iterate docs of a "db.collection" namespace (sync only —
|
|
27
|
+
* pymongo-style cursor.toArray is async, so JS backfill for the native
|
|
28
|
+
* driver is async; provide scanAsync). */
|
|
29
|
+
hostScan(mapping) { return []; }
|
|
30
|
+
|
|
31
|
+
async hostScanAsync(mapping, batchSize) {
|
|
32
|
+
const [dbName, collName] = mapping.pattern.split('.');
|
|
33
|
+
const out = [];
|
|
34
|
+
try {
|
|
35
|
+
const coll = this.client.db(dbName).collection(collName);
|
|
36
|
+
const cursor = coll.find({});
|
|
37
|
+
while (await cursor.hasNext()) {
|
|
38
|
+
const doc = await cursor.next();
|
|
39
|
+
const { _id, ...rest } = doc;
|
|
40
|
+
out.push([String(_id), rest]);
|
|
41
|
+
if (out.length >= (batchSize || 200)) break;
|
|
42
|
+
}
|
|
43
|
+
} catch (_) { /* skip */ }
|
|
44
|
+
return out;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
shadowDoc() { return null; }
|
|
48
|
+
|
|
49
|
+
/** Explicit row shadow: routes into the registered collection. */
|
|
50
|
+
shadowRow(ns, doc, op = 'UPSERT') {
|
|
51
|
+
if (!this.shadowWrites) return;
|
|
52
|
+
try {
|
|
53
|
+
const m = this.mappings.find((x) => x.pattern === ns);
|
|
54
|
+
const coll = m ? m.collection : ns.split('.').pop();
|
|
55
|
+
if (doc === null || doc === undefined) {
|
|
56
|
+
this.engine.put('__mongo_shadow__', `${ns}:del:${op}`,
|
|
57
|
+
{ ns, _op: 'DELETE' });
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
const { _id, ...rest } = doc;
|
|
61
|
+
this.engine.put(coll, String(_id ?? op), { ...rest, _ns: ns, _op: op });
|
|
62
|
+
} catch (_) { /* never break the host */ }
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
class WrappedMongoClient {
|
|
67
|
+
constructor(client, opts = {}) {
|
|
68
|
+
const dbName = opts.dbName || 'default';
|
|
69
|
+
const engine = openEngine({
|
|
70
|
+
dbName, nedbdUrl: opts.nedbdUrl, nedbdToken: opts.nedbdToken,
|
|
71
|
+
dagPath: opts.dagPath, dagTmk: opts.dagTmk, native: opts.native,
|
|
72
|
+
});
|
|
73
|
+
this._client = client;
|
|
74
|
+
this.nedb = new MongoSurface(client, dbName, engine);
|
|
75
|
+
}
|
|
76
|
+
_raw() { return this._client; }
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function wrapMongo(client, opts) {
|
|
80
|
+
const w = new WrappedMongoClient(client, opts);
|
|
81
|
+
return new Proxy(w, {
|
|
82
|
+
get(target, prop, receiver) {
|
|
83
|
+
if (prop in target || prop === 'nedb') return Reflect.get(target, prop, receiver);
|
|
84
|
+
const v = Reflect.get(target._client, prop);
|
|
85
|
+
return typeof v === 'function' ? v.bind(target._client) : v;
|
|
86
|
+
},
|
|
87
|
+
set(target, prop, value) { target._client[prop] = value; return true; },
|
|
88
|
+
});
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
module.exports = { wrapMongo, WrappedMongoClient, MongoSurface };
|
package/wrap/redis.js
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
// nedb/wrap/redis.js — wrapRedis: one-line causal provenance for Redis (JS).
|
|
3
|
+
//
|
|
4
|
+
// const { wrapRedis } = require('nedb-engine/wrap');
|
|
5
|
+
// const redis = require('redis');
|
|
6
|
+
//
|
|
7
|
+
// const r = wrapRedis(redis.createClient(), { dbName: 'rideshare' });
|
|
8
|
+
//
|
|
9
|
+
// r.nedb.register('driver:*', 'driver');
|
|
10
|
+
// r.nedb.backfill();
|
|
11
|
+
// r.nedb.shadowWrites = true;
|
|
12
|
+
//
|
|
13
|
+
// await r.set('driver:d1', JSON.stringify({ name: 'Bob' })); // shadowed
|
|
14
|
+
// r.nedb.query('FROM driver'); // NQL on top
|
|
15
|
+
// r.nedb.verify(); // → true
|
|
16
|
+
//
|
|
17
|
+
// Works with any client whose write methods are functions on the connection
|
|
18
|
+
// (node-redis v4, ioredis, redis-mock). Write commands are intercepted by
|
|
19
|
+
// wrapping the method — the surface-1 behavior is unchanged.
|
|
20
|
+
//
|
|
21
|
+
// © INTERCHAINED LLC × Claude Sonnet 4.6
|
|
22
|
+
'use strict';
|
|
23
|
+
|
|
24
|
+
const { WrapSurface, openEngine } = require('./surface');
|
|
25
|
+
|
|
26
|
+
// Redis write commands we shadow (mirrors the Python _WRITE_CMDS set).
|
|
27
|
+
const WRITE_CMDS = new Set([
|
|
28
|
+
'set', 'setnx', 'setex', 'psetex', 'getset', 'getdel', 'getex',
|
|
29
|
+
'mset', 'msetnx', 'hset', 'hmset', 'hsetnx', 'hincrby', 'hincrbyfloat', 'hdel',
|
|
30
|
+
'lpush', 'rpush', 'lset', 'linsert', 'ltrim', 'lpop', 'rpop',
|
|
31
|
+
'sadd', 'srem', 'smove', 'zadd', 'zincrby', 'zrem',
|
|
32
|
+
'del', 'unlink', 'rename', 'renamenx', 'append', 'incr', 'incrby', 'decr', 'decrby',
|
|
33
|
+
]);
|
|
34
|
+
|
|
35
|
+
class RedisSurface extends WrapSurface {
|
|
36
|
+
constructor(client, dbName, engine) {
|
|
37
|
+
super(dbName, engine);
|
|
38
|
+
this.client = client;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// ── host scan: iterate keys matching the pattern, read their values ──────
|
|
42
|
+
hostScan(mapping, batchSize) {
|
|
43
|
+
// node-redis v4 / ioredis both expose scan + get/hgetall; a sync-scan
|
|
44
|
+
// fallback covers in-memory fakes.
|
|
45
|
+
const out = [];
|
|
46
|
+
const scanSync = typeof this.client.scanKeys === 'function'
|
|
47
|
+
? this.client.scanKeys(mapping.pattern)
|
|
48
|
+
: (this.client.keys ? this.client.keys(mapping.pattern) : []);
|
|
49
|
+
for (const key of scanSync || []) {
|
|
50
|
+
try {
|
|
51
|
+
const raw = mapping.valueType === 'hash'
|
|
52
|
+
? this.client.hgetall(key)
|
|
53
|
+
: this.client.get(key);
|
|
54
|
+
if (raw !== null && raw !== undefined) out.push([key, raw]);
|
|
55
|
+
} catch (_) { /* skip unreadable */ }
|
|
56
|
+
if (out.length >= (batchSize || 200)) break;
|
|
57
|
+
}
|
|
58
|
+
return out;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// ── host write → NEDB doc ──────────────────────────────────────────────────
|
|
62
|
+
// args = the host call's arguments AFTER the key.
|
|
63
|
+
// __replace: true → surface puts doc as-is (full replace); default merges.
|
|
64
|
+
shadowDoc(mapping, cmd, args) {
|
|
65
|
+
if (cmd === 'hset') {
|
|
66
|
+
// hset key field value | hset key {obj}
|
|
67
|
+
if (typeof args[0] === 'object' && args[0] !== null) return mapping.parseValue(args[0]);
|
|
68
|
+
return { [args[0]]: args[1] }; // merged over existing by the surface
|
|
69
|
+
}
|
|
70
|
+
if (cmd === 'set' || cmd === 'setex' || cmd === 'psetex' || cmd === 'setnx' || cmd === 'getset') {
|
|
71
|
+
return { ...mapping.parseValue(args[0]), __replace: true };
|
|
72
|
+
}
|
|
73
|
+
if (cmd === 'incr' || cmd === 'incrby' || cmd === 'decr' || cmd === 'decrby') {
|
|
74
|
+
return { _v: String(args[0] === undefined ? '' : args[0]) };
|
|
75
|
+
}
|
|
76
|
+
if (cmd === 'del' || cmd === 'unlink') {
|
|
77
|
+
return { _deleted: true, __replace: true };
|
|
78
|
+
}
|
|
79
|
+
// other write types: store the command as metadata (merged)
|
|
80
|
+
return { [`_redis_${cmd}`]: String(args[0] === undefined ? '' : args[0]) };
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
class WrappedRedis {
|
|
85
|
+
constructor(client, opts = {}) {
|
|
86
|
+
const dbName = opts.dbName || 'default';
|
|
87
|
+
const engine = openEngine({
|
|
88
|
+
dbName, nedbdUrl: opts.nedbdUrl, nedbdToken: opts.nedbdToken,
|
|
89
|
+
dagPath: opts.dagPath, dagTmk: opts.dagTmk, native: opts.native,
|
|
90
|
+
});
|
|
91
|
+
this._client = client;
|
|
92
|
+
this.nedb = new RedisSurface(client, dbName, engine);
|
|
93
|
+
this._installInterception();
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
_installInterception() {
|
|
97
|
+
const surface = this.nedb;
|
|
98
|
+
const client = this._client;
|
|
99
|
+
for (const cmd of WRITE_CMDS) {
|
|
100
|
+
const orig = client[cmd];
|
|
101
|
+
if (typeof orig !== 'function' || orig.__nedbWrapped) continue;
|
|
102
|
+
const wrapped = function (...args) {
|
|
103
|
+
const result = orig.apply(client, args);
|
|
104
|
+
try {
|
|
105
|
+
const key = args[0];
|
|
106
|
+
if (typeof key === 'string' && surface.shadowWrites) surface.shadow(cmd, key, ...args.slice(1));
|
|
107
|
+
} catch (_) { /* never break the host call */ }
|
|
108
|
+
return result;
|
|
109
|
+
};
|
|
110
|
+
wrapped.__nedbWrapped = true;
|
|
111
|
+
try { client[cmd] = wrapped; } catch (_) { /* frozen client — skip */ }
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// passthrough for everything else
|
|
116
|
+
__getattrPrivate(name) { return this._client[name]; }
|
|
117
|
+
get _raw() { return this._client; }
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
function wrapRedis(client, opts) {
|
|
121
|
+
const w = new WrappedRedis(client, opts);
|
|
122
|
+
// Proxy property access to the underlying client so `await r.get(...)`
|
|
123
|
+
// works naturally, while `.nedb` stays on the wrapper.
|
|
124
|
+
return new Proxy(w, {
|
|
125
|
+
get(target, prop, receiver) {
|
|
126
|
+
if (prop in target || prop === 'nedb' || prop === '_client') return Reflect.get(target, prop, receiver);
|
|
127
|
+
const v = target._client[prop];
|
|
128
|
+
return typeof v === 'function' ? v.bind(target._client) : v;
|
|
129
|
+
},
|
|
130
|
+
set(target, prop, value) { target._client[prop] = value; return true; },
|
|
131
|
+
});
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
module.exports = { wrapRedis, WrappedRedis, RedisSurface, WRITE_CMDS };
|
package/wrap/sql.js
ADDED
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
// nedb/wrap/sql.js — wrapSqlite / wrapMysql / wrapPg: causal provenance for
|
|
3
|
+
// SQL databases (JS).
|
|
4
|
+
//
|
|
5
|
+
// const { wrapSqlite } = require('nedb-engine/wrap');
|
|
6
|
+
// const db = wrapSqlite(mySqliteConnection, { dbName: 'app' });
|
|
7
|
+
//
|
|
8
|
+
// db.nedb.register('drivers', 'driver');
|
|
9
|
+
// db.nedb.backfill();
|
|
10
|
+
// db.nedb.shadowWrites = true;
|
|
11
|
+
//
|
|
12
|
+
// SQLite: execute() is intercepted — INSERT/UPDATE/DELETE on registered
|
|
13
|
+
// tables are shadowed automatically after they succeed.
|
|
14
|
+
// MySQL/Postgres (DB-API style or promise clients): shadowing is explicit —
|
|
15
|
+
// after your INSERT/UPDATE call db.nedb.shadowRow(table, pk, rowObj);
|
|
16
|
+
// row=null chains a DELETE tombstone.
|
|
17
|
+
//
|
|
18
|
+
// © INTERCHAINED LLC × Claude Sonnet 4.6
|
|
19
|
+
'use strict';
|
|
20
|
+
|
|
21
|
+
const { WrapSurface, openEngine } = require('./surface');
|
|
22
|
+
|
|
23
|
+
class SqlSurface extends WrapSurface {
|
|
24
|
+
constructor(conn, dbName, engine, kind) {
|
|
25
|
+
super(dbName, engine);
|
|
26
|
+
this.conn = conn;
|
|
27
|
+
this.kind = kind;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** Default host scan: SELECT * FROM <table> — works on sync sqlite3 and
|
|
31
|
+
* promise clients that expose .all/.query. Override for exotic drivers. */
|
|
32
|
+
hostScan(mapping, batchSize) {
|
|
33
|
+
const table = mapping.pattern;
|
|
34
|
+
const out = [];
|
|
35
|
+
const push = (cols, rows) => {
|
|
36
|
+
for (const row of rows || []) {
|
|
37
|
+
if (Array.isArray(row)) out.push([String(out.length + 1), Object.fromEntries(cols.map((c, i) => [c, row[i]]))]);
|
|
38
|
+
else out.push([String(row[cols[0]] ?? out.length + 1), row]);
|
|
39
|
+
}
|
|
40
|
+
};
|
|
41
|
+
if (typeof this.conn.all === 'function') { // node-sqlite3 style
|
|
42
|
+
// sync reads aren't available; caller should backfill via callback mode.
|
|
43
|
+
return out;
|
|
44
|
+
}
|
|
45
|
+
if (typeof this.conn.prepare === 'function') { // better-sqlite3
|
|
46
|
+
const stmt = this.conn.prepare(`SELECT rowid AS __nedb_rowid, * FROM "${table}"`);
|
|
47
|
+
for (const row of stmt.iterate()) {
|
|
48
|
+
const { __nedb_rowid, ...rest } = row;
|
|
49
|
+
out.push([String(__nedb_rowid), rest]);
|
|
50
|
+
}
|
|
51
|
+
return out;
|
|
52
|
+
}
|
|
53
|
+
if (typeof this.conn.exec === 'function' && this.conn.prepare === undefined) {
|
|
54
|
+
// node:sqlite (built-in) — similar to better-sqlite3
|
|
55
|
+
try {
|
|
56
|
+
const stmt = this.conn.prepare(`SELECT rowid AS __nedb_rowid, * FROM "${table}"`);
|
|
57
|
+
for (const row of stmt.iterate ? stmt.iterate() : []) {
|
|
58
|
+
const { __nedb_rowid, ...rest } = row;
|
|
59
|
+
out.push([String(__nedb_rowid), rest]);
|
|
60
|
+
}
|
|
61
|
+
return out;
|
|
62
|
+
} catch (_) { /* fall through */ }
|
|
63
|
+
}
|
|
64
|
+
if (typeof this.conn.query === 'function') { // mysql2 / pg promise
|
|
65
|
+
// handled by host adapters with async backfill — sync scan unsupported
|
|
66
|
+
return out;
|
|
67
|
+
}
|
|
68
|
+
return out;
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
shadowDoc() { return null; }
|
|
72
|
+
|
|
73
|
+
/** Explicit row shadow (mysql/pg): routes into the registered collection. */
|
|
74
|
+
shadowRow(table, pk, row, op = 'UPSERT') {
|
|
75
|
+
if (!this.shadowWrites) return;
|
|
76
|
+
try {
|
|
77
|
+
if (row === null || row === undefined) {
|
|
78
|
+
this.engine.put('__sql_shadow__', `${table}:del:${pk}`,
|
|
79
|
+
{ table, pk: String(pk), _op: 'DELETE' });
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
const m = this.mappings.find((x) => x.pattern === table);
|
|
83
|
+
const coll = m ? m.collection : '__sql_shadow__';
|
|
84
|
+
this.engine.put(coll, String(pk), { ...row, _table: table, _op: op });
|
|
85
|
+
} catch (_) { /* never break the host */ }
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
class WrappedSqlConn {
|
|
90
|
+
constructor(conn, opts, kind) {
|
|
91
|
+
const dbName = opts.dbName || 'default';
|
|
92
|
+
const engine = openEngine({
|
|
93
|
+
dbName, nedbdUrl: opts.nedbdUrl, nedbdToken: opts.nedbdToken,
|
|
94
|
+
dagPath: opts.dagPath, dagTmk: opts.dagTmk, native: opts.native,
|
|
95
|
+
});
|
|
96
|
+
this._conn = conn;
|
|
97
|
+
this.nedb = new SqlSurface(conn, dbName, engine, kind);
|
|
98
|
+
if (kind === 'sqlite') this._installSqliteInterception();
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
/** sqlite3/better-sqlite/node:sqlite — wrap exec/run/execute for shadowing. */
|
|
102
|
+
_installSqliteInterception() {
|
|
103
|
+
const surface = this.nedb;
|
|
104
|
+
const conn = this._conn;
|
|
105
|
+
const methodNames = ['execute', 'run', 'exec'].filter((m) => typeof conn[m] === 'function');
|
|
106
|
+
for (const name of methodNames) {
|
|
107
|
+
const orig = conn[name];
|
|
108
|
+
if (orig.__nedbWrapped) continue;
|
|
109
|
+
const wrapped = function (sql, ...rest) {
|
|
110
|
+
const result = orig.call(conn, sql, ...rest);
|
|
111
|
+
try {
|
|
112
|
+
if (surface.shadowWrites && /^\s*(INSERT|UPDATE|DELETE|REPLACE)/i.test(String(sql))) {
|
|
113
|
+
surface.shadow('sql', sql, 'sql', sql, result);
|
|
114
|
+
}
|
|
115
|
+
} catch (_) {}
|
|
116
|
+
return result;
|
|
117
|
+
};
|
|
118
|
+
wrapped.__nedbWrapped = true;
|
|
119
|
+
try { conn[name] = wrapped; } catch (_) {}
|
|
120
|
+
}
|
|
121
|
+
// transactional helpers pass through
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
_raw() { return this._conn; }
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
function _wrapWithProxy(wrapper, conn) {
|
|
128
|
+
return new Proxy(wrapper, {
|
|
129
|
+
get(target, prop, receiver) {
|
|
130
|
+
if (prop in target || prop === 'nedb') return Reflect.get(target, prop, receiver);
|
|
131
|
+
const v = Reflect.get(conn, prop);
|
|
132
|
+
return typeof v === 'function' ? v.bind(conn) : v;
|
|
133
|
+
},
|
|
134
|
+
set(target, prop, value) { conn[prop] = value; return true; },
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
function wrapSqlite(conn, opts) {
|
|
139
|
+
return _wrapWithProxy(new WrappedSqlConn(conn, opts || {}, 'sqlite'), conn);
|
|
140
|
+
}
|
|
141
|
+
function wrapMysql(conn, opts) {
|
|
142
|
+
return _wrapWithProxy(new WrappedSqlConn(conn, opts || {}, 'mysql'), conn);
|
|
143
|
+
}
|
|
144
|
+
function wrapPg(conn, opts) {
|
|
145
|
+
const w = new WrappedSqlConn(conn, opts || {}, 'pg');
|
|
146
|
+
const p = _wrapWithProxy(w, conn);
|
|
147
|
+
// pg uses .query — shadowRow is explicit on the surface
|
|
148
|
+
return p;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
module.exports = { wrapSqlite, wrapMysql, wrapPg, WrappedSqlConn, SqlSurface };
|
package/wrap/surface.js
ADDED
|
@@ -0,0 +1,330 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
// nedb/wrap/surface.js — the engine-agnostic NEDB layer-2 surface (JS).
|
|
3
|
+
//
|
|
4
|
+
// One contract, three engines:
|
|
5
|
+
// dag — embedded Rust core (NedbCore from the napi-rs addon)
|
|
6
|
+
// nedbd — HTTP nedbd server (v1 AOF, v2 DAG, v3 --dag-v3)
|
|
7
|
+
// memory — a caller-supplied engine object (tests, custom backends)
|
|
8
|
+
//
|
|
9
|
+
// Host adapters (wrapRedis/wrapSqlite/wrapMysql/wrapMongo/wrapPg) supply:
|
|
10
|
+
// hostScan(mapping) → iterate existing host records
|
|
11
|
+
// shadowDoc(mapping, args) → host write → NEDB doc (or null)
|
|
12
|
+
//
|
|
13
|
+
// © INTERCHAINED LLC × Claude Sonnet 4.6
|
|
14
|
+
'use strict';
|
|
15
|
+
|
|
16
|
+
const fs = require('fs');
|
|
17
|
+
const path = require('path');
|
|
18
|
+
|
|
19
|
+
// ── engine resolution ────────────────────────────────────────────────────────
|
|
20
|
+
|
|
21
|
+
/** Acquire the NedbCore class: caller-supplied, native addon, or throw. */
|
|
22
|
+
function resolveNative(provided) {
|
|
23
|
+
if (provided) return provided;
|
|
24
|
+
try {
|
|
25
|
+
return require('../index.js').NedbCore;
|
|
26
|
+
} catch (_) {
|
|
27
|
+
throw new Error(
|
|
28
|
+
'nedb-engine native addon not found — rebuild with `npm run build` ' +
|
|
29
|
+
'or pass an engine explicitly (wrap(x, { engine }) / { nedbdUrl })');
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Open an engine handle for a wrap_* surface.
|
|
35
|
+
* openEngine({ nedbdUrl, nedbdToken, dagPath, engine }) → handle
|
|
36
|
+
* The handle exposes: put/get/query/createIndex/delete/link/unlink/
|
|
37
|
+
* neighbors/inbound/verify()/head/seq + engineKind.
|
|
38
|
+
*/
|
|
39
|
+
function openEngine(opts = {}) {
|
|
40
|
+
if (opts.engine) return normalizeEngine(opts.engine, 'supplied');
|
|
41
|
+
|
|
42
|
+
if (opts.nedbdUrl) return new NedbdProxy(opts.nedbdUrl, opts.dbName, opts.nedbdToken);
|
|
43
|
+
|
|
44
|
+
const NedbCore = resolveNative(opts.native);
|
|
45
|
+
const core = opts.dagPath
|
|
46
|
+
? NedbCore.open(opts.dagPath, opts.dagTmk || null)
|
|
47
|
+
: new NedbCore();
|
|
48
|
+
return normalizeEngine(core, 'dag-embedded');
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** Normalize any engine-ish object into the canonical duck-typed contract. */
|
|
52
|
+
function normalizeEngine(core, kind) {
|
|
53
|
+
// HTTP proxy shape → pass through as-is
|
|
54
|
+
if (core instanceof NedbdProxy) return core;
|
|
55
|
+
return new NativeEngine(core, kind);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
// ── Native engine handle (embedded Rust DAG) ────────────────────────────────
|
|
59
|
+
|
|
60
|
+
class NativeEngine {
|
|
61
|
+
constructor(core, kind) {
|
|
62
|
+
this.core = core;
|
|
63
|
+
this.kind = kind || 'dag-embedded';
|
|
64
|
+
}
|
|
65
|
+
put(coll, id, doc) {
|
|
66
|
+
const body = { ...doc };
|
|
67
|
+
for (const k of ['caused_by', 'valid_from', 'valid_to']) {
|
|
68
|
+
if (body[k] === undefined) delete body[k];
|
|
69
|
+
}
|
|
70
|
+
const node = this.core.put(coll, id, JSON.stringify(body));
|
|
71
|
+
return typeof node === 'string' ? JSON.parse(node) : node;
|
|
72
|
+
}
|
|
73
|
+
get(coll, id, asOf) {
|
|
74
|
+
const node = asOf === undefined || asOf === null
|
|
75
|
+
? this.core.get(coll, id)
|
|
76
|
+
: this.core.get(coll, id, asOf);
|
|
77
|
+
return node ? (typeof node === 'string' ? JSON.parse(node) : node) : null;
|
|
78
|
+
}
|
|
79
|
+
query(nql) {
|
|
80
|
+
const rows = this.core.query(nql);
|
|
81
|
+
return rows.map((r) => (typeof r === 'string' ? JSON.parse(r) : r));
|
|
82
|
+
}
|
|
83
|
+
createIndex(coll, field, kind) { this.core.createIndex(coll, field, kind || 'eq'); }
|
|
84
|
+
delete(coll, id) { this.core.delete(coll, id); }
|
|
85
|
+
link(frm, rel, to) { this.core.link(frm, rel, to); }
|
|
86
|
+
unlink(frm, rel, to) { this.core.unlink(frm, rel, to); }
|
|
87
|
+
neighbors(frm, rel, asOf) {
|
|
88
|
+
return asOf === undefined ? this.core.neighbors(frm, rel)
|
|
89
|
+
: this.core.neighbors(frm, rel, asOf);
|
|
90
|
+
}
|
|
91
|
+
inbound(to, rel, asOf) {
|
|
92
|
+
return asOf === undefined ? this.core.inbound(to, rel)
|
|
93
|
+
: this.core.inbound(to, rel, asOf);
|
|
94
|
+
}
|
|
95
|
+
verify() { return this.core.verify() === true; }
|
|
96
|
+
get head() { return this.core.head(); }
|
|
97
|
+
get seq() { return this.core.seq(); }
|
|
98
|
+
checkpoint() { this.core.flush(); return this.head; }
|
|
99
|
+
tip() { const t = this.core.tip ? this.core.tip() : null; return t ? JSON.parse(t) : null; }
|
|
100
|
+
since(afterSeq, limit) {
|
|
101
|
+
if (!this.core.since) throw new Error('changefeed requires the DAG backend');
|
|
102
|
+
// napi binding: after_seq is u64 (BigInt), limit is usize (Number)
|
|
103
|
+
const after = typeof afterSeq === 'bigint' ? afterSeq : BigInt(afterSeq);
|
|
104
|
+
const lim = limit === undefined ? 0 : Number(limit);
|
|
105
|
+
return JSON.parse(this.core.since(after, lim));
|
|
106
|
+
}
|
|
107
|
+
scanStatus() {
|
|
108
|
+
if (!this.core.scanStatus) throw new Error('scanStatus requires the DAG backend');
|
|
109
|
+
return JSON.parse(this.core.scanStatus());
|
|
110
|
+
}
|
|
111
|
+
flush() { if (this.core.flush) this.core.flush(); }
|
|
112
|
+
get engineKind() { return this.kind; }
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// ── HTTP nedbd handle ────────────────────────────────────────────────────────
|
|
116
|
+
|
|
117
|
+
class NedbdProxy {
|
|
118
|
+
constructor(baseUrl, dbName, token) {
|
|
119
|
+
this.base = baseUrl.replace(/\/$/, '');
|
|
120
|
+
this.name = dbName;
|
|
121
|
+
this.token = token || null;
|
|
122
|
+
this._ensureDb();
|
|
123
|
+
}
|
|
124
|
+
_headers() {
|
|
125
|
+
const h = { 'Content-Type': 'application/json', Accept: 'application/json' };
|
|
126
|
+
if (this.token) h.Authorization = `Bearer ${this.token}`;
|
|
127
|
+
return h;
|
|
128
|
+
}
|
|
129
|
+
_req(method, p, body) {
|
|
130
|
+
const http = require('http');
|
|
131
|
+
const url = new URL(this.base + p);
|
|
132
|
+
const payload = body === undefined ? null : JSON.stringify(body);
|
|
133
|
+
const res = http.request({
|
|
134
|
+
hostname: url.hostname, port: url.port || 80, path: url.pathname + url.search,
|
|
135
|
+
method, headers: { ...this._headers(), ...(payload ? { 'Content-Length': Buffer.byteLength(payload) } : {}) },
|
|
136
|
+
});
|
|
137
|
+
// synchronous-feeling via Atomics.wait on a SharedArrayBuffer flag
|
|
138
|
+
const flag = new Int32Array(new SharedArrayBuffer(4));
|
|
139
|
+
let out = { status: 0, text: '' };
|
|
140
|
+
const r2 = res;
|
|
141
|
+
r2.on('response', (r) => {
|
|
142
|
+
let t = '';
|
|
143
|
+
r.on('data', (c) => { t += c; });
|
|
144
|
+
r.on('end', () => { out = { status: r.statusCode, text: t }; Atomics.store(flag, 0, 1); Atomics.notify(flag, 0); });
|
|
145
|
+
});
|
|
146
|
+
r2.on('error', (e) => { out = { status: 0, text: String(e) }; Atomics.store(flag, 0, 1); Atomics.notify(flag, 0); });
|
|
147
|
+
if (payload) r2.write(payload);
|
|
148
|
+
r2.end();
|
|
149
|
+
Atomics.wait(flag, 0, 0);
|
|
150
|
+
if (out.status === 0) throw new Error(`nedbd ${method} ${p} failed: ${out.text}`);
|
|
151
|
+
let parsed;
|
|
152
|
+
try { parsed = JSON.parse(out.text); } catch { parsed = { raw: out.text }; }
|
|
153
|
+
if (out.status >= 400) throw new Error(`nedbd ${method} ${p} → HTTP ${out.status}: ${out.text.slice(0, 200)}`);
|
|
154
|
+
return parsed;
|
|
155
|
+
}
|
|
156
|
+
_db(suffix) { return `/v1/databases/${this.name}${suffix || ''}`; }
|
|
157
|
+
_ensureDb() {
|
|
158
|
+
try { this._req('GET', this._db()); } catch (e) {
|
|
159
|
+
if (String(e).includes('404')) this._req('POST', '/v1/databases', { name: this.name });
|
|
160
|
+
else throw e;
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
put(coll, id, doc, kw = {}) {
|
|
164
|
+
const payload = { coll, id, doc };
|
|
165
|
+
for (const k of ['client', 'nonce', 'idem', 'evidence', 'confidence', 'valid_from', 'valid_to', 'caused_by']) {
|
|
166
|
+
if (kw[k] !== undefined && kw[k] !== null) payload[k] = kw[k];
|
|
167
|
+
}
|
|
168
|
+
const r = this._req('POST', this._db('/put'), payload);
|
|
169
|
+
return r.doc !== undefined ? r.doc : doc;
|
|
170
|
+
}
|
|
171
|
+
get(coll, id, asOf) {
|
|
172
|
+
const clause = asOf !== undefined && asOf !== null ? ` AS OF ${asOf}` : '';
|
|
173
|
+
const rows = this.query(`FROM ${coll}${clause} WHERE _id = "${id}"`);
|
|
174
|
+
return rows.length ? rows[0] : null;
|
|
175
|
+
}
|
|
176
|
+
query(nql) { return this._req('POST', this._db('/query'), { nql }).rows || []; }
|
|
177
|
+
createIndex(coll, field, kind) { this._req('POST', this._db('/index'), { coll, field, kind: kind || 'eq' }); }
|
|
178
|
+
delete(coll, id) { this._req('DELETE', `/v1/databases/${this.name}/rows/${coll}/${id}`); }
|
|
179
|
+
link(frm, rel, to) {
|
|
180
|
+
try { this._req('POST', this._db('/link'), { frm, rel, to }); }
|
|
181
|
+
catch (e) {
|
|
182
|
+
if (String(e).includes('404') || String(e).toLowerCase().includes('not found')) {
|
|
183
|
+
this.put('__links__', `${frm}|${rel}|${to}`, { _from: frm, _rel: rel, _to: to });
|
|
184
|
+
} else throw e;
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
unlink(frm, rel, to) {
|
|
188
|
+
try { this._req('DELETE', `/v1/databases/${this.name}/links/${frm}/${rel}/${to}`); }
|
|
189
|
+
catch (_) { try { this._req('DELETE', `/v1/databases/${this.name}/rows/__links__/${frm}|${rel}|${to}`); } catch (_) {} }
|
|
190
|
+
}
|
|
191
|
+
neighbors(frm, rel, asOf) {
|
|
192
|
+
const clause = asOf !== undefined && asOf !== null ? ` AS OF ${asOf}` : '';
|
|
193
|
+
const [c] = frm.split(':');
|
|
194
|
+
const rows = this.query(`FROM ${c}${clause} WHERE _id = "${frm.split(':')[1] || ''}" TRAVERSE ${rel}`);
|
|
195
|
+
return rows.filter((r) => r._id).map((r) => `${r._coll || c}:${r._id}`);
|
|
196
|
+
}
|
|
197
|
+
inbound(to, rel, asOf) {
|
|
198
|
+
const clause = asOf !== undefined && asOf !== null ? ` AS OF ${asOf}` : '';
|
|
199
|
+
const [c] = to.split(':');
|
|
200
|
+
try {
|
|
201
|
+
const rows = this.query(`FROM ${c} WHERE _id = "${to.split(':')[1] || ''}" TRAVERSE ${rel} REVERSE`);
|
|
202
|
+
return rows.filter((r) => r._id).map((r) => `${r._coll || c}:${r._id}`);
|
|
203
|
+
} catch (_) { return []; }
|
|
204
|
+
}
|
|
205
|
+
verify() { return this._req('GET', this._db('/verify')).ok === true; }
|
|
206
|
+
get head() { return this._req('GET', this._db()).head || '0'.repeat(64); }
|
|
207
|
+
get seq() { return this._req('GET', this._db()).seq || 0; }
|
|
208
|
+
checkpoint() { return this._req('POST', this._db('/checkpoint')).head || this.head; }
|
|
209
|
+
get engineKind() { return 'nedbd-http'; }
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
// ── collection mapping (key/table glob → NEDB collection) ───────────────────
|
|
213
|
+
|
|
214
|
+
class CollectionMapping {
|
|
215
|
+
constructor(pattern, collection, opts = {}) {
|
|
216
|
+
this.pattern = pattern;
|
|
217
|
+
this.collection = collection;
|
|
218
|
+
this.idExtractor = opts.idExtractor || ((k) => k.split(':').pop());
|
|
219
|
+
this.valueParser = opts.valueParser || CollectionMapping.defaultParse;
|
|
220
|
+
this.valueType = opts.valueType || 'string';
|
|
221
|
+
}
|
|
222
|
+
static defaultParse(v) {
|
|
223
|
+
if (v === null || v === undefined) return { _v: null };
|
|
224
|
+
if (typeof v === 'object') return v;
|
|
225
|
+
const s = String(v);
|
|
226
|
+
try {
|
|
227
|
+
const p = JSON.parse(s);
|
|
228
|
+
return (p && typeof p === 'object') ? p : { _v: p };
|
|
229
|
+
} catch { return { _v: s }; }
|
|
230
|
+
}
|
|
231
|
+
matches(key) {
|
|
232
|
+
// translate a glob to a RegExp (tiny, no deps)
|
|
233
|
+
const rx = new RegExp('^' + this.pattern
|
|
234
|
+
.replace(/[.+^${}()|[\]\\]/g, '\\$&')
|
|
235
|
+
.replace(/\*/g, '.*').replace(/\?/g, '.') + '$');
|
|
236
|
+
return rx.test(key);
|
|
237
|
+
}
|
|
238
|
+
extractId(key) { return this.idExtractor(key); }
|
|
239
|
+
parseValue(v) { return this.valueParser(v); }
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
// ── the shared surface (the `.nedb` attribute) ──────────────────────────────
|
|
243
|
+
|
|
244
|
+
class WrapSurface {
|
|
245
|
+
constructor(dbName, engine) {
|
|
246
|
+
this.dbName = dbName;
|
|
247
|
+
this.engine = engine;
|
|
248
|
+
this.mappings = [];
|
|
249
|
+
this.shadowWrites = false;
|
|
250
|
+
this.backfilled = false;
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
// host hooks — override in host adapters
|
|
254
|
+
hostScan(/* mapping, batchSize */) { return []; }
|
|
255
|
+
shadowDoc(/* mapping, cmd, args */) { return null; }
|
|
256
|
+
|
|
257
|
+
register(pattern, collection, opts) {
|
|
258
|
+
this.mappings.push(new CollectionMapping(pattern, collection, opts));
|
|
259
|
+
return this;
|
|
260
|
+
}
|
|
261
|
+
mappingFor(key) { return this.mappings.find((m) => m.matches(key)) || null; }
|
|
262
|
+
|
|
263
|
+
backfill(opts = {}) {
|
|
264
|
+
const mappings = opts.pattern
|
|
265
|
+
? [new CollectionMapping(opts.pattern, opts.collection || opts.pattern.split(':')[0], opts)]
|
|
266
|
+
: this.mappings;
|
|
267
|
+
let total = 0;
|
|
268
|
+
for (const m of mappings) {
|
|
269
|
+
for (const [key, raw] of this.hostScan(m, opts.batchSize || 200)) {
|
|
270
|
+
try {
|
|
271
|
+
const doc = m.parseValue(raw);
|
|
272
|
+
doc._source = 'backfill';
|
|
273
|
+
this.engine.put(m.collection, m.extractId(key), doc);
|
|
274
|
+
total += 1;
|
|
275
|
+
} catch (_) { /* skip unreadable */ }
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
this.backfilled = true;
|
|
279
|
+
return total;
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
shadow(cmd, key, ...rest) {
|
|
283
|
+
if (!this.shadowWrites) return;
|
|
284
|
+
try {
|
|
285
|
+
const m = this.mappingFor(key);
|
|
286
|
+
if (!m) {
|
|
287
|
+
this.engine.put('__shadow_raw__', key, { cmd, key, _source: 'shadow_raw' });
|
|
288
|
+
return;
|
|
289
|
+
}
|
|
290
|
+
// rest = the host call's arguments AFTER the key (value(s) / fields)
|
|
291
|
+
const doc = this.shadowDoc(m, cmd, rest);
|
|
292
|
+
if (!doc) return;
|
|
293
|
+
doc._source = 'shadow';
|
|
294
|
+
const id = m.extractId(key);
|
|
295
|
+
// merge over the existing doc (hset/incr are incremental by nature;
|
|
296
|
+
// set replaces — the adapter marks replacement with doc.__replace)
|
|
297
|
+
const prev = this.engine.get(m.collection, id);
|
|
298
|
+
const merged = doc.__replace || !prev ? doc : { ...prev, ...doc };
|
|
299
|
+
delete merged.__replace;
|
|
300
|
+
this.engine.put(m.collection, id, merged);
|
|
301
|
+
} catch (e) { if (process.env.NEDB_WRAP_DEBUG) console.error('[nedb shadow]', e); }
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
// ── full NEDB API ──────────────────────────────────────────────────────────
|
|
305
|
+
put(coll, id, doc, kw) { return this.engine.put(coll, id, doc, kw); }
|
|
306
|
+
get(coll, id, asOf) { return this.engine.get(coll, id, asOf); }
|
|
307
|
+
query(nql) { return this.engine.query(nql); }
|
|
308
|
+
createIndex(coll, field, kind) { this.engine.createIndex(coll, field, kind); }
|
|
309
|
+
delete(coll, id) { return this.engine.delete(coll, id); }
|
|
310
|
+
link(frm, rel, to) { return this.engine.link(frm, rel, to); }
|
|
311
|
+
unlink(frm, rel, to) { return this.engine.unlink(frm, rel, to); }
|
|
312
|
+
neighbors(frm, rel, asOf) { return this.engine.neighbors(frm, rel, asOf); }
|
|
313
|
+
inbound(to, rel, asOf) { return this.engine.inbound(to, rel, asOf); }
|
|
314
|
+
verify() { return this.engine.verify(); }
|
|
315
|
+
get head() { return this.engine.head; }
|
|
316
|
+
get seq() { return this.engine.seq; }
|
|
317
|
+
checkpoint() { return this.engine.checkpoint(); }
|
|
318
|
+
tip() { return this.engine.tip ? this.engine.tip() : null; }
|
|
319
|
+
since(afterSeq, limit) { return this.engine.since(afterSeq, limit); }
|
|
320
|
+
scanStatus() { return this.engine.scanStatus(); }
|
|
321
|
+
get engineKind() { return this.engine.engineKind; }
|
|
322
|
+
|
|
323
|
+
[Symbol.for('nodejs.util.inspect.custom')]() {
|
|
324
|
+
return `<WrapSurface db=${JSON.stringify(this.dbName)} engine=${this.engineKind} mappings=${this.mappings.length}>`;
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
module.exports = {
|
|
329
|
+
openEngine, NativeEngine, NedbdProxy, CollectionMapping, WrapSurface, resolveNative,
|
|
330
|
+
};
|