@semics-tech/mongolite 1.3.0 → 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +80 -21
- package/dist/cjs/collection.js +9 -6
- package/dist/cjs/cursors/findCursor.js +6 -4
- package/dist/cjs/index.js +108 -1
- package/dist/cjs/mongo-client.js +18 -1
- package/dist/cjs/server.js +164 -0
- package/dist/cjs/sync/commands.js +126 -0
- package/dist/cjs/sync/diff.js +174 -0
- package/dist/cjs/sync/httpSink.js +159 -0
- package/dist/cjs/sync/index.js +40 -0
- package/dist/cjs/sync/mongoExecutor.js +306 -0
- package/dist/cjs/sync/mongoSink.js +161 -0
- package/dist/cjs/sync/outbox.js +358 -0
- package/dist/cjs/sync/protocol.js +177 -0
- package/dist/cjs/sync/replicator.js +747 -0
- package/dist/cjs/sync/shadow.js +178 -0
- package/dist/cjs/sync/types.js +2 -0
- package/dist/cloudflare.d.ts +1 -1
- package/dist/cloudflare.js.map +1 -1
- package/dist/collection.js +9 -6
- package/dist/collection.js.map +1 -1
- package/dist/cursors/findCursor.d.ts +2 -1
- package/dist/cursors/findCursor.js +6 -4
- package/dist/cursors/findCursor.js.map +1 -1
- package/dist/index.d.ts +68 -2
- package/dist/index.js +96 -1
- package/dist/index.js.map +1 -1
- package/dist/mongo-client.d.ts +13 -0
- package/dist/mongo-client.js +18 -1
- package/dist/mongo-client.js.map +1 -1
- package/dist/server.d.ts +97 -0
- package/dist/server.js +158 -0
- package/dist/server.js.map +1 -0
- package/dist/sync/commands.d.ts +49 -0
- package/dist/sync/commands.js +118 -0
- package/dist/sync/commands.js.map +1 -0
- package/dist/sync/diff.d.ts +55 -0
- package/dist/sync/diff.js +169 -0
- package/dist/sync/diff.js.map +1 -0
- package/dist/sync/httpSink.d.ts +93 -0
- package/dist/sync/httpSink.js +155 -0
- package/dist/sync/httpSink.js.map +1 -0
- package/dist/sync/index.d.ts +17 -0
- package/dist/sync/index.js +10 -0
- package/dist/sync/index.js.map +1 -0
- package/dist/sync/mongoExecutor.d.ts +37 -0
- package/dist/sync/mongoExecutor.js +302 -0
- package/dist/sync/mongoExecutor.js.map +1 -0
- package/dist/sync/mongoSink.d.ts +109 -0
- package/dist/sync/mongoSink.js +123 -0
- package/dist/sync/mongoSink.js.map +1 -0
- package/dist/sync/outbox.d.ts +120 -0
- package/dist/sync/outbox.js +355 -0
- package/dist/sync/outbox.js.map +1 -0
- package/dist/sync/protocol.d.ts +93 -0
- package/dist/sync/protocol.js +168 -0
- package/dist/sync/protocol.js.map +1 -0
- package/dist/sync/replicator.d.ts +177 -0
- package/dist/sync/replicator.js +744 -0
- package/dist/sync/replicator.js.map +1 -0
- package/dist/sync/shadow.d.ts +54 -0
- package/dist/sync/shadow.js +173 -0
- package/dist/sync/shadow.js.map +1 -0
- package/dist/sync/types.d.ts +388 -0
- package/dist/sync/types.js +2 -0
- package/dist/sync/types.js.map +1 -0
- package/package.json +16 -2
package/README.md
CHANGED
|
@@ -12,6 +12,7 @@ A MongoDB-like client backed by SQLite. Use a familiar MongoDB API with the simp
|
|
|
12
12
|
- You want a MongoDB-style API without running a MongoDB server
|
|
13
13
|
- You need a lightweight, embedded database for local apps, CLIs, or testing
|
|
14
14
|
- You want simple file-based persistence with zero infrastructure overhead
|
|
15
|
+
- You want a persistent, high-performance local cache in front of a remote MongoDB, one that survives restarts and network interruptions, without giving up Mongo's query semantics (including things that are natural in a document model but awkward in SQL, like layered/override queries)
|
|
15
16
|
|
|
16
17
|
## Features
|
|
17
18
|
|
|
@@ -24,6 +25,8 @@ A MongoDB-like client backed by SQLite. Use a familiar MongoDB API with the simp
|
|
|
24
25
|
- **Update operators** — `$set`, `$inc`, `$push`, `$pull`, `$addToSet`, `$mul`, and more
|
|
25
26
|
- **Indexing** — create, list, and drop indexes including unique and compound indexes
|
|
26
27
|
- **Change streams** — real-time change tracking via `collection.watch()`
|
|
28
|
+
- **MongoDB sync** — replicate local writes upstream, surviving restarts and offline periods
|
|
29
|
+
- **HTTP sync** — post the same changes to a remote API when MongoDB is unreachable
|
|
27
30
|
- **JSON safety** — validates documents before insert, and a corrupted row can't break queries for the rest of the collection
|
|
28
31
|
- **TypeScript** — fully typed with strict mode
|
|
29
32
|
|
|
@@ -83,28 +86,29 @@ raw SQL are a menu item away when you want them. See [the CLI docs](./docs/CLI.m
|
|
|
83
86
|
|
|
84
87
|
## How does it compare?
|
|
85
88
|
|
|
86
|
-
|
|
|
87
|
-
|
|
88
|
-
| MongoDB query API (`$set`, `$elemMatch`, aggregation...) | ✅
|
|
89
|
-
| Runs in the browser
|
|
90
|
-
| Runs on the edge (Cloudflare Durable Objects)
|
|
91
|
-
| Zero infrastructure (no server process)
|
|
92
|
-
| TypeScript, strict mode
|
|
93
|
-
| Actively maintained
|
|
89
|
+
| | MongoLite | lowdb | better-sqlite3 (raw) | NeDB | PouchDB | MongoDB |
|
|
90
|
+
| -------------------------------------------------------- | ----------- | ------------------------ | -------------------- | -------------------- | ------------------------ | ------- |
|
|
91
|
+
| MongoDB query API (`$set`, `$elemMatch`, aggregation...) | ✅ | ❌ (plain object access) | ❌ (raw SQL) | ✅ | ❌ (Mango/CouchDB-style) | ✅ |
|
|
92
|
+
| Runs in the browser | ✅ (sql.js) | ✅ | ❌ (native binding) | ✅ | ✅ (IndexedDB) | ❌ |
|
|
93
|
+
| Runs on the edge (Cloudflare Durable Objects) | ✅ | ❌ | ❌ (native binding) | ❌ | ❌ | ❌ |
|
|
94
|
+
| Zero infrastructure (no server process) | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ |
|
|
95
|
+
| TypeScript, strict mode | ✅ | ✅ | ✅ | ⚠️ (community types) | ⚠️ (community types) | ✅ |
|
|
96
|
+
| Actively maintained | ✅ | ✅ | ✅ | ❌ (unmaintained) | ⚠️ (slow-moving) | ✅ |
|
|
94
97
|
|
|
95
98
|
MongoLite's niche: the MongoDB query API you already know, running anywhere SQLite runs — a local file, an in-memory test database, the browser, or a Cloudflare Durable Object — without standing up a MongoDB server.
|
|
96
99
|
|
|
97
100
|
## Documentation
|
|
98
101
|
|
|
99
|
-
| Topic
|
|
100
|
-
|
|
101
|
-
| [API Reference](./docs/API.md)
|
|
102
|
-
| [Change Streams](./docs/CHANGE_STREAMS.md)
|
|
103
|
-
| [
|
|
104
|
-
| [
|
|
105
|
-
| [
|
|
106
|
-
| [
|
|
107
|
-
| [
|
|
102
|
+
| Topic | Description |
|
|
103
|
+
| -------------------------------------------------- | ---------------------------------------------------------------------- |
|
|
104
|
+
| [API Reference](./docs/API.md) | Full API docs: methods, query operators, update operators |
|
|
105
|
+
| [Change Streams](./docs/CHANGE_STREAMS.md) | Real-time change tracking with `collection.watch()` |
|
|
106
|
+
| [Syncing to MongoDB](./docs/SYNC.md) | Replication to an upstream MongoDB, directly or through an HTTP API |
|
|
107
|
+
| [JSON Safety](./docs/JSON_SAFETY.md) | Document validation and corrupted data recovery |
|
|
108
|
+
| [CLI](./docs/CLI.md) | `npx mongolite` — guided explorer for browsing and querying a database |
|
|
109
|
+
| [Query Debugger](./docs/DEBUGGER.md) | The older command-driven REPL (`npx mongolite --repl`) |
|
|
110
|
+
| [Benchmarks](./docs/BENCHMARKS.md) | Performance benchmarks and storage characteristics |
|
|
111
|
+
| [Cloudflare Durable Objects](./docs/CLOUDFLARE.md) | Using MongoLite inside a Cloudflare Durable Object |
|
|
108
112
|
|
|
109
113
|
## Backend Examples
|
|
110
114
|
|
|
@@ -125,9 +129,9 @@ await client.close();
|
|
|
125
129
|
```typescript
|
|
126
130
|
const client = new MongoLite({
|
|
127
131
|
filePath: './myapp.sqlite',
|
|
128
|
-
WAL: true,
|
|
129
|
-
busyTimeout: 5000,
|
|
130
|
-
shared: true,
|
|
132
|
+
WAL: true, // Write-Ahead Logging. Default: true
|
|
133
|
+
busyTimeout: 5000, // ms to wait for a lock before SQLITE_BUSY. Default: 5000
|
|
134
|
+
shared: true, // reuse one connection per file in this process. Default: false
|
|
131
135
|
readOnly: false,
|
|
132
136
|
verbose: false,
|
|
133
137
|
});
|
|
@@ -149,7 +153,7 @@ await a.close(); // b keeps working — the handle is reference counted
|
|
|
149
153
|
await b.close(); // last holder out actually closes it
|
|
150
154
|
```
|
|
151
155
|
|
|
152
|
-
Connections are only shared when the resolved path
|
|
156
|
+
Connections are only shared when the resolved path _and_ every setting that
|
|
153
157
|
changes the handle's behaviour match. `:memory:` databases are never shared,
|
|
154
158
|
since two `:memory:` opens are two unrelated databases.
|
|
155
159
|
|
|
@@ -193,6 +197,7 @@ await client.close();
|
|
|
193
197
|
```
|
|
194
198
|
|
|
195
199
|
<a id="native-nodesqlite-vs-better-sqlite3"></a>
|
|
200
|
+
|
|
196
201
|
> **Which one should I use?** `node:sqlite` (the default) is still marked
|
|
197
202
|
> experimental upstream but requires no native build step, which is why it's
|
|
198
203
|
> the default here. `better-sqlite3` is the older, more battle-tested native
|
|
@@ -248,6 +253,60 @@ export class MyDurableObject extends DurableObject {
|
|
|
248
253
|
|
|
249
254
|
> See [docs/CLOUDFLARE.md](./docs/CLOUDFLARE.md) for the full guide, supported operations, and limitations.
|
|
250
255
|
|
|
256
|
+
## Syncing to MongoDB
|
|
257
|
+
|
|
258
|
+
Point MongoLite at an upstream MongoDB deployment and every insert, update and
|
|
259
|
+
delete is replicated to it. The local database becomes a local-first write layer
|
|
260
|
+
that converges upstream — through restarts, network outages and offline periods.
|
|
261
|
+
|
|
262
|
+
Requires the optional `mongodb` peer dependency (`npm install mongodb`).
|
|
263
|
+
|
|
264
|
+
```typescript
|
|
265
|
+
const client = new MongoLite('./app.sqlite');
|
|
266
|
+
await client.connect();
|
|
267
|
+
|
|
268
|
+
const sync = client.syncToMongo({
|
|
269
|
+
connectionString: 'mongodb+srv://user:pass@cluster.example.com/app',
|
|
270
|
+
collections: ['users', 'orders'], // Omit to replicate everything
|
|
271
|
+
});
|
|
272
|
+
|
|
273
|
+
await sync.start();
|
|
274
|
+
|
|
275
|
+
// Ordinary writes — replication happens in the background.
|
|
276
|
+
await client.collection('users').insertOne({ name: 'Alice', age: 30 });
|
|
277
|
+
|
|
278
|
+
await sync.stop({ flush: true }); // Push anything still queued on shutdown
|
|
279
|
+
```
|
|
280
|
+
|
|
281
|
+
Changes are captured by SQLite triggers into a durable outbox **inside the same
|
|
282
|
+
transaction as the write itself**, so a change is either committed locally _and_
|
|
283
|
+
queued for replication, or neither. A background worker coalesces the queue, applies
|
|
284
|
+
it upstream, and only then advances a persisted checkpoint — so an outage parks the
|
|
285
|
+
backlog rather than losing it, and a restart resumes exactly where it stopped.
|
|
286
|
+
|
|
287
|
+
Upstream is treated as the source of truth. Each push is a conditional write against
|
|
288
|
+
the version it last saw, and carries only the fields that actually changed, so a
|
|
289
|
+
concurrent edit by another writer is detected rather than silently overwritten.
|
|
290
|
+
|
|
291
|
+
Where the local instance cannot reach MongoDB directly, `syncToHttp` posts the same
|
|
292
|
+
operations to a remote API instead, and `@semics-tech/mongolite/server` applies them on
|
|
293
|
+
the far side:
|
|
294
|
+
|
|
295
|
+
```typescript
|
|
296
|
+
const sync = client.syncToHttp({
|
|
297
|
+
baseUrl: 'https://api.example.com',
|
|
298
|
+
database: 'app',
|
|
299
|
+
getAuthHeaders: async () => ({ Authorization: `Bearer ${await getAccessToken()}` }),
|
|
300
|
+
});
|
|
301
|
+
```
|
|
302
|
+
|
|
303
|
+
The version predicates travel with each operation and conflicts come back in the
|
|
304
|
+
response, so the HTTP hop is pure transport — replication behaves the same either way.
|
|
305
|
+
|
|
306
|
+
> See [docs/SYNC.md](./docs/SYNC.md) for the full guide: syncing through an HTTP API,
|
|
307
|
+
> advanced authentication (X.509, mTLS, AWS IAM, Azure managed identity), filtering and
|
|
308
|
+
> reshaping documents, dead-letter handling, multiple upstreams, and custom sinks.
|
|
309
|
+
|
|
251
310
|
## Development
|
|
252
311
|
|
|
253
312
|
```bash
|
package/dist/cjs/collection.js
CHANGED
|
@@ -1099,11 +1099,7 @@ class MongoLiteCollection {
|
|
|
1099
1099
|
*/
|
|
1100
1100
|
async countDocuments(filter = {}) {
|
|
1101
1101
|
await this.ensureTable();
|
|
1102
|
-
|
|
1103
|
-
const whereClause = new findCursor_js_1.FindCursor(this.db, this.name, filter)['buildGuardedWhereClause'](filter, paramsForCount);
|
|
1104
|
-
const countSql = `SELECT COUNT(*) as count FROM "${this.name}" WHERE ${whereClause}`;
|
|
1105
|
-
const result = await this.db.get(countSql, paramsForCount);
|
|
1106
|
-
return result?.count || 0;
|
|
1102
|
+
return this.find(filter).count();
|
|
1107
1103
|
}
|
|
1108
1104
|
/**
|
|
1109
1105
|
* Finds rows whose stored `data` column is not valid JSON.
|
|
@@ -1285,7 +1281,14 @@ class MongoLiteCollection {
|
|
|
1285
1281
|
if (!upsert) {
|
|
1286
1282
|
return { acknowledged: true, matchedCount: 0, modifiedCount: 0, upsertedId: null };
|
|
1287
1283
|
}
|
|
1288
|
-
|
|
1284
|
+
// An upsert adopts the `_id` the filter pinned down, matching MongoDB. Without
|
|
1285
|
+
// this the inserted document gets a freshly generated id and the caller's
|
|
1286
|
+
// `replaceOne({ _id }, ..., { upsert: true })` silently creates the wrong row.
|
|
1287
|
+
const filterId = filter._id;
|
|
1288
|
+
const seeded = typeof filterId === 'string'
|
|
1289
|
+
? { ...replacement, _id: filterId }
|
|
1290
|
+
: replacement;
|
|
1291
|
+
const result = await this.insertOne(seeded);
|
|
1289
1292
|
return {
|
|
1290
1293
|
acknowledged: true,
|
|
1291
1294
|
matchedCount: 0,
|
|
@@ -123,15 +123,16 @@ function restoreDates(obj) {
|
|
|
123
123
|
* Represents a cursor for find operations, allowing chaining of limit, skip, and sort.
|
|
124
124
|
*/
|
|
125
125
|
class FindCursor {
|
|
126
|
-
constructor(db, collectionName,
|
|
126
|
+
constructor(db, collectionName, filter, options = {}) {
|
|
127
127
|
this.db = db;
|
|
128
128
|
this.collectionName = collectionName;
|
|
129
|
+
this.filter = filter;
|
|
129
130
|
this.options = options;
|
|
130
131
|
this.limitCount = null;
|
|
131
132
|
this.skipCount = null;
|
|
132
133
|
this.sortCriteria = null;
|
|
133
134
|
this.projectionFields = null;
|
|
134
|
-
this.queryParts = this.buildSelectQuery(
|
|
135
|
+
this.queryParts = this.buildSelectQuery(filter);
|
|
135
136
|
}
|
|
136
137
|
parseJsonPath(path) {
|
|
137
138
|
return `'$.${path.replace(/\./g, '.')}'`;
|
|
@@ -840,9 +841,10 @@ class FindCursor {
|
|
|
840
841
|
* @returns A promise that resolves to the count of matching documents.
|
|
841
842
|
*/
|
|
842
843
|
async count() {
|
|
843
|
-
const
|
|
844
|
+
const params = [];
|
|
845
|
+
const whereClause = this.buildGuardedWhereClause(this.filter, params);
|
|
844
846
|
const countSql = `SELECT COUNT(*) as count FROM "${this.collectionName}" WHERE ${whereClause}`;
|
|
845
|
-
const result = await this.db.get(countSql);
|
|
847
|
+
const result = await this.db.get(countSql, params);
|
|
846
848
|
return result?.count ?? 0; // Return 0 if result is undefined
|
|
847
849
|
}
|
|
848
850
|
}
|
package/dist/cjs/index.js
CHANGED
|
@@ -14,7 +14,7 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
|
|
14
14
|
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
|
15
15
|
};
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
|
-
exports.MongoLite = exports.NodeSqliteAdapter = exports.BrowserSqliteAdapter = exports.CloudflareDurableObjectAdapter = exports.ChangeStream = exports.MongoLiteCollection = void 0;
|
|
17
|
+
exports.MongoLite = exports.projectToLocalShape = exports.isEmptyDiff = exports.applyDiff = exports.computeDiff = exports.buildWriteCommand = exports.SYNC_PROTOCOL_VERSION = exports.SyncRequestRejected = exports.HttpUpstreamSink = exports.MongoUpstreamSink = exports.SyncShadow = exports.SyncOutbox = exports.SyncReplicator = exports.NodeSqliteAdapter = exports.BrowserSqliteAdapter = exports.CloudflareDurableObjectAdapter = exports.ChangeStream = exports.MongoLiteCollection = void 0;
|
|
18
18
|
const node_sqlite_js_1 = require("./adapters/node-sqlite.js");
|
|
19
19
|
const mongo_client_js_1 = require("./mongo-client.js");
|
|
20
20
|
var collection_js_1 = require("./collection.js");
|
|
@@ -28,6 +28,20 @@ var browser_js_1 = require("./adapters/browser.js");
|
|
|
28
28
|
Object.defineProperty(exports, "BrowserSqliteAdapter", { enumerable: true, get: function () { return browser_js_1.BrowserSqliteAdapter; } });
|
|
29
29
|
var node_sqlite_js_2 = require("./adapters/node-sqlite.js");
|
|
30
30
|
Object.defineProperty(exports, "NodeSqliteAdapter", { enumerable: true, get: function () { return node_sqlite_js_2.NodeSqliteAdapter; } });
|
|
31
|
+
const index_js_1 = require("./sync/index.js");
|
|
32
|
+
var index_js_2 = require("./sync/index.js");
|
|
33
|
+
Object.defineProperty(exports, "SyncReplicator", { enumerable: true, get: function () { return index_js_2.SyncReplicator; } });
|
|
34
|
+
Object.defineProperty(exports, "SyncOutbox", { enumerable: true, get: function () { return index_js_2.SyncOutbox; } });
|
|
35
|
+
Object.defineProperty(exports, "SyncShadow", { enumerable: true, get: function () { return index_js_2.SyncShadow; } });
|
|
36
|
+
Object.defineProperty(exports, "MongoUpstreamSink", { enumerable: true, get: function () { return index_js_2.MongoUpstreamSink; } });
|
|
37
|
+
Object.defineProperty(exports, "HttpUpstreamSink", { enumerable: true, get: function () { return index_js_2.HttpUpstreamSink; } });
|
|
38
|
+
Object.defineProperty(exports, "SyncRequestRejected", { enumerable: true, get: function () { return index_js_2.SyncRequestRejected; } });
|
|
39
|
+
Object.defineProperty(exports, "SYNC_PROTOCOL_VERSION", { enumerable: true, get: function () { return index_js_2.SYNC_PROTOCOL_VERSION; } });
|
|
40
|
+
Object.defineProperty(exports, "buildWriteCommand", { enumerable: true, get: function () { return index_js_2.buildWriteCommand; } });
|
|
41
|
+
Object.defineProperty(exports, "computeDiff", { enumerable: true, get: function () { return index_js_2.computeDiff; } });
|
|
42
|
+
Object.defineProperty(exports, "applyDiff", { enumerable: true, get: function () { return index_js_2.applyDiff; } });
|
|
43
|
+
Object.defineProperty(exports, "isEmptyDiff", { enumerable: true, get: function () { return index_js_2.isEmptyDiff; } });
|
|
44
|
+
Object.defineProperty(exports, "projectToLocalShape", { enumerable: true, get: function () { return index_js_2.projectToLocalShape; } });
|
|
31
45
|
/**
|
|
32
46
|
* MongoLite class is the main entry point for interacting with the SQLite-backed database.
|
|
33
47
|
*
|
|
@@ -61,5 +75,98 @@ class MongoLite extends mongo_client_js_1.MongoLite {
|
|
|
61
75
|
super(new node_sqlite_js_1.NodeSqliteAdapter(dbPathOrOptions), options);
|
|
62
76
|
}
|
|
63
77
|
}
|
|
78
|
+
/**
|
|
79
|
+
* Replicates every insert, update and delete in this database to an upstream
|
|
80
|
+
* MongoDB deployment.
|
|
81
|
+
*
|
|
82
|
+
* Changes are captured by SQLite triggers into a durable outbox inside this
|
|
83
|
+
* database, in the same transaction as the write itself, then streamed
|
|
84
|
+
* upstream in batches. Replication survives process restarts and upstream
|
|
85
|
+
* outages: the outbox holds the backlog and a checkpoint records exactly how
|
|
86
|
+
* far it got.
|
|
87
|
+
*
|
|
88
|
+
* Requires the optional `mongodb` peer dependency.
|
|
89
|
+
*
|
|
90
|
+
* @example
|
|
91
|
+
* ```typescript
|
|
92
|
+
* const sync = client.syncToMongo({
|
|
93
|
+
* connectionString: 'mongodb+srv://user:pass@cluster.example.com/app',
|
|
94
|
+
* collections: ['users', 'orders'],
|
|
95
|
+
* });
|
|
96
|
+
*
|
|
97
|
+
* sync.on('error', (err) => console.error('replication degraded:', err));
|
|
98
|
+
* await sync.start();
|
|
99
|
+
*
|
|
100
|
+
* // …later, on shutdown
|
|
101
|
+
* await sync.stop({ flush: true });
|
|
102
|
+
* ```
|
|
103
|
+
*/
|
|
104
|
+
syncToMongo(options) {
|
|
105
|
+
const { connectionString, database, writeConcern, driverOptions, driver, authSource, authMechanism, authMechanismProperties, username, password, tls, tlsCAFile, tlsCertificateKeyFile, tlsCertificateKeyFilePassword, tlsCRLFile, tlsAllowInvalidHostnames, tlsAllowInvalidCertificates, ...syncOptions } = options;
|
|
106
|
+
const sink = new index_js_1.MongoUpstreamSink({
|
|
107
|
+
connectionString,
|
|
108
|
+
database,
|
|
109
|
+
writeConcern,
|
|
110
|
+
driverOptions,
|
|
111
|
+
driver,
|
|
112
|
+
authSource,
|
|
113
|
+
authMechanism,
|
|
114
|
+
authMechanismProperties,
|
|
115
|
+
username,
|
|
116
|
+
password,
|
|
117
|
+
tls,
|
|
118
|
+
tlsCAFile,
|
|
119
|
+
tlsCertificateKeyFile,
|
|
120
|
+
tlsCertificateKeyFilePassword,
|
|
121
|
+
tlsCRLFile,
|
|
122
|
+
tlsAllowInvalidHostnames,
|
|
123
|
+
tlsAllowInvalidCertificates,
|
|
124
|
+
idMapping: syncOptions.idMapping,
|
|
125
|
+
// The sink builds the conditional writes, so it needs the same answer the
|
|
126
|
+
// replicator uses when deciding whether to track shadows at all.
|
|
127
|
+
versioning: syncOptions.versioning,
|
|
128
|
+
});
|
|
129
|
+
return this.createSync(sink, syncOptions);
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Replicates changes to a remote HTTP API, which applies them to MongoDB on this
|
|
133
|
+
* client's behalf. For deployments where the local instance cannot reach the database
|
|
134
|
+
* directly.
|
|
135
|
+
*
|
|
136
|
+
* The wire format carries MongoDB-shaped commands along with the version predicates,
|
|
137
|
+
* so replication behaves the same as over a direct connection — the upstream stays the
|
|
138
|
+
* source of truth and concurrent writers are still detected. Mount
|
|
139
|
+
* `@semics-tech/mongolite/server` on the API to receive the messages.
|
|
140
|
+
*
|
|
141
|
+
* @example
|
|
142
|
+
* ```typescript
|
|
143
|
+
* const sync = client.syncToHttp({
|
|
144
|
+
* baseUrl: 'https://api.example.com',
|
|
145
|
+
* database: 'app',
|
|
146
|
+
* headers: { 'luna-environment': 'prod' },
|
|
147
|
+
* getAuthHeaders: async () => ({ Authorization: `Bearer ${await getAccessToken()}` }),
|
|
148
|
+
* });
|
|
149
|
+
*
|
|
150
|
+
* await sync.start();
|
|
151
|
+
* ```
|
|
152
|
+
*/
|
|
153
|
+
syncToHttp(options) {
|
|
154
|
+
const { baseUrl, database, pathPrefix, getAuthHeaders, headers, cookies, fetch: fetchImpl, timeoutMs, includeCommands, ...syncOptions } = options;
|
|
155
|
+
const sink = new index_js_1.HttpUpstreamSink({
|
|
156
|
+
baseUrl,
|
|
157
|
+
database,
|
|
158
|
+
pathPrefix,
|
|
159
|
+
getAuthHeaders,
|
|
160
|
+
headers,
|
|
161
|
+
cookies,
|
|
162
|
+
fetch: fetchImpl,
|
|
163
|
+
timeoutMs,
|
|
164
|
+
includeCommands,
|
|
165
|
+
idMapping: syncOptions.idMapping,
|
|
166
|
+
versioning: syncOptions.versioning,
|
|
167
|
+
replicator: syncOptions.name,
|
|
168
|
+
});
|
|
169
|
+
return this.createSync(sink, syncOptions);
|
|
170
|
+
}
|
|
64
171
|
}
|
|
65
172
|
exports.MongoLite = MongoLite;
|
package/dist/cjs/mongo-client.js
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
3
|
exports.MongoLite = void 0;
|
|
4
4
|
const collection_js_1 = require("./collection.js");
|
|
5
|
+
const replicator_js_1 = require("./sync/replicator.js");
|
|
5
6
|
/**
|
|
6
7
|
* Core MongoLite client. Accepts any `IDatabaseAdapter` implementation.
|
|
7
8
|
*
|
|
@@ -42,11 +43,27 @@ class MongoLite {
|
|
|
42
43
|
listCollections() {
|
|
43
44
|
return {
|
|
44
45
|
toArray: async () => {
|
|
45
|
-
|
|
46
|
+
// `__mongolite*` tables are internal bookkeeping (change log, sync
|
|
47
|
+
// outbox and checkpoints), not user collections.
|
|
48
|
+
const result = await this.db.all(`SELECT name FROM sqlite_master
|
|
49
|
+
WHERE type='table' AND name NOT LIKE 'sqlite_%' AND name NOT LIKE '__mongolite%'`);
|
|
46
50
|
return result.map((row) => row.name);
|
|
47
51
|
},
|
|
48
52
|
};
|
|
49
53
|
}
|
|
54
|
+
/**
|
|
55
|
+
* Starts replicating this database's changes to an upstream {@link SyncSink}.
|
|
56
|
+
*
|
|
57
|
+
* Node.js users replicating to MongoDB should prefer the `syncToMongo`
|
|
58
|
+
* overload on the package-root `MongoLite`, which takes a connection string
|
|
59
|
+
* directly. This method is the backend-agnostic form — pass any sink to
|
|
60
|
+
* replicate over HTTP, into a queue, or to a test double.
|
|
61
|
+
*
|
|
62
|
+
* The returned replicator is **not** started; call `start()` on it.
|
|
63
|
+
*/
|
|
64
|
+
createSync(sink, options = {}) {
|
|
65
|
+
return new replicator_js_1.SyncReplicator(this.db, sink, { verbose: this.options.verbose, ...options });
|
|
66
|
+
}
|
|
50
67
|
/**
|
|
51
68
|
* Returns a typed collection handle.
|
|
52
69
|
* @param name The collection (table) name.
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.SyncReceiver = exports.SyncProtocolError = exports.SYNC_PROTOCOL_VERSION = void 0;
|
|
4
|
+
exports.createSyncReceiver = createSyncReceiver;
|
|
5
|
+
/**
|
|
6
|
+
* Server-side entry point: receives sync messages from a MongoLite client and applies
|
|
7
|
+
* them to MongoDB.
|
|
8
|
+
*
|
|
9
|
+
* Install this on the API that sits in front of your database, for deployments where
|
|
10
|
+
* the local instance cannot reach MongoDB directly.
|
|
11
|
+
*
|
|
12
|
+
* ```ts
|
|
13
|
+
* import { MongoClient } from 'mongodb';
|
|
14
|
+
* import { createSyncReceiver } from '@semics-tech/mongolite/server';
|
|
15
|
+
*
|
|
16
|
+
* const mongo = await new MongoClient(process.env.MONGO_URL!).connect();
|
|
17
|
+
* const receiver = createSyncReceiver({
|
|
18
|
+
* client: mongo,
|
|
19
|
+
* database: 'app',
|
|
20
|
+
* allowedCollections: ['users', 'orders'],
|
|
21
|
+
* });
|
|
22
|
+
*
|
|
23
|
+
* app.post('/sync/app/_sync', async (req, res) => {
|
|
24
|
+
* const result = await receiver.apply(req.body);
|
|
25
|
+
* res.status(result.status).type('application/json').send(result.body);
|
|
26
|
+
* });
|
|
27
|
+
* ```
|
|
28
|
+
*
|
|
29
|
+
* This module imports nothing SQLite-related, so it is safe to install on a server that
|
|
30
|
+
* has no local database of its own.
|
|
31
|
+
*/
|
|
32
|
+
const commands_js_1 = require("./sync/commands.js");
|
|
33
|
+
const mongoExecutor_js_1 = require("./sync/mongoExecutor.js");
|
|
34
|
+
const protocol_js_1 = require("./sync/protocol.js");
|
|
35
|
+
var protocol_js_2 = require("./sync/protocol.js");
|
|
36
|
+
Object.defineProperty(exports, "SYNC_PROTOCOL_VERSION", { enumerable: true, get: function () { return protocol_js_2.SYNC_PROTOCOL_VERSION; } });
|
|
37
|
+
Object.defineProperty(exports, "SyncProtocolError", { enumerable: true, get: function () { return protocol_js_2.SyncProtocolError; } });
|
|
38
|
+
const JSON_HEADERS = { 'content-type': 'application/json' };
|
|
39
|
+
class SyncReceiver {
|
|
40
|
+
constructor(options) {
|
|
41
|
+
this.options = options;
|
|
42
|
+
if (!options.client)
|
|
43
|
+
throw new Error('createSyncReceiver requires a connected `client`.');
|
|
44
|
+
this.db = options.client.db(options.database);
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* Applies a batch of operations.
|
|
48
|
+
*
|
|
49
|
+
* @param body The raw request body — a string, or an already-parsed object.
|
|
50
|
+
* @returns Status and Extended JSON body to send back.
|
|
51
|
+
*/
|
|
52
|
+
async apply(body) {
|
|
53
|
+
try {
|
|
54
|
+
const decoded = (0, protocol_js_1.decodeBody)(body);
|
|
55
|
+
const { replicator, operations } = (0, protocol_js_1.parseApplyRequest)(decoded);
|
|
56
|
+
const maxOperations = this.options.maxOperations ?? 1000;
|
|
57
|
+
if (operations.length > maxOperations) {
|
|
58
|
+
throw new protocol_js_1.SyncProtocolError(`Batch of ${operations.length} exceeds the limit of ${maxOperations}`, 413);
|
|
59
|
+
}
|
|
60
|
+
const collections = [...new Set(operations.map((op) => op.collection))];
|
|
61
|
+
this.assertCollectionsAllowed(collections);
|
|
62
|
+
await this.authorise({ replicator, collections, operationCount: operations.length });
|
|
63
|
+
const writes = operations.map((op, index) => ({
|
|
64
|
+
index,
|
|
65
|
+
op,
|
|
66
|
+
command: this.commandFor(op, decoded, index),
|
|
67
|
+
}));
|
|
68
|
+
const result = await (0, mongoExecutor_js_1.executeWrites)(this.db, writes, {
|
|
69
|
+
writeConcern: this.options.writeConcern ?? { w: 'majority' },
|
|
70
|
+
versioning: this.options.versioning ?? true,
|
|
71
|
+
idMapping: this.options.idMapping,
|
|
72
|
+
ObjectId: this.options.ObjectId,
|
|
73
|
+
});
|
|
74
|
+
return this.ok({
|
|
75
|
+
protocol: protocol_js_1.SYNC_PROTOCOL_VERSION,
|
|
76
|
+
applied: result.applied,
|
|
77
|
+
failures: result.failures,
|
|
78
|
+
conflicts: result.conflicts,
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
catch (err) {
|
|
82
|
+
return this.error(err);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
/** Reads documents back by `_id`, so a client can refresh its shadow after a conflict. */
|
|
86
|
+
async fetch(body) {
|
|
87
|
+
try {
|
|
88
|
+
const { collection, documentIds } = (0, protocol_js_1.parseFetchRequest)((0, protocol_js_1.decodeBody)(body));
|
|
89
|
+
this.assertCollectionsAllowed([collection]);
|
|
90
|
+
await this.authorise({
|
|
91
|
+
replicator: 'fetch',
|
|
92
|
+
collections: [collection],
|
|
93
|
+
operationCount: documentIds.length,
|
|
94
|
+
});
|
|
95
|
+
const documents = await (0, mongoExecutor_js_1.fetchDocuments)(this.db, collection, documentIds, this.options.idMapping, this.options.ObjectId);
|
|
96
|
+
return this.ok({ protocol: protocol_js_1.SYNC_PROTOCOL_VERSION, documents });
|
|
97
|
+
}
|
|
98
|
+
catch (err) {
|
|
99
|
+
return this.error(err);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
// ---------------------------------------------------------------- internals
|
|
103
|
+
/**
|
|
104
|
+
* Produces the command to execute for an operation.
|
|
105
|
+
*
|
|
106
|
+
* By default this **rebuilds** the command from the operation's own fields rather than
|
|
107
|
+
* using whatever the client sent. For a well-behaved client the two are identical;
|
|
108
|
+
* for a hostile one the difference is the whole security boundary.
|
|
109
|
+
*/
|
|
110
|
+
commandFor(op, decoded, index) {
|
|
111
|
+
if (this.options.trustClientCommands) {
|
|
112
|
+
const wire = decoded.operations?.[index];
|
|
113
|
+
const command = wire?.command;
|
|
114
|
+
if (command && typeof command === 'object')
|
|
115
|
+
return command;
|
|
116
|
+
}
|
|
117
|
+
return (0, commands_js_1.buildWriteCommand)(op, {
|
|
118
|
+
idMapping: this.options.idMapping,
|
|
119
|
+
versioning: this.options.versioning ?? true,
|
|
120
|
+
ObjectId: this.options.ObjectId,
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
assertCollectionsAllowed(collections) {
|
|
124
|
+
const allowed = this.options.allowedCollections;
|
|
125
|
+
if (!allowed)
|
|
126
|
+
return;
|
|
127
|
+
const permitted = new Set(allowed);
|
|
128
|
+
for (const collection of collections) {
|
|
129
|
+
if (!permitted.has(collection)) {
|
|
130
|
+
throw new protocol_js_1.SyncProtocolError(`Collection "${collection}" is not permitted`, 403);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
async authorise(context) {
|
|
135
|
+
if (!this.options.verifyRequest)
|
|
136
|
+
return;
|
|
137
|
+
const verdict = await this.options.verifyRequest(context);
|
|
138
|
+
if (verdict === false) {
|
|
139
|
+
throw new protocol_js_1.SyncProtocolError('Request rejected', 403);
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
ok(payload) {
|
|
143
|
+
return { status: 200, body: (0, protocol_js_1.encodeBody)(payload), headers: JSON_HEADERS };
|
|
144
|
+
}
|
|
145
|
+
error(err) {
|
|
146
|
+
const status = err instanceof protocol_js_1.SyncProtocolError ? err.status : 500;
|
|
147
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
148
|
+
return {
|
|
149
|
+
status,
|
|
150
|
+
// 5xx bodies stay vague: an unexpected error can carry connection strings or
|
|
151
|
+
// document contents, and this response goes to a client.
|
|
152
|
+
body: (0, protocol_js_1.encodeBody)({
|
|
153
|
+
protocol: protocol_js_1.SYNC_PROTOCOL_VERSION,
|
|
154
|
+
error: status >= 500 ? 'Internal error applying sync batch' : message,
|
|
155
|
+
}),
|
|
156
|
+
headers: JSON_HEADERS,
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
exports.SyncReceiver = SyncReceiver;
|
|
161
|
+
/** Creates a receiver for sync messages posted by a MongoLite client. */
|
|
162
|
+
function createSyncReceiver(options) {
|
|
163
|
+
return new SyncReceiver(options);
|
|
164
|
+
}
|