@semics-tech/mongolite 1.3.0 → 1.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. package/README.md +80 -21
  2. package/dist/cjs/collection.js +9 -6
  3. package/dist/cjs/cursors/findCursor.js +6 -4
  4. package/dist/cjs/index.js +108 -1
  5. package/dist/cjs/mongo-client.js +18 -1
  6. package/dist/cjs/server.js +164 -0
  7. package/dist/cjs/sync/commands.js +126 -0
  8. package/dist/cjs/sync/diff.js +174 -0
  9. package/dist/cjs/sync/httpSink.js +159 -0
  10. package/dist/cjs/sync/index.js +40 -0
  11. package/dist/cjs/sync/mongoExecutor.js +306 -0
  12. package/dist/cjs/sync/mongoSink.js +161 -0
  13. package/dist/cjs/sync/outbox.js +358 -0
  14. package/dist/cjs/sync/protocol.js +177 -0
  15. package/dist/cjs/sync/replicator.js +747 -0
  16. package/dist/cjs/sync/shadow.js +178 -0
  17. package/dist/cjs/sync/types.js +2 -0
  18. package/dist/cloudflare.d.ts +1 -1
  19. package/dist/cloudflare.js.map +1 -1
  20. package/dist/collection.js +9 -6
  21. package/dist/collection.js.map +1 -1
  22. package/dist/cursors/findCursor.d.ts +2 -1
  23. package/dist/cursors/findCursor.js +6 -4
  24. package/dist/cursors/findCursor.js.map +1 -1
  25. package/dist/index.d.ts +68 -2
  26. package/dist/index.js +96 -1
  27. package/dist/index.js.map +1 -1
  28. package/dist/mongo-client.d.ts +13 -0
  29. package/dist/mongo-client.js +18 -1
  30. package/dist/mongo-client.js.map +1 -1
  31. package/dist/server.d.ts +97 -0
  32. package/dist/server.js +158 -0
  33. package/dist/server.js.map +1 -0
  34. package/dist/sync/commands.d.ts +49 -0
  35. package/dist/sync/commands.js +118 -0
  36. package/dist/sync/commands.js.map +1 -0
  37. package/dist/sync/diff.d.ts +55 -0
  38. package/dist/sync/diff.js +169 -0
  39. package/dist/sync/diff.js.map +1 -0
  40. package/dist/sync/httpSink.d.ts +93 -0
  41. package/dist/sync/httpSink.js +155 -0
  42. package/dist/sync/httpSink.js.map +1 -0
  43. package/dist/sync/index.d.ts +17 -0
  44. package/dist/sync/index.js +10 -0
  45. package/dist/sync/index.js.map +1 -0
  46. package/dist/sync/mongoExecutor.d.ts +37 -0
  47. package/dist/sync/mongoExecutor.js +302 -0
  48. package/dist/sync/mongoExecutor.js.map +1 -0
  49. package/dist/sync/mongoSink.d.ts +109 -0
  50. package/dist/sync/mongoSink.js +123 -0
  51. package/dist/sync/mongoSink.js.map +1 -0
  52. package/dist/sync/outbox.d.ts +120 -0
  53. package/dist/sync/outbox.js +355 -0
  54. package/dist/sync/outbox.js.map +1 -0
  55. package/dist/sync/protocol.d.ts +93 -0
  56. package/dist/sync/protocol.js +168 -0
  57. package/dist/sync/protocol.js.map +1 -0
  58. package/dist/sync/replicator.d.ts +177 -0
  59. package/dist/sync/replicator.js +744 -0
  60. package/dist/sync/replicator.js.map +1 -0
  61. package/dist/sync/shadow.d.ts +54 -0
  62. package/dist/sync/shadow.js +173 -0
  63. package/dist/sync/shadow.js.map +1 -0
  64. package/dist/sync/types.d.ts +388 -0
  65. package/dist/sync/types.js +2 -0
  66. package/dist/sync/types.js.map +1 -0
  67. package/package.json +16 -2
package/README.md CHANGED
@@ -12,6 +12,7 @@ A MongoDB-like client backed by SQLite. Use a familiar MongoDB API with the simp
12
12
  - You want a MongoDB-style API without running a MongoDB server
13
13
  - You need a lightweight, embedded database for local apps, CLIs, or testing
14
14
  - You want simple file-based persistence with zero infrastructure overhead
15
+ - You want a persistent, high-performance local cache in front of a remote MongoDB, one that survives restarts and network interruptions, without giving up Mongo's query semantics (including things that are natural in a document model but awkward in SQL, like layered/override queries)
15
16
 
16
17
  ## Features
17
18
 
@@ -24,6 +25,8 @@ A MongoDB-like client backed by SQLite. Use a familiar MongoDB API with the simp
24
25
  - **Update operators** — `$set`, `$inc`, `$push`, `$pull`, `$addToSet`, `$mul`, and more
25
26
  - **Indexing** — create, list, and drop indexes including unique and compound indexes
26
27
  - **Change streams** — real-time change tracking via `collection.watch()`
28
+ - **MongoDB sync** — replicate local writes upstream, surviving restarts and offline periods
29
+ - **HTTP sync** — post the same changes to a remote API when MongoDB is unreachable
27
30
  - **JSON safety** — validates documents before insert, and a corrupted row can't break queries for the rest of the collection
28
31
  - **TypeScript** — fully typed with strict mode
29
32
 
@@ -83,28 +86,29 @@ raw SQL are a menu item away when you want them. See [the CLI docs](./docs/CLI.m
83
86
 
84
87
  ## How does it compare?
85
88
 
86
- | | MongoLite | lowdb | better-sqlite3 (raw) | NeDB | PouchDB | MongoDB |
87
- |---|---|---|---|---|---|---|
88
- | MongoDB query API (`$set`, `$elemMatch`, aggregation...) | ✅ | ❌ (plain object access) | ❌ (raw SQL) | ✅ | ❌ (Mango/CouchDB-style) | ✅ |
89
- | Runs in the browser | ✅ (sql.js) | ✅ | ❌ (native binding) | ✅ | ✅ (IndexedDB) | ❌ |
90
- | Runs on the edge (Cloudflare Durable Objects) | ✅ | ❌ | ❌ (native binding) | ❌ | ❌ | ❌ |
91
- | Zero infrastructure (no server process) | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ |
92
- | TypeScript, strict mode | ✅ | ✅ | ✅ | ⚠️ (community types) | ⚠️ (community types) | ✅ |
93
- | Actively maintained | ✅ | ✅ | ✅ | ❌ (unmaintained) | ⚠️ (slow-moving) | ✅ |
89
+ | | MongoLite | lowdb | better-sqlite3 (raw) | NeDB | PouchDB | MongoDB |
90
+ | -------------------------------------------------------- | ----------- | ------------------------ | -------------------- | -------------------- | ------------------------ | ------- |
91
+ | MongoDB query API (`$set`, `$elemMatch`, aggregation...) | ✅ | ❌ (plain object access) | ❌ (raw SQL) | ✅ | ❌ (Mango/CouchDB-style) | ✅ |
92
+ | Runs in the browser | ✅ (sql.js) | ✅ | ❌ (native binding) | ✅ | ✅ (IndexedDB) | ❌ |
93
+ | Runs on the edge (Cloudflare Durable Objects) | ✅ | ❌ | ❌ (native binding) | ❌ | ❌ | ❌ |
94
+ | Zero infrastructure (no server process) | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ |
95
+ | TypeScript, strict mode | ✅ | ✅ | ✅ | ⚠️ (community types) | ⚠️ (community types) | ✅ |
96
+ | Actively maintained | ✅ | ✅ | ✅ | ❌ (unmaintained) | ⚠️ (slow-moving) | ✅ |
94
97
 
95
98
  MongoLite's niche: the MongoDB query API you already know, running anywhere SQLite runs — a local file, an in-memory test database, the browser, or a Cloudflare Durable Object — without standing up a MongoDB server.
96
99
 
97
100
  ## Documentation
98
101
 
99
- | Topic | Description |
100
- |-------|-------------|
101
- | [API Reference](./docs/API.md) | Full API docs: methods, query operators, update operators |
102
- | [Change Streams](./docs/CHANGE_STREAMS.md) | Real-time change tracking with `collection.watch()` |
103
- | [JSON Safety](./docs/JSON_SAFETY.md) | Document validation and corrupted data recovery |
104
- | [CLI](./docs/CLI.md) | `npx mongolite` — guided explorer for browsing and querying a database |
105
- | [Query Debugger](./docs/DEBUGGER.md) | The older command-driven REPL (`npx mongolite --repl`) |
106
- | [Benchmarks](./docs/BENCHMARKS.md) | Performance benchmarks and storage characteristics |
107
- | [Cloudflare Durable Objects](./docs/CLOUDFLARE.md) | Using MongoLite inside a Cloudflare Durable Object |
102
+ | Topic | Description |
103
+ | -------------------------------------------------- | ---------------------------------------------------------------------- |
104
+ | [API Reference](./docs/API.md) | Full API docs: methods, query operators, update operators |
105
+ | [Change Streams](./docs/CHANGE_STREAMS.md) | Real-time change tracking with `collection.watch()` |
106
+ | [Syncing to MongoDB](./docs/SYNC.md) | Replication to an upstream MongoDB, directly or through an HTTP API |
107
+ | [JSON Safety](./docs/JSON_SAFETY.md) | Document validation and corrupted data recovery |
108
+ | [CLI](./docs/CLI.md) | `npx mongolite` — guided explorer for browsing and querying a database |
109
+ | [Query Debugger](./docs/DEBUGGER.md) | The older command-driven REPL (`npx mongolite --repl`) |
110
+ | [Benchmarks](./docs/BENCHMARKS.md) | Performance benchmarks and storage characteristics |
111
+ | [Cloudflare Durable Objects](./docs/CLOUDFLARE.md) | Using MongoLite inside a Cloudflare Durable Object |
108
112
 
109
113
  ## Backend Examples
110
114
 
@@ -125,9 +129,9 @@ await client.close();
125
129
  ```typescript
126
130
  const client = new MongoLite({
127
131
  filePath: './myapp.sqlite',
128
- WAL: true, // Write-Ahead Logging. Default: true
129
- busyTimeout: 5000, // ms to wait for a lock before SQLITE_BUSY. Default: 5000
130
- shared: true, // reuse one connection per file in this process. Default: false
132
+ WAL: true, // Write-Ahead Logging. Default: true
133
+ busyTimeout: 5000, // ms to wait for a lock before SQLITE_BUSY. Default: 5000
134
+ shared: true, // reuse one connection per file in this process. Default: false
131
135
  readOnly: false,
132
136
  verbose: false,
133
137
  });
@@ -149,7 +153,7 @@ await a.close(); // b keeps working — the handle is reference counted
149
153
  await b.close(); // last holder out actually closes it
150
154
  ```
151
155
 
152
- Connections are only shared when the resolved path *and* every setting that
156
+ Connections are only shared when the resolved path _and_ every setting that
153
157
  changes the handle's behaviour match. `:memory:` databases are never shared,
154
158
  since two `:memory:` opens are two unrelated databases.
155
159
 
@@ -193,6 +197,7 @@ await client.close();
193
197
  ```
194
198
 
195
199
  <a id="native-nodesqlite-vs-better-sqlite3"></a>
200
+
196
201
  > **Which one should I use?** `node:sqlite` (the default) is still marked
197
202
  > experimental upstream but requires no native build step, which is why it's
198
203
  > the default here. `better-sqlite3` is the older, more battle-tested native
@@ -248,6 +253,60 @@ export class MyDurableObject extends DurableObject {
248
253
 
249
254
  > See [docs/CLOUDFLARE.md](./docs/CLOUDFLARE.md) for the full guide, supported operations, and limitations.
250
255
 
256
+ ## Syncing to MongoDB
257
+
258
+ Point MongoLite at an upstream MongoDB deployment and every insert, update and
259
+ delete is replicated to it. The local database becomes a local-first write layer
260
+ that converges upstream — through restarts, network outages and offline periods.
261
+
262
+ Requires the optional `mongodb` peer dependency (`npm install mongodb`).
263
+
264
+ ```typescript
265
+ const client = new MongoLite('./app.sqlite');
266
+ await client.connect();
267
+
268
+ const sync = client.syncToMongo({
269
+ connectionString: 'mongodb+srv://user:pass@cluster.example.com/app',
270
+ collections: ['users', 'orders'], // Omit to replicate everything
271
+ });
272
+
273
+ await sync.start();
274
+
275
+ // Ordinary writes — replication happens in the background.
276
+ await client.collection('users').insertOne({ name: 'Alice', age: 30 });
277
+
278
+ await sync.stop({ flush: true }); // Push anything still queued on shutdown
279
+ ```
280
+
281
+ Changes are captured by SQLite triggers into a durable outbox **inside the same
282
+ transaction as the write itself**, so a change is either committed locally _and_
283
+ queued for replication, or neither. A background worker coalesces the queue, applies
284
+ it upstream, and only then advances a persisted checkpoint — so an outage parks the
285
+ backlog rather than losing it, and a restart resumes exactly where it stopped.
286
+
287
+ Upstream is treated as the source of truth. Each push is a conditional write against
288
+ the version it last saw, and carries only the fields that actually changed, so a
289
+ concurrent edit by another writer is detected rather than silently overwritten.
290
+
291
+ Where the local instance cannot reach MongoDB directly, `syncToHttp` posts the same
292
+ operations to a remote API instead, and `@semics-tech/mongolite/server` applies them on
293
+ the far side:
294
+
295
+ ```typescript
296
+ const sync = client.syncToHttp({
297
+ baseUrl: 'https://api.example.com',
298
+ database: 'app',
299
+ getAuthHeaders: async () => ({ Authorization: `Bearer ${await getAccessToken()}` }),
300
+ });
301
+ ```
302
+
303
+ The version predicates travel with each operation and conflicts come back in the
304
+ response, so the HTTP hop is pure transport — replication behaves the same either way.
305
+
306
+ > See [docs/SYNC.md](./docs/SYNC.md) for the full guide: syncing through an HTTP API,
307
+ > advanced authentication (X.509, mTLS, AWS IAM, Azure managed identity), filtering and
308
+ > reshaping documents, dead-letter handling, multiple upstreams, and custom sinks.
309
+
251
310
  ## Development
252
311
 
253
312
  ```bash
@@ -1099,11 +1099,7 @@ class MongoLiteCollection {
1099
1099
  */
1100
1100
  async countDocuments(filter = {}) {
1101
1101
  await this.ensureTable();
1102
- const paramsForCount = [];
1103
- const whereClause = new findCursor_js_1.FindCursor(this.db, this.name, filter)['buildGuardedWhereClause'](filter, paramsForCount);
1104
- const countSql = `SELECT COUNT(*) as count FROM "${this.name}" WHERE ${whereClause}`;
1105
- const result = await this.db.get(countSql, paramsForCount);
1106
- return result?.count || 0;
1102
+ return this.find(filter).count();
1107
1103
  }
1108
1104
  /**
1109
1105
  * Finds rows whose stored `data` column is not valid JSON.
@@ -1285,7 +1281,14 @@ class MongoLiteCollection {
1285
1281
  if (!upsert) {
1286
1282
  return { acknowledged: true, matchedCount: 0, modifiedCount: 0, upsertedId: null };
1287
1283
  }
1288
- const result = await this.insertOne(replacement);
1284
+ // An upsert adopts the `_id` the filter pinned down, matching MongoDB. Without
1285
+ // this the inserted document gets a freshly generated id and the caller's
1286
+ // `replaceOne({ _id }, ..., { upsert: true })` silently creates the wrong row.
1287
+ const filterId = filter._id;
1288
+ const seeded = typeof filterId === 'string'
1289
+ ? { ...replacement, _id: filterId }
1290
+ : replacement;
1291
+ const result = await this.insertOne(seeded);
1289
1292
  return {
1290
1293
  acknowledged: true,
1291
1294
  matchedCount: 0,
@@ -123,15 +123,16 @@ function restoreDates(obj) {
123
123
  * Represents a cursor for find operations, allowing chaining of limit, skip, and sort.
124
124
  */
125
125
  class FindCursor {
126
- constructor(db, collectionName, initialFilter, options = {}) {
126
+ constructor(db, collectionName, filter, options = {}) {
127
127
  this.db = db;
128
128
  this.collectionName = collectionName;
129
+ this.filter = filter;
129
130
  this.options = options;
130
131
  this.limitCount = null;
131
132
  this.skipCount = null;
132
133
  this.sortCriteria = null;
133
134
  this.projectionFields = null;
134
- this.queryParts = this.buildSelectQuery(initialFilter);
135
+ this.queryParts = this.buildSelectQuery(filter);
135
136
  }
136
137
  parseJsonPath(path) {
137
138
  return `'$.${path.replace(/\./g, '.')}'`;
@@ -840,9 +841,10 @@ class FindCursor {
840
841
  * @returns A promise that resolves to the count of matching documents.
841
842
  */
842
843
  async count() {
843
- const whereClause = this.buildWhereClause({}, []);
844
+ const params = [];
845
+ const whereClause = this.buildGuardedWhereClause(this.filter, params);
844
846
  const countSql = `SELECT COUNT(*) as count FROM "${this.collectionName}" WHERE ${whereClause}`;
845
- const result = await this.db.get(countSql);
847
+ const result = await this.db.get(countSql, params);
846
848
  return result?.count ?? 0; // Return 0 if result is undefined
847
849
  }
848
850
  }
package/dist/cjs/index.js CHANGED
@@ -14,7 +14,7 @@ var __exportStar = (this && this.__exportStar) || function(m, exports) {
14
14
  for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
15
15
  };
16
16
  Object.defineProperty(exports, "__esModule", { value: true });
17
- exports.MongoLite = exports.NodeSqliteAdapter = exports.BrowserSqliteAdapter = exports.CloudflareDurableObjectAdapter = exports.ChangeStream = exports.MongoLiteCollection = void 0;
17
+ exports.MongoLite = exports.projectToLocalShape = exports.isEmptyDiff = exports.applyDiff = exports.computeDiff = exports.buildWriteCommand = exports.SYNC_PROTOCOL_VERSION = exports.SyncRequestRejected = exports.HttpUpstreamSink = exports.MongoUpstreamSink = exports.SyncShadow = exports.SyncOutbox = exports.SyncReplicator = exports.NodeSqliteAdapter = exports.BrowserSqliteAdapter = exports.CloudflareDurableObjectAdapter = exports.ChangeStream = exports.MongoLiteCollection = void 0;
18
18
  const node_sqlite_js_1 = require("./adapters/node-sqlite.js");
19
19
  const mongo_client_js_1 = require("./mongo-client.js");
20
20
  var collection_js_1 = require("./collection.js");
@@ -28,6 +28,20 @@ var browser_js_1 = require("./adapters/browser.js");
28
28
  Object.defineProperty(exports, "BrowserSqliteAdapter", { enumerable: true, get: function () { return browser_js_1.BrowserSqliteAdapter; } });
29
29
  var node_sqlite_js_2 = require("./adapters/node-sqlite.js");
30
30
  Object.defineProperty(exports, "NodeSqliteAdapter", { enumerable: true, get: function () { return node_sqlite_js_2.NodeSqliteAdapter; } });
31
+ const index_js_1 = require("./sync/index.js");
32
+ var index_js_2 = require("./sync/index.js");
33
+ Object.defineProperty(exports, "SyncReplicator", { enumerable: true, get: function () { return index_js_2.SyncReplicator; } });
34
+ Object.defineProperty(exports, "SyncOutbox", { enumerable: true, get: function () { return index_js_2.SyncOutbox; } });
35
+ Object.defineProperty(exports, "SyncShadow", { enumerable: true, get: function () { return index_js_2.SyncShadow; } });
36
+ Object.defineProperty(exports, "MongoUpstreamSink", { enumerable: true, get: function () { return index_js_2.MongoUpstreamSink; } });
37
+ Object.defineProperty(exports, "HttpUpstreamSink", { enumerable: true, get: function () { return index_js_2.HttpUpstreamSink; } });
38
+ Object.defineProperty(exports, "SyncRequestRejected", { enumerable: true, get: function () { return index_js_2.SyncRequestRejected; } });
39
+ Object.defineProperty(exports, "SYNC_PROTOCOL_VERSION", { enumerable: true, get: function () { return index_js_2.SYNC_PROTOCOL_VERSION; } });
40
+ Object.defineProperty(exports, "buildWriteCommand", { enumerable: true, get: function () { return index_js_2.buildWriteCommand; } });
41
+ Object.defineProperty(exports, "computeDiff", { enumerable: true, get: function () { return index_js_2.computeDiff; } });
42
+ Object.defineProperty(exports, "applyDiff", { enumerable: true, get: function () { return index_js_2.applyDiff; } });
43
+ Object.defineProperty(exports, "isEmptyDiff", { enumerable: true, get: function () { return index_js_2.isEmptyDiff; } });
44
+ Object.defineProperty(exports, "projectToLocalShape", { enumerable: true, get: function () { return index_js_2.projectToLocalShape; } });
31
45
  /**
32
46
  * MongoLite class is the main entry point for interacting with the SQLite-backed database.
33
47
  *
@@ -61,5 +75,98 @@ class MongoLite extends mongo_client_js_1.MongoLite {
61
75
  super(new node_sqlite_js_1.NodeSqliteAdapter(dbPathOrOptions), options);
62
76
  }
63
77
  }
78
+ /**
79
+ * Replicates every insert, update and delete in this database to an upstream
80
+ * MongoDB deployment.
81
+ *
82
+ * Changes are captured by SQLite triggers into a durable outbox inside this
83
+ * database, in the same transaction as the write itself, then streamed
84
+ * upstream in batches. Replication survives process restarts and upstream
85
+ * outages: the outbox holds the backlog and a checkpoint records exactly how
86
+ * far it got.
87
+ *
88
+ * Requires the optional `mongodb` peer dependency.
89
+ *
90
+ * @example
91
+ * ```typescript
92
+ * const sync = client.syncToMongo({
93
+ * connectionString: 'mongodb+srv://user:pass@cluster.example.com/app',
94
+ * collections: ['users', 'orders'],
95
+ * });
96
+ *
97
+ * sync.on('error', (err) => console.error('replication degraded:', err));
98
+ * await sync.start();
99
+ *
100
+ * // …later, on shutdown
101
+ * await sync.stop({ flush: true });
102
+ * ```
103
+ */
104
+ syncToMongo(options) {
105
+ const { connectionString, database, writeConcern, driverOptions, driver, authSource, authMechanism, authMechanismProperties, username, password, tls, tlsCAFile, tlsCertificateKeyFile, tlsCertificateKeyFilePassword, tlsCRLFile, tlsAllowInvalidHostnames, tlsAllowInvalidCertificates, ...syncOptions } = options;
106
+ const sink = new index_js_1.MongoUpstreamSink({
107
+ connectionString,
108
+ database,
109
+ writeConcern,
110
+ driverOptions,
111
+ driver,
112
+ authSource,
113
+ authMechanism,
114
+ authMechanismProperties,
115
+ username,
116
+ password,
117
+ tls,
118
+ tlsCAFile,
119
+ tlsCertificateKeyFile,
120
+ tlsCertificateKeyFilePassword,
121
+ tlsCRLFile,
122
+ tlsAllowInvalidHostnames,
123
+ tlsAllowInvalidCertificates,
124
+ idMapping: syncOptions.idMapping,
125
+ // The sink builds the conditional writes, so it needs the same answer the
126
+ // replicator uses when deciding whether to track shadows at all.
127
+ versioning: syncOptions.versioning,
128
+ });
129
+ return this.createSync(sink, syncOptions);
130
+ }
131
+ /**
132
+ * Replicates changes to a remote HTTP API, which applies them to MongoDB on this
133
+ * client's behalf. For deployments where the local instance cannot reach the database
134
+ * directly.
135
+ *
136
+ * The wire format carries MongoDB-shaped commands along with the version predicates,
137
+ * so replication behaves the same as over a direct connection — the upstream stays the
138
+ * source of truth and concurrent writers are still detected. Mount
139
+ * `@semics-tech/mongolite/server` on the API to receive the messages.
140
+ *
141
+ * @example
142
+ * ```typescript
143
+ * const sync = client.syncToHttp({
144
+ * baseUrl: 'https://api.example.com',
145
+ * database: 'app',
146
+ * headers: { 'luna-environment': 'prod' },
147
+ * getAuthHeaders: async () => ({ Authorization: `Bearer ${await getAccessToken()}` }),
148
+ * });
149
+ *
150
+ * await sync.start();
151
+ * ```
152
+ */
153
+ syncToHttp(options) {
154
+ const { baseUrl, database, pathPrefix, getAuthHeaders, headers, cookies, fetch: fetchImpl, timeoutMs, includeCommands, ...syncOptions } = options;
155
+ const sink = new index_js_1.HttpUpstreamSink({
156
+ baseUrl,
157
+ database,
158
+ pathPrefix,
159
+ getAuthHeaders,
160
+ headers,
161
+ cookies,
162
+ fetch: fetchImpl,
163
+ timeoutMs,
164
+ includeCommands,
165
+ idMapping: syncOptions.idMapping,
166
+ versioning: syncOptions.versioning,
167
+ replicator: syncOptions.name,
168
+ });
169
+ return this.createSync(sink, syncOptions);
170
+ }
64
171
  }
65
172
  exports.MongoLite = MongoLite;
@@ -2,6 +2,7 @@
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  exports.MongoLite = void 0;
4
4
  const collection_js_1 = require("./collection.js");
5
+ const replicator_js_1 = require("./sync/replicator.js");
5
6
  /**
6
7
  * Core MongoLite client. Accepts any `IDatabaseAdapter` implementation.
7
8
  *
@@ -42,11 +43,27 @@ class MongoLite {
42
43
  listCollections() {
43
44
  return {
44
45
  toArray: async () => {
45
- const result = await this.db.all(`SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'`);
46
+ // `__mongolite*` tables are internal bookkeeping (change log, sync
47
+ // outbox and checkpoints), not user collections.
48
+ const result = await this.db.all(`SELECT name FROM sqlite_master
49
+ WHERE type='table' AND name NOT LIKE 'sqlite_%' AND name NOT LIKE '__mongolite%'`);
46
50
  return result.map((row) => row.name);
47
51
  },
48
52
  };
49
53
  }
54
+ /**
55
+ * Starts replicating this database's changes to an upstream {@link SyncSink}.
56
+ *
57
+ * Node.js users replicating to MongoDB should prefer the `syncToMongo`
58
+ * overload on the package-root `MongoLite`, which takes a connection string
59
+ * directly. This method is the backend-agnostic form — pass any sink to
60
+ * replicate over HTTP, into a queue, or to a test double.
61
+ *
62
+ * The returned replicator is **not** started; call `start()` on it.
63
+ */
64
+ createSync(sink, options = {}) {
65
+ return new replicator_js_1.SyncReplicator(this.db, sink, { verbose: this.options.verbose, ...options });
66
+ }
50
67
  /**
51
68
  * Returns a typed collection handle.
52
69
  * @param name The collection (table) name.
@@ -0,0 +1,164 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.SyncReceiver = exports.SyncProtocolError = exports.SYNC_PROTOCOL_VERSION = void 0;
4
+ exports.createSyncReceiver = createSyncReceiver;
5
+ /**
6
+ * Server-side entry point: receives sync messages from a MongoLite client and applies
7
+ * them to MongoDB.
8
+ *
9
+ * Install this on the API that sits in front of your database, for deployments where
10
+ * the local instance cannot reach MongoDB directly.
11
+ *
12
+ * ```ts
13
+ * import { MongoClient } from 'mongodb';
14
+ * import { createSyncReceiver } from '@semics-tech/mongolite/server';
15
+ *
16
+ * const mongo = await new MongoClient(process.env.MONGO_URL!).connect();
17
+ * const receiver = createSyncReceiver({
18
+ * client: mongo,
19
+ * database: 'app',
20
+ * allowedCollections: ['users', 'orders'],
21
+ * });
22
+ *
23
+ * app.post('/sync/app/_sync', async (req, res) => {
24
+ * const result = await receiver.apply(req.body);
25
+ * res.status(result.status).type('application/json').send(result.body);
26
+ * });
27
+ * ```
28
+ *
29
+ * This module imports nothing SQLite-related, so it is safe to install on a server that
30
+ * has no local database of its own.
31
+ */
32
+ const commands_js_1 = require("./sync/commands.js");
33
+ const mongoExecutor_js_1 = require("./sync/mongoExecutor.js");
34
+ const protocol_js_1 = require("./sync/protocol.js");
35
+ var protocol_js_2 = require("./sync/protocol.js");
36
+ Object.defineProperty(exports, "SYNC_PROTOCOL_VERSION", { enumerable: true, get: function () { return protocol_js_2.SYNC_PROTOCOL_VERSION; } });
37
+ Object.defineProperty(exports, "SyncProtocolError", { enumerable: true, get: function () { return protocol_js_2.SyncProtocolError; } });
38
+ const JSON_HEADERS = { 'content-type': 'application/json' };
39
+ class SyncReceiver {
40
+ constructor(options) {
41
+ this.options = options;
42
+ if (!options.client)
43
+ throw new Error('createSyncReceiver requires a connected `client`.');
44
+ this.db = options.client.db(options.database);
45
+ }
46
+ /**
47
+ * Applies a batch of operations.
48
+ *
49
+ * @param body The raw request body — a string, or an already-parsed object.
50
+ * @returns Status and Extended JSON body to send back.
51
+ */
52
+ async apply(body) {
53
+ try {
54
+ const decoded = (0, protocol_js_1.decodeBody)(body);
55
+ const { replicator, operations } = (0, protocol_js_1.parseApplyRequest)(decoded);
56
+ const maxOperations = this.options.maxOperations ?? 1000;
57
+ if (operations.length > maxOperations) {
58
+ throw new protocol_js_1.SyncProtocolError(`Batch of ${operations.length} exceeds the limit of ${maxOperations}`, 413);
59
+ }
60
+ const collections = [...new Set(operations.map((op) => op.collection))];
61
+ this.assertCollectionsAllowed(collections);
62
+ await this.authorise({ replicator, collections, operationCount: operations.length });
63
+ const writes = operations.map((op, index) => ({
64
+ index,
65
+ op,
66
+ command: this.commandFor(op, decoded, index),
67
+ }));
68
+ const result = await (0, mongoExecutor_js_1.executeWrites)(this.db, writes, {
69
+ writeConcern: this.options.writeConcern ?? { w: 'majority' },
70
+ versioning: this.options.versioning ?? true,
71
+ idMapping: this.options.idMapping,
72
+ ObjectId: this.options.ObjectId,
73
+ });
74
+ return this.ok({
75
+ protocol: protocol_js_1.SYNC_PROTOCOL_VERSION,
76
+ applied: result.applied,
77
+ failures: result.failures,
78
+ conflicts: result.conflicts,
79
+ });
80
+ }
81
+ catch (err) {
82
+ return this.error(err);
83
+ }
84
+ }
85
+ /** Reads documents back by `_id`, so a client can refresh its shadow after a conflict. */
86
+ async fetch(body) {
87
+ try {
88
+ const { collection, documentIds } = (0, protocol_js_1.parseFetchRequest)((0, protocol_js_1.decodeBody)(body));
89
+ this.assertCollectionsAllowed([collection]);
90
+ await this.authorise({
91
+ replicator: 'fetch',
92
+ collections: [collection],
93
+ operationCount: documentIds.length,
94
+ });
95
+ const documents = await (0, mongoExecutor_js_1.fetchDocuments)(this.db, collection, documentIds, this.options.idMapping, this.options.ObjectId);
96
+ return this.ok({ protocol: protocol_js_1.SYNC_PROTOCOL_VERSION, documents });
97
+ }
98
+ catch (err) {
99
+ return this.error(err);
100
+ }
101
+ }
102
+ // ---------------------------------------------------------------- internals
103
+ /**
104
+ * Produces the command to execute for an operation.
105
+ *
106
+ * By default this **rebuilds** the command from the operation's own fields rather than
107
+ * using whatever the client sent. For a well-behaved client the two are identical;
108
+ * for a hostile one the difference is the whole security boundary.
109
+ */
110
+ commandFor(op, decoded, index) {
111
+ if (this.options.trustClientCommands) {
112
+ const wire = decoded.operations?.[index];
113
+ const command = wire?.command;
114
+ if (command && typeof command === 'object')
115
+ return command;
116
+ }
117
+ return (0, commands_js_1.buildWriteCommand)(op, {
118
+ idMapping: this.options.idMapping,
119
+ versioning: this.options.versioning ?? true,
120
+ ObjectId: this.options.ObjectId,
121
+ });
122
+ }
123
+ assertCollectionsAllowed(collections) {
124
+ const allowed = this.options.allowedCollections;
125
+ if (!allowed)
126
+ return;
127
+ const permitted = new Set(allowed);
128
+ for (const collection of collections) {
129
+ if (!permitted.has(collection)) {
130
+ throw new protocol_js_1.SyncProtocolError(`Collection "${collection}" is not permitted`, 403);
131
+ }
132
+ }
133
+ }
134
+ async authorise(context) {
135
+ if (!this.options.verifyRequest)
136
+ return;
137
+ const verdict = await this.options.verifyRequest(context);
138
+ if (verdict === false) {
139
+ throw new protocol_js_1.SyncProtocolError('Request rejected', 403);
140
+ }
141
+ }
142
+ ok(payload) {
143
+ return { status: 200, body: (0, protocol_js_1.encodeBody)(payload), headers: JSON_HEADERS };
144
+ }
145
+ error(err) {
146
+ const status = err instanceof protocol_js_1.SyncProtocolError ? err.status : 500;
147
+ const message = err instanceof Error ? err.message : String(err);
148
+ return {
149
+ status,
150
+ // 5xx bodies stay vague: an unexpected error can carry connection strings or
151
+ // document contents, and this response goes to a client.
152
+ body: (0, protocol_js_1.encodeBody)({
153
+ protocol: protocol_js_1.SYNC_PROTOCOL_VERSION,
154
+ error: status >= 500 ? 'Internal error applying sync batch' : message,
155
+ }),
156
+ headers: JSON_HEADERS,
157
+ };
158
+ }
159
+ }
160
+ exports.SyncReceiver = SyncReceiver;
161
+ /** Creates a receiver for sync messages posted by a MongoLite client. */
162
+ function createSyncReceiver(options) {
163
+ return new SyncReceiver(options);
164
+ }